1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
188}
189
190#[derive(Debug, Clone, Default)]
217pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
218
219impl Pause {
220 #[must_use]
222 pub fn new() -> Self {
223 Self::default()
224 }
225
226 pub fn park(&self) {
228 self.0.store(true, Ordering::SeqCst);
229 }
230
231 pub fn park_because(&self, reason: impl Into<String>) {
237 self.park_for(ParkCause::Interrupt {
238 reason: reason.into(),
239 });
240 }
241
242 pub fn park_for(&self, cause: ParkCause) {
245 let mut reason_guard = self
246 .1
247 .lock()
248 .unwrap_or_else(std::sync::PoisonError::into_inner);
249 if reason_guard.is_none() {
250 *reason_guard = Some(cause);
251 }
252 drop(reason_guard);
253 self.park();
254 }
255
256 #[must_use]
258 pub fn parked(&self) -> bool {
259 self.0.load(Ordering::SeqCst)
260 }
261
262 #[must_use]
264 pub fn reason(&self) -> Option<String> {
265 self.cause().map(|c| c.label())
266 }
267
268 #[must_use]
270 pub fn cause(&self) -> Option<ParkCause> {
271 self.1
272 .lock()
273 .unwrap_or_else(std::sync::PoisonError::into_inner)
274 .clone()
275 }
276}
277
278pub struct Runner {
280 pub state: RunState,
282 roles: ResolvedRoles,
283 sem: Arc<Semaphore>,
284 pause: Pause,
288 interrupt: Pause,
294}
295
296async fn review_base(
301 repo: &Path,
302 remote: &str,
303 base_branch: &str,
304 base_commit: &str,
305 branch: &str,
306) -> String {
307 review_base_checked(repo, remote, base_branch, base_commit, branch)
308 .await
309 .0
310}
311
312async fn review_base_checked(
317 repo: &Path,
318 remote: &str,
319 base_branch: &str,
320 base_commit: &str,
321 branch: &str,
322) -> (String, bool) {
323 let tracking = format!("{remote}/{base_branch}");
324 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
325 if git::rev_exists(repo, &tracking).await
326 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
327 && !mb.is_empty()
328 {
329 return (mb, fresh);
330 }
331 let mb = git::merge_base(repo, base_commit, branch)
332 .await
333 .ok()
334 .filter(|mb| !mb.is_empty())
335 .unwrap_or_else(|| base_commit.to_owned());
336 (mb, false)
337}
338
339pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
343 if !is_review_run(state) {
344 return;
345 }
346 let base = review_base(
347 &state.repo,
348 &state.config.merge.remote,
349 &state.base_branch,
350 &state.base_commit,
351 branch,
352 )
353 .await;
354 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
355 && !subjects.is_empty()
356 && state.reviewed_commits.as_ref() != Some(&subjects)
357 {
358 state.reviewed_commits = Some(subjects);
359 state.save().ok();
360 }
361}
362
363async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
367 let (base, trusted) = review_base_checked(
368 &state.repo,
369 &state.config.merge.remote,
370 &state.base_branch,
371 &state.base_commit,
372 branch,
373 )
374 .await;
375 if !trusted {
376 return None;
377 }
378 git::subjects(&state.repo, &state.base_commit, &base)
379 .await
380 .ok()
381}
382
383pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
387 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
388 if is_leak(computed) {
389 return false;
390 }
391 let cur = current.trim();
392 cur.is_empty()
393 || cur.starts_with(REVIEW_PROMPT_OPENING)
394 || cur.starts_with("chore: land candidate")
395 || cur.starts_with("magi: candidate")
396 || is_leak(cur)
397}
398
399async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
428 let tracking = format!("{remote}/{branch}");
429 let fetched = git::fetch(repo, remote, branch).await;
430 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
431 let local_exists = git::branch_exists(repo, branch).await?;
432 if !fresh {
433 if !local_exists {
434 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
435 }
436 tracing::warn!(
437 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
438 );
439 return Ok(());
440 }
441 let remote_sha = git::rev_parse(repo, &tracking).await?;
442 if !local_exists {
443 git::git(repo, &["branch", branch, &tracking]).await?;
444 return Ok(());
445 }
446 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
447 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
448 return Ok(());
449 }
450 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
451 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
457 .await?
458 {
459 crate::reconcile::Reconciliation::Pushed => {
460 tracing::warn!(
461 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
462 short(&local_sha),
463 short(&remote_sha)
464 );
465 return Ok(());
466 }
467 crate::reconcile::Reconciliation::Placeholder => {}
468 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
469 }
470 }
471 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
472 if !out.ok() {
473 bail!(
474 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
475 short(&local_sha),
476 short(&remote_sha),
477 out.stderr
478 );
479 }
480 tracing::warn!(
481 "local `{branch}` was stale: fast-forwarded {} -> {}",
482 short(&local_sha),
483 short(&remote_sha)
484 );
485 Ok(())
486}
487
488async fn resolve_base(
489 repo: &Path,
490 base_branch: &str,
491 remote: &str,
492 coordination: Option<&crate::config::Coordination>,
493) -> Result<crate::run::FetchedBase> {
494 let _lease = crate::repo_lock::SeatLease::operation_with(
495 repo,
496 coordination,
497 &crate::repo_lock::WaitPolicy::current(),
498 )
499 .await?;
500 let tracking = format!("{remote}/{base_branch}");
501 let fetched = git::fetch(repo, remote, base_branch).await;
502 if let Ok(out) = &fetched
503 && out.ok()
504 && git::rev_exists(repo, &tracking).await
505 {
506 return Ok(crate::run::FetchedBase {
507 remote: remote.to_owned(),
508 branch: base_branch.to_owned(),
509 commit: git::rev_parse(repo, &tracking).await?,
510 fetched_at: jiff::Timestamp::now(),
511 });
512 }
513 let why = match &fetched {
514 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
515 Ok(_) => format!("{remote} has no {base_branch}"),
516 Err(e) => e.to_string(),
517 };
518 bail!(
521 "cannot read {tracking} ({why}); refusing to branch off the local \
522 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
523 base / remote in magi.toml. This is launcher preparation: check the \
524 launcher's Git administration access, network and repository coordination \
525 before retrying; an unattended seat cannot approve those permissions"
526 )
527}
528
529struct FixClaim {
546 path: PathBuf,
547}
548
549impl FixClaim {
550 fn acquire(dir: &Path) -> Result<Self> {
551 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
552 let path = dir.join("fix.lock");
553 let retry = crate::lock_file::Retry::new(None);
554 match Self::create(&path, retry) {
555 Ok(claim) => Ok(claim),
556 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
557 if Self::reclaim_if_dead(&path) {
558 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
559 } else {
560 bail!(
561 "another repair or `magi fix` is already running for this run ({} exists)",
562 path.display()
563 )
564 }
565 }
566 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
567 }
568 }
569
570 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
571 let mut f = retry.create(path)?;
572 use std::io::Write as _;
573 writeln!(f, "{}", std::process::id())?;
575 Ok(Self {
576 path: path.to_owned(),
577 })
578 }
579
580 fn reclaim_if_dead(path: &Path) -> bool {
584 let dead = std::fs::read_to_string(path)
585 .ok()
586 .and_then(|body| body.trim().parse::<u32>().ok())
587 .is_some_and(|pid| !crate::proc::pid_alive(pid));
588 dead && std::fs::remove_file(path).is_ok()
589 }
590}
591
592impl Drop for FixClaim {
593 fn drop(&mut self) {
594 let _ = std::fs::remove_file(&self.path);
595 }
596}
597
598struct DriverExecution<'a> {
601 runner: &'a mut Runner,
602 driver: crate::driver::DriverGuard,
603}
604
605impl Drop for DriverExecution<'_> {
606 fn drop(&mut self) {
607 self.driver.release_local(&mut self.runner.state);
608 }
609}
610
611impl Runner {
612 pub async fn start(
614 repo: &Path,
615 instruction: String,
616 config: Config,
617 origin: Origin,
618 ) -> Result<Self> {
619 Self::start_naming(repo, instruction, "", config, origin).await
620 }
621
622 pub async fn start_naming(
626 repo: &Path,
627 instruction: String,
628 also_scan: &str,
629 config: Config,
630 origin: Origin,
631 ) -> Result<Self> {
632 let repo = git::toplevel(repo).await?;
633 let missing = agent::missing_programs(&config.agents);
634 if !missing.is_empty() {
635 bail!(
636 "these agent programs are not on PATH: {}. Fix the roster in \
637 magi.toml or install them.",
638 missing.join(", ")
639 );
640 }
641 let base_branch =
642 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643 .await?;
644 let fetched_base = resolve_base(
645 &repo,
646 &base_branch,
647 &config.merge.remote,
648 config.coordination.as_ref(),
649 )
650 .await?;
651 let base_commit = fetched_base.commit.clone();
652 let roles = config.resolve_roles()?;
653 let max_parallel = config.graph.max_parallel.max(1);
654 let seeds = refs::resolve(
657 &repo,
658 &base_commit,
659 &config.merge.remote,
660 &format!("{also_scan}\n{instruction}"),
661 )
662 .await;
663 refs::plan(&repo, &seeds).await?;
664 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
665 state.fetched_base = Some(fetched_base);
669 state.origin = Some(origin);
670 for seed in &seeds {
671 state.event(
672 "seed",
673 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
674 );
675 }
676 state.seeds = seeds;
677 state.event("start", format!("run {} created", state.id));
678 state.save()?;
679 Ok(Self {
680 state,
681 roles,
682 sem: Arc::new(Semaphore::new(max_parallel)),
683 pause: Pause::new(),
684 interrupt: Pause::new(),
685 })
686 }
687
688 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
702 Self::review_taking_over(repo, branch, config, None, origin).await
703 }
704
705 pub async fn review_taking_over(
712 repo: &Path,
713 branch: &str,
714 config: Config,
715 takeover: Option<crate::handover::Takeover>,
716 origin: Origin,
717 ) -> Result<Self> {
718 let repo = git::toplevel(repo).await?;
719 let missing = agent::missing_programs(&config.agents);
720 if !missing.is_empty() {
721 bail!(
722 "these agent programs are not on PATH: {}. Fix the roster in \
723 magi.toml or install them.",
724 missing.join(", ")
725 );
726 }
727 let base_branch =
728 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
729 .await?;
730 if base_branch == branch {
731 bail!("`{branch}` is the base branch; there is nothing to review against");
732 }
733 let fetched_base = resolve_base(
734 &repo,
735 &base_branch,
736 &config.merge.remote,
737 config.coordination.as_ref(),
738 )
739 .await?;
740 let base_commit = fetched_base.commit.clone();
741
742 let roles = config.resolve_roles()?;
743 let max_parallel = config.graph.max_parallel.max(1);
744 let mut state = RunState::new(
745 repo.clone(),
746 base_branch,
747 base_commit.clone(),
748 String::new(),
749 config,
750 );
751 state.fetched_base = Some(fetched_base);
752 state.origin = Some(origin);
753
754 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
761 earlier: Vec::new(),
762 home: crate::run::home(),
763 choice: None,
764 task_instruction: None,
765 });
766 let wait = repository_wait(&state, None);
767 let coordination_config = state.config.coordination.clone();
768 let coordination =
769 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
770 .await?;
771 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
772 if let Some(released) = &released {
773 state.event(
774 "release",
775 format!(
776 "took `{branch}` over from run {}: its worktree was released: {}",
777 crate::run::short_of(&released.old_id),
778 released.audit
779 ),
780 );
781 }
782 if let Some(choice) = takeover.choice.as_ref()
786 && let Err(e) =
787 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
788 .await
789 {
790 if let Some(released) = &released {
791 released.restore(&repo, branch).await;
792 }
793 return Err(e.context("applying the owner's answer about the diverged branch"));
794 }
795 for id in takeover.earlier.iter().rev() {
801 if let Ok(old) = RunState::load_under(id, &takeover.home)
802 && let Some(head) = old.pr_head.clone()
803 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
804 {
805 state.pr_head = Some(head);
806 break;
807 }
808 }
809 drop(coordination);
810 let opened = Self::open_review(
811 &repo,
812 branch,
813 state,
814 roles,
815 max_parallel,
816 base_commit,
817 &takeover,
818 )
819 .await;
820 if opened.is_err()
821 && let Some(released) = &released
822 {
823 let _coordination = crate::repo_lock::SeatLease::operation_with(
824 &repo,
825 coordination_config.as_ref(),
826 &wait,
827 )
828 .await?;
829 released.restore(&repo, branch).await;
830 }
831 opened
832 }
833
834 pub async fn recover_pr(
836 repo: &Path,
837 task: &str,
838 selection: crate::pr_recovery::Selection,
839 mut config: Config,
840 ) -> Result<Self> {
841 selection.configure(&mut config)?;
842 selection.check_task(task, &selection.instruction)?;
843 let base_branch =
844 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
845 .await?;
846 let fetched_base = resolve_base(
847 repo,
848 &base_branch,
849 &config.merge.remote,
850 config.coordination.as_ref(),
851 )
852 .await?;
853 let base_commit = fetched_base.commit.clone();
854 selection
855 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
856 .await?;
857 Box::pin(selection.verify_source(repo)).await?;
858 if let Some(proof) = &selection.source_proof {
859 anyhow::ensure!(
860 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
861 "source rebase base is outside the fetched landing base"
862 );
863 }
864 let roles = config.resolve_roles()?;
865 let max_parallel = config.graph.max_parallel.max(1);
866 let mut state = RunState::new(
867 repo.to_path_buf(),
868 base_branch,
869 base_commit.clone(),
870 selection.instruction.clone(),
871 config,
872 );
873 state.fetched_base = Some(fetched_base);
874 state.origin = Some(crate::run::Origin::queue(task));
875 state.pr_head = Some(selection.branch.clone());
876 state.pr_recovery = Some(selection.clone());
877 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
878 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
879 }
880
881 async fn open_selected(
882 repo: &Path,
883 state: RunState,
884 roles: ResolvedRoles,
885 max_parallel: usize,
886 base_commit: String,
887 selection: crate::pr_recovery::Selection,
888 ) -> Result<Self> {
889 let mut state = state;
890 let branch = state.branch_for('A');
891 let wait = repository_wait(&state, None);
892 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
893 git::git(repo, &["branch", &branch, selection.source()]).await?;
894 drop(coordination);
895 state.event(
896 "recovery",
897 format!(
898 "selected implementation {} for {} (public lease {})",
899 selection.source(),
900 selection.url,
901 selection.head
902 ),
903 );
904 let takeover = crate::handover::Takeover {
905 earlier: Vec::new(),
906 home: crate::run::home(),
907 choice: None,
908 task_instruction: Some(selection.instruction.clone()),
909 };
910 state.save()?;
911 let id = state.id.clone();
912 let opened = Self::open_review(
913 repo,
914 &branch,
915 state,
916 roles,
917 max_parallel,
918 base_commit,
919 &takeover,
920 )
921 .await;
922 match opened {
923 Ok(runner) => Ok(runner),
924 Err(e) => {
925 if let Ok(mut partial) = RunState::load(&id) {
926 partial.status = RunStatus::Blocked;
927 partial.driver_exited = true;
928 partial.event(
929 "recovery",
930 format!("startup failed; retained source and candidate {branch}: {e:#}"),
931 );
932 partial.save()?;
933 }
934 Err(crate::pr_recovery::StartupFailed {
935 run: id,
936 cause: format!("{e:#}"),
937 }
938 .into())
939 }
940 }
941 }
942
943 async fn inherit(
948 repo: &Path,
949 branch: &str,
950 base_commit: &str,
951 takeover: &crate::handover::Takeover,
952 ) -> Option<crate::recovery::RecoveryRecord> {
953 use crate::recovery::{Relation, assess};
954 let newest = takeover.earlier.last()?;
955 let mut chosen = None;
956 for id in takeover.earlier.iter().rev() {
957 if let Ok(old) = RunState::load_under(id, &takeover.home)
958 && !old.reviews.is_empty()
959 && (old.winner().is_some_and(|w| w.branch == branch)
960 || old.pr_head.as_deref() == Some(branch))
961 {
962 chosen = Some(old);
963 break;
964 }
965 }
966 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
967 .await
968 .unwrap_or_default();
969 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
970 Some(last) if last.head == tip => Relation::Same,
971 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
972 _ => Relation::Diverged,
973 };
974 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
975 Some(assess(from, chosen.as_ref(), base_commit, relation))
976 }
977
978 async fn open_review(
981 repo: &Path,
982 branch: &str,
983 mut state: RunState,
984 roles: ResolvedRoles,
985 max_parallel: usize,
986 base_commit: String,
987 takeover: &crate::handover::Takeover,
988 ) -> Result<Self> {
989 let wait = repository_wait(&state, None);
990 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
991 if state.pr_recovery.is_none() {
992 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
993 }
994 let start = review_base(
997 repo,
998 &state.config.merge.remote,
999 &state.base_branch,
1000 &base_commit,
1001 branch,
1002 )
1003 .await;
1004 let log = git::log_oneline(repo, &start, branch)
1005 .await
1006 .unwrap_or_default();
1007 let log_text = if log.trim().is_empty() {
1008 "(no commit messages)"
1009 } else {
1010 log.trim()
1011 };
1012 let mut instruction = format!(
1013 "Review the work already on branch `{branch}`. There is no task \
1014 statement: what the change claims to do is whatever its commits \
1015 say.\n\n{log_text}"
1016 );
1017 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1021 if let Some(task) = takeover
1022 .task_instruction
1023 .as_deref()
1024 .filter(|t| !t.trim().is_empty())
1025 && rec.reviewed_head.is_some()
1026 {
1027 instruction = format!(
1028 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1029 task.trim()
1030 );
1031 rec.criteria_inherited = true;
1032 }
1033 state.event("recovery", rec.summary());
1034 state.recovery = Some(rec);
1035 }
1036 if let Some(selected) = &state.pr_recovery {
1037 instruction = selected.instruction.clone();
1038 }
1039 state.task_instruction = takeover.task_instruction.clone();
1040 state.instruction = instruction;
1041 state.reviewed_commits = Some(
1042 git::subjects(repo, &start, branch)
1043 .await
1044 .unwrap_or_default(),
1045 );
1046
1047 let worktree = state.worktree_root().join("under-review");
1050 if let Some(parent) = worktree.parent() {
1051 tokio::fs::create_dir_all(parent).await.ok();
1052 }
1053 let path = worktree.to_string_lossy().to_string();
1054 git::git(repo, &["worktree", "add", &path, branch])
1055 .await
1056 .with_context(|| {
1057 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1058 })?;
1059
1060 let setup = match ensure_setup_config(&mut state, repo).await {
1061 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1062 Err(e) => Err(e),
1063 };
1064 if let Err(e) = setup {
1065 git::worktree_remove_clean(repo, &worktree).await.ok();
1066 return Err(e);
1067 }
1068 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1069 .await
1070 .unwrap_or(0);
1071 if commits == 0 {
1072 git::worktree_remove_clean(repo, &worktree).await.ok();
1073 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1074 }
1075 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1076 .await
1077 .map(|f| f.len())
1078 .unwrap_or(0);
1079 if files == 0
1080 && let (Ok(head_tree), Ok(base_tree)) = (
1081 git::tree_of(&worktree, "HEAD").await,
1082 git::tree_of(&worktree, &base_commit).await,
1083 )
1084 && head_tree == base_tree
1085 {
1086 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1087 git::worktree_remove_clean(repo, &worktree).await.ok();
1088 bail!(
1089 "`{branch}` at {} has a tree identical to base {}; this usually means \
1090 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1091 against `{}/{branch}`) rather than an empty change",
1092 short(&head),
1093 short(&base_commit),
1094 state.config.merge.remote
1095 );
1096 }
1097 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1098 .await
1099 .unwrap_or_default();
1100
1101 state.candidates.push(Candidate {
1102 index: 0,
1103 label: 'A',
1104 agent: EXISTING_BRANCH.to_owned(),
1107 branch: branch.to_owned(),
1108 worktree,
1109 summary: String::new(),
1110 stat,
1111 files,
1112 commits,
1113 empty: false,
1114 failed: None,
1115 verified_noop: None,
1116 duration_ms: 0,
1117 folded: false,
1118 });
1119 state.tally = Some(Tally {
1120 first_choice: BTreeMap::from([('A', 0)]),
1121 borda: BTreeMap::new(),
1122 winner: 'A',
1123 rankings: 0,
1124 unanimous_initial: false,
1125 deliberated: false,
1126 changed_votes: 0,
1127 unanimous_final: false,
1128 tie_break: None,
1129 judges: 0,
1133 present: 0,
1134 quorum: 0,
1135 met_quorum: true,
1136 uncontested: Some("review-only run: nothing competed".to_owned()),
1137 });
1138 state.status = RunStatus::Reviewing;
1139 state.prep_complete = Some(true);
1140 state.event(
1141 "start",
1142 format!(
1143 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1144 state.id
1145 ),
1146 );
1147 state.save()?;
1148 Ok(Self {
1149 state,
1150 roles,
1151 sem: Arc::new(Semaphore::new(max_parallel)),
1152 pause: Pause::new(),
1153 interrupt: Pause::new(),
1154 })
1155 }
1156
1157 pub fn resume(id: &str) -> Result<Self> {
1159 let state = RunState::load(id)?;
1160 crate::pr_head::ensure_owner(&state)?;
1161 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1162 bail!(
1163 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1164 );
1165 }
1166 if let Some(to) = &state.released_to {
1167 bail!(
1168 "run {} cannot be resumed: its worktree was released to run {}",
1169 state.short(),
1170 crate::run::short_of(to)
1171 );
1172 }
1173 let roles = state.config.resolve_roles()?;
1174 if let Some(id) = &state.config.graph.selected_implementer
1175 && state.candidates.iter().any(|c| c.agent != *id)
1176 {
1177 bail!(
1178 "replacement implementer pin does not match the saved candidates; task must stay held"
1179 );
1180 }
1181 let max_parallel = state.config.graph.max_parallel.max(1);
1182 Ok(Self {
1183 state,
1184 roles,
1185 sem: Arc::new(Semaphore::new(max_parallel)),
1186 pause: Pause::new(),
1187 interrupt: Pause::new(),
1188 })
1189 }
1190
1191 pub async fn execute(&mut self) -> Result<()> {
1198 if let Some(selection) = &self.state.pr_recovery
1199 && (selection.source_proof.is_some() || selection.source_run.is_some())
1200 {
1201 Box::pin(selection.verify_source(&self.state.repo)).await?;
1202 }
1203 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1206 || self.state.land_approval.is_some()
1207 || self.state.merge_decision.is_some()
1208 || !self.state.merge_repairs.is_empty()
1209 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1210 {
1211 let claim = FixClaim::acquire(&self.state.dir())?;
1212 let disk = RunState::load(&self.state.id)?;
1213 let checkpoint = |s: &RunState| {
1214 (
1215 s.merge_repairs.len(),
1216 s.merge_repairs
1217 .last()
1218 .map(|r| (r.question.clone(), r.phase)),
1219 s.land_approval
1220 .as_ref()
1221 .map(|a| (a.question.clone(), a.head.clone())),
1222 )
1223 };
1224 if checkpoint(&disk) != checkpoint(&self.state) {
1225 bail!(
1226 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1227 );
1228 }
1229 Some(claim)
1230 } else {
1231 None
1232 };
1233 let mut execution = self.claim_driver()?;
1234 let runner = &mut *execution.runner;
1235 let wait = runner.repository_wait();
1236 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1239 let result = match result {
1240 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1241 record_repository_block(&mut runner.state, &e);
1242 runner.park_here()?;
1243 Ok(())
1244 }
1245 other => other,
1246 };
1247 execution.driver.finish(&mut runner.state)?;
1248 let ended = if result.is_err() {
1249 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1250 } else {
1251 crate::notices::run_ended(&runner.state)
1252 };
1253 if let Some(notice) = ended {
1254 crate::notices::raise(notice);
1255 }
1256 result
1257 }
1258
1259 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1260 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1261 }
1262
1263 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1264 let wait = self.repository_wait();
1265 coordinate_operation(&mut self.state, cwd, &wait).await
1266 }
1267
1268 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1270 let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1271 Ok(owner) => owner,
1272 Err(error) => {
1273 let detail = format!("worktree maintenance or ownership update: {error:#}");
1274 self.state.resource_block = Some(crate::run::ResourceBlock {
1278 node: "start".into(),
1279 kind: "worktree maintenance".into(),
1280 detail: detail.clone(),
1281 at: Timestamp::now(),
1282 });
1283 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1284 }
1285 };
1286 let disk = RunState::load(&self.state.id)?;
1287 crate::pr_head::ensure_owner(&disk)?;
1288 if disk.liveness(false) == crate::run::Liveness::Live {
1289 bail!("run already has a live driver; wait for its execution to end");
1290 }
1291 if let Some(to) = &disk.released_to {
1292 bail!(
1293 "run {} cannot continue: its worktree was released to run {}",
1294 self.state.short(),
1295 crate::run::short_of(to)
1296 );
1297 }
1298 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1299 let mut execution = DriverExecution {
1300 runner: self,
1301 driver,
1302 };
1303 let _owner = owner;
1306 let state = &mut execution.runner.state;
1307 state.parked = false;
1312 state.resource_wait = None;
1313 state.resource_block = None;
1314 state.clear_active();
1321 let pid = std::process::id();
1336 state.driver_pid = Some(pid);
1337 state.driver_started_at = crate::proc::process_started_at(pid);
1338 state.driver_exited = false;
1339 state.begin_pass();
1340 execution.driver.bind(state);
1341 state.save()?;
1342 execution.driver.record(state)?;
1343 Ok(execution)
1344 }
1345
1346 async fn execute_graph(&mut self) -> Result<()> {
1347 self.state.configure_recovery_resume()?;
1348 self.persist_candidates().await?;
1351 if self
1352 .state
1353 .merge_decision
1354 .as_ref()
1355 .is_some_and(|decision| !decision.closed)
1356 && self.state.status != RunStatus::Landing
1357 {
1358 if crate::merge_decision::resumable_hold(&self.state) {
1359 self.state.merge = Some(MergeOutcome {
1360 mode: MergeMode::Pr,
1361 ok: false,
1362 detail: self
1363 .state
1364 .merge_decision
1365 .as_ref()
1366 .unwrap()
1367 .target
1368 .pr
1369 .clone(),
1370 empty: false,
1371 });
1372 self.run_land().await?;
1373 self.settle_questions();
1374 return Ok(());
1375 }
1376 bail!(
1377 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1378 );
1379 }
1380 if self.state.config.graph.selected_implementer.is_some()
1381 && self.state.status == RunStatus::Blocked
1382 && self.state.tally.is_none()
1383 && self.state.candidate_preservation.is_empty()
1384 && self.state.interrupted_implementations.is_empty()
1385 {
1386 bail!(
1387 "selected implementation is held; explicitly release the owning task to retry it"
1388 );
1389 }
1390
1391 if self.state.merge_repairs.last().is_some_and(|r| {
1392 matches!(
1393 r.phase,
1394 crate::run::MergeRepairPhase::Requested
1395 | crate::run::MergeRepairPhase::Fixing
1396 | crate::run::MergeRepairPhase::Reviewing
1397 )
1398 }) {
1399 if self.repair_merge_review().await? {
1400 self.run_land().await?;
1401 }
1402 self.settle_questions();
1403 return Ok(());
1404 }
1405 if self.state.pr_publication_pending {
1408 self.merge().await?;
1409 if self.resource_halt()? {
1410 return Ok(());
1411 }
1412 self.state.save()?;
1413 self.settle_questions();
1414 return Ok(());
1415 }
1416 if self.state.status == RunStatus::Stalled {
1429 if self.recover_stall().await? {
1430 self.finish_after_tally().await?;
1431 } else {
1432 self.state.save()?;
1434 }
1435 return Ok(());
1436 }
1437 if self.state.status == RunStatus::Landing
1447 || crate::handover::landing_checkpoint(&self.state)
1448 {
1449 self.state.land_sync_pending = true;
1450 self.state.save()?;
1451 self.run_land().await?;
1452 self.settle_questions();
1457 return Ok(());
1458 }
1459 if self.state.github_text.is_some()
1464 && self.state.merge.is_none()
1465 && self.state.status == RunStatus::Gating
1466 {
1467 self.merge().await?;
1468 if self.resource_halt()? {
1469 return Ok(());
1470 }
1471 self.state.save()?;
1472 return Ok(());
1473 }
1474 if let Err(e) = self.prep().await {
1475 if e.is::<crate::repo_lock::ResourceBlocked>() {
1476 return Err(e);
1477 }
1478 self.state.status = RunStatus::Blocked;
1479 self.state
1480 .prep_failure
1481 .get_or_insert_with(|| format!("{e:#}"));
1482 self.state
1483 .event("prep", format!("preparation stopped: {e:#}"));
1484 self.state.save()?;
1485 return Err(e);
1486 }
1487 if self.park_here()? {
1488 return Ok(());
1489 }
1490 self.advise().await?;
1491 if self.park_here()? {
1492 return Ok(());
1493 }
1494 self.implement().await?;
1495 if self.park_here()? {
1496 return Ok(());
1497 }
1498 if self.state.status == RunStatus::VerifiedNoop {
1502 return Ok(());
1503 }
1504 self.judge().await?;
1505 if self.park_here()? {
1506 return Ok(());
1507 }
1508 self.deliberate().await?;
1509 if self.park_here()? {
1510 return Ok(());
1511 }
1512 self.vote().await?;
1513 if self.park_here()? {
1514 return Ok(());
1515 }
1516 self.tally()?;
1517 if self.state.status == RunStatus::Stalled {
1522 self.state.save()?;
1526 return Ok(());
1527 }
1528 self.finish_after_tally().await?;
1529 Ok(())
1530 }
1531
1532 fn stop_for_resource(&mut self) -> Result<()> {
1537 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1539 {
1540 self.state.status = RunStatus::Blocked;
1541 }
1542 self.state.save()
1543 }
1544
1545 fn resource_halt(&mut self) -> Result<bool> {
1547 if self.state.resource_block.is_none() {
1548 return Ok(false);
1549 }
1550 self.stop_for_resource()?;
1551 Ok(true)
1552 }
1553
1554 fn park_here(&mut self) -> Result<bool> {
1561 let parking = self.pause.parked() || self.interrupt.parked();
1562 if !parking && self.state.resource_block.is_some() {
1563 self.stop_for_resource()?;
1564 return Ok(true);
1565 }
1566 if !self.pause.parked() && !self.interrupt.parked() {
1571 return Ok(false);
1572 }
1573 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1574 let why = match cause.as_ref().map(ParkCause::label) {
1575 Some(reason) => format!(
1576 "parked after `{}` ({reason}) — resume to carry on from here",
1577 self.state.status.as_str()
1578 ),
1579 None => format!(
1580 "parked after `{}` — resume to carry on from here",
1581 self.state.status.as_str()
1582 ),
1583 };
1584 self.state.event("park", why);
1585 self.state
1586 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1587 self.state.parked = true;
1588 self.state.save()?;
1589 Ok(true)
1590 }
1591
1592 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1594 if self.state.prep_failure.is_none() {
1595 return Ok(());
1596 }
1597 let failed_at = self
1598 .state
1599 .events
1600 .iter()
1601 .rev()
1602 .find(|event| event.node == "prep")
1603 .map_or(self.state.updated_at, |event| event.at);
1604 anyhow::ensure!(
1605 approval_at > failed_at,
1606 "preparation retry requires a new explicit release after the failure"
1607 );
1608 self.state.prep_failure = None;
1609 self.state.prep_complete = Some(false);
1610 self.state
1611 .event("prep-retry", "operator approved retry of preparation");
1612 Ok(())
1613 }
1614
1615 pub fn on_pause(&mut self, pause: Pause) {
1617 self.pause = pause;
1618 }
1619
1620 pub fn watch_interrupt(&mut self, pause: Pause) {
1626 self.interrupt = pause;
1627 }
1628
1629 fn settle_questions(&mut self) {
1649 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1650 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1651 }
1652 }
1653
1654 async fn finish_after_tally(&mut self) -> Result<()> {
1657 self.graft().await?;
1660 if self.resource_halt()? {
1661 return Ok(());
1662 }
1663 self.fold_losers().await?;
1664 self.sync_to_base().await?;
1669 if self.resource_halt()? {
1670 return Ok(());
1671 }
1672 if self.state.status == RunStatus::AlreadyInBase {
1673 return Ok(());
1674 }
1675 self.review_loop().await?;
1676 if self.resource_halt()? {
1677 return Ok(());
1678 }
1679 self.sync_to_base().await?;
1680 if self.resource_halt()? {
1681 return Ok(());
1682 }
1683 if self.state.status == RunStatus::AlreadyInBase {
1684 return Ok(());
1685 }
1686 self.gate().await?;
1687 self.merge().await?;
1688 if self.resource_halt()? {
1689 return Ok(());
1690 }
1691 self.state.save()?;
1692 Ok(())
1693 }
1694
1695 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1702 let result = match ensure_setup_config(&mut self.state, repo).await {
1703 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1704 Err(e) => Err(e),
1705 };
1706 if let Err(e) = result {
1707 git::worktree_remove_clean(repo, wt).await.ok();
1708 if in_prep {
1709 self.rollback_prep(repo).await;
1710 }
1711 self.state.status = RunStatus::Blocked;
1712 self.state.event(
1713 "setup",
1714 format!("worktree setup failed in {}: {e:#}", wt.display()),
1715 );
1716 self.state.save()?;
1717 return Err(e);
1718 }
1719 Ok(())
1720 }
1721
1722 async fn rollback_prep(&mut self, repo: &Path) {
1724 let root = self.state.worktree_root();
1725 for c in std::mem::take(&mut self.state.candidates) {
1726 if matches!(
1727 git::worktree_remove_clean(repo, &c.worktree).await,
1728 Ok(true)
1729 ) {
1730 git::branch_delete(repo, &c.branch).await.ok();
1731 } else {
1732 self.state.candidates.push(c);
1733 }
1734 }
1735 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1736 for j in 1..=seats {
1737 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1738 .await
1739 .ok();
1740 }
1741 for k in 1..=self.state.config.graph.advisors {
1742 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1743 .await
1744 .ok();
1745 }
1746 }
1747
1748 async fn prep(&mut self) -> Result<()> {
1749 if let Some(reason) = &self.state.prep_failure {
1750 bail!(
1751 "candidate preparation previously failed: {reason}; correct the source before retrying"
1752 );
1753 }
1754 if self
1755 .state
1756 .prep_complete
1757 .unwrap_or(!self.state.candidates.is_empty())
1758 {
1759 return Ok(());
1760 }
1761 let repo = self.state.repo.clone();
1762 let _coordination = self.coordinate(&repo).await?;
1765 self.rollback_prep(&repo).await;
1766 if !self.state.candidates.is_empty() {
1767 bail!(
1768 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1769 );
1770 }
1771 self.state.prep_complete = Some(false);
1772 self.state.status = RunStatus::Prep;
1773 self.state.save()?;
1774 let repo = self.state.repo.clone();
1775 let base = self.state.base_commit.clone();
1776 let plan = refs::plan(&repo, &self.state.seeds).await?;
1777 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1778 let root = self.state.worktree_root();
1779 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1780
1781 let hooks_dir = self.state.dir().join("hooks");
1784 if self.state.config.blind.commit_msg_hook {
1785 std::fs::create_dir_all(&hooks_dir)
1786 .with_context(|| format!("create {}", hooks_dir.display()))?;
1787 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1788 let path = hooks_dir.join("commit-msg");
1789 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1790 make_executable(&path)?;
1791 git::acquire_worktree_config(&repo).await?;
1799 self.state.enabled_worktree_config = true;
1800 }
1801
1802 ensure_setup_config(&mut self.state, &repo).await?;
1805
1806 for (index, (spec, label)) in self
1807 .roles
1808 .implementers
1809 .clone()
1810 .into_iter()
1811 .zip(labels)
1812 .enumerate()
1813 {
1814 let branch = self.state.branch_for(label);
1815 let worktree = root.join(format!("cand-{label}"));
1816 self.state.event(
1820 "prep-intent",
1821 serde_json::json!({
1822 "branch": branch,
1823 "worktree": worktree,
1824 "start": start,
1825 "picks": plan.picks,
1826 })
1827 .to_string(),
1828 );
1829 self.state.save()?;
1830 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1831 if self.state.config.blind.commit_msg_hook {
1832 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1833 }
1834 git::local_exclude(&worktree, "/.magi/").await?;
1835 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1836 git::worktree_remove_clean(&repo, &worktree).await.ok();
1839 git::branch_delete(&repo, &branch).await.ok();
1840 self.rollback_prep(&repo).await;
1841 self.state.status = RunStatus::Blocked;
1842 self.state
1843 .event("prep", format!("worktree setup failed: {e:#}"));
1844 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1845 self.state.save()?;
1846 return Err(e);
1847 }
1848 self.state.candidates.push(Candidate {
1849 index,
1850 label,
1851 agent: spec.id.clone(),
1852 branch,
1853 worktree,
1854 summary: String::new(),
1855 stat: String::new(),
1856 files: 0,
1857 commits: 0,
1858 empty: false,
1859 failed: None,
1860 verified_noop: None,
1861 duration_ms: 0,
1862 folded: false,
1863 });
1864 self.state.save()?;
1865 for pick in &plan.picks {
1866 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1867 {
1868 self.state.status = RunStatus::Blocked;
1869 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1870 self.state
1871 .event("prep", format!("cannot apply referenced commit: {e}"));
1872 self.state.save()?;
1873 return Err(e);
1874 }
1875 }
1876 }
1877
1878 for j in 1..=self.roles.judges.len() {
1879 let wt = root.join(format!("judge-{j}"));
1880 if !wt.exists() {
1881 git::worktree_add_detached(&repo, &wt, &base).await?;
1882 self.setup_seat_worktree(&repo, &wt, true).await?;
1883 }
1884 }
1885
1886 if self.state.config.graph.advise {
1894 for k in 1..=self.state.config.graph.advisors {
1895 let wt = root.join(format!("advisor-{k}"));
1896 if !wt.exists() {
1897 git::worktree_add_detached(&repo, &wt, &base).await?;
1898 self.setup_seat_worktree(&repo, &wt, true).await?;
1899 }
1900 }
1901 }
1902
1903 let authors: Vec<&str> = self
1908 .roles
1909 .implementers
1910 .iter()
1911 .map(|a| a.id.as_str())
1912 .collect();
1913 let overlap: Vec<String> = self
1914 .roles
1915 .judges
1916 .iter()
1917 .enumerate()
1918 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1919 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1920 .collect();
1921 if !overlap.is_empty() {
1922 let note = format!(
1923 "{} also authored a candidate; blind, but the panel is less \
1924 independent than {} distinct agents would be",
1925 overlap.join(", "),
1926 self.roles.judges.len()
1927 );
1928 self.state.event("prep", note);
1929 }
1930
1931 self.state.event(
1932 "prep",
1933 format!(
1934 "{} candidates, {} judges, base {} ({})",
1935 self.state.candidates.len(),
1936 self.roles.judges.len(),
1937 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1938 self.state.base_branch
1939 ),
1940 );
1941 self.state.prep_complete = Some(true);
1942 self.state.status = RunStatus::Implementing;
1943 self.state.save()?;
1944 Ok(())
1945 }
1946
1947 async fn advise(&mut self) -> Result<()> {
1980 let implement_untouched = self
1981 .state
1982 .candidates
1983 .iter()
1984 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1985 if !self.state.config.graph.advise || self.state.advise_attempted {
1986 return Ok(());
1987 }
1988 if !implement_untouched {
1989 self.state.event(
1990 "advise",
1991 "skipping the design-deliberation stage: at least one \
1992 candidate already shows implementation progress, so this \
1993 run is past the point the stage exists to run before"
1994 .to_owned(),
1995 );
1996 self.state.advise_attempted = true;
1997 self.state.save()?;
1998 return Ok(());
1999 }
2000 let run_id = self.state.id.clone();
2001 let prompts = self.state.config.prompts.clone();
2002 let instruction = self.state.instruction.clone();
2003 let language = self.state.config.graph.language.clone();
2004 let root = self.state.worktree_root();
2005 let n = self.state.config.graph.advisors;
2006 let where_recorded = self.state.dir().join("run.json");
2007
2008 let seats = match self.state.config.advisors() {
2009 Ok(seats) if !seats.is_empty() => seats,
2010 Ok(_) => {
2011 self.state.event(
2012 "advise",
2013 format!(
2014 "[graph] advisors is 0; skipping the design-deliberation \
2015 stage and continuing without a synthesis brief (see {})",
2016 where_recorded.display()
2017 ),
2018 );
2019 self.state.advise_attempted = true;
2020 self.state.save()?;
2021 return Ok(());
2022 }
2023 Err(e) => {
2024 self.state.event(
2025 "advise",
2026 format!(
2027 "could not resolve advisor seats ({e:#}); continuing \
2028 without a design-deliberation brief (see {})",
2029 where_recorded.display()
2030 ),
2031 );
2032 self.state.advise_attempted = true;
2033 self.state.save()?;
2034 return Ok(());
2035 }
2036 };
2037
2038 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2039 let artifacts = agent::artifacts_dir(&self.state.dir());
2040 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2041
2042 let mut jobs = Vec::new();
2043 for (i, spec) in seats.iter().cloned().enumerate() {
2044 let seat_key = format!("advisor-{}", i + 1);
2045 let seat = self.seat(&seat_key, &spec.id);
2046 jobs.push(SeatJob {
2047 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2048 spec,
2049 seat,
2050 cwd: worktrees[i % worktrees.len()].clone(),
2051 timeout,
2052 allow_write: false,
2053 sessions: false,
2054 artifacts: artifacts.clone(),
2055 stem: seat_key,
2056 handover: None,
2057 });
2058 }
2059
2060 self.state.event(
2061 "advise",
2062 format!(
2063 "{} advisor seat(s) sketching a design in parallel",
2064 jobs.len()
2065 ),
2066 );
2067 let mut quota_losses = Vec::new();
2068 let cache = self.state.config.cache_dir();
2069 let ctx = WaveCtx {
2070 carry_seats: false,
2071 pauses: Some((&self.pause, &self.interrupt)),
2072 run: &run_id,
2073 node: "advise",
2074 prompts: &prompts,
2075 cache: cache.as_deref(),
2076 round: None,
2077 };
2078 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2079 let results = ask_json_wave::<Proposal>(
2080 jobs,
2081 Arc::clone(&self.sem),
2082 self.state.config.graph.retries,
2083 &advisor_roster,
2084 &ctx,
2085 &mut quota_losses,
2086 &mut self.state,
2087 &|p: &Proposal| p.validate(),
2088 )
2089 .await;
2090 if self.state.resource_block.is_some() {
2091 return self.stop_for_resource();
2092 }
2093 self.state.quota.extend(quota_losses);
2094
2095 let mut records = Vec::with_capacity(results.len());
2096 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2097 let agent_id = seat.agent.clone();
2098 self.state.seats.insert(seat.key.clone(), seat);
2099 match res {
2100 Ok((proposal, out)) => {
2101 self.state
2102 .event("advise", format!("advisor-{} proposed a design", i + 1));
2103 records.push(advise::AdvisorRecord::proposed(
2104 i + 1,
2105 agent_id,
2106 proposal,
2107 out.duration_ms,
2108 ));
2109 }
2110 Err(e) => {
2111 self.state.event(
2112 "advise",
2113 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2114 );
2115 records.push(advise::AdvisorRecord::failed(
2116 i + 1,
2117 agent_id,
2118 e.to_string(),
2119 ));
2120 }
2121 }
2122 }
2123
2124 let mut advice = advise::Advice {
2125 records,
2126 synthesis: None,
2127 };
2128 if advice.proposals().is_empty() {
2129 self.state.event(
2130 "advise",
2131 "no advisor produced a usable proposal; continuing without a \
2132 synthesis brief"
2133 .to_owned(),
2134 );
2135 } else {
2136 match self
2137 .synthesize_brief(
2138 &advice,
2139 &instruction,
2140 &language,
2141 &worktrees[0],
2142 &artifacts,
2143 &run_id,
2144 &prompts,
2145 cache.as_deref(),
2146 )
2147 .await
2148 {
2149 Ok(Some(text)) => {
2150 self.state.event(
2151 "advise",
2152 "synthesized a design brief for the implementer".to_owned(),
2153 );
2154 advice.synthesis = Some(text);
2155 }
2156 Ok(None) => {
2157 self.state.event(
2158 "advise",
2159 "the synthesis seat produced nothing usable; continuing \
2160 without a design brief"
2161 .to_owned(),
2162 );
2163 }
2164 Err(e) => {
2165 if e.is::<crate::repo_lock::ResourceBlocked>() {
2166 return Err(e);
2167 }
2168 self.state.event(
2169 "advise",
2170 format!("could not synthesize a design brief: {e:#}"),
2171 );
2172 }
2173 }
2174 }
2175 advise::apply_reflection(&mut advice);
2176
2177 self.state.advice = Some(advice);
2178 self.state.advise_attempted = true;
2179 self.state.save()?;
2180 Ok(())
2181 }
2182
2183 #[allow(clippy::too_many_arguments)]
2195 async fn synthesize_brief(
2196 &mut self,
2197 advice: &advise::Advice,
2198 instruction: &str,
2199 language: &str,
2200 cwd: &Path,
2201 artifacts: &Path,
2202 run_id: &str,
2203 prompts: &Prompts,
2204 cache: Option<&Path>,
2205 ) -> Result<Option<String>> {
2206 let chain = agent::pick_chain(
2207 &self.state.config.agents,
2208 self.state.config.roles.synthesizer.as_ref(),
2209 &agent::installed,
2210 "synthesizer",
2211 )?;
2212 let proposals = advice.proposals();
2213 let mut prompt = prompt::with_overlay(
2214 prompt::synthesize_brief(instruction, &proposals, language),
2215 prompts.overlay("advise"),
2216 );
2217 if cache.is_some() {
2218 prompt.push('\n');
2223 prompt.push_str(&prompt::build_cache_note("advise", false));
2224 }
2225 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2226 let mut last = None;
2231 for (n, spec) in chain.iter().enumerate() {
2232 if n > 0 {
2233 self.state
2234 .event("advise", format!("synthesis falling back to {}", spec.id));
2235 }
2236 let mut seat = self.seat("advise-synthesis", &spec.id);
2237 let wait = self.repository_wait();
2238 let outcome = wait
2239 .scope(agent::invoke(
2240 spec,
2241 &mut seat,
2242 &Invocation {
2243 scope: crate::agent::InvocationScope::Repository,
2244 coordination: self.state.config.coordination.as_ref(),
2245 fetched_base: None,
2246 cwd,
2247 prompt: &prompt,
2248 timeout,
2249 allow_write: false,
2250 unsandboxed: false,
2251 sessions: false,
2252 artifacts,
2253 stem: &if n == 0 {
2254 "advise-synthesis".to_owned()
2255 } else {
2256 format!("advise-synthesis-{}", spec.id)
2257 },
2258 run: run_id,
2259 node: "advise",
2260 cache_dir: None,
2261 attachments: &[],
2262 writable: &[],
2263 },
2264 ))
2265 .await;
2266 if outcome.is_ok() {
2267 self.state.seats.insert(seat.key.clone(), seat);
2268 }
2269 let advance = agent::chain_advances(&outcome);
2270 last = Some(outcome);
2271 if !advance {
2272 break;
2273 }
2274 }
2275 let out = last.expect("a chain holds at least one agent")?;
2278 if !out.usable() {
2279 return Ok(None);
2280 }
2281 let text =
2282 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2283 Ok((!text.trim().is_empty()).then_some(text))
2284 }
2285
2286 async fn implement(&mut self) -> Result<()> {
2289 self.persist_candidates().await?;
2290 let run_id = self.state.id.clone();
2295 let prompts = self.state.config.prompts.clone();
2296 let todo: Vec<usize> = self
2297 .state
2298 .candidates
2299 .iter()
2300 .enumerate()
2301 .filter(|(_, c)| {
2302 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2303 && c.failed.is_none()
2304 && !c.empty
2305 })
2306 .map(|(i, _)| i)
2307 .collect();
2308 if todo.is_empty() {
2309 return self.after_implement();
2310 }
2311 self.state.status = RunStatus::Implementing;
2312
2313 let language = self.state.config.graph.language.clone();
2314 let github_language = self.state.config.graph.github_language.clone();
2315 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2316 let sessions = self.state.config.graph.sessions;
2317 let artifacts = agent::artifacts_dir(&self.state.dir());
2318 let brief = self
2322 .state
2323 .advice
2324 .as_ref()
2325 .and_then(|a| a.synthesis.as_deref())
2326 .map(str::to_owned);
2327 let attachments = self.state.attachments.clone();
2328
2329 let mut jobs = Vec::new();
2330 for &i in &todo {
2331 let (index, label, worktree) = {
2332 let c = &self.state.candidates[i];
2333 (c.index, c.label, c.worktree.clone())
2334 };
2335 let spec = self.roles.implementers[index].clone();
2336 let seat_key = format!("impl-{label}");
2337 let seat = self.seat(&seat_key, &spec.id);
2338 let instruction = seeded_instruction(&self.state);
2339 jobs.push(SeatJob {
2340 spec,
2341 seat,
2342 prompt: prompt::implement_in(
2343 &instruction,
2344 &worktree.to_string_lossy(),
2345 &language,
2346 &github_language,
2347 brief.as_deref(),
2348 &attachments,
2349 ),
2350 cwd: worktree,
2351 timeout,
2352 allow_write: true,
2353 sessions,
2354 artifacts: artifacts.clone(),
2355 stem: format!("impl-{label}"),
2356 handover: None,
2357 });
2358 }
2359
2360 self.state.event(
2361 "implement",
2362 format!("{} candidates in parallel", jobs.len()),
2363 );
2364 let mut sent = jobs.clone();
2370 let cache = self.state.config.cache_dir();
2371 let ctx = WaveCtx {
2372 carry_seats: false,
2373 pauses: Some((&self.pause, &self.interrupt)),
2374 run: &run_id,
2375 node: "implement",
2376 prompts: &prompts,
2377 cache: cache.as_deref(),
2378 round: None,
2379 };
2380 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2381 if !wave_blocked(&results) {
2382 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2383 .await;
2384 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2385 .await?;
2386 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2387 .await;
2388 }
2389 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2390
2391 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2392 && results
2393 .iter()
2394 .any(|(_, _, out)| FailClass::of(out).is_some());
2395 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2396 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2397 if self
2401 .state
2402 .candidate_preservation
2403 .contains_key(&self.state.candidates[i].label)
2404 {
2405 self.state.seats.insert(seat.key.clone(), seat);
2406 }
2407 continue;
2408 }
2409 let seat_key = seat.key.clone();
2410 let agent = seat.agent.clone();
2419 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2420 self.state.seats.insert(seat.key.clone(), seat);
2421 let label = self.state.candidates[i].label;
2422 self.state.interrupted_implementations.remove(&label);
2423
2424 let (summary, duration, failed, verified_claim) = match out {
2425 AgentOutcome::Ok(o) => {
2426 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2427 let failed = (!o.usable()).then(|| {
2428 if o.timed_out {
2429 "agent timed out".to_owned()
2430 } else {
2431 format!(
2432 "agent exited with {}",
2433 crate::notices::exit_status(o.exit_code)
2434 )
2435 }
2436 });
2437 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2438 (text, o.duration_ms, failed, verified_claim)
2439 }
2440 AgentOutcome::Dropped(o) => {
2446 let why = o
2447 .dropped
2448 .as_ref()
2449 .map(|d| d.why.as_str())
2450 .unwrap_or("the CLI ended the stream without delivering its answer");
2451 (
2452 String::new(),
2453 o.duration_ms,
2454 Some(format!("the CLI dropped the stream ({why})")),
2455 None,
2456 )
2457 }
2458 AgentOutcome::Quota(o) => {
2459 self.state.quota.push(QuotaLoss {
2460 seat: seat_key,
2461 node: "implement".to_owned(),
2462 at: Timestamp::now(),
2463 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2464 });
2465 (
2466 String::new(),
2467 o.duration_ms,
2468 Some("rate limited (quota); produced no change".to_owned()),
2469 None,
2470 )
2471 }
2472 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2473 (String::new(), 0, Some(e), None)
2474 }
2475 };
2476
2477 let c = &mut self.state.candidates[i];
2480 if !exhausted_the_fallback_chain {
2481 c.agent = agent;
2482 }
2483 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2484 c.duration_ms = duration;
2485 c.failed = failed;
2486 c.verified_noop = verified_claim;
2487 self.state
2488 .candidate_preservation
2489 .insert(label, "awaiting persistence".to_owned());
2490 }
2491 self.state.save()?;
2492 self.persist_candidates().await?;
2493
2494 if resource_blocked {
2495 return self.stop_for_resource();
2496 }
2497 if selected_failed {
2498 self.state.status = RunStatus::Blocked;
2499 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2500 self.state.save()?;
2501 bail!(
2502 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2503 );
2504 }
2505 self.after_implement()
2506 }
2507
2508 async fn persist_candidates(&mut self) -> Result<()> {
2510 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2511 for label in labels {
2512 let i = self
2513 .state
2514 .candidates
2515 .iter()
2516 .position(|c| c.label == label)
2517 .context("pending candidate is missing")?;
2518 let c = self.state.candidates[i].clone();
2519 let interrupted = self.state.interrupted_implementations.contains(&label);
2520 let result = async {
2521 let wait = crate::repo_lock::WaitPolicy::new(
2522 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2523 || false,
2524 );
2525 let _coordination =
2526 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2527 if let Err(e) = git::snapshot_uncommitted(
2530 &c.worktree,
2531 &self
2532 .state
2533 .dir()
2534 .join("artifacts")
2535 .join(format!("candidate-{label}-recovery")),
2536 )
2537 .await
2538 {
2539 self.state.event(
2540 "preserve",
2541 format!("candidate {label} snapshot incomplete: {e:#}"),
2542 );
2543 }
2544 let rescued = git::rescue_commit(
2545 &c.worktree,
2546 &format!("magi: candidate {label} (uncommitted work)"),
2547 )
2548 .await?;
2549 self.state.note_withheld("implement", &rescued.withheld);
2550 let base = &self.state.base_commit;
2551 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2552 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2553 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2554 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2555 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2556 if c.empty && commits > 0 && !patch.trim().is_empty()
2557 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2558 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2559 || self.state.tally.is_some())
2560 {
2561 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2565 }
2566 let c = &mut self.state.candidates[i];
2567 c.commits = commits;
2568 c.stat = stat;
2569 c.files = files;
2570 let no_changes = commits == 0 || patch.trim().is_empty();
2571 c.empty = no_changes && !interrupted;
2572 if interrupted && no_changes {
2573 c.failed = None;
2574 c.verified_noop = None;
2575 }
2576 if !c.empty {
2577 c.failed = None;
2578 c.verified_noop = None;
2579 }
2580 self.state.event(
2581 "preserve",
2582 format!("candidate {label}: {files} files, {commits} commits"),
2583 );
2584 Ok::<_, anyhow::Error>(())
2585 }
2586 .await;
2587 match result {
2588 Ok(()) => {
2589 self.state.candidate_preservation.remove(&label);
2590 if !interrupted || self.state.candidates[i].files > 0 {
2591 self.state.interrupted_implementations.remove(&label);
2592 }
2593 }
2594 Err(e) => {
2595 let reason = format!("candidate {label} preservation failed: {e:#}");
2596 self.state.event("preserve", &reason);
2597 self.state.candidate_preservation.insert(label, reason);
2598 }
2599 }
2600 if self.state.candidate_preservation.is_empty()
2601 && self.state.status == RunStatus::Blocked
2602 && self.state.candidates.iter().all(|c| c.failed.is_none())
2603 {
2604 self.state.status = RunStatus::Implementing;
2608 self.state.event(
2609 "preserve",
2610 "candidate persistence recovered; implementation can continue",
2611 );
2612 }
2613 self.state.save()?;
2614 }
2615 if !self.state.candidate_preservation.is_empty() {
2616 self.state.status = RunStatus::Blocked;
2617 self.state.save()?;
2618 bail!(
2619 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2620 );
2621 }
2622 Ok(())
2623 }
2624
2625 async fn resume_undelivered(
2653 &mut self,
2654 results: &mut [(usize, SeatState, AgentOutcome)],
2655 sent: &[SeatJob],
2656 prompts: &Prompts,
2657 run_id: &str,
2658 ) {
2659 for (wi, seat, out) in results.iter_mut() {
2660 let Some(dropped) = (match &*out {
2661 AgentOutcome::Dropped(o) => o.dropped.clone(),
2662 _ => None,
2663 }) else {
2664 continue;
2665 };
2666 let Some(job) = sent.get(*wi) else { continue };
2667 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2669 self.state.event(
2670 "implement",
2671 format!(
2672 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2673 work is in the tree",
2674 seat.key, dropped.output_tokens, dropped.why
2675 ),
2676 );
2677 continue;
2678 }
2679 if !has_context(&job.spec, seat, job.sessions) {
2687 self.state.event(
2688 "implement",
2689 format!(
2690 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2691 is no session left to resume",
2692 seat.key, dropped.output_tokens, dropped.why
2693 ),
2694 );
2695 continue;
2696 }
2697 self.state.event(
2698 "implement",
2699 format!(
2700 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2701 conversation",
2702 seat.key, dropped.output_tokens, dropped.why
2703 ),
2704 );
2705 let mut retry = job.clone();
2706 retry.seat = seat.clone();
2707 retry.prompt = prompt::resume_after_drop(&dropped.why);
2708 retry.timeout = retry_budget(job.timeout, true);
2709 retry.stem = format!("{}-resume", job.stem);
2710 let cache = self.state.config.cache_dir();
2711 let ctx = WaveCtx {
2712 carry_seats: false,
2713 pauses: Some((&self.pause, &self.interrupt)),
2714 run: run_id,
2715 node: "implement",
2716 prompts,
2717 cache: cache.as_deref(),
2718 round: None,
2719 };
2720 let (resumed_seat, resumed) =
2721 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2722 *seat = resumed_seat;
2723 *out = resumed;
2724 }
2725 }
2726
2727 async fn resume_seat_handovers(
2792 &mut self,
2793 results: &mut [(usize, SeatState, AgentOutcome)],
2794 sent: &mut [SeatJob],
2795 prompts: &Prompts,
2796 run_id: &str,
2797 ) -> Result<()> {
2798 let instruction = seeded_instruction(&self.state);
2799 let language = self.state.config.graph.language.clone();
2800 let github_language = self.state.config.graph.github_language.clone();
2801 let brief = self
2802 .state
2803 .advice
2804 .as_ref()
2805 .and_then(|a| a.synthesis.as_deref())
2806 .map(str::to_owned);
2807 let attachments = self.state.attachments.clone();
2808 for (wi, seat, out) in results.iter_mut() {
2809 let others: BTreeSet<String> = sent
2812 .iter()
2813 .enumerate()
2814 .filter(|(j, _)| j != wi)
2815 .map(|(_, j)| j.spec.id.clone())
2816 .collect();
2817 let Some(job) = sent.get_mut(*wi) else {
2818 continue;
2819 };
2820 let start = self
2825 .roles
2826 .implementer_roster
2827 .iter()
2828 .position(|s| s.id == job.spec.id)
2829 .unwrap_or(0);
2830 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2831 let mut fallback_attempt = 0usize;
2832 let mut prev: Option<FailClass> = None;
2833 while let Some(cur) = FailClass::of(&*out) {
2834 if !should_hand_over(prev.as_ref(), &cur) {
2835 break;
2836 }
2837 let Some(next) =
2838 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2839 .cloned()
2840 else {
2841 break;
2842 };
2843 tried.insert(next.id.clone());
2844 fallback_attempt += 1;
2845
2846 let coordination = self.coordinate(&job.cwd).await?;
2847 if let Ok(r) = git::rescue_commit(
2848 &job.cwd,
2849 &format!(
2850 "magi: candidate {} (uncommitted work before {} fallback)",
2851 seat.key,
2852 if cur == FailClass::Quota {
2853 "quota"
2854 } else {
2855 "handover"
2856 }
2857 ),
2858 )
2859 .await
2860 {
2861 self.state.note_withheld("implement", &r.withheld);
2862 }
2863 drop(coordination);
2864
2865 record_handover(
2866 &mut self.state,
2867 "implement",
2868 &seat.key,
2869 &seat.agent,
2870 &next.id,
2871 &cur,
2872 &fail_reason(&*out),
2873 );
2874 prev = Some(cur.clone());
2875
2876 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2877 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2878 job.spec = next.clone();
2886 let mut retry = job.clone();
2887 retry.seat = new_seat;
2888 retry.prompt = prompt::implement_in(
2889 &instruction,
2890 &job.cwd.to_string_lossy(),
2891 &language,
2892 &github_language,
2893 brief.as_deref(),
2894 &attachments,
2895 );
2896 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2897 let cache = self.state.config.cache_dir();
2898 let ctx = WaveCtx {
2899 carry_seats: false,
2900 pauses: Some((&self.pause, &self.interrupt)),
2901 run: run_id,
2902 node: "implement",
2903 prompts,
2904 cache: cache.as_deref(),
2905 round: None,
2906 };
2907 let (fallback_seat, fallback_out) = run_one(
2908 retry,
2909 Arc::clone(&self.sem),
2910 &ctx,
2911 &mut self.state,
2912 fallback_attempt,
2913 )
2914 .await;
2915 *seat = fallback_seat;
2916 *out = fallback_out;
2917 }
2918 }
2919 Ok(())
2920 }
2921
2922 async fn resume_unconfirmed_commands(
2946 &mut self,
2947 results: &mut [(usize, SeatState, AgentOutcome)],
2948 sent: &[SeatJob],
2949 prompts: &Prompts,
2950 run_id: &str,
2951 ) {
2952 for (wi, seat, out) in results.iter_mut() {
2953 let AgentOutcome::Ok(o) = &*out else {
2954 continue;
2955 };
2956 if !has_unconfirmed_command(&o.commands) {
2957 continue;
2958 }
2959 let Some(job) = sent.get(*wi) else { continue };
2960 if !has_context(&job.spec, seat, job.sessions) {
2961 self.state.event(
2962 "implement",
2963 format!(
2964 "{}: the reply named a command whose own CLI never confirmed the exit \
2965 status of, but there is no session left to resume",
2966 seat.key
2967 ),
2968 );
2969 continue;
2970 }
2971 self.state.event(
2972 "implement",
2973 format!(
2974 "{}: the reply named a command whose own CLI never confirmed the exit \
2975 status of; resuming the conversation",
2976 seat.key
2977 ),
2978 );
2979 let mut retry = job.clone();
2980 retry.seat = seat.clone();
2981 retry.prompt = prompt::resume_incomplete(
2982 "a command in your last reply had no confirmed exit status",
2983 );
2984 retry.timeout = retry_budget(job.timeout, true);
2985 retry.stem = format!("{}-confirm", job.stem);
2986 let cache = self.state.config.cache_dir();
2987 let ctx = WaveCtx {
2988 carry_seats: false,
2989 pauses: Some((&self.pause, &self.interrupt)),
2990 run: run_id,
2991 node: "implement",
2992 prompts,
2993 cache: cache.as_deref(),
2994 round: None,
2995 };
2996 let (resumed_seat, resumed) =
2997 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2998 *seat = resumed_seat;
2999 *out = resumed;
3000 }
3001 }
3002
3003 async fn continue_fix_report(
3024 &mut self,
3025 mut seat: SeatState,
3026 parse_err: String,
3027 job: &SeatJob,
3028 prompts: &Prompts,
3029 run_id: &str,
3030 round: usize,
3031 ) -> (
3032 SeatState,
3033 Option<FixReport>,
3034 Option<String>,
3035 ContinuationRecord,
3036 ) {
3037 let mut last_err = parse_err;
3038 let mut cumulative_wait_ms = 0u64;
3039 let mut attempts = 0usize;
3040 loop {
3041 if !has_context(&job.spec, &seat, job.sessions) {
3042 self.state.event(
3043 "fix",
3044 format!(
3045 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3046 session left to resume into"
3047 ),
3048 );
3049 let outcome = if attempts == 0 {
3050 ContinuationOutcome::NoSession
3051 } else {
3052 ContinuationOutcome::Exhausted
3053 };
3054 return (
3055 seat,
3056 None,
3057 Some(format!("unparsable fix report: {last_err}")),
3058 ContinuationRecord {
3059 attempts,
3060 cumulative_wait_ms,
3061 outcome,
3062 },
3063 );
3064 }
3065 if attempts >= MAX_FIX_CONTINUATIONS {
3066 self.state.event(
3067 "fix",
3068 format!(
3069 "round {round}: fixer's reply still had no adoption report after \
3070 {attempts} continuation(s) ({last_err}); giving up"
3071 ),
3072 );
3073 return (
3074 seat,
3075 None,
3076 Some(format!(
3077 "unparsable fix report after {attempts} continuation(s): {last_err}"
3078 )),
3079 ContinuationRecord {
3080 attempts,
3081 cumulative_wait_ms,
3082 outcome: ContinuationOutcome::Exhausted,
3083 },
3084 );
3085 }
3086 attempts += 1;
3087 self.state.event(
3088 "fix",
3089 format!(
3090 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3091 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3092 ),
3093 );
3094 let mut retry = job.clone();
3095 retry.seat = seat.clone();
3096 retry.prompt = prompt::resume_incomplete(&last_err);
3097 retry.timeout = retry_budget(job.timeout, true);
3098 retry.stem = format!("{}-continue{attempts}", job.stem);
3099 let cache = self.state.config.cache_dir();
3100 let ctx = WaveCtx {
3101 carry_seats: false,
3102 pauses: Some((&self.pause, &self.interrupt)),
3103 run: run_id,
3104 node: "fix",
3105 prompts,
3106 cache: cache.as_deref(),
3107 round: Some(round),
3108 };
3109 let (resumed_seat, resumed_out) = run_one(
3110 retry,
3111 Arc::clone(&self.sem),
3112 &ctx,
3113 &mut self.state,
3114 attempts,
3115 )
3116 .await;
3117 seat = resumed_seat;
3118 match resumed_out {
3119 AgentOutcome::Ok(o) => {
3120 cumulative_wait_ms += o.duration_ms;
3121 match verdict::extract_json::<FixReport>(&o.text) {
3122 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3123 self.state.event(
3124 "fix",
3125 format!(
3126 "round {round}: fixer's adoption report recovered after \
3127 {attempts} continuation(s)"
3128 ),
3129 );
3130 return (
3131 seat,
3132 Some(report),
3133 None,
3134 ContinuationRecord {
3135 attempts,
3136 cumulative_wait_ms,
3137 outcome: ContinuationOutcome::Resumed,
3138 },
3139 );
3140 }
3141 Ok(_) => {
3149 last_err = "the reply parsed, but it reported a command whose own CLI \
3150 never confirmed an exit status"
3151 .to_owned();
3152 }
3153 Err(e) => last_err = e.to_string(),
3154 }
3155 }
3156 AgentOutcome::Quota(o) => {
3157 cumulative_wait_ms += o.duration_ms;
3158 self.state.quota.push(QuotaLoss {
3159 seat: seat.key.clone(),
3160 node: "fix".to_owned(),
3161 at: Timestamp::now(),
3162 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3163 });
3164 self.state.event(
3165 "fix",
3166 format!(
3167 "round {round}: continuation rate limited (quota); not retrying now"
3168 ),
3169 );
3170 return (
3171 seat,
3172 None,
3173 Some("rate limited (quota) while recovering the fix report".to_owned()),
3174 ContinuationRecord {
3175 attempts,
3176 cumulative_wait_ms,
3177 outcome: ContinuationOutcome::QuotaLost,
3178 },
3179 );
3180 }
3181 AgentOutcome::Dropped(o) => {
3182 cumulative_wait_ms += o.duration_ms;
3183 let why = o
3184 .dropped
3185 .as_ref()
3186 .map(|d| d.why.as_str())
3187 .unwrap_or("the CLI ended the stream without delivering its answer");
3188 last_err = format!("the CLI dropped the stream ({why})");
3189 }
3190 AgentOutcome::Failed(e) => last_err = e,
3191 AgentOutcome::ResourceBlocked(e) => {
3194 return (
3195 seat,
3196 None,
3197 Some(format!("the continuation was not started: {e}")),
3198 ContinuationRecord {
3199 attempts: attempts.saturating_sub(1),
3200 cumulative_wait_ms,
3201 outcome: ContinuationOutcome::NoSession,
3202 },
3203 );
3204 }
3205 }
3206 }
3207 }
3208
3209 fn after_implement(&mut self) -> Result<()> {
3210 if !self.state.candidate_preservation.is_empty() {
3211 bail!("candidate preservation incomplete; judging is unsafe");
3212 }
3213 if self.state.leaks.is_empty() {
3215 let cfg = self.state.config.blind.clone();
3216 let mut leaks = Vec::new();
3217 for c in &self.state.candidates {
3218 let Some(patch) =
3219 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3220 else {
3221 continue;
3222 };
3223 leaks.extend(blind::scan(
3224 &format!("candidate {} patch", c.label),
3225 &patch,
3226 &cfg.vendor_tokens,
3227 ));
3228 }
3229 if !leaks.is_empty() {
3230 let summary = leaks
3231 .iter()
3232 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3233 .collect::<Vec<_>>()
3234 .join(", ");
3235 match cfg.on_leak {
3236 LeakPolicy::Fail => {
3237 self.state.status = RunStatus::Failed;
3238 self.state
3239 .event("blind", format!("vendor text in a patch: {summary}"));
3240 self.state.leaks = leaks;
3241 self.state.save()?;
3242 self.settle_questions();
3243 bail!(
3244 "blind.on_leak = \"fail\" and vendor text reached a \
3245 judged patch: {summary}"
3246 );
3247 }
3248 LeakPolicy::Redact => self.state.event(
3249 "blind",
3250 format!("redacting vendor text for judging: {summary}"),
3251 ),
3252 LeakPolicy::Warn => self.state.event(
3253 "blind",
3254 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3255 ),
3256 }
3257 self.state.leaks = leaks;
3258 }
3259 }
3260
3261 if self.state.viable().is_empty() {
3262 if self.state.all_candidates_verified_noop() {
3263 self.state.status = RunStatus::VerifiedNoop;
3274 self.state.save()?;
3275 self.settle_questions();
3276 return Ok(());
3277 }
3278 self.state.status = RunStatus::Failed;
3279 self.state.save()?;
3280 self.settle_questions();
3281 bail!("no candidate produced a change; nothing to judge");
3282 }
3283 self.state.status = RunStatus::Judging;
3284 self.state.save()?;
3285 Ok(())
3286 }
3287
3288 async fn judge(&mut self) -> Result<()> {
3291 let run_id = self.state.id.clone();
3296 let prompts = self.state.config.prompts.clone();
3297 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3298 return Ok(());
3299 }
3300 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3301 if viable.len() == 1 {
3302 self.state.judge_skipped = true;
3309 self.state.event(
3310 "judge",
3311 format!(
3312 "only candidate {} produced a change; judging skipped",
3313 viable[0].label
3314 ),
3315 );
3316 self.state.save()?;
3317 return Ok(());
3318 }
3319 self.state.status = RunStatus::Judging;
3320
3321 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3322 let language = self.state.config.graph.language.clone();
3323 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3324 let sessions = self.state.config.graph.sessions;
3325 let artifacts = agent::artifacts_dir(&self.state.dir());
3326 let root = self.state.worktree_root();
3327 let base_short = short(&self.state.base_commit);
3328
3329 let mut jobs = Vec::new();
3330 let mut orders = Vec::new();
3331 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3332 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3333 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3334 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3335 let seat_key = format!("judge-{}", j + 1);
3336 let seat = self.seat(&seat_key, &spec.id);
3337 jobs.push(SeatJob {
3338 prompt: prompt::judge(
3339 &self.state.instruction,
3340 &views,
3341 self.roles.judges.len(),
3342 &base_short,
3343 &language,
3344 ),
3345 spec,
3346 seat,
3347 cwd: root.join(format!("judge-{}", j + 1)),
3348 timeout,
3349 allow_write: false,
3350 sessions,
3351 artifacts: artifacts.clone(),
3352 stem: format!("judge-{}", j + 1),
3353 handover: None,
3354 });
3355 }
3356
3357 self.state.event(
3358 "judge",
3359 format!(
3360 "{} judges ranking {} candidates blind",
3361 jobs.len(),
3362 viable.len()
3363 ),
3364 );
3365 let labels_for_check = labels.clone();
3366 let mut quota_losses = Vec::new();
3367 let cache = self.state.config.cache_dir();
3368 let ctx = WaveCtx {
3369 carry_seats: false,
3370 pauses: Some((&self.pause, &self.interrupt)),
3371 run: &run_id,
3372 node: "judge",
3373 prompts: &prompts,
3374 cache: cache.as_deref(),
3375 round: None,
3376 };
3377 let results = ask_json_wave::<Ranking>(
3378 jobs,
3379 Arc::clone(&self.sem),
3380 self.state.config.graph.retries,
3381 &self.roles.judge_roster,
3382 &ctx,
3383 &mut quota_losses,
3384 &mut self.state,
3385 &move |r: &Ranking| r.validate(&labels_for_check),
3386 )
3387 .await;
3388 if self.state.resource_block.is_some() {
3389 return self.stop_for_resource();
3390 }
3391 self.state.quota.extend(quota_losses);
3392
3393 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3394 let agent_id = seat.agent.clone();
3395 self.state.seats.insert(seat.key.clone(), seat);
3396 let mut record = Judgement {
3397 judge: j + 1,
3398 seat: format!("judge-{}", j + 1),
3399 agent: agent_id,
3400 ranking: Vec::new(),
3401 reasons: BTreeMap::new(),
3402 confidence: None,
3403 strengths: BTreeMap::new(),
3404 order: orders[j].clone(),
3405 failed: None,
3406 duration_ms: 0,
3407 };
3408 match res {
3409 Ok((ranking, out)) => {
3410 record.ranking = ranking.normalized();
3411 record.reasons = ranking.reasons;
3412 record.strengths = ranking.strengths;
3413 record.confidence = ranking.confidence;
3414 record.duration_ms = out.duration_ms;
3415 self.state.event(
3416 "judge",
3417 format!(
3418 "judge {} ranked {}",
3419 j + 1,
3420 record.ranking.iter().collect::<String>()
3421 ),
3422 );
3423 }
3424 Err(e) => {
3425 record.failed = Some(e.to_string());
3426 self.state
3427 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3428 }
3429 }
3430 self.state.judgements.push(record);
3431 self.state.save()?;
3432 }
3433 Ok(())
3434 }
3435
3436 async fn deliberate(&mut self) -> Result<()> {
3439 let run_id = self.state.id.clone();
3444 let prompts = self.state.config.prompts.clone();
3445 if !self.state.deliberation.is_empty() {
3446 return Ok(());
3447 }
3448 let tops: Vec<char> = self
3449 .state
3450 .judgements
3451 .iter()
3452 .filter_map(|j| j.ranking.first().copied())
3453 .collect();
3454 let rounds = self.state.config.graph.deliberate_rounds;
3455 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3456 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3457 self.state.event(
3458 "deliberate",
3459 format!("judges agreed on {} outright; no deliberation", tops[0]),
3460 );
3461 }
3462 self.state.status = RunStatus::Voting;
3463 self.state.save()?;
3464 return Ok(());
3465 }
3466
3467 self.state.status = RunStatus::Deliberating;
3468 self.state.event(
3469 "deliberate",
3470 format!(
3471 "split: first choices were {} — opening {rounds} round(s)",
3472 tops.iter().collect::<String>()
3473 ),
3474 );
3475
3476 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3477 let language = self.state.config.graph.language.clone();
3478 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3479 let sessions = self.state.config.graph.sessions;
3480 let artifacts = agent::artifacts_dir(&self.state.dir());
3481 let root = self.state.worktree_root();
3482 let base_short = short(&self.state.base_commit);
3483
3484 for round in 1..=rounds {
3488 let mut turns: Vec<DeliberationTurn> = Vec::new();
3489 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3490 if self.state.judgements[j].failed.is_some() {
3491 continue;
3492 }
3493 let seat_key = format!("judge-{}", j + 1);
3494 let spec = self.occupant(&seat_key, spec);
3495 let mut seat = self.seat(&seat_key, &spec.id);
3496 let transcript = self.transcript(&turns, j);
3497 let build = |context: Option<&str>| {
3498 prompt::deliberate(
3499 &self.state.instruction,
3500 context,
3501 &transcript,
3502 round,
3503 rounds,
3504 &language,
3505 )
3506 };
3507 let block = self.candidate_block(&viable, &base_short);
3508 let full = build(Some(&block));
3509 let text = if has_context(&spec, &seat, sessions) {
3510 build(None)
3511 } else {
3512 full.clone()
3513 };
3514 let job = SeatJob {
3515 spec,
3516 seat: seat.clone(),
3517 prompt: text,
3518 cwd: root.join(format!("judge-{}", j + 1)),
3519 timeout,
3520 allow_write: false,
3521 sessions,
3522 artifacts: artifacts.clone(),
3523 stem: format!("delib-{round}-judge-{}", j + 1),
3524 handover: Some(full),
3525 };
3526 let cache = self.state.config.cache_dir();
3527 let ctx = WaveCtx {
3528 carry_seats: false,
3529 pauses: Some((&self.pause, &self.interrupt)),
3530 run: &run_id,
3531 node: "deliberate",
3532 prompts: &prompts,
3533 cache: cache.as_deref(),
3534 round: None,
3535 };
3536 let mut losses = Vec::new();
3540 let mut results = ask_wave_with::<String>(
3541 vec![job],
3542 Arc::clone(&self.sem),
3543 0,
3544 &self.roles.judge_roster,
3545 &ctx,
3546 &mut losses,
3547 &mut self.state,
3548 &|text: &str| {
3549 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3550 },
3551 )
3552 .await;
3553 self.state.quota.extend(losses);
3554 let (updated, res, _) = results.pop().expect("one job in, one result out");
3555 seat = updated;
3556 let agent_id = seat.agent.clone();
3557 self.state.seats.insert(seat.key.clone(), seat);
3558 let body = match res {
3559 Ok((body, _)) => body,
3560 Err(e) => {
3563 self.state
3564 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3565 continue;
3566 }
3567 };
3568 let tentative = verdict::extract_json::<Position>(&body)
3569 .ok()
3570 .and_then(|p| p.tentative)
3571 .and_then(|s| s.trim().chars().next())
3572 .map(|c| c.to_ascii_uppercase());
3573 self.state.event(
3574 "deliberate",
3575 format!(
3576 "round {round}: judge {} now favours {}",
3577 j + 1,
3578 tentative.map_or("—".to_owned(), |c| c.to_string())
3579 ),
3580 );
3581 turns.push(DeliberationTurn {
3582 judge: j + 1,
3583 agent: agent_id,
3584 body: blind::sanitize_prose(&body, &self.state.config.blind),
3585 tentative,
3586 });
3587 }
3588 self.state
3589 .deliberation
3590 .push(DeliberationRound { round, turns });
3591 self.state.save()?;
3592 }
3593
3594 self.state.status = RunStatus::Voting;
3595 self.state.save()?;
3596 Ok(())
3597 }
3598
3599 async fn vote(&mut self) -> Result<()> {
3602 let run_id = self.state.id.clone();
3607 let prompts = self.state.config.prompts.clone();
3608 if !self.state.votes.is_empty() {
3609 return Ok(());
3610 }
3611 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3612 if viable.len() == 1 {
3613 return Ok(());
3614 }
3615 self.state.status = RunStatus::Voting;
3616
3617 let language = self.state.config.graph.language.clone();
3618 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3619 let sessions = self.state.config.graph.sessions;
3620 let artifacts = agent::artifacts_dir(&self.state.dir());
3621 let root = self.state.worktree_root();
3622 let base_short = short(&self.state.base_commit);
3623 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3624
3625 let mut jobs = Vec::new();
3626 let mut seats_at = Vec::new();
3627 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3628 if self
3629 .state
3630 .judgements
3631 .get(j)
3632 .is_some_and(|r| r.failed.is_some())
3633 {
3634 continue;
3635 }
3636 let seat_key = format!("judge-{}", j + 1);
3637 let spec = self.occupant(&seat_key, spec);
3638 let seat = self.seat(&seat_key, &spec.id);
3639 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3640 let text = if has_context(&spec, &seat, sessions) {
3641 prompt::final_vote(&viable, &language)
3642 } else {
3643 full.clone()
3644 };
3645 jobs.push(SeatJob {
3646 spec,
3647 seat,
3648 prompt: text,
3649 cwd: root.join(format!("judge-{}", j + 1)),
3650 timeout,
3651 allow_write: false,
3652 sessions,
3653 artifacts: artifacts.clone(),
3654 stem: format!("vote-judge-{}", j + 1),
3655 handover: Some(full),
3656 });
3657 seats_at.push(j);
3658 }
3659
3660 self.state.event(
3661 "vote",
3662 format!(
3663 "collecting {} final votes one by one, privately",
3664 jobs.len()
3665 ),
3666 );
3667 let allowed = viable.clone();
3668 let mut quota_losses = Vec::new();
3669 let cache = self.state.config.cache_dir();
3670 let ctx = WaveCtx {
3671 carry_seats: false,
3672 pauses: Some((&self.pause, &self.interrupt)),
3673 run: &run_id,
3674 node: "vote",
3675 prompts: &prompts,
3676 cache: cache.as_deref(),
3677 round: None,
3678 };
3679 let results = ask_json_wave::<FinalVote>(
3680 jobs,
3681 Arc::clone(&self.sem),
3682 self.state.config.graph.retries,
3683 &self.roles.judge_roster,
3684 &ctx,
3685 &mut quota_losses,
3686 &mut self.state,
3687 &move |v: &FinalVote| match v.label() {
3688 Some(c) if allowed.contains(&c) => Ok(()),
3689 other => bail!("vote {other:?} is not one of {allowed:?}"),
3690 },
3691 )
3692 .await;
3693 if self.state.resource_block.is_some() {
3694 return self.stop_for_resource();
3695 }
3696 self.state.quota.extend(quota_losses);
3697
3698 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3699 let agent_id = seat.agent.clone();
3700 self.state.seats.insert(seat.key.clone(), seat);
3701 let initial = self
3702 .state
3703 .judgements
3704 .get(j)
3705 .and_then(|r| r.ranking.first().copied());
3706 let mut record = VoteRecord {
3707 judge: j + 1,
3708 agent: agent_id,
3709 vote: None,
3710 reason: String::new(),
3711 changed: false,
3712 };
3713 match res {
3714 Ok((v, _)) => {
3715 record.vote = v.label();
3716 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3717 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3718 self.state.event(
3719 "vote",
3720 format!(
3721 "judge {} voted {}{}",
3722 j + 1,
3723 record.vote.unwrap_or('?'),
3724 if record.changed { " (changed)" } else { "" }
3725 ),
3726 );
3727 }
3728 Err(e) => {
3729 self.state
3730 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3731 }
3732 }
3733 self.state.votes.push(record);
3734 self.state.save()?;
3735 }
3736 Ok(())
3737 }
3738
3739 fn tally(&mut self) -> Result<()> {
3742 if self.state.tally.is_some() {
3743 return Ok(());
3744 }
3745 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3746 let tops: Vec<char> = self
3747 .state
3748 .judgements
3749 .iter()
3750 .filter_map(|j| j.ranking.first().copied())
3751 .collect();
3752 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3753
3754 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3757 let mut cast: Vec<char> = Vec::new();
3758 for (i, j) in self.state.judgements.iter().enumerate() {
3759 let vote = self
3760 .state
3761 .votes
3762 .iter()
3763 .find(|v| v.judge == i + 1)
3764 .and_then(|v| v.vote)
3765 .or_else(|| j.ranking.first().copied());
3766 if let Some(v) = vote {
3767 *first_choice.entry(v).or_insert(0) += 1;
3768 cast.push(v);
3769 }
3770 }
3771
3772 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3773 for j in &self.state.judgements {
3774 let n = j.ranking.len();
3775 for (pos, label) in j.ranking.iter().enumerate() {
3776 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3777 }
3778 }
3779
3780 let best = first_choice.values().copied().max().unwrap_or(0);
3781 let mut leaders: Vec<char> = first_choice
3782 .iter()
3783 .filter(|(_, v)| **v == best)
3784 .map(|(k, _)| *k)
3785 .collect();
3786 let mut tie_break = None;
3787 if leaders.len() > 1 {
3788 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3789 let borda_leaders: Vec<char> = leaders
3790 .iter()
3791 .copied()
3792 .filter(|l| borda[l] == top_borda)
3793 .collect();
3794 tie_break = Some(if borda_leaders.len() == 1 {
3795 format!(
3796 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3797 leaders.len()
3798 )
3799 } else {
3800 format!(
3801 "{} way tie on both first-choice votes and Borda points, broken by label order",
3802 leaders.len()
3803 )
3804 });
3805 leaders = borda_leaders;
3806 leaders.sort_unstable();
3807 }
3808 let winner = *leaders
3809 .first()
3810 .or(viable.first())
3811 .context("no candidate to declare a winner from")?;
3812
3813 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3814 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3815 let deliberated = !self.state.deliberation.is_empty();
3816
3817 let quota_seats: std::collections::BTreeSet<&str> =
3821 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3822 let mut present = 0usize;
3823 for (i, j) in self.state.judgements.iter().enumerate() {
3824 if quota_seats.contains(j.seat.as_str()) {
3825 continue;
3826 }
3827 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3828 let voted = self
3829 .state
3830 .votes
3831 .iter()
3832 .any(|v| v.judge == i + 1 && v.vote.is_some());
3833 if ranked || voted {
3834 present += 1;
3835 }
3836 }
3837 let needs_quorum = viable.len() > 1;
3843 let judges_total = if needs_quorum {
3844 self.roles.judges.len()
3845 } else {
3846 0
3847 };
3848 let quorum = if needs_quorum {
3849 judges_total / 2 + 1
3850 } else {
3851 0
3852 };
3853 let met_quorum = !needs_quorum || present >= quorum;
3854 let uncontested = (!needs_quorum).then(|| {
3855 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3856 });
3857
3858 self.state.event(
3859 "tally",
3860 match &uncontested {
3861 Some(reason) => format!("winner {winner} — {reason}"),
3862 None => format!(
3863 "winner {winner} — votes {} | initial {} | {} changed | \
3864 {present}/{judges_total} judges{}",
3865 first_choice
3866 .iter()
3867 .map(|(k, v)| format!("{k}:{v}"))
3868 .collect::<Vec<_>>()
3869 .join(" "),
3870 if unanimous_initial {
3871 "unanimous"
3872 } else {
3873 "split"
3874 },
3875 changed_votes,
3876 if met_quorum {
3877 String::new()
3878 } else {
3879 format!(" — below quorum ({quorum} required)")
3880 },
3881 ),
3882 },
3883 );
3884 if !met_quorum {
3885 self.state.event(
3886 "stall",
3887 format!(
3888 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3889 the run stops here, resumable"
3890 ),
3891 );
3892 }
3893 self.state.tally = Some(Tally {
3894 first_choice,
3895 borda,
3896 winner,
3897 rankings: tops.len(),
3898 unanimous_initial,
3899 deliberated,
3900 changed_votes,
3901 unanimous_final,
3902 tie_break,
3903 judges: judges_total,
3904 present,
3905 quorum,
3906 met_quorum,
3907 uncontested,
3908 });
3909 self.state.status = if met_quorum {
3910 RunStatus::Reviewing
3911 } else {
3912 RunStatus::Stalled
3913 };
3914 self.state.save()?;
3915 Ok(())
3916 }
3917
3918 #[allow(clippy::too_many_lines)]
3939 async fn recover_stall(&mut self) -> Result<bool> {
3940 let run_id = self.state.id.clone();
3945 let prompts = self.state.config.prompts.clone();
3946 let quota_seats: BTreeSet<&str> =
3951 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3952 let absent: Vec<String> = self
3953 .state
3954 .judgements
3955 .iter()
3956 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3957 .map(|j| j.seat.clone())
3958 .collect();
3959 if absent.is_empty() {
3960 return Ok(false);
3961 }
3962 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3963 if viable.len() <= 1 {
3964 return Ok(false);
3965 }
3966 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3967 let language = self.state.config.graph.language.clone();
3968 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3969 let sessions = self.state.config.graph.sessions;
3970 let artifacts = agent::artifacts_dir(&self.state.dir());
3971 let root = self.state.worktree_root();
3972 let base_short = short(&self.state.base_commit);
3973 let candidates: Vec<Candidate> = viable.clone();
3974
3975 let mut positions: Vec<usize> = absent
3977 .iter()
3978 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3979 .collect();
3980 if positions.is_empty() {
3981 return Ok(false);
3982 }
3983 positions.sort_unstable();
3984 positions.dedup();
3985
3986 let mut judge_jobs = Vec::new();
3988 for &j in &positions {
3989 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3990 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3991 let seat_key = format!("judge-{}", j + 1);
3992 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3993 let seat = self.seat(&seat_key, &spec.id);
3994 judge_jobs.push(SeatJob {
3995 spec,
3996 seat,
3997 prompt: prompt::judge(
3998 &self.state.instruction,
3999 &views,
4000 self.roles.judges.len(),
4001 &base_short,
4002 &language,
4003 ),
4004 cwd: root.join(seat_key),
4005 timeout,
4006 allow_write: false,
4007 sessions,
4008 artifacts: artifacts.clone(),
4009 stem: format!("judge-{}-recover", j + 1),
4010 handover: None,
4011 });
4012 }
4013
4014 let labels_for_check = labels.clone();
4015 let mut judge_losses = Vec::new();
4016 let retries = self.state.config.graph.retries;
4017 let cache = self.state.config.cache_dir();
4018 let ctx = WaveCtx {
4019 carry_seats: false,
4020 pauses: Some((&self.pause, &self.interrupt)),
4021 run: &run_id,
4022 node: "judge",
4023 prompts: &prompts,
4024 cache: cache.as_deref(),
4025 round: None,
4026 };
4027 let results = ask_json_wave::<Ranking>(
4028 judge_jobs,
4029 Arc::clone(&self.sem),
4030 retries,
4031 &self.roles.judge_roster,
4032 &ctx,
4033 &mut judge_losses,
4034 &mut self.state,
4035 &move |r: &Ranking| r.validate(&labels_for_check),
4036 )
4037 .await;
4038 if self.state.resource_block.is_some() {
4039 self.stop_for_resource()?;
4040 return Ok(false);
4041 }
4042
4043 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4045 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4046 let agent_id = seat.agent.clone();
4047 self.state.seats.insert(seat.key.clone(), seat);
4048 let record = &mut self.state.judgements[j];
4049 match res {
4050 Ok((ranking, out)) => {
4051 record.agent = agent_id;
4052 record.ranking = ranking.normalized();
4053 record.reasons = ranking.reasons;
4054 record.strengths = ranking.strengths;
4055 record.confidence = ranking.confidence;
4056 record.failed = None;
4057 record.duration_ms = out.duration_ms;
4058 recovered.insert(j);
4059 self.state.event(
4060 "recover",
4061 format!("judge {} ranked again after the limit", j + 1),
4062 );
4063 }
4064 Err(e) => {
4065 self.state
4066 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4067 }
4068 }
4069 }
4070
4071 let mut vote_jobs = Vec::new();
4073 let mut vote_pos: Vec<usize> = Vec::new();
4074 for &j in &recovered {
4075 let seat_key = format!("judge-{}", j + 1);
4076 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4077 let seat = self.seat(&seat_key, &spec.id);
4078 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4079 let text = if has_context(&spec, &seat, sessions) {
4080 prompt::final_vote(&labels, &language)
4081 } else {
4082 full.clone()
4083 };
4084 vote_jobs.push(SeatJob {
4085 spec,
4086 seat,
4087 prompt: text,
4088 cwd: root.join(seat_key),
4089 timeout,
4090 allow_write: false,
4091 sessions,
4092 artifacts: artifacts.clone(),
4093 stem: format!("vote-judge-{}-recover", j + 1),
4094 handover: Some(full),
4095 });
4096 vote_pos.push(j);
4097 }
4098 let allowed = labels.clone();
4099 let mut vote_losses = Vec::new();
4100 let vote_retries = self.state.config.graph.retries;
4101 let vote_cache = self.state.config.cache_dir();
4102 let ctx = WaveCtx {
4103 carry_seats: false,
4104 pauses: Some((&self.pause, &self.interrupt)),
4105 run: &run_id,
4106 node: "vote",
4107 prompts: &prompts,
4108 cache: vote_cache.as_deref(),
4109 round: None,
4110 };
4111 let votes = ask_json_wave::<FinalVote>(
4112 vote_jobs,
4113 Arc::clone(&self.sem),
4114 vote_retries,
4115 &self.roles.judge_roster,
4116 &ctx,
4117 &mut vote_losses,
4118 &mut self.state,
4119 &move |v: &FinalVote| match v.label() {
4120 Some(c) if allowed.contains(&c) => Ok(()),
4121 other => bail!("vote {other:?} is not one of {allowed:?}"),
4122 },
4123 )
4124 .await;
4125 if self.state.resource_block.is_some() {
4126 self.stop_for_resource()?;
4127 return Ok(false);
4128 }
4129 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4130 let agent_id = seat.agent.clone();
4131 self.state.seats.insert(seat.key.clone(), seat);
4132 match res {
4133 Ok((v, _)) => {
4134 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4135 rec.vote = v.label();
4136 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4137 } else {
4138 self.state.votes.push(VoteRecord {
4139 judge: j + 1,
4140 agent: agent_id,
4141 vote: v.label(),
4142 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4143 changed: false,
4144 });
4145 }
4146 self.state.event(
4147 "recover",
4148 format!("judge {} voted again after the limit", j + 1),
4149 );
4150 }
4151 Err(e) => {
4152 self.state
4153 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4154 }
4155 }
4156 }
4157
4158 let recovered_keys: BTreeSet<String> = recovered
4162 .iter()
4163 .map(|&j| format!("judge-{}", j + 1))
4164 .collect();
4165 self.state
4166 .quota
4167 .retain(|q| !recovered_keys.contains(&q.seat));
4168 for loss in judge_losses.into_iter().chain(vote_losses) {
4172 if recovered_keys.contains(&loss.seat) {
4173 continue;
4174 }
4175 self.state.quota.retain(|q| q.seat != loss.seat);
4176 self.state.quota.push(loss);
4177 }
4178
4179 self.state.tally = None;
4181 self.tally()?;
4182 Ok(self
4183 .state
4184 .tally
4185 .as_ref()
4186 .map(|t| t.met_quorum)
4187 .unwrap_or(false))
4188 }
4189
4190 async fn graft(&mut self) -> Result<()> {
4206 let Err(e) = self.graft_inner().await else {
4207 return Ok(());
4208 };
4209 if e.is::<crate::repo_lock::ResourceBlocked>() {
4210 return Err(e);
4211 }
4212 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4213 self.state.status = RunStatus::Blocked;
4214 self.state.event("graft", format!("blocked: {e:#}"));
4215 self.state.save()?;
4216 return Err(e);
4217 }
4218 if let Some(rec) = self.state.graft.clone()
4223 && !rec.outcome.settled()
4224 {
4225 if let Some(winner) = self.state.winner().cloned() {
4226 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4227 if re.is::<crate::repo_lock::ResourceBlocked>() {
4228 return Err(re);
4229 }
4230 self.state.status = RunStatus::Blocked;
4231 self.state.event("graft", format!("blocked: {re:#}"));
4232 self.state.save()?;
4233 return Err(re);
4234 }
4235 }
4236 if let Some(g) = self.state.graft.as_mut() {
4237 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4238 }
4239 }
4240 self.state.event("graft", format!("skipped: {e:#}"));
4241 if let Err(e) = self.state.save() {
4242 tracing::warn!("save after graft: {e:#}");
4243 }
4244 Ok(())
4245 }
4246
4247 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4250 let _coordination = self.coordinate(wt).await?;
4251 crate::recovery::restore_repair(wt, pre)
4252 .await
4253 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4254 }
4255
4256 fn graft_skipped(&mut self, why: &str) {
4258 let message = format!("skipped: {why}");
4259 if !self
4260 .state
4261 .events
4262 .iter()
4263 .any(|e| e.node == "graft" && e.message == message)
4264 {
4265 self.state.event("graft", message);
4266 }
4267 }
4268
4269 async fn graft_inner(&mut self) -> Result<()> {
4270 if let Some(rec) = self.state.graft.clone()
4273 && !rec.outcome.settled()
4274 {
4275 if self.state.reviews.is_empty()
4279 && let Some(winner) = self.state.winner().cloned()
4280 {
4281 self.graft_restore(&winner.worktree, &rec.pre).await?;
4282 }
4283 if rec.outcome == GraftOutcome::WaitingForRestore {
4284 self.state.graft = None;
4285 self.state.save()?;
4286 } else {
4287 if let Some(g) = self.state.graft.as_mut() {
4288 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4289 }
4290 self.state.event(
4291 "graft",
4292 "rolled back: the previous round was interrupted; the winner is untouched",
4293 );
4294 self.state.save()?;
4295 return Ok(());
4296 }
4297 }
4298 let plan = match graft::plan(&self.state) {
4299 Ok(plan) => plan,
4300 Err(skip) => {
4301 if skip != graft::Skip::AlreadyDone {
4302 self.graft_skipped(skip.reason());
4303 self.state.save()?;
4304 }
4305 return Ok(());
4306 }
4307 };
4308 let Some(winner) = self.state.winner().cloned() else {
4309 return Ok(());
4310 };
4311 let wt = winner.worktree.clone();
4312 if !git::is_clean(&wt).await? {
4313 self.graft_skipped("the winner's worktree has uncommitted changes");
4314 self.state.save()?;
4315 return Ok(());
4316 }
4317 let pre = git::rev_parse(&wt, "HEAD").await?;
4318 self.state.graft = Some(GraftRecord {
4321 pre: pre.clone(),
4322 post: None,
4323 lines: None,
4324 limit: None,
4325 agent: None,
4326 outcome: GraftOutcome::Running,
4327 requested: graft::requested(&plan),
4328 adoption: None,
4329 adoption_error: None,
4330 });
4331 self.state.event(
4332 "graft",
4333 format!(
4334 "porting strengths of candidate(s) {} onto {}",
4335 plan.strengths
4336 .iter()
4337 .map(|(l, _, _)| l.to_string())
4338 .collect::<Vec<_>>()
4339 .join(", "),
4340 plan.winner
4341 ),
4342 );
4343 self.state.save()?;
4344
4345 let fix_prompt = prompt::graft(
4346 &self.state.instruction,
4347 plan.winner,
4348 &plan.strengths,
4349 &self.state.config.graph.language,
4350 );
4351 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4352 let sessions = self.state.config.graph.sessions;
4353 let artifacts = agent::artifacts_dir(&self.state.dir());
4354 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4355 let (_, seat, out) = self
4356 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4357 SeatJob {
4358 prompt: fix_prompt.clone(),
4359 spec,
4360 seat,
4361 cwd: wt.clone(),
4362 timeout,
4363 allow_write: true,
4364 sessions,
4365 artifacts: artifacts.clone(),
4366 stem: "graft".to_owned(),
4367 handover: None,
4368 }
4369 })
4370 .await;
4371 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4372 self.state.graft.as_mut().expect("graft record").outcome =
4375 GraftOutcome::WaitingForRestore;
4376 return self.stop_for_resource();
4377 }
4378 let mut why: Option<String> = None;
4379 match &out {
4380 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4381 why = Some("the fixer produced nothing usable".to_owned());
4382 }
4383 AgentOutcome::Ok(_) => {}
4384 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4385 AgentOutcome::Quota(o) => {
4386 self.state.quota.push(QuotaLoss {
4387 seat: seat.key.clone(),
4388 node: "graft".to_owned(),
4389 at: Timestamp::now(),
4390 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4391 });
4392 why = Some("rate limited (quota)".to_owned());
4393 }
4394 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4395 why = Some(format!(
4396 "the fixer failed: {}",
4397 e.lines()
4398 .next()
4399 .unwrap_or("")
4400 .chars()
4401 .take(160)
4402 .collect::<String>()
4403 ));
4404 }
4405 }
4406 let agent_id = seat.agent.clone();
4407 self.state.seats.insert(seat.key.clone(), seat);
4408
4409 let mut post = None;
4410 let mut lines = None;
4411 let _coordination = self.coordinate(&wt).await?;
4412 if why.is_none() {
4413 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4414 if let Ok(r) = &rescued {
4415 self.state.note_withheld("graft", &r.withheld);
4416 }
4417 let head = git::rev_parse(&wt, "HEAD").await?;
4418 let dirty = match &rescued {
4422 Ok(r) => git::status_porcelain(&wt)
4423 .await
4424 .map(|st| {
4425 st.lines().any(|l| {
4426 !r.withheld
4427 .iter()
4428 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4429 })
4430 })
4431 .unwrap_or(true),
4432 Err(_) => true,
4433 };
4434 if dirty {
4435 why = Some("the fixer left changes that could not be committed".to_owned());
4436 } else if head == pre {
4437 why = Some("the fixer changed nothing".to_owned());
4438 } else {
4439 lines = git::changed_lines(&wt, &pre, &head).await;
4440 post = Some(head);
4441 }
4442 }
4443 let (adoption, adoption_error) = match &out {
4444 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4445 _ => (None, None),
4446 };
4447 let requested = graft::requested(&plan);
4448 let outcome = match why {
4449 None => {
4450 self.state.event(
4451 "graft",
4452 format!(
4453 "applied: {}",
4454 lines.map_or_else(
4455 || "changed lines unavailable".to_owned(),
4456 |n| format!("{n} changed lines")
4457 )
4458 ),
4459 );
4460 GraftOutcome::Applied
4461 }
4462 Some(why) => {
4463 crate::recovery::restore_repair(&wt, &pre)
4466 .await
4467 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4468 self.state.event(
4469 "graft",
4470 format!("rolled back: {why}; the winner is untouched"),
4471 );
4472 GraftOutcome::RolledBack(why)
4473 }
4474 };
4475 self.state.graft = Some(GraftRecord {
4476 pre,
4477 post,
4478 lines,
4479 limit: None,
4480 agent: Some(agent_id),
4481 outcome,
4482 requested,
4483 adoption,
4484 adoption_error,
4485 });
4486 self.state.save()?;
4487 Ok(())
4488 }
4489
4490 async fn fold_losers(&mut self) -> Result<()> {
4493 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4494 return Ok(());
4495 };
4496 if !self.state.candidate_preservation.is_empty() {
4497 bail!("candidate preservation incomplete; refusing cleanup");
4498 }
4499 let repo = self.state.repo.clone();
4500 let _coordination = self.coordinate(&repo).await?;
4501 let mut folded = Vec::new();
4502 for i in 0..self.state.candidates.len() {
4503 let c = &self.state.candidates[i];
4504 if c.label == winner || c.folded {
4505 continue;
4506 }
4507 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4508 remove_preserved_worktree(&repo, &wt).await?;
4509 if git::branch_exists(&repo, &branch).await?
4510 && !git::branch_delete(&repo, &branch).await?
4511 {
4512 bail!("candidate branch removal refused");
4513 }
4514 self.state.candidates[i].folded = true;
4515 self.state.save()?;
4516 folded.push(label.to_string());
4517 }
4518 let root = self.state.worktree_root();
4520 for j in 1..=self.roles.judges.len() {
4521 let wt = root.join(format!("judge-{j}"));
4522 if wt.exists() {
4523 remove_preserved_worktree(&repo, &wt).await?;
4524 }
4525 }
4526 if self.state.config.graph.advise {
4529 for k in 1..=self.state.config.graph.advisors {
4530 let wt = root.join(format!("advisor-{k}"));
4531 if wt.exists() {
4532 remove_preserved_worktree(&repo, &wt).await?;
4533 }
4534 }
4535 }
4536 if !folded.is_empty() {
4537 self.state
4538 .event("fold", format!("folded candidates {}", folded.join(", ")));
4539 self.state.save()?;
4540 }
4541 Ok(())
4542 }
4543
4544 async fn sync_to_base(&mut self) -> Result<()> {
4584 if self.state.status == RunStatus::AlreadyInBase {
4585 return Ok(());
4586 }
4587 let conflicted = self
4588 .state
4589 .base_sync
4590 .as_ref()
4591 .is_some_and(|s| s.conflict.is_some());
4592 let Some(winner) = self.state.winner().cloned() else {
4593 return Ok(());
4594 };
4595
4596 let repo = self.state.repo.clone();
4597 let remote = self.state.config.merge.remote.clone();
4598 let base_branch = self.state.base_branch.clone();
4599 let tracking = format!("{remote}/{base_branch}");
4600
4601 git::fetch(&repo, &remote, &base_branch).await.ok();
4602 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4607 return Ok(());
4608 };
4609
4610 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4611 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4612 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4613
4614 if (behind > 0 || conflicted || head == tip)
4623 && self
4624 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4625 .await?
4626 {
4627 return Ok(());
4628 }
4629 if conflicted {
4630 return Ok(());
4631 }
4632
4633 if behind == 0 {
4634 if let Some(from) = self
4643 .state
4644 .rebase_fixes
4645 .iter()
4646 .rev()
4647 .find_map(|r| r.from.clone())
4648 && from != head
4649 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4650 .await
4651 .is_ok_and(|o| o.ok())
4652 {
4653 git::sync_to_head(&winner.worktree).await?;
4654 }
4655 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4661 if let Some(why) = &conflict {
4662 self.state.status = RunStatus::Blocked;
4663 self.state.event("land", why.clone());
4664 }
4665 self.state.base_sync = Some(BaseSync {
4666 tip,
4667 behind: 0,
4668 attempts,
4669 conflict,
4670 already_in: None,
4671 });
4672 self.state.save()?;
4673 return Ok(());
4674 }
4675
4676 if attempts >= BASE_SYNC_ROUNDS {
4677 let why = format!(
4678 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4679 rebase(s); rebasing again would only race it",
4680 winner.branch
4681 );
4682 self.state.status = RunStatus::Blocked;
4683 self.state.base_sync = Some(BaseSync {
4684 tip,
4685 behind,
4686 attempts,
4687 conflict: Some(why.clone()),
4688 already_in: None,
4689 });
4690 self.state.event("land", why);
4691 self.state.save()?;
4692 return Ok(());
4693 }
4694
4695 self.state.event(
4696 "land",
4697 format!(
4698 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4699 winner.branch
4700 ),
4701 );
4702 self.state.save()?;
4703
4704 let branch_tracking = format!("{remote}/{}", winner.branch);
4710 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4711 let remote_tip = if self.state.task_pr_branch().is_none()
4712 && matches!(&fetched_branch, Ok(o) if o.ok())
4713 {
4714 git::rev_parse(&repo, &branch_tracking).await.ok()
4715 } else {
4716 None
4717 };
4718 if let Some(theirs) = &remote_tip
4722 && !git::is_ancestor(&repo, theirs, &head).await
4723 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4724 .await
4725 .is_ok_and(|missing| missing.is_empty())
4726 {
4727 let why = format!(
4728 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4729 them",
4730 short(theirs),
4731 winner.branch
4732 );
4733 self.state.status = RunStatus::Blocked;
4734 self.state.base_sync = Some(BaseSync {
4735 tip,
4736 behind,
4737 attempts,
4738 conflict: Some(why.clone()),
4739 already_in: None,
4740 });
4741 self.state.event("land", why);
4742 self.state.save()?;
4743 return Ok(());
4744 }
4745
4746 let scratch = self.state.dir().join("base-sync");
4747 let rebased = match crate::rebase::rebase_with_fixer(
4748 &mut self.state,
4749 &scratch,
4750 &winner.branch,
4751 &tracking,
4752 )
4753 .await
4754 {
4755 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4756 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4757 Err(e) => Err(e),
4758 };
4759 let attempts = attempts + 1;
4760 match rebased {
4761 Ok(None) => {
4762 git::sync_to_head(&winner.worktree).await?;
4766 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4767 let mut conflict = None;
4768 if let Some(pinned) = &remote_tip {
4769 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4770 match pushed {
4771 Ok(o) if o.ok() => self.state.event(
4772 "land",
4773 format!("pushed rebased {} to {remote}", winner.branch),
4774 ),
4775 Ok(o) => {
4776 conflict = Some(format!(
4777 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4778 winner.branch,
4779 short(pinned),
4780 o.stderr.chars().take(600).collect::<String>()
4781 ));
4782 }
4783 Err(e) => {
4784 conflict = Some(format!(
4785 "rebased {} locally but could not push it: {e:#}",
4786 winner.branch
4787 ));
4788 }
4789 }
4790 }
4791 if let Some(why) = &conflict {
4792 self.state.status = RunStatus::Blocked;
4793 self.state.event("land", why.clone());
4794 }
4795 self.state.base_sync = Some(BaseSync {
4796 tip: tip.clone(),
4797 behind: 0,
4798 attempts,
4799 conflict,
4800 already_in: None,
4801 });
4802 self.state
4803 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4804 }
4805 Ok(Some(conflict)) => {
4806 let why = format!(
4807 "{} conflicts with {tracking} and did not rebase: {}",
4808 winner.branch,
4809 conflict.chars().take(600).collect::<String>()
4810 );
4811 self.state.status = RunStatus::Blocked;
4812 self.state.base_sync = Some(BaseSync {
4813 tip,
4814 behind,
4815 attempts,
4816 conflict: Some(why.clone()),
4817 already_in: None,
4818 });
4819 self.state.event("land", why);
4820 }
4821 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4825 self.state.save()?;
4826 return Ok(());
4827 }
4828 Err(e) => {
4829 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4830 self.state.status = RunStatus::Blocked;
4831 self.state.base_sync = Some(BaseSync {
4832 tip,
4833 behind,
4834 attempts,
4835 conflict: Some(why.clone()),
4836 already_in: None,
4837 });
4838 self.state.event("land", why);
4839 }
4840 }
4841 self.state.save()?;
4842 Ok(())
4843 }
4844
4845 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4851 if self.state.task_pr_branch().is_some() {
4854 return None;
4855 }
4856 let repo = self.state.repo.clone();
4857 let remote = self.state.config.merge.remote.clone();
4858 let fetched = git::fetch(&repo, &remote, branch).await;
4859 if !matches!(&fetched, Ok(o) if o.ok()) {
4860 return None;
4861 }
4862 let branch_tracking = format!("{remote}/{branch}");
4863 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4864 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4865 return None;
4866 }
4867 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4868 .await
4869 .is_ok_and(|missing| missing.is_empty())
4870 {
4871 return Some(format!(
4872 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4873 them",
4874 short(&theirs)
4875 ));
4876 }
4877 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4878 Ok(o) if o.ok() => {
4879 self.state
4880 .event("land", format!("pushed rebased {branch} to {remote}"));
4881 None
4882 }
4883 Ok(o) => Some(format!(
4884 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4885 someone may have pushed): {}",
4886 short(&theirs),
4887 o.stderr.chars().take(600).collect::<String>()
4888 )),
4889 Err(e) => Some(format!(
4890 "{branch} is rebased locally but could not be pushed: {e:#}"
4891 )),
4892 }
4893 }
4894
4895 async fn settle_already_in(
4912 &mut self,
4913 branch: &str,
4914 tip: &str,
4915 head: &str,
4916 attempts: usize,
4917 behind: usize,
4918 ) -> Result<bool> {
4919 let repo = self.state.repo.clone();
4920 let remote = self.state.config.merge.remote.clone();
4921 let start = self.state.base_commit.clone();
4922 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4923 Ok(Some(e)) => e,
4924 Ok(None) => return Ok(false),
4925 Err(e) => {
4926 tracing::warn!("already-in-base check for {branch}: {e:#}");
4927 return Ok(false);
4928 }
4929 };
4930 let mut verified = vec![head.to_owned()];
4931 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4932 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4933 if matches!(&fetched, Ok(o) if o.ok())
4934 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4935 && theirs != head
4936 {
4937 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4938 Ok(Some(_)) => verified.push(theirs),
4939 _ => return Ok(false),
4940 }
4941 }
4942 let base_branch = self.state.base_branch.clone();
4943 let message = format!(
4944 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4945 land",
4946 evidence.names(),
4947 evidence.proof.as_str()
4948 );
4949 let closed =
4950 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4951 .await;
4952 match closed {
4953 Ok(Ok(url)) => self
4954 .state
4955 .event("land", format!("closed {url}: superseded on {base_branch}")),
4956 Ok(Err(why)) => self
4957 .state
4958 .event("land", format!("did not close a pull request: {why}")),
4959 Err(e) => {
4960 let why = format!(
4961 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4962 not be closed ({e:#}); resume to retry"
4963 );
4964 self.state.status = RunStatus::Blocked;
4965 self.state.base_sync = Some(BaseSync {
4966 tip: tip.to_owned(),
4967 behind,
4968 attempts,
4969 conflict: Some(why.clone()),
4970 already_in: None,
4971 });
4972 self.state.event("land", why);
4973 self.state.save()?;
4974 return Ok(true);
4975 }
4976 }
4977 self.state.status = RunStatus::AlreadyInBase;
4978 self.state.base_sync = Some(BaseSync {
4979 tip: tip.to_owned(),
4980 behind,
4981 attempts,
4982 conflict: None,
4983 already_in: Some(evidence),
4984 });
4985 self.state.event("land", message);
4986 self.state.save()?;
4987 self.settle_questions();
4988 Ok(true)
4989 }
4990
4991 fn landing_base(&self) -> String {
5001 self.state
5002 .base_sync
5003 .as_ref()
5004 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5005 }
5006
5007 pub async fn fix_selected(
5040 &mut self,
5041 ids: &[String],
5042 reason: &str,
5043 allow_stale: bool,
5044 ) -> Result<()> {
5045 let reason = reason.trim();
5046 if reason.is_empty() {
5047 bail!("a fix request needs a reason — that is the operator's own record of why");
5048 }
5049 if ids.is_empty() {
5050 bail!("no finding id given");
5051 }
5052 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5053 bail!(
5054 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5055 has already concluded — can be given a targeted fix. A run still \
5056 in progress should simply be resumed; a `merged` run's branch has \
5057 already landed, so its answer is a fresh `magi review <branch>`, \
5058 not reopening this run's own record",
5059 self.state.id,
5060 self.state.status.as_str()
5061 );
5062 }
5063 let Some(winner) = self.state.winner().cloned() else {
5064 bail!("run {} has no winning candidate to fix", self.state.id);
5065 };
5066 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5067 bail!(
5068 "branch `{}` no longer exists; this run cannot be extended",
5069 winner.branch
5070 );
5071 }
5072 let home = crate::run::home();
5073 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5074 bail!(
5075 "run {} is currently being worked on by another magi process",
5076 self.state.id
5077 );
5078 }
5079 let _claim = FixClaim::acquire(&self.state.dir())?;
5085
5086 let mut seen = BTreeSet::new();
5090 let mut findings = Vec::new();
5091 let mut missing = Vec::new();
5092 for id in ids {
5093 if !seen.insert(id.clone()) {
5094 continue;
5095 }
5096 match self.state.finding(id) {
5097 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5098 id: f.id.clone(),
5099 severity: f.severity,
5100 reviewer_vote: rec.vote,
5101 round: round.round,
5102 round_head: round.head.clone(),
5103 reviewer: rec.reviewer,
5104 agent: rec.agent.clone(),
5105 file: f.file.clone(),
5106 line: f.line,
5107 title: f.title.clone(),
5108 detail: f.detail.clone(),
5109 outcome: OperatorFixOutcome::Pending,
5110 }),
5111 None => missing.push(id.clone()),
5112 }
5113 }
5114 if !missing.is_empty() {
5115 bail!(
5116 "unknown finding id(s): {}; nothing was changed",
5117 missing.join(", ")
5118 );
5119 }
5120
5121 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5122 let stale_details: Vec<(String, String)> = findings
5123 .iter()
5124 .filter(|f| f.round_head != head_at_request)
5125 .map(|f| (f.id.clone(), f.round_head.clone()))
5126 .collect();
5127 let stale = !stale_details.is_empty();
5128 if stale && !allow_stale {
5129 bail!(
5130 "the branch has moved since some finding(s) were raised — {} — now \
5131 at {}; pass --allow-stale to fix anyway, or re-run review first",
5132 stale_details
5133 .iter()
5134 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5135 .collect::<Vec<_>>()
5136 .join(", "),
5137 short(&head_at_request)
5138 );
5139 }
5140
5141 let request = OperatorFixRequest {
5142 requested_at: Timestamp::now(),
5143 reason: reason.to_owned(),
5144 findings,
5145 head_at_request: head_at_request.clone(),
5146 allow_stale,
5147 stale,
5148 fix: None,
5149 result_head: None,
5150 follow_up_review_run: None,
5151 };
5152 self.state.event(
5153 "fix",
5154 format!(
5155 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5156 request.findings.len(),
5157 request
5158 .findings
5159 .iter()
5160 .map(|f| f.id.as_str())
5161 .collect::<Vec<_>>()
5162 .join(", "),
5163 ),
5164 );
5165 self.state.operator_fixes.push(request);
5172 self.state.save()?;
5173 let request_index = self.state.operator_fixes.len() - 1;
5174
5175 let repo = self.state.repo.clone();
5184 let coordination = self.coordinate(&repo).await?;
5185 if winner.worktree.exists() {
5186 let dirty = git::git(
5189 &winner.worktree,
5190 &["status", "--porcelain", "--untracked-files=all"],
5191 )
5192 .await?;
5193 let only_withheld = dirty.lines().all(|l| {
5194 l.strip_prefix("?? ")
5195 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5196 });
5197 if !only_withheld {
5198 bail!(
5199 "`{}` has uncommitted changes; refusing to touch it — commit or \
5200 discard them first",
5201 winner.worktree.display()
5202 );
5203 }
5204 git::worktree_remove(&self.state.repo, &winner.worktree)
5205 .await
5206 .ok();
5207 }
5208 let fix_worktree = self.state.worktree_root().join("operator-fix");
5209 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5210 git::git(
5211 &self.state.repo,
5212 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5213 )
5214 .await
5215 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5216 if !git::is_clean(&fix_worktree).await? {
5217 git::worktree_remove(&self.state.repo, &fix_worktree)
5218 .await
5219 .ok();
5220 bail!(
5221 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5222 winner.branch
5223 );
5224 }
5225 let repo_for_setup = self.state.repo.clone();
5226 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5227 Ok(()) => {
5228 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5229 }
5230 Err(e) => Err(e),
5231 };
5232 if let Err(e) = setup {
5233 git::worktree_remove(&self.state.repo, &fix_worktree)
5234 .await
5235 .ok();
5236 return Err(e);
5237 }
5238
5239 let run_id = self.state.id.clone();
5240 let prompts = self.state.config.prompts.clone();
5241 let language = self.state.config.graph.language.clone();
5242 let sessions = self.state.config.graph.sessions;
5243 let artifacts = agent::artifacts_dir(&self.state.dir());
5244 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5245 .findings
5246 .iter()
5247 .map(|f| Finding {
5248 id: f.id.clone(),
5249 severity: f.severity,
5250 file: f.file.clone(),
5251 line: f.line,
5252 title: f.title.clone(),
5253 detail: f.detail.clone(),
5254 })
5255 .collect();
5256 let fix_prompt = prompt::operator_fix_in(
5257 &self.state.instruction,
5258 &finding_list,
5259 reason,
5260 &stale_details,
5261 &head_at_request,
5262 &language,
5263 &self.state.config.graph.github_language,
5264 );
5265 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5266 drop(coordination);
5267 let (job, seat, out) = self
5268 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5269 prompt: fix_prompt.clone(),
5270 spec,
5271 seat,
5272 cwd: fix_worktree.clone(),
5273 timeout,
5274 allow_write: true,
5275 sessions,
5276 artifacts: artifacts.clone(),
5277 stem: "operator-fix".to_owned(),
5278 handover: None,
5279 })
5280 .await;
5281 let agent_id = seat.agent.clone();
5282
5283 let mut fix = FixRecord {
5284 agent: agent_id,
5285 addressed: Vec::new(),
5286 rejected: Vec::new(),
5287 notes: String::new(),
5288 committed: false,
5289 failed: None,
5290 duration_ms: 0,
5291 continuation: None,
5292 };
5293 let mut final_seat = seat.clone();
5294 match out {
5295 AgentOutcome::Ok(o) => {
5296 fix.duration_ms = o.duration_ms;
5297 let parsed = verdict::extract_json::<FixReport>(&o.text);
5298 let incomplete_reason = match &parsed {
5299 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5300 "the reply parsed, but it reported a command whose own CLI \
5301 never confirmed an exit status"
5302 .to_owned(),
5303 ),
5304 Ok(_) => None,
5305 Err(e) => Some(e.to_string()),
5306 };
5307 match incomplete_reason {
5308 None => {
5309 let report = parsed.expect("checked Ok above");
5310 fix.addressed = report.addressed;
5311 fix.rejected = report.rejected;
5312 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5313 }
5314 Some(reason) => {
5315 let (resumed_seat, resolved, failure, cont) = self
5316 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5317 .await;
5318 fix.duration_ms += cont.cumulative_wait_ms;
5319 fix.continuation = Some(cont);
5320 final_seat = resumed_seat;
5321 match resolved {
5322 Some(report) => {
5323 fix.addressed = report.addressed;
5324 fix.rejected = report.rejected;
5325 fix.notes =
5326 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5327 }
5328 None => fix.failed = failure,
5329 }
5330 }
5331 }
5332 }
5333 AgentOutcome::Dropped(o) => {
5334 fix.duration_ms = o.duration_ms;
5335 let why = o
5336 .dropped
5337 .as_ref()
5338 .map(|d| d.why.as_str())
5339 .unwrap_or("the CLI ended the stream without delivering its answer");
5340 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5341 }
5342 AgentOutcome::Quota(o) => {
5343 self.state.quota.push(QuotaLoss {
5344 seat: final_seat.key.clone(),
5345 node: "fix".to_owned(),
5346 at: Timestamp::now(),
5347 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5348 });
5349 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5350 }
5351 AgentOutcome::Failed(e) => fix.failed = Some(e),
5352 AgentOutcome::ResourceBlocked(e) => {
5353 fix.failed = Some(format!("the fixer was not started: {e}"));
5354 }
5355 }
5356 if fix.continuation.is_none() {
5357 fix.continuation = Some(ContinuationRecord::not_needed());
5358 }
5359 self.state.seats.insert(final_seat.key.clone(), final_seat);
5360
5361 let rescue_message = format!(
5362 "magi: operator-selected fix ({}) (uncommitted work)",
5363 self.state.operator_fixes[request_index]
5364 .findings
5365 .iter()
5366 .map(|f| f.id.as_str())
5367 .collect::<Vec<_>>()
5368 .join(", ")
5369 );
5370 let repo = self.state.repo.clone();
5371 let coordination = self.coordinate(&repo).await?;
5372 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5373 self.state.note_withheld("fix", &r.withheld);
5374 }
5375 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5376 fix.committed = after != head_at_request;
5377 git::worktree_remove(&self.state.repo, &fix_worktree)
5378 .await
5379 .ok();
5380 drop(coordination);
5381
5382 self.state.event(
5383 "fix",
5384 match &fix.failed {
5385 Some(reason) => format!(
5386 "operator fix: adoption report was lost ({reason}); {}",
5387 if fix.committed {
5388 "committed"
5389 } else {
5390 "NO new commit"
5391 }
5392 ),
5393 None => format!(
5394 "operator fix: {} addressed, {} rejected, {}",
5395 fix.addressed.len(),
5396 fix.rejected.len(),
5397 if fix.committed {
5398 "committed"
5399 } else {
5400 "NO new commit"
5401 }
5402 ),
5403 },
5404 );
5405
5406 for f in &mut self.state.operator_fixes[request_index].findings {
5413 f.outcome = if fix.failed.is_some() {
5414 OperatorFixOutcome::Unreported
5415 } else if fix.addressed.contains(&f.id) {
5416 OperatorFixOutcome::Addressed
5417 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5418 OperatorFixOutcome::Rejected { why: r.why.clone() }
5419 } else {
5420 OperatorFixOutcome::Unreported
5421 };
5422 }
5423
5424 let committed = fix.committed;
5425 if committed {
5426 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5427 }
5428 self.state.operator_fixes[request_index].fix = Some(fix);
5429 self.state.save()?;
5432
5433 if committed {
5434 self.state.event(
5435 "fix",
5436 format!(
5437 "operator fix committed {}; opening a follow-up review-only run",
5438 short(&after)
5439 ),
5440 );
5441 let origin =
5444 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5445 match Self::review(
5446 &self.state.repo,
5447 &winner.branch,
5448 self.state.config.clone(),
5449 origin,
5450 )
5451 .await
5452 {
5453 Ok(mut follow_up) => {
5454 follow_up.state.event(
5455 "start",
5456 format!(
5457 "requested by an operator fix on run {} for finding(s) {}",
5458 self.state.id,
5459 self.state.operator_fixes[request_index]
5460 .findings
5461 .iter()
5462 .map(|f| f.id.as_str())
5463 .collect::<Vec<_>>()
5464 .join(", "),
5465 ),
5466 );
5467 follow_up.state.save()?;
5468 let follow_up_id = follow_up.state.id.clone();
5469 let adopted = match crate::direct::adopt(
5472 &follow_up.state,
5473 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5474 ) {
5475 Ok(a) => a,
5476 Err(e) => {
5477 self.state.event(
5480 "fix",
5481 format!(
5482 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5483 ),
5484 );
5485 self.state.save()?;
5486 return Ok(());
5487 }
5488 };
5489 let executed = follow_up.execute().await;
5490 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5491 if let Some(a) = adopted {
5492 a.finish(&follow_up.state, executed);
5493 }
5494 if let Some(e) = failure {
5495 self.state.event(
5496 "fix",
5497 format!(
5498 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5499 ),
5500 );
5501 }
5502 self.state.operator_fixes[request_index].follow_up_review_run =
5503 Some(follow_up_id);
5504 }
5505 Err(e) => {
5506 self.state.event(
5507 "fix",
5508 format!("committed the fix but could not open a follow-up review: {e:#}"),
5509 );
5510 }
5511 }
5512 self.state.save()?;
5513 }
5514
5515 Ok(())
5516 }
5517
5518 async fn ask_fixer(
5538 &mut self,
5539 winner: &Candidate,
5540 node: &'static str,
5541 round: Option<usize>,
5542 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5543 ) -> (SeatJob, SeatState, AgentOutcome) {
5544 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5545 self.ask_fixer_among(attempts, node, round, None, build)
5546 .await
5547 }
5548
5549 async fn ask_fixer_among(
5554 &mut self,
5555 attempts: Vec<(AgentSpec, String)>,
5556 node: &'static str,
5557 round: Option<usize>,
5558 restore: Option<(&Path, &str)>,
5559 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5560 ) -> (SeatJob, SeatState, AgentOutcome) {
5561 let run_id = self.state.id.clone();
5562 let prompts = self.state.config.prompts.clone();
5563 let cache = self.state.config.cache_dir();
5564 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5565 let mut last = None;
5566 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5567 let seat = self.seat(&key, &spec.id);
5568 let mut job = build(spec.clone(), seat);
5569 if i > 0 {
5570 job.stem = format!("{}-{}", job.stem, spec.id);
5571 }
5572 let ctx = WaveCtx {
5573 carry_seats: false,
5574 pauses: Some((&self.pause, &self.interrupt)),
5575 run: &run_id,
5576 node,
5577 prompts: &prompts,
5578 cache: cache.as_deref(),
5579 round,
5580 };
5581 let (seat, out) =
5582 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5583 let advances = match &out {
5586 AgentOutcome::Ok(o) => agent::output_advances(o),
5587 AgentOutcome::ResourceBlocked(_) => false,
5588 _ => true,
5589 };
5590 if let Some(next) = ids.get(i + 1)
5591 && advances
5592 {
5593 self.state.seats.insert(seat.key.clone(), seat.clone());
5594 let class =
5595 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5596 let reason = match &out {
5597 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5598 other => fail_reason(other),
5599 };
5600 if let Some((wt, rev)) = restore
5604 && let Err(e) = self.graft_restore(wt, rev).await
5605 {
5606 let why =
5607 format!("could not restore the worktree before the next fixer: {e:#}");
5608 tracing::warn!("{why}");
5609 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5610 AgentOutcome::ResourceBlocked(why)
5611 } else {
5612 AgentOutcome::Failed(why)
5613 };
5614 last = Some((job, seat, out));
5615 break;
5616 }
5617 record_handover(
5618 &mut self.state,
5619 node,
5620 &seat.key,
5621 &spec.id,
5622 next,
5623 &class,
5624 &reason,
5625 );
5626 continue;
5627 }
5628 last = Some((job, seat, out));
5629 break;
5630 }
5631 last.expect("the fixer chain always has an entry")
5632 }
5633
5634 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5645 use crate::recovery::{Mode, Repair};
5646 if !self.state.reviews.is_empty() {
5647 return Ok(());
5648 }
5649 let Some(mut rec) = self.state.recovery.clone() else {
5650 return Ok(());
5651 };
5652 if rec.mode != Mode::VerifyRepair {
5653 return Ok(());
5654 }
5655 let wt = winner.worktree.clone();
5656 let mut head = git::rev_parse(&wt, "HEAD").await?;
5657 match rec.repair.clone() {
5658 Repair::Done { .. } => return Ok(()),
5659 Repair::WaitingForRestore { before } => {
5660 self.graft_restore(&wt, &before).await?;
5661 head = before;
5662 rec.repair = Repair::NotStarted;
5663 self.state.recovery = Some(rec.clone());
5664 self.state.save()?;
5665 }
5666 Repair::Started { before } => {
5667 if let Err(e) = self.graft_restore(&wt, &before).await {
5671 if e.is::<crate::repo_lock::ResourceBlocked>() {
5672 return Err(e);
5673 }
5674 return self
5675 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5676 .await;
5677 }
5678 rec.repair = Repair::Done {
5679 before: before.clone(),
5680 after: before,
5681 committed: false,
5682 failed: Some("interrupted; rolled back".to_owned()),
5683 };
5684 self.state
5685 .event("recovery", "the repair was interrupted and rolled back");
5686 self.state.recovery = Some(rec);
5687 self.state.save()?;
5688 return Ok(());
5689 }
5690 Repair::NotStarted => {}
5691 }
5692 if crate::recovery::recheck(&mut rec, &head) {
5693 self.state.event("recovery", rec.summary());
5694 self.state.recovery = Some(rec);
5695 self.state.save()?;
5696 return Ok(());
5697 }
5698 rec.repair = Repair::Started {
5699 before: head.clone(),
5700 };
5701 self.state.recovery = Some(rec.clone());
5702 self.state.save()?;
5703
5704 let language = self.state.config.graph.language.clone();
5705 let sessions = self.state.config.graph.sessions;
5706 let artifacts = agent::artifacts_dir(&self.state.dir());
5707 let fix_prompt = prompt::recovery_fix(
5708 &self.state.instruction,
5709 &rec,
5710 self.state.config.graph.review_rounds,
5711 &language,
5712 );
5713 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5714 self.state.event(
5715 "recovery",
5716 format!(
5717 "repair: the fixer starts from the failed verification on {}",
5718 short(&head)
5719 ),
5720 );
5721 let (_job, seat, out) = self
5722 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5723 prompt: fix_prompt.clone(),
5724 spec,
5725 seat,
5726 cwd: wt.clone(),
5727 timeout,
5728 allow_write: true,
5729 sessions,
5730 artifacts: artifacts.clone(),
5731 stem: "fix-recovery".to_owned(),
5732 handover: None,
5733 })
5734 .await;
5735 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5736 rec.repair = Repair::WaitingForRestore { before: head };
5737 self.state.recovery = Some(rec);
5738 return self.stop_for_resource();
5739 }
5740 let failed = match &out {
5741 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5742 AgentOutcome::Quota(o) => {
5743 self.state.quota.push(QuotaLoss {
5744 seat: seat.key.clone(),
5745 node: "fix".to_owned(),
5746 at: Timestamp::now(),
5747 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5748 });
5749 Some("rate limited (quota); the fixer could not run".to_owned())
5750 }
5751 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5752 AgentOutcome::Failed(e) => Some(e.clone()),
5753 };
5754 self.state.seats.insert(seat.key.clone(), seat);
5755 let coordination = self.coordinate(&wt).await?;
5756 let failed = match failed {
5757 Some(why) => {
5759 if let Err(e) = git::restore_to(&wt, &head).await {
5760 return self
5761 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5762 .await;
5763 }
5764 Some(why)
5765 }
5766 None => {
5767 if let Ok(r) = git::rescue_commit(
5768 &wt,
5769 "magi: recovery repair of the failed verification (uncommitted work)",
5770 )
5771 .await
5772 {
5773 self.state.note_withheld("fix", &r.withheld);
5774 }
5775 None
5776 }
5777 };
5778 let after = git::rev_parse(&wt, "HEAD").await?;
5779 drop(coordination);
5780 self.state.event(
5781 "recovery",
5782 match &failed {
5783 Some(why) => format!("repair: nothing kept ({why})"),
5784 None if after != head => {
5785 format!("repair: committed {}", short(&after))
5786 }
5787 None => "repair: the fixer made no commit".to_owned(),
5788 },
5789 );
5790 rec.repair = Repair::Done {
5791 before: head.clone(),
5792 committed: after != head,
5793 after,
5794 failed,
5795 };
5796 self.state.recovery = Some(rec);
5797 self.state.save()?;
5798 Ok(())
5799 }
5800
5801 async fn stop_recovery(
5804 &mut self,
5805 mut rec: crate::recovery::RecoveryRecord,
5806 head: &str,
5807 why: String,
5808 ) -> Result<()> {
5809 rec.repair = crate::recovery::Repair::Started {
5811 before: head.to_owned(),
5812 };
5813 self.state.recovery = Some(rec);
5814 self.state
5815 .event("recovery", format!("repair could not be undone: {why}"));
5816 self.state.status = RunStatus::Blocked;
5817 self.state.save()?;
5818 bail!("recovery repair could not be rolled back: {why}")
5819 }
5820
5821 async fn review_loop(&mut self) -> Result<()> {
5822 if self
5827 .state
5828 .base_sync
5829 .as_ref()
5830 .is_some_and(|s| s.conflict.is_some())
5831 {
5832 return Ok(());
5833 }
5834 let run_id = self.state.id.clone();
5839 let prompts = self.state.config.prompts.clone();
5840 let Some(winner) = self.state.winner().cloned() else {
5841 return Ok(());
5842 };
5843 let max_rounds = self.state.config.graph.review_rounds;
5844 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5854 if status == RunStatus::Gating {
5857 self.record_contested_handoff();
5858 }
5859 self.state.status = status;
5860 self.state.save()?;
5861 return Ok(());
5862 }
5863 self.state.status = RunStatus::Reviewing;
5864 if self
5874 .state
5875 .reviews
5876 .last()
5877 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5878 {
5879 let shell = self.state.config.shell();
5880 return self
5881 .stop_reviewing(
5882 "the last round's own verification never resolved",
5883 &shell,
5884 &winner.worktree,
5885 )
5886 .await;
5887 }
5888
5889 self.recovery_repair(&winner).await?;
5890 if self.resource_halt()? {
5891 return Ok(());
5892 }
5893
5894 let repo = self.state.repo.clone();
5895 let root = self.state.worktree_root();
5896 let language = self.state.config.graph.language.clone();
5897 let sessions = self.state.config.graph.sessions;
5898 let artifacts = self.merge_repair_artifacts();
5899 let base = self.landing_base();
5900 let base_short = short(&base);
5901 let reviewers = self.roles.reviewers.clone();
5902 let shell = self.state.config.shell();
5903
5904 for round in (self.state.reviews.len() + 1)..=max_rounds {
5905 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5906 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5907 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5908 let prev_verification = self
5917 .state
5918 .reviews
5919 .last()
5920 .and_then(|r| r.verification_summary(&head));
5921 let prior_section = match (&self.state.recovery, round) {
5924 (Some(rec), 1) if self.state.reviews.is_empty() => {
5925 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5926 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5927 Some(crate::recovery::head_of(
5928 &git::diff(&winner.worktree, reviewed, "HEAD")
5929 .await
5930 .unwrap_or_default(),
5931 crate::recovery::SINCE_DIFF_MAX,
5932 ))
5933 }
5934 _ => None,
5935 };
5936 Some(prompt::recovery_review(rec, since.as_deref()))
5937 }
5938 _ => None,
5939 };
5940
5941 let mut jobs = Vec::new();
5945 let coordination = self.coordinate(&repo).await?;
5946 for (r, spec) in reviewers.iter().cloned().enumerate() {
5947 let wt = root.join(format!("review-{}", r + 1));
5948 if wt.exists() {
5951 git::reset_detached(&wt, &head).await?;
5952 } else {
5953 git::worktree_add_detached(&repo, &wt, &head).await?;
5954 }
5955 self.setup_seat_worktree(&repo, &wt, false).await?;
5956 let seat_key = format!("review-{}", r + 1);
5957 let spec = pick_start_spec(
5960 &self.roles.reviewer_roster,
5961 spec,
5962 self.state.seat_history.get(&seat_key),
5963 );
5964 let seat = self.seat(&seat_key, &spec.id);
5965 jobs.push(SeatJob {
5966 prompt: prompt::review(&prompt::ReviewCtx {
5967 instruction: &self.state.instruction,
5968 branch: &winner.branch,
5969 base_short: &base_short,
5970 stat: &stat,
5971 patch: &patch,
5972 verification: prev_verification.as_ref(),
5973 prior: prior_section.as_deref(),
5974 reviewers: reviewers.len(),
5975 round,
5976 rounds: max_rounds,
5977 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5980 lens: Lens::for_seat(r),
5981 language: &language,
5982 github_language: &self.state.config.graph.github_language,
5983 }),
5984 spec,
5985 seat,
5986 cwd: wt,
5987 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5988 allow_write: false,
5989 sessions,
5990 artifacts: artifacts.clone(),
5991 stem: format!("review-{round}-{}", r + 1),
5992 handover: None,
5993 });
5994 }
5995
5996 drop(coordination);
5997 self.state.event(
5998 "review",
5999 format!(
6000 "round {round}: {} reviewers on {}",
6001 jobs.len(),
6002 short(&head)
6003 ),
6004 );
6005 let mut quota_losses = Vec::new();
6006 let review_retries = self.state.config.graph.retries;
6007 let review_cache = self.state.config.cache_dir();
6008 let ctx = WaveCtx {
6009 carry_seats: true,
6010 pauses: Some((&self.pause, &self.interrupt)),
6011 run: &run_id,
6012 node: "review",
6013 prompts: &prompts,
6014 cache: review_cache.as_deref(),
6015 round: Some(round),
6016 };
6017 let results = ask_json_wave::<Review>(
6018 jobs,
6019 Arc::clone(&self.sem),
6020 review_retries,
6021 &self.roles.reviewer_roster,
6022 &ctx,
6023 &mut quota_losses,
6024 &mut self.state,
6025 &|_: &Review| Ok(()),
6026 )
6027 .await;
6028 if self.state.resource_block.is_some() {
6029 return self.stop_for_resource();
6030 }
6031 let round_quota_missing = quota_losses.len();
6035 self.state.quota.extend(quota_losses);
6036
6037 let mut records = Vec::new();
6038 let mut all_findings = Vec::new();
6039 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6040 let agent_id = seat.agent.clone();
6041 self.state.seats.insert(seat.key.clone(), seat);
6042 let mut record = ReviewRecord {
6043 reviewer: r + 1,
6044 agent: agent_id,
6045 summary: String::new(),
6046 findings: Vec::new(),
6047 vote: None,
6048 failed: None,
6049 duration_ms: 0,
6050 attempts,
6056 };
6057 match res {
6058 Ok((review, out)) => {
6059 record.summary =
6067 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6068 record.vote = Some(review.vote);
6069 record.duration_ms = out.duration_ms;
6070 for (n, mut f) in review.findings.into_iter().enumerate() {
6071 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6074 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6075 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6076 f.file = f
6082 .file
6083 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6084 all_findings.push(f.clone());
6085 record.findings.push(f);
6086 }
6087 self.state.event(
6088 "review",
6089 format!(
6090 "round {round}: reviewer {} voted {} with {} finding(s)",
6091 r + 1,
6092 review.vote.label(),
6093 record.findings.len()
6094 ),
6095 );
6096 }
6097 Err(e) => {
6098 record.failed = Some(e.to_string());
6099 self.state.event(
6100 "review",
6101 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6102 );
6103 }
6104 }
6105 records.push(record);
6106 }
6107
6108 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6115 let vote_split =
6116 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6117 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6118 if vote_split {
6119 self.state.event(
6120 "review",
6121 format!(
6122 "round {round}: votes split ({}) — one round of reconsideration",
6123 initial_votes
6124 .iter()
6125 .map(|v| v.label())
6126 .collect::<Vec<_>>()
6127 .join(", ")
6128 ),
6129 );
6130 let panel: Vec<ReviewSeatReport<'_>> = records
6133 .iter()
6134 .filter_map(|r| {
6135 r.vote.map(|vote| ReviewSeatReport {
6136 reviewer: r.reviewer,
6137 vote,
6138 summary: &r.summary,
6139 findings: &r.findings,
6140 })
6141 })
6142 .collect();
6143
6144 let mut jobs = Vec::new();
6145 let mut seats_at = Vec::new();
6146 for (r, spec) in reviewers.iter().cloned().enumerate() {
6147 if records[r].vote.is_none() {
6151 continue;
6152 }
6153 let wt = root.join(format!("review-{}", r + 1));
6154 let seat_key = format!("review-{}", r + 1);
6155 let spec = self.occupant(&seat_key, spec);
6156 let seat = self.seat(&seat_key, &spec.id);
6157 let build = |with_patch: bool| {
6165 prompt::review_reconsider(&ReviewReconsiderCtx {
6166 instruction: &self.state.instruction,
6167 reviewer: r + 1,
6168 lens: Lens::for_seat(r),
6169 panel: &panel,
6170 patch: with_patch.then_some(ReviewPatch {
6171 branch: &winner.branch,
6172 base_short: &base_short,
6173 stat: &stat,
6174 patch: &patch,
6175 }),
6176 round,
6177 rounds: max_rounds,
6178 language: &language,
6179 })
6180 };
6181 let full = build(true);
6182 let prompt = if has_context(&spec, &seat, sessions) {
6183 build(false)
6184 } else {
6185 full.clone()
6186 };
6187 jobs.push(SeatJob {
6188 prompt,
6189 spec,
6190 seat,
6191 cwd: wt,
6192 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6193 allow_write: false,
6194 sessions,
6195 artifacts: artifacts.clone(),
6196 stem: format!("review-{round}-reconsider-{}", r + 1),
6197 handover: Some(full),
6198 });
6199 seats_at.push(r);
6200 }
6201
6202 let mut recon_quota_losses = Vec::new();
6203 let recon_cache = self.state.config.cache_dir();
6204 let recon_ctx = WaveCtx {
6205 carry_seats: true,
6206 pauses: Some((&self.pause, &self.interrupt)),
6207 run: &run_id,
6208 node: "review",
6209 prompts: &prompts,
6210 cache: recon_cache.as_deref(),
6211 round: Some(round),
6212 };
6213 let recon_results = ask_json_wave::<ReviewRevote>(
6214 jobs,
6215 Arc::clone(&self.sem),
6216 review_retries,
6217 &self.roles.reviewer_roster,
6218 &recon_ctx,
6219 &mut recon_quota_losses,
6220 &mut self.state,
6221 &|_: &ReviewRevote| Ok(()),
6222 )
6223 .await;
6224 if self.state.resource_block.is_some() {
6225 return self.stop_for_resource();
6226 }
6227 self.state.quota.extend(recon_quota_losses);
6228
6229 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6230 let agent_id = seat.agent.clone();
6231 self.state.seats.insert(seat.key.clone(), seat);
6232 let mut rec = ReviewRevoteRecord {
6233 reviewer: r + 1,
6234 agent: agent_id,
6235 vote: None,
6236 reason: String::new(),
6237 failed: None,
6238 };
6239 match res {
6240 Ok((rv, _)) => {
6241 rec.vote = Some(rv.vote);
6242 rec.reason =
6243 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6244 self.state.event(
6245 "review",
6246 format!(
6247 "round {round}: reviewer {} revoted {}",
6248 r + 1,
6249 rv.vote.label()
6250 ),
6251 );
6252 }
6253 Err(e) => {
6254 rec.failed = Some(e.to_string());
6255 self.state.event(
6256 "review",
6257 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6258 );
6259 }
6260 }
6261 reconsideration.push(rec);
6262 }
6263 } else if initial_votes.len() > 1 {
6264 self.state.event(
6265 "review",
6266 format!(
6267 "round {round}: votes agreed ({}) — no reconsideration",
6268 initial_votes[0].label()
6269 ),
6270 );
6271 }
6272
6273 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6274 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6275 let defer_e2e =
6286 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6287 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6288 let reason =
6289 format!("{blocking} blocking finding(s) already required a fix this round");
6290 self.state.event(
6291 "verify",
6292 format!(
6293 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6294 {}); it will run once a round has none left",
6295 short(&head)
6296 ),
6297 );
6298 (Vec::new(), false, true, Some(reason))
6299 } else {
6300 let e2e_commands = self.state.config.verify.e2e.clone();
6301 let cache_dir = self.state.config.cache_dir();
6302 let context = format!("round {round}");
6303 let wait = self.repository_wait();
6304 let (e2e, verify_retried) = with_cache_lease(
6305 &mut self.state,
6306 cache_dir.as_deref(),
6307 "e2e",
6308 "e2e",
6309 &winner.worktree,
6310 &head,
6311 verify_timeout,
6312 &context,
6313 &wait,
6314 |state, budget| {
6315 let shell = shell.clone();
6316 let e2e_commands = e2e_commands.clone();
6317 let worktree = winner.worktree.clone();
6318 let context = context.clone();
6319 async move {
6320 run_e2e_with_retry(
6321 state,
6322 &shell,
6323 &e2e_commands,
6324 &worktree,
6325 budget,
6326 &context,
6327 )
6328 .await
6329 }
6330 },
6331 )
6332 .await?;
6333 (e2e, verify_retried, false, None)
6334 };
6335
6336 let expected = records.len();
6337 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6338 let incomplete = answered < expected;
6339 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6340 let policy = self.state.config.graph.incomplete_review;
6341 let clean = round_is_clean(
6342 blocking,
6343 e2e_ok,
6344 answered,
6345 expected,
6346 round_quota_missing,
6347 policy,
6348 );
6349
6350 let mut round_record = ReviewRound {
6351 round,
6352 head: head.clone(),
6353 verified_head: None,
6354 verified_at: None,
6355 reviews: records,
6356 e2e,
6357 verify_retried,
6358 e2e_deferred,
6359 e2e_defer_reason,
6360 fix: None,
6361 blocking,
6362 answered,
6363 expected,
6364 clean,
6365 progressed: false,
6366 vote_split,
6367 reconsideration,
6368 verdict: None,
6369 };
6370 round_record.verdict = ReviewVote::worst(
6374 round_record
6375 .final_votes()
6376 .into_iter()
6377 .map(|(_, _, vote)| vote),
6378 );
6379 if !matches!(
6390 round_record.e2e_status(),
6391 E2eStatus::Deferred | E2eStatus::NotConfigured
6392 ) {
6393 round_record.verified_head = Some(head.clone());
6394 round_record.verified_at = Some(Timestamp::now());
6395 }
6396 let this_round_verification = round_record.verification_summary(&head);
6397
6398 if incomplete {
6399 let missing: Vec<String> = round_record
6400 .reviews
6401 .iter()
6402 .filter(|r| r.failed.is_some())
6403 .map(|r| format!("review-{}", r.reviewer))
6404 .collect();
6405 self.state.event(
6406 "review",
6407 format!(
6408 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6409 missing.join(", ")
6410 ),
6411 );
6412 }
6413
6414 if clean {
6415 self.state.event(
6416 "review",
6417 if incomplete && policy == IncompleteReviewPolicy::Warn {
6418 format!(
6419 "round {round}: clean (warn policy, incomplete panel) — no \
6420 blocking findings from the seats that answered, verification green"
6421 )
6422 } else if incomplete {
6423 format!(
6424 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6425 quorum) — no blocking findings from the seats that answered, \
6426 verification green",
6427 expected - answered
6428 )
6429 } else {
6430 format!("round {round}: clean — no blocking findings, verification green")
6431 },
6432 );
6433 self.state.reviews.push(round_record);
6434 self.state.status = RunStatus::Gating;
6435 self.state.save()?;
6436 return Ok(());
6437 }
6438
6439 if incomplete && blocking == 0 && e2e_ok {
6447 self.state.reviews.push(round_record);
6448 self.state.save()?;
6449 if round == max_rounds {
6450 self.state.status = RunStatus::Blocked;
6451 self.state.event(
6452 "review",
6453 format!(
6454 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6455 refusing to call it clean",
6456 expected - answered
6457 ),
6458 );
6459 return Ok(());
6460 }
6461 continue;
6462 }
6463
6464 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6476 self.state.reviews.push(round_record);
6477 return self
6478 .stop_reviewing(
6479 "the round's own verification could not run",
6480 &shell,
6481 &winner.worktree,
6482 )
6483 .await;
6484 }
6485
6486 if round == max_rounds {
6487 self.state.reviews.push(round_record);
6488 return self
6489 .stop_reviewing(
6490 &format!(
6491 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6492 ),
6493 &shell,
6494 &winner.worktree,
6495 )
6496 .await;
6497 }
6498
6499 let blocking_findings: Vec<_> = all_findings
6502 .iter()
6503 .filter(|f| f.severity.blocks())
6504 .cloned()
6505 .collect();
6506 let fix_prompt = prompt::fix_in(
6507 &self.state.instruction,
6508 &blocking_findings,
6509 this_round_verification.as_ref(),
6510 round,
6511 max_rounds,
6512 &language,
6513 &self.state.config.graph.github_language,
6514 );
6515 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6516 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6517 let (job, seat, out) = self
6518 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6519 prompt: fix_prompt.clone(),
6520 spec,
6521 seat,
6522 cwd: winner.worktree.clone(),
6523 timeout,
6524 allow_write: true,
6525 sessions,
6526 artifacts: artifacts.clone(),
6527 stem: format!("fix-{round}"),
6528 handover: None,
6529 })
6530 .await;
6531 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6534 return self.stop_for_resource();
6535 }
6536 let agent_id = seat.agent.clone();
6537
6538 let mut fix = FixRecord {
6539 agent: agent_id,
6540 addressed: Vec::new(),
6541 rejected: Vec::new(),
6542 notes: String::new(),
6543 committed: false,
6544 failed: None,
6545 duration_ms: 0,
6546 continuation: None,
6547 };
6548 let mut continuation = ContinuationRecord::not_needed();
6549 let mut final_seat = seat.clone();
6550 match out {
6551 AgentOutcome::Ok(o) => {
6552 fix.duration_ms = o.duration_ms;
6553 let parsed = verdict::extract_json::<FixReport>(&o.text);
6554 let incomplete_reason = match &parsed {
6561 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6562 "the reply parsed, but it reported a command whose own CLI never \
6563 confirmed an exit status"
6564 .to_owned(),
6565 ),
6566 Ok(_) => None,
6567 Err(e) => Some(e.to_string()),
6568 };
6569 match incomplete_reason {
6570 None => {
6571 let report = parsed.expect("checked Ok above");
6572 fix.addressed = report.addressed;
6573 fix.rejected = report.rejected;
6574 fix.notes =
6575 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6576 }
6577 Some(reason) => {
6578 let (resumed_seat, resolved, failure, cont) = self
6579 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6580 .await;
6581 fix.duration_ms += cont.cumulative_wait_ms;
6582 continuation = cont;
6583 final_seat = resumed_seat;
6584 match resolved {
6585 Some(report) => {
6586 fix.addressed = report.addressed;
6587 fix.rejected = report.rejected;
6588 fix.notes = blind::sanitize_prose(
6589 &report.notes,
6590 &self.state.config.blind,
6591 );
6592 }
6593 None => fix.failed = failure,
6594 }
6595 }
6596 }
6597 }
6598 AgentOutcome::Dropped(o) => {
6600 fix.duration_ms = o.duration_ms;
6601 let why = o
6602 .dropped
6603 .as_ref()
6604 .map(|d| d.why.as_str())
6605 .unwrap_or("the CLI ended the stream without delivering its answer");
6606 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6607 }
6608 AgentOutcome::Quota(o) => {
6609 self.state.quota.push(QuotaLoss {
6610 seat: final_seat.key.clone(),
6611 node: "fix".to_owned(),
6612 at: Timestamp::now(),
6613 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6614 });
6615 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6616 }
6617 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6618 }
6619 fix.continuation = Some(continuation);
6620 self.state.seats.insert(final_seat.key.clone(), final_seat);
6621 let coordination = self.coordinate(&winner.worktree).await?;
6622 if let Ok(r) = git::rescue_commit(
6623 &winner.worktree,
6624 &format!("magi: review round {round} fixes (uncommitted work)"),
6625 )
6626 .await
6627 {
6628 self.state.note_withheld("fix", &r.withheld);
6629 }
6630 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6631 fix.committed = after != before;
6632 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6640 drop(coordination);
6641 let progressed = diff_after != patch;
6642 let commit_note = if fix.committed {
6643 "committed"
6644 } else {
6645 "NO new commit"
6646 };
6647 let tree_note = if progressed {
6648 "changed vs base"
6649 } else {
6650 "unchanged vs base"
6651 };
6652 self.state.event(
6653 "fix",
6654 match &fix.failed {
6655 Some(reason) => {
6661 format!(
6662 "round {round}: fixer's adoption report was lost ({reason}); \
6663 {commit_note}, tree {tree_note}"
6664 )
6665 }
6666 None => format!(
6667 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6668 {tree_note}{}",
6669 fix.addressed.len(),
6670 fix.rejected.len(),
6671 if continuation.outcome == ContinuationOutcome::Resumed {
6672 format!(
6673 " (adoption report recovered after {} continuation(s))",
6674 continuation.attempts
6675 )
6676 } else {
6677 String::new()
6678 },
6679 ),
6680 },
6681 );
6682 round_record.fix = Some(fix);
6683 round_record.progressed = progressed;
6684 self.state.reviews.push(round_record);
6685 self.state.save()?;
6686
6687 if matches!(
6700 continuation.outcome,
6701 ContinuationOutcome::Exhausted
6702 | ContinuationOutcome::QuotaLost
6703 | ContinuationOutcome::NoSession
6704 ) {
6705 return self
6706 .stop_reviewing(
6707 "the fixer's adoption report never came back, even after resuming its \
6708 own seat; refusing to start another round against the same worktree \
6709 while that is unresolved",
6710 &shell,
6711 &winner.worktree,
6712 )
6713 .await;
6714 }
6715
6716 let streak = self
6717 .state
6718 .reviews
6719 .iter()
6720 .rev()
6721 .take_while(|r| !r.progressed)
6722 .count();
6723 if streak >= STAGNANT_LIMIT {
6724 return self
6725 .stop_reviewing(
6726 &format!(
6727 "the tree has not moved against base for {streak} round(s) in a row"
6728 ),
6729 &shell,
6730 &winner.worktree,
6731 )
6732 .await;
6733 }
6734 }
6735 Ok(())
6736 }
6737
6738 fn record_contested_handoff(&mut self) {
6772 if self.state.contested_handoff.is_some() {
6773 return;
6774 }
6775 let Some(contested) = self
6776 .state
6777 .reviews
6778 .last()
6779 .and_then(ReviewRound::contested_handoff)
6780 else {
6781 return;
6782 };
6783 self.state.event(
6784 "review",
6785 format!(
6786 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6787 wait for the owner's approval",
6788 contested.findings.len(),
6789 contested.rejecters.len()
6790 ),
6791 );
6792 self.state.contested_handoff = Some(contested);
6793 }
6794
6795 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6796 let round_idx = self.state.reviews.len() - 1;
6797 let needs_catchup_run = matches!(
6805 self.state.reviews[round_idx].e2e_status(),
6806 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6807 );
6808 if needs_catchup_run {
6809 let round = self.state.reviews[round_idx].round;
6810 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6811 let commands = self.state.config.verify.e2e.clone();
6812 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6813 let cache_dir = self.state.config.cache_dir();
6814 let context = format!(
6815 "round {round}: verification unresolved, catching up before the final decision"
6816 );
6817 let wait = self.repository_wait();
6818 let (outcomes, verify_retried) = with_cache_lease(
6819 &mut self.state,
6820 cache_dir.as_deref(),
6821 "e2e",
6822 "e2e",
6823 worktree,
6824 &attempted_head,
6825 timeout,
6826 &context,
6827 &wait,
6828 |state, budget| {
6829 let shell = shell.to_vec();
6830 let commands = commands.clone();
6831 let context = context.clone();
6832 async move {
6833 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6834 .await
6835 }
6836 },
6837 )
6838 .await?;
6839 let last = &mut self.state.reviews[round_idx];
6840 last.e2e = outcomes;
6841 last.verify_retried = verify_retried;
6842 last.verified_head = Some(attempted_head);
6849 last.verified_at = Some(Timestamp::now());
6850 if verify_inconclusive(&last.e2e) {
6851 self.state.save()?;
6858 return Ok(());
6859 }
6860 last.e2e_deferred = false;
6861 }
6862 let last = &self.state.reviews[round_idx];
6863 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6864
6865 match last.e2e_status() {
6866 E2eStatus::Failed => {
6867 let red: Vec<String> = last
6868 .e2e
6869 .iter()
6870 .filter(|o| !o.ok())
6871 .map(|o| {
6872 format!(
6873 "`{}` -> {:?}\n{}",
6874 o.command,
6875 o.code,
6876 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6877 )
6878 })
6879 .collect();
6880 self.state
6881 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6882 self.state.status = RunStatus::Blocked;
6883 }
6884 E2eStatus::ResourceBlocked => {
6889 self.state.event(
6890 "review",
6891 format!(
6892 "{why}; e2e could not run (shared build cache unavailable); not \
6893 deciding yet"
6894 ),
6895 );
6896 }
6897 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6898 self.state.event(
6899 "review",
6900 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6901 );
6902 self.record_contested_handoff();
6903 self.state.status = RunStatus::Gating;
6904 }
6905 }
6906 self.state.save()?;
6907 Ok(())
6908 }
6909
6910 async fn gate(&mut self) -> Result<()> {
6913 if self.state.status == RunStatus::Failed
6925 || self
6926 .state
6927 .base_sync
6928 .as_ref()
6929 .is_some_and(|s| s.conflict.is_some())
6930 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6931 != Some(RunStatus::Gating)
6932 {
6933 return Ok(());
6934 }
6935 if self.state.gate_ran {
6936 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6947 self.state.status = RunStatus::Blocked;
6948 self.state.save()?;
6949 }
6950 return Ok(());
6951 }
6952 let Some(winner) = self.state.winner().cloned() else {
6953 return Ok(());
6954 };
6955 self.state.status = RunStatus::Gating;
6956 let mut outcomes = self.run_gate(&winner).await?;
6957 loop {
6958 if verify_inconclusive(&outcomes) {
6969 self.state.save()?;
6970 return Ok(());
6971 }
6972 if outcomes.iter().all(CommandOutcome::ok) {
6973 break;
6974 }
6975 match self.gate_fix_round(&winner, &outcomes).await? {
6976 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6977 GateFix::Stop => break,
6978 GateFix::Defer => {
6979 self.state.save()?;
6980 return Ok(());
6981 }
6982 }
6983 }
6984 let passed = outcomes.iter().all(CommandOutcome::ok);
6985 self.state.gate = outcomes;
6986 self.state.gate_ran = true;
6987 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6988 if !passed {
6989 self.state.status = RunStatus::Blocked;
6990 let spent = self.state.gate_fixes.len();
6991 self.state.event(
6992 "gate",
6993 if spent == 0 {
6994 "gate failed; not merging".to_owned()
6995 } else {
6996 format!("gate failed after {spent} gate-fix round(s); not merging")
6997 },
6998 );
6999 }
7000 self.state.save()?;
7001 Ok(())
7002 }
7003
7004 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7014 let commands = state.config.verify.pre_gate.clone();
7015 if commands.is_empty() {
7016 return;
7017 }
7018 let shell = state.config.shell();
7019 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7020 let (outcomes, _) = run_commands(
7021 state,
7022 "pre_gate",
7023 "pre_gate",
7024 0,
7025 &shell,
7026 &commands,
7027 &winner.worktree,
7028 timeout,
7029 )
7030 .await;
7031 for o in &outcomes {
7032 if !o.ok() {
7033 tracing::warn!(
7034 "pre_gate `{}` failed ({:?}); the gate decides",
7035 o.command,
7036 o.code
7037 );
7038 }
7039 state.event(
7040 "pre_gate",
7041 format!(
7042 "`{}` -> {}",
7043 o.command,
7044 if o.ok() {
7045 "pass".to_owned()
7046 } else {
7047 format!(
7048 "FAIL ({:?})\n{}",
7049 o.code,
7050 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7051 )
7052 }
7053 ),
7054 );
7055 }
7056 state.pre_gate = outcomes;
7057 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7058 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7059 Ok(head) => {
7060 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7061 state.pre_gate_commit = Some(head);
7062 }
7063 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7064 },
7065 Ok(false) => {}
7066 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7067 }
7068 if let Err(e) = state.save() {
7069 tracing::warn!("could not persist the pre_gate record: {e:#}");
7070 }
7071 }
7072
7073 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7076 let wait = self.repository_wait();
7077 run_landing_gate(&mut self.state, winner, &wait).await
7078 }
7079
7080 async fn gate_fix_round(
7093 &mut self,
7094 winner: &Candidate,
7095 outcomes: &[CommandOutcome],
7096 ) -> Result<GateFix> {
7097 let cap = self.state.config.graph.gate_fix_rounds;
7098 let spent = self.state.gate_fixes.len();
7099 if spent >= cap {
7100 if cap > 0 {
7101 self.state.event(
7102 "gate",
7103 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7104 );
7105 }
7106 return Ok(GateFix::Stop);
7107 }
7108 if !gate_fixable(outcomes) {
7109 self.state.event(
7110 "gate",
7111 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7112 command or similar); not spending a fix round on it",
7113 );
7114 return Ok(GateFix::Stop);
7115 }
7116 let min_free = self.state.config.disk.min_free_bytes;
7117 if min_free > 0 {
7118 match crate::disk::free_bytes(&winner.worktree) {
7119 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7120 Ok(free) => {
7121 self.state.event(
7122 "gate",
7123 format!(
7124 "only {free} bytes free ({min_free} required by `[disk] \
7125 min_free_bytes`); not spending a fix round on a failure the disk \
7126 may explain"
7127 ),
7128 );
7129 return Ok(GateFix::Stop);
7130 }
7131 Err(e) => {
7132 self.state.event(
7133 "gate",
7134 format!("free disk space could not be measured ({e:#}); no fix round"),
7135 );
7136 return Ok(GateFix::Stop);
7137 }
7138 }
7139 }
7140
7141 let attempt = spent + 1;
7142 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7143 let base = self.landing_base();
7144 let fix_prompt = prompt::gate_fix_in(
7145 &self.state.instruction,
7146 &failed,
7147 attempt,
7148 cap,
7149 &self.state.config.graph.language,
7150 &self.state.config.graph.github_language,
7151 );
7152 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7153 let sessions = self.state.config.graph.sessions;
7154 let artifacts = self.merge_repair_artifacts();
7155 self.state.event(
7156 "gate",
7157 format!("gate failed; gate-fix round {attempt} of {cap}"),
7158 );
7159 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7160 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7161 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7164 head: before.clone(),
7165 passed: false,
7166 outcomes: Vec::new(),
7167 });
7168 self.state.save()?;
7169 let (_, seat, out) = self
7170 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7171 prompt: fix_prompt.clone(),
7172 spec,
7173 seat,
7174 cwd: winner.worktree.clone(),
7175 timeout,
7176 allow_write: true,
7177 sessions,
7178 artifacts: artifacts.clone(),
7179 stem: format!("gate-fix-{attempt}"),
7180 handover: None,
7181 })
7182 .await;
7183 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7186 return Ok(GateFix::Defer);
7187 }
7188 let mut record = GateFixRecord {
7189 agent: seat.agent.clone(),
7190 failed,
7191 notes: String::new(),
7192 committed: false,
7193 error: None,
7194 };
7195 match out {
7196 AgentOutcome::Ok(o) => {
7197 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7200 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7201 }
7202 }
7203 AgentOutcome::Dropped(_) => {
7204 record.error = Some("the CLI dropped the stream".to_owned());
7205 }
7206 AgentOutcome::Quota(o) => {
7207 self.state.quota.push(QuotaLoss {
7208 seat: seat.key.clone(),
7209 node: "gate-fix".to_owned(),
7210 at: Timestamp::now(),
7211 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7212 });
7213 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7214 }
7215 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7216 }
7217 self.state.seats.insert(seat.key.clone(), seat);
7218 let coordination = self.coordinate(&winner.worktree).await?;
7219 if let Ok(r) = git::rescue_commit(
7220 &winner.worktree,
7221 &format!("magi: gate fix {attempt} (uncommitted work)"),
7222 )
7223 .await
7224 {
7225 self.state.note_withheld("gate-fix", &r.withheld);
7226 }
7227 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7228 record.committed = after != before;
7229 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7230 let note = record.error.clone();
7231 self.state.gate_fixes.push(record);
7232 self.state.save()?;
7233 if !changed {
7234 self.state.event(
7235 "gate",
7236 match note {
7237 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7238 None => format!("gate-fix round {attempt}: the tree did not change"),
7239 },
7240 );
7241 return Ok(GateFix::Stop);
7242 }
7243 self.state.event(
7244 "gate",
7245 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7246 );
7247
7248 drop(coordination);
7249 let wait = self.repository_wait();
7250 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7251 }
7252
7253 async fn guarded_pr_message(
7260 &mut self,
7261 winner: &Candidate,
7262 facts: Option<&BranchFacts>,
7263 posting: bool,
7264 ) -> Result<Option<PrMessage>> {
7265 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7266 let head = match final_head.as_deref() {
7267 Some(h) => crate::pr_outcome::FinalHead::At(h),
7268 None => crate::pr_outcome::FinalHead::Unknown,
7269 };
7270 let summarized = if posting {
7272 Some(self.summarize_change(winner, final_head.as_deref()).await)
7273 } else {
7274 None
7275 };
7276 if self.state.resource_block.is_some() {
7281 self.state.event(
7282 "pr-summary",
7283 "a shared resource was busy; nothing was published, resume to summarize again",
7284 );
7285 return Ok(None);
7286 }
7287 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7288 anyhow::bail!(
7289 "the final commit moved during public description generation; retry before publishing"
7290 );
7291 }
7292 let note = match (&summarized, &self.state.change_summary) {
7293 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7294 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7295 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7296 (None, _) => ChangeNote::NotAsked,
7297 };
7298 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7299 if !posting {
7300 let identity = crate::scrub::Identity::current();
7301 return Ok(Some(PrMessage {
7302 title: crate::scrub::scrub(&pr.title, &identity),
7303 body: crate::scrub::scrub(&pr.body, &identity),
7304 }));
7305 }
7306 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7309 if let Some(known) = self
7310 .state
7311 .github_text
7312 .clone()
7313 .filter(|g| g.fingerprint == raw_fp)
7314 {
7315 return self.settle_withheld(winner, &pr, known).await;
7316 }
7317 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7318 let mut violations =
7319 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7320 let identity = crate::scrub::Identity::current();
7321 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7322 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7323 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7324 {
7325 violations.push(crate::github_text::Violation::SensitiveData);
7326 }
7327 let has_language = violations
7331 .iter()
7332 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7333 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7334 self.state.event(
7335 "github-text",
7336 if has_language {
7337 format!("description rejected: {violations:?}; requesting one rewrite")
7338 } else {
7339 format!("description rejected: {violations:?}; asking the owner")
7340 },
7341 );
7342 let mut rewritten = false;
7343 let mut rejected_rewrite = None;
7344 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7345 {
7346 let output = text;
7347 if let Some(title) = summary_title(&output) {
7348 let raw_body = summary_without_title(&output);
7351 let clean_title = crate::scrub::scrub(&title, &identity);
7352 let mut body = crate::scrub::scrub(&raw_body, &identity);
7353 let redacted = clean_title != title || body != raw_body;
7354 let title = clean_title;
7355 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7356 if !complete {
7357 self.state.event(
7358 "github-text",
7359 "description rewrite omitted required findings",
7360 );
7361 }
7362 if !body.trim().is_empty() {
7363 body.push_str("\n\n");
7364 body.push_str(&public_pr_evidence(&self.state, head));
7365 }
7366 let marker = format!("magi:run/{}", self.state.id);
7367 if !body.trim().is_empty() && !body.contains(&marker) {
7368 body.push_str(&format!("\n\n{marker}"));
7369 }
7370 let rewrite = PrMessage {
7371 title: title.clone(),
7372 body: body.clone(),
7373 };
7374 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7375 let mut failed = crate::github_text::check_for(
7376 &self.state.config.graph,
7377 &title,
7378 &body,
7379 redecision,
7380 );
7381 if (!crate::github_text::shareable(&title)
7382 || !crate::github_text::shareable(&body))
7383 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7384 {
7385 failed.push(crate::github_text::Violation::SensitiveData);
7386 }
7387 if complete
7388 && !title.trim().is_empty()
7389 && !body.trim().is_empty()
7390 && !failed.is_empty()
7391 {
7392 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7393 title: title.clone(),
7394 body: body.clone(),
7395 failed: failed
7396 .iter()
7397 .map(|v| crate::github_text::category(*v).to_owned())
7398 .collect(),
7399 });
7400 }
7401 if complete
7402 && !title.trim().is_empty()
7403 && !raw_body.trim().is_empty()
7404 && failed.is_empty()
7405 {
7406 decision = redecision;
7407 pr = PrMessage { title, body };
7408 rewritten = true;
7409 self.state
7410 .event("github-text", "description rewrite passed");
7411 if redacted {
7412 self.state.event(
7413 "github-text",
7414 "sensitive data removed from description rewrite",
7415 );
7416 }
7417 }
7418 }
7419 }
7420 if !rewritten {
7421 self.state.event(
7422 "github-text",
7423 "description rewrite unavailable or rejected; asking the owner",
7424 );
7425 if let Some(w) =
7426 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7427 {
7428 self.retire_github_text_question("the withheld text changed");
7431 let record = crate::run::GithubTextAsk {
7432 rewrite: rejected_rewrite,
7433 fingerprint: raw_fp,
7434 title: w.title,
7435 body: w.body,
7436 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7437 question: None,
7438 asks: 0,
7439 resolved: false,
7440 chosen_title: None,
7441 choice: None,
7442 };
7443 return self.settle_withheld(winner, &pr, record).await;
7444 }
7445 }
7446 }
7447 let (title, body) =
7448 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7449 Ok(Some(PrMessage { title, body }))
7450 }
7451
7452 async fn summarize_change(
7463 &mut self,
7464 winner: &Candidate,
7465 head: Option<&str>,
7466 ) -> std::result::Result<(), String> {
7467 let Some(head) = head else {
7468 return Err("the final commit could not be read".to_owned());
7469 };
7470 let remote = self.state.config.merge.remote.clone();
7471 let tracking = format!("{remote}/{}", self.state.base_branch);
7472 if !git::rev_exists(&winner.worktree, &tracking).await {
7473 return Err(format!("the remote base `{tracking}` could not be read"));
7474 }
7475 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7476 Ok(b) if !b.is_empty() => b,
7477 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7478 };
7479 let evidence = crate::pr_outcome::summary_evidence(
7480 &self.state,
7481 crate::pr_outcome::FinalHead::At(head),
7482 );
7483 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7484 if self
7485 .state
7486 .change_summary
7487 .as_ref()
7488 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7489 {
7490 return Ok(());
7491 }
7492 if self.state.change_summary.take().is_some() {
7493 self.state.event(
7494 "pr-summary",
7495 "the final diff or evidence changed; the earlier change summary was discarded",
7496 );
7497 }
7498 if let Err(e) = write_artifact(
7500 &self.state,
7501 &format!("public-description-{fingerprint}.md"),
7502 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7503 ) {
7504 tracing::warn!("could not save public description evidence: {e:#}");
7505 }
7506 let chain = match agent::pick_chain(
7507 &self.state.config.agents,
7508 self.state.config.roles.synthesizer.as_ref(),
7509 &agent::installed,
7510 "synthesizer",
7511 ) {
7512 Ok(chain) => chain,
7513 Err(e) => {
7514 tracing::warn!("no agent for the change summary: {e:#}");
7515 self.state.event("pr-summary", "no summary agent available");
7516 return Err("no agent was available to summarize it".to_owned());
7517 }
7518 };
7519 let subjects = git::subjects(&winner.worktree, &base, head)
7520 .await
7521 .unwrap_or_default();
7522 let stat = git::diff_stat(&winner.worktree, &base, head)
7523 .await
7524 .unwrap_or_default();
7525 let stat: String = stat.chars().take(6_000).collect();
7526 let mut prompt = crate::prompt::change_summary_in(
7527 &base,
7528 head,
7529 &subjects,
7530 stat.trim(),
7531 &self.state.config.graph.github_language,
7532 );
7533 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7534 let sessions = self.state.config.graph.sessions;
7535 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7536 let prompts = self.state.config.prompts.clone();
7537 let cache = self.state.config.cache_dir();
7538 let run = self.state.id.clone();
7539 for (i, spec) in chain.into_iter().enumerate() {
7540 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7541 let job = SeatJob {
7542 spec: spec.clone(),
7543 seat,
7544 cwd: winner.worktree.clone(),
7545 prompt: prompt.clone(),
7546 timeout,
7547 allow_write: false,
7548 sessions,
7549 artifacts: agent::artifacts_dir(&self.state.dir()),
7550 stem: if i == 0 {
7551 "change-summary".to_owned()
7552 } else {
7553 format!("change-summary-{}", spec.id)
7554 },
7555 handover: None,
7556 };
7557 let ctx = WaveCtx {
7558 run: &run,
7559 node: "pr-summary",
7560 prompts: &prompts,
7561 cache: cache.as_deref(),
7562 round: None,
7563 carry_seats: false,
7564 pauses: Some((&self.pause, &self.interrupt)),
7565 };
7566 let (seat, outcome) =
7567 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7568 self.state.seats.insert(seat.key.clone(), seat);
7569 let text = match outcome {
7570 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7571 AgentOutcome::ResourceBlocked(why) => {
7572 if self.state.resource_block.is_none() {
7575 self.state.resource_block = Some(crate::run::ResourceBlock {
7576 node: "pr-summary".to_owned(),
7577 kind: "shared resource".to_owned(),
7578 detail: why,
7579 at: Timestamp::now(),
7580 });
7581 }
7582 return Err("a shared resource was busy".to_owned());
7583 }
7584 _ => continue,
7585 };
7586 let blind = &self.state.config.blind;
7587 let Some(title) = summary_title(&text) else {
7588 continue;
7589 };
7590 let body: String = summary_without_title(&text)
7591 .lines()
7592 .filter(|l| !l.contains("magi:run/"))
7593 .collect::<Vec<_>>()
7594 .join("\n");
7595 let body = blind::sanitize_prose(body.trim(), blind);
7596 let title = blind::sanitize_prose(&title, blind);
7597 if body.trim().is_empty() || title.trim().is_empty() {
7598 continue;
7599 }
7600 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7601 continue;
7602 }
7603 if git::rev_parse(&winner.worktree, "HEAD")
7605 .await
7606 .ok()
7607 .as_deref()
7608 != Some(head)
7609 {
7610 return Err("the branch moved while the summary was written".to_owned());
7611 }
7612 if git::merge_base(&winner.worktree, &tracking, head)
7613 .await
7614 .ok()
7615 .as_deref()
7616 != Some(base.as_str())
7617 || crate::pr_outcome::summary_evidence(
7618 &self.state,
7619 crate::pr_outcome::FinalHead::At(head),
7620 ) != evidence
7621 {
7622 return Err("the summary evidence changed while it was written".to_owned());
7623 }
7624 self.state.change_summary = Some(crate::run::ChangeSummary {
7625 head: head.to_owned(),
7626 base,
7627 evidence: fingerprint,
7628 title,
7629 body,
7630 });
7631 self.state.event(
7632 "pr-summary",
7633 format!("change summary written by {} for the final diff", spec.id),
7634 );
7635 if let Err(e) = self.state.save() {
7636 tracing::warn!("could not save the change summary: {e:#}");
7637 }
7638 return Ok(());
7639 }
7640 self.state
7641 .event("pr-summary", "no agent produced a change summary");
7642 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7643 }
7644
7645 async fn rewrite_pr_text(
7653 &mut self,
7654 winner: &Candidate,
7655 pr: &PrMessage,
7656 violations: &[crate::github_text::Violation],
7657 ) -> Option<String> {
7658 let prompt = format!(
7659 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7660 pr.title, pr.body
7661 );
7662 let sessions = self.state.config.graph.sessions;
7663 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7664 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7666 let key = format!("impl-{}", winner.label);
7667 if let Some(spec) = self
7668 .state
7669 .config
7670 .agents
7671 .iter()
7672 .find(|a| a.id == winner.agent)
7673 .cloned()
7674 && let Some(existing) = self.state.seats.get(&key)
7675 && existing.agent == spec.id
7676 && agent::has_session(spec.kind, existing, sessions)
7677 {
7678 attempts.push((spec, existing.clone(), true));
7679 } else {
7680 let chain = agent::pick_chain(
7681 &self.state.config.agents,
7682 self.state.config.roles.language_judge.as_ref(),
7683 &agent::installed,
7684 "language_judge",
7685 );
7686 match chain {
7687 Ok(chain) => {
7688 for spec in chain {
7689 let seat = crate::fixer::seat_for(
7690 &mut self.state,
7691 "github-text-rewrite",
7692 &spec.id,
7693 );
7694 attempts.push((spec, seat, false));
7695 }
7696 }
7697 Err(e) => {
7698 tracing::warn!("no agent for the description rewrite: {e:#}");
7699 self.state
7700 .event("github-text", "no rewrite agent available");
7701 return None;
7702 }
7703 }
7704 }
7705 let prompts = self.state.config.prompts.clone();
7706 let cache = self.state.config.cache_dir();
7707 let run = self.state.id.clone();
7708 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7709 let job = SeatJob {
7710 spec: spec.clone(),
7711 seat,
7712 cwd: winner.worktree.clone(),
7713 prompt: prompt.clone(),
7714 timeout: retry_budget(timeout_full, resumed),
7715 allow_write: false,
7716 sessions,
7717 artifacts: agent::artifacts_dir(&self.state.dir()),
7718 stem: if i == 0 {
7719 "github-text-rewrite".to_owned()
7720 } else {
7721 format!("github-text-rewrite-{}", spec.id)
7722 },
7723 handover: None,
7724 };
7725 let ctx = WaveCtx {
7726 run: &run,
7727 node: "github-text",
7728 prompts: &prompts,
7729 cache: cache.as_deref(),
7730 round: None,
7731 carry_seats: false,
7732 pauses: Some((&self.pause, &self.interrupt)),
7733 };
7734 let (seat, outcome) =
7735 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7736 self.state.seats.insert(seat.key.clone(), seat);
7737 match outcome {
7738 AgentOutcome::Ok(o)
7739 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7740 {
7741 return Some(o.text);
7742 }
7743 AgentOutcome::ResourceBlocked(_) => return None,
7744 _ => {}
7745 }
7746 }
7747 None
7748 }
7749
7750 fn retire_github_text_question(&mut self, why: &str) {
7752 let Some(id) = self
7753 .state
7754 .github_text
7755 .as_ref()
7756 .and_then(|g| g.question.clone())
7757 else {
7758 return;
7759 };
7760 let _ = ask::Questions::open().update(&id, |q| {
7761 q.abandon(why);
7762 Ok(())
7763 });
7764 }
7765
7766 async fn settle_withheld(
7770 &mut self,
7771 winner: &Candidate,
7772 pr: &PrMessage,
7773 mut g: crate::run::GithubTextAsk,
7774 ) -> Result<Option<PrMessage>> {
7775 use crate::github_text as gt;
7776 let store = ask::Questions::open();
7777 let timeout = self.state.config.graph.answer_timeout;
7778 if !g.resolved {
7779 let mut standing = g
7780 .question
7781 .as_ref()
7782 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7783 if let Some(q) = standing.as_ref()
7784 && q.status.open()
7785 && gt::expired(q, timeout)
7786 {
7787 let why = format!("no answer within {}s of asking", timeout.max(1));
7788 standing = store
7789 .update(&q.id, |q| {
7790 q.abandon(&why);
7791 Ok(())
7792 })
7793 .ok()
7794 .map(|(q, ())| q);
7795 }
7796 let mut ask_again: Option<Option<Vec<String>>> = None;
7797 match standing {
7798 None if g.asks <= 1 => {
7801 g.asks = 0;
7802 ask_again = Some(None);
7803 }
7804 None => {
7805 g.resolved = true;
7806 g.chosen_title = None;
7807 g.choice = Some(GithubTextChoice::Fallback);
7808 }
7809 Some(q) if q.status.open() => {
7810 self.park_on_github_text(&g)?;
7811 return Ok(None);
7812 }
7813 Some(q) => match gt::read_reply(&q) {
7814 gt::Reply::Fallback => {
7815 g.resolved = true;
7816 g.chosen_title = None;
7817 g.choice = Some(GithubTextChoice::Fallback);
7818 }
7819 gt::Reply::Original => {
7820 g.resolved = true;
7821 g.chosen_title = None;
7822 g.choice = Some(GithubTextChoice::Original);
7823 }
7824 gt::Reply::Rewrite => {
7825 g.resolved = true;
7826 g.chosen_title = None;
7827 g.choice = Some(if g.rewrite.is_some() {
7828 GithubTextChoice::Rewrite
7829 } else {
7830 GithubTextChoice::Fallback
7831 });
7832 }
7833 gt::Reply::Title(text) => {
7834 let first = text
7835 .lines()
7836 .map(str::trim)
7837 .find(|l| !l.is_empty())
7838 .unwrap_or("");
7839 let decision = if first.is_empty() {
7840 None
7841 } else {
7842 let candidate = PrMessage {
7843 title: first.to_owned(),
7844 body: String::new(),
7845 };
7846 self.judge_pr_language(&winner.worktree, &candidate).await
7847 };
7848 match gt::vet_title(
7849 &text,
7850 decision,
7851 self.state.config.graph.github_english(),
7852 ) {
7853 Ok(title) => {
7854 g.resolved = true;
7855 g.chosen_title = Some(title);
7856 g.choice = Some(GithubTextChoice::MyTitle);
7857 }
7858 Err(cats) if g.asks < 2 => {
7859 ask_again = Some(Some(
7860 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7861 ))
7862 }
7863 Err(_) => {
7864 g.resolved = true;
7865 g.chosen_title = None;
7866 g.choice = Some(GithubTextChoice::Fallback);
7867 }
7868 }
7869 }
7870 },
7871 }
7872 if let Some(rejected) = ask_again {
7873 let w = gt::Withheld {
7876 title: g.title,
7877 body: g.body,
7878 categories: g.categories.clone(),
7879 };
7880 let artifact = match crate::run::write_artifact(
7883 &self.state,
7884 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7885 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7886 ) {
7887 Ok(p) => Some(p),
7888 Err(e) => {
7889 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7890 None
7891 }
7892 };
7893 let (detail, comparison) = gt::question_content(
7894 &w,
7895 &pr.title,
7896 &pr.body,
7897 &self.state.id,
7898 rejected.as_deref(),
7899 artifact.as_deref(),
7900 g.rewrite.as_ref(),
7901 );
7902 let mut q = ask::Question::new(
7903 self.state.id.clone(),
7904 gt::ASK_NODE.to_owned(),
7905 gt::ASK_SEAT.to_owned(),
7906 gt::question_summary(&self.state.config.graph.language, &w),
7907 detail,
7908 gt::question_choices(&w, g.rewrite.as_ref()),
7909 );
7910 q.posting_comparison = Some(comparison);
7911 q.answer_timeout = timeout;
7912 g.question = Some(q.id.clone());
7915 g.asks += 1;
7916 self.state.github_text = Some(g.clone());
7917 self.state.event(
7918 "github-text",
7919 format!("asking the owner about the withheld text ({})", q.short()),
7920 );
7921 self.state.save()?;
7922 store
7923 .put_executable(&mut q)
7924 .context("file the posting-gate question")?;
7925 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7926 tracing::warn!(
7927 "could not notify about posting-gate question {}: {e:#}",
7928 q.short()
7929 );
7930 }
7931 self.park_on_github_text(&g)?;
7932 return Ok(None);
7933 }
7934 }
7935 let resolved = g.chosen_title.clone();
7936 let original = g.choice == Some(GithubTextChoice::Original);
7937 self.state.event(
7939 "github-text",
7940 if original {
7941 format!(
7942 "posting the original text for: {}",
7943 match (g.title, g.body) {
7944 (true, true) => "title, description",
7945 (true, false) => "title",
7946 _ => "description",
7947 }
7948 )
7949 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7950 "posting the saved rewrite title and description".to_owned()
7951 } else if resolved.is_some() {
7952 "posting the owner's replacement title".to_owned()
7953 } else {
7954 "posting the neutral text for the withheld fields".to_owned()
7955 },
7956 );
7957 self.state.github_text = Some(g.clone());
7958 self.state.save()?;
7959 if g.choice == Some(GithubTextChoice::Rewrite)
7960 && let Some(r) = g.rewrite.as_ref()
7961 {
7962 return Ok(Some(PrMessage {
7963 title: r.title.clone(),
7964 body: r.body.clone(),
7965 }));
7966 }
7967 let w = gt::Withheld {
7968 title: g.title,
7969 body: g.body,
7970 categories: g.categories.clone(),
7971 };
7972 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7973 if let Some(t) = resolved {
7974 title = t;
7975 }
7976 let english = gt::LanguageDecision {
7978 title_english: true,
7979 body_english: true,
7980 body_complete: true,
7981 };
7982 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7983 if original {
7984 if g.title {
7987 title = pr.title.clone();
7988 }
7989 if g.body {
7990 body = pr.body.clone();
7991 }
7992 }
7993 Ok(Some(PrMessage { title, body }))
7994 }
7995
7996 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7997 self.state.github_text = Some(g.clone());
7998 self.state.end_pass_parked(ParkCause::Question);
7999 self.state.parked = true;
8000 self.state.event(
8001 "github-text",
8002 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8003 );
8004 self.state.save()
8005 }
8006
8007 async fn judge_pr_language(
8010 &mut self,
8011 cwd: &Path,
8012 pr: &PrMessage,
8013 ) -> Option<crate::github_text::LanguageDecision> {
8014 self.state.config.roles.language_judge.as_ref()?;
8015 if !self.state.config.graph.language_gate() {
8016 return None;
8017 }
8018 let decision =
8019 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8020 self.state.event(
8021 "github-text",
8022 if decision.is_some() {
8023 "language decided by the language judge"
8024 } else {
8025 "language judge unavailable; using built-in heuristics"
8026 },
8027 );
8028 decision
8029 }
8030
8031 async fn merge(&mut self) -> Result<()> {
8032 if self
8047 .state
8048 .base_sync
8049 .as_ref()
8050 .is_some_and(|s| s.conflict.is_some())
8051 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8052 != Some(RunStatus::Gating)
8053 || !self.state.gate_allows_progress()
8061 {
8062 return Ok(());
8063 }
8064 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8073 return Ok(());
8074 }
8075 let Some(winner) = self.state.winner().cloned() else {
8076 return Ok(());
8077 };
8078 let repo = self.state.repo.clone();
8079 let base = self.state.base_branch.clone();
8080 let mode = self.state.config.merge.mode;
8081 let style = self.state.config.merge.style;
8082 let facts = if is_review_run(&self.state) {
8083 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8084 let start = review_base(
8085 &repo,
8086 &self.state.config.merge.remote,
8087 &base,
8088 &self.state.base_commit,
8089 &winner.branch,
8090 )
8091 .await;
8092 branch_facts(&repo, &start, &winner.branch).await
8093 } else {
8094 None
8095 };
8096 let leaked = if is_review_run(&self.state) {
8099 leaked_subjects(&self.state, &winner.branch).await
8100 } else {
8101 Some(Vec::new())
8102 };
8103 let Some(pr) = self
8107 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8108 .await?
8109 else {
8110 return Ok(());
8111 };
8112 let message = pr.commit_message();
8113
8114 let outcome = match mode {
8115 MergeMode::None => MergeOutcome {
8116 mode,
8117 ok: true,
8118 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8119 empty: false,
8120 },
8121 MergeMode::Pr | MergeMode::Local
8122 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8123 {
8124 MergeOutcome {
8125 mode,
8126 ok: false,
8127 detail: empty_candidate_detail(&self.state, &base),
8128 empty: true,
8129 }
8130 }
8131 MergeMode::Local => {
8132 let on = git::current_branch(&repo).await?;
8133 if on.as_deref() != Some(base.as_str()) {
8134 MergeOutcome {
8135 mode,
8136 ok: false,
8137 detail: format!(
8138 "{} has {} checked out, not the base branch {base}",
8139 repo.display(),
8140 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8141 ),
8142 empty: false,
8143 }
8144 } else if !git::is_clean(&repo).await? {
8145 MergeOutcome {
8146 mode,
8147 ok: false,
8148 detail: format!("{} is dirty; refusing to merge", repo.display()),
8149 empty: false,
8150 }
8151 } else {
8152 let out = match style {
8153 MergeStyle::Merge => {
8154 git::merge_no_ff(&repo, &winner.branch, &message).await?
8155 }
8156 MergeStyle::Squash => {
8157 git::merge_squash(&repo, &winner.branch, &message).await?
8158 }
8159 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8160 };
8161 MergeOutcome {
8162 mode,
8163 ok: out.ok(),
8164 detail: if out.ok() { out.stdout } else { out.stderr },
8165 empty: false,
8166 }
8167 }
8168 }
8169 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8170 self.state.pr_publication_pending = true;
8171 self.state.save()?;
8172 let out =
8173 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8174 self.state.pr_publication_pending = out.is_err();
8175 MergeOutcome {
8176 mode,
8177 ok: out.is_ok(),
8178 detail: match out {
8179 Ok(url) => url,
8180 Err(e) => format!("{e:#}"),
8181 },
8182 empty: false,
8183 }
8184 }
8185 MergeMode::Pr => {
8186 let remote = self.state.config.merge.remote.clone();
8187 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8188 if !pushed.ok() {
8189 MergeOutcome {
8190 mode,
8191 ok: false,
8192 detail: pushed.stderr,
8193 empty: false,
8194 }
8195 } else {
8196 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8202 let out = match pr_merge_plan(found) {
8203 PrPlan::Create => {
8204 gh_pr_create(
8205 &winner.worktree,
8206 &base,
8207 &winner.branch,
8208 &pr.title,
8209 &pr.body,
8210 )
8211 .await
8212 }
8213 PrPlan::Adopt { url, title } => {
8214 self.state
8215 .event("merge", format!("Pr: adopted open pull request {url}"));
8216 if self.state.github_text.as_ref().is_some_and(|g| {
8217 g.resolved
8218 && g.choice == Some(GithubTextChoice::Rewrite)
8219 && g.rewrite.is_some()
8220 }) {
8221 land::set_pr_text_verbatim(
8224 &winner.worktree,
8225 &url,
8226 &pr.title,
8227 &pr.body,
8228 )
8229 .await
8230 .map(|()| url)
8231 } else {
8232 if title != pr.title
8233 && (!is_review_run(&self.state)
8234 || leaked
8235 .as_deref()
8236 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8237 && let Err(e) =
8238 if self.state.github_text.as_ref().is_some_and(|g| {
8239 g.resolved
8240 && g.title
8241 && g.choice == Some(GithubTextChoice::Original)
8242 }) {
8243 land::set_pr_title_verbatim(
8245 &winner.worktree,
8246 &url,
8247 &pr.title,
8248 )
8249 .await
8250 } else {
8251 land::set_pr_title(
8252 &mut self.state,
8253 &winner.worktree,
8254 &url,
8255 &pr.title,
8256 )
8257 .await
8258 }
8259 {
8260 tracing::warn!("could not refresh title of {url}: {e:#}");
8261 self.state
8262 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8263 }
8264 Ok(url)
8265 }
8266 }
8267 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8268 };
8269 match out {
8270 Ok(url) => MergeOutcome {
8271 mode,
8272 ok: true,
8273 detail: url,
8274 empty: false,
8275 },
8276 Err(e) => MergeOutcome {
8277 mode,
8278 ok: false,
8279 detail: e.to_string(),
8280 empty: false,
8281 },
8282 }
8283 }
8284 }
8285 };
8286
8287 self.state.status = match (mode, outcome.ok) {
8288 (MergeMode::None, _) => RunStatus::Ready,
8289 (_, true) if outcome.empty => RunStatus::Ready,
8290 (MergeMode::Pr, true) => RunStatus::Ready,
8291 (_, true) => RunStatus::Merged,
8292 (_, false) => RunStatus::Blocked,
8293 };
8294 self.state.event(
8295 "merge",
8296 format!(
8297 "{:?}: {}",
8298 mode,
8299 outcome.detail.lines().next().unwrap_or("")
8300 ),
8301 );
8302 self.state.merge = Some(outcome);
8303 self.state.save()?;
8304
8305 if self.state.config.graph.land
8311 && mode == MergeMode::Pr
8312 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8313 {
8314 self.run_land().await?;
8315 }
8316 self.settle_questions();
8321 Ok(())
8322 }
8323
8324 async fn run_land(&mut self) -> Result<()> {
8335 crate::pr_head::ensure_owner(&self.state)?;
8336 let url = self
8337 .state
8338 .pr
8339 .as_ref()
8340 .map(|pr| pr.url.clone())
8341 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8342 .unwrap_or_default();
8343 let url = url.lines().next().unwrap_or("").trim().to_owned();
8344 if !url.starts_with("http") {
8345 return Ok(());
8346 }
8347 loop {
8350 match land::land(&mut self.state, &url).await {
8351 Ok(_)
8352 if self
8353 .state
8354 .merge_repairs
8355 .last()
8356 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8357 {
8358 if self.repair_merge_review().await? {
8359 continue;
8360 }
8361 }
8362 Ok(pr) if self.state.parked => {
8363 let _ = pr;
8367 }
8368 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8369 self.state.status = RunStatus::Superseded;
8370 self.state.save()?;
8371 }
8372 Ok(pr) => {
8373 self.state.status = match pr.state {
8374 land::PrLifecycle::Merged => RunStatus::Merged,
8375 _ => RunStatus::Blocked,
8376 };
8377 if bump::should_release_bump(self.state.status)
8384 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8385 {
8386 self.state
8392 .event("bump", format!("release bump skipped: {e:#}"));
8393 }
8394 if self.state.status == RunStatus::Merged {
8397 crate::followup::after_merge(&mut self.state, &pr.url).await;
8398 }
8399 self.state.save()?;
8400 }
8401 Err(e) => {
8402 if e.is::<crate::repo_lock::ResourceBlocked>() {
8403 return Err(e);
8404 }
8405 self.state.status = RunStatus::Blocked;
8406 self.state.event("land", format!("gave up: {e}"));
8407 self.state.save()?;
8408 }
8409 }
8410 break;
8411 }
8412 Ok(())
8413 }
8414
8415 async fn repair_merge_review(&mut self) -> Result<bool> {
8418 use crate::run::MergeRepairPhase as Phase;
8419 crate::pr_head::ensure_owner(&self.state)?;
8420 let i = self.state.merge_repairs.len() - 1;
8421 let request = self.state.merge_repairs[i].clone();
8422 if request.phase == Phase::Fixing {
8423 let restored = match self.state.winner().cloned() {
8424 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8425 None => Err(anyhow::anyhow!("repair has no winning branch")),
8426 };
8427 let why = match restored {
8428 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8429 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8430 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8431 };
8432 self.hold_merge_repair(i, why)?;
8433 return Ok(false);
8434 }
8435 let failure = if i >= self.state.config.graph.land_rounds
8436 || self.state.config.graph.review_rounds == 0
8437 {
8438 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8439 } else {
8440 None
8441 };
8442 if let Some(why) = failure {
8443 self.hold_merge_repair(i, why)?;
8444 return Ok(false);
8445 }
8446 if request.phase == Phase::Requested {
8447 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8448 Ok(pr) => pr,
8449 Err(e) => {
8450 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8451 return Ok(false);
8452 }
8453 };
8454 self.state.merge_repairs[i].phase = Phase::Fixing;
8455 self.state.land_approval = None;
8456 self.state.land_armed_head = None;
8457 self.state.save()?;
8458 let evidence = serde_json::to_string_pretty(&request)?;
8461 let reason = format!(
8462 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8463 );
8464 let budget = self.state.config.graph.land_rounds;
8465 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8466 match result {
8467 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8468 Ok(land::Fixed::Failed(why)) => {
8469 self.hold_merge_repair(
8470 i,
8471 format!(
8472 "repair failed: {why}; inspect the branch and run an explicit review"
8473 ),
8474 )?;
8475 return Ok(false);
8476 }
8477 Err(e) => {
8478 self.hold_merge_repair(
8479 i,
8480 format!(
8481 "repair failed: {e:#}; inspect the branch and run an explicit review"
8482 ),
8483 )?;
8484 return Ok(false);
8485 }
8486 }
8487 self.begin_merge_repair_review(i)?;
8491 }
8492 self.review_loop().await?;
8493 self.gate().await?;
8494 if self.resource_halt()? || self.state.parked {
8495 return Ok(false);
8496 }
8497 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8498 != Some(RunStatus::Gating)
8499 || !self.state.gate_allows_progress()
8500 {
8501 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8502 return Ok(false);
8503 }
8504 let winner = self
8507 .state
8508 .winner()
8509 .cloned()
8510 .context("repair has no winning branch")?;
8511 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8512 if self.state.gate_head.as_deref() != Some(after.as_str())
8513 || !git::is_clean(&winner.worktree).await?
8514 {
8515 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8516 return Ok(false);
8517 }
8518 let expected = self
8519 .state
8520 .pr_pushed_head
8521 .as_deref()
8522 .unwrap_or(&request.head)
8523 .to_owned();
8524 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8525 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8528 let refspec = format!("{after}:refs/heads/{public}");
8529 let push = git::git_raw(
8530 &winner.worktree,
8531 &["push", &lease, &self.state.config.merge.remote, &refspec],
8532 )
8533 .await;
8534 match push {
8535 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8536 Ok(out) => {
8537 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8538 return Ok(false);
8539 }
8540 Err(e) => {
8541 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8542 return Ok(false);
8543 }
8544 }
8545 self.state.merge_repairs[i].phase = Phase::Complete;
8546 self.state.status = RunStatus::Landing;
8547 self.state.event(
8548 "repair",
8549 "new review and gate completed; fresh owner merge decision required",
8550 );
8551 self.state.save()?;
8552 Ok(true)
8553 }
8554
8555 fn merge_repair_artifacts(&self) -> PathBuf {
8556 let artifacts = agent::artifacts_dir(&self.state.dir());
8557 match self
8558 .state
8559 .merge_repairs
8560 .last()
8561 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8562 {
8563 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8564 None => artifacts,
8565 }
8566 }
8567
8568 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8569 self.state.reviews.clear();
8570 self.state.gate.clear();
8571 self.state.gate_ran = false;
8572 self.state.gate_head = None;
8573 self.state.gate_fixes.clear();
8574 self.state.gate_repair_e2e = None;
8575 self.state.pre_gate.clear();
8576 self.state.pre_gate_commit = None;
8577 self.state.contested_handoff = None;
8578 self.state.land_reruns = None;
8579 self.state.land_approval = None;
8580 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8581 self.state.status = RunStatus::Reviewing;
8582 self.state.save()
8583 }
8584
8585 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8586 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8587 self.state.merge_repairs[index].failure = Some(why.clone());
8588 self.state.status = RunStatus::Blocked;
8589 self.state.event("repair", why);
8590 self.state.save()
8591 }
8592
8593 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8597 if let Some(existing) = self.state.seats.get(key)
8598 && existing.agent == agent
8599 {
8600 return existing.clone();
8601 }
8602 let fresh = if self.state.seats.contains_key(key) {
8605 handover_seat(key, agent, self.state.next_seat_seed())
8606 } else {
8607 SeatState::new(key, agent, self.state.seed)
8608 };
8609 self.state.seats.insert(key.to_owned(), fresh.clone());
8610 fresh
8611 }
8612
8613 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8619 match self.state.seats.get(key) {
8620 Some(s) if s.agent != spec.id => {
8621 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8622 }
8623 _ => spec,
8624 }
8625 }
8626
8627 fn view(&self, c: &Candidate) -> CandidateView {
8629 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8630 .unwrap_or_default();
8631 let (patch, _) = blind::sanitize_patch(
8632 &format!("candidate {} patch", c.label),
8633 &raw,
8634 &self.state.config.blind,
8635 );
8636 CandidateView {
8637 label: c.label,
8638 branch: c.branch.clone(),
8639 summary: c.summary.clone(),
8640 stat: c.stat.clone(),
8641 patch,
8642 }
8643 }
8644
8645 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8647 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8648 prompt::judge(
8649 "(see above)",
8650 &views,
8651 self.roles.judges.len(),
8652 base_short,
8653 "en",
8654 )
8655 }
8656
8657 fn vote_prompt_full(
8663 &self,
8664 j: usize,
8665 labels: &[char],
8666 language: &str,
8667 candidates: &[Candidate],
8668 base_short: &str,
8669 ) -> String {
8670 let mut text = format!(
8671 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8672 prompt::final_vote(labels, language),
8673 self.state.instruction,
8674 self.candidate_block(candidates, base_short)
8675 );
8676 if let Some(own) = self
8677 .state
8678 .judgements
8679 .get(j)
8680 .filter(|r| !r.ranking.is_empty())
8681 {
8682 let reasons = own
8683 .reasons
8684 .iter()
8685 .map(|(k, v)| format!("- {k}: {v}"))
8686 .collect::<Vec<_>>()
8687 .join("\n");
8688 text.push_str(&format!(
8689 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8690 own.ranking.iter().collect::<String>(),
8691 if reasons.is_empty() {
8692 ""
8693 } else {
8694 ", because:\n"
8695 }
8696 ));
8697 }
8698 if !self.state.deliberation.is_empty() {
8699 text.push_str("\n# What was argued before this vote\n");
8700 for t in self.transcript(&[], j) {
8701 text.push_str(&format!(
8702 "\n## {}{}\n\n{}\n",
8703 t.who,
8704 if t.is_self { " (you)" } else { "" },
8705 t.body.trim()
8706 ));
8707 }
8708 }
8709 text
8710 }
8711
8712 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8719 let mut turns = Vec::new();
8720 for j in &self.state.judgements {
8721 if j.ranking.is_empty() {
8722 continue;
8723 }
8724 let reasons = j
8725 .reasons
8726 .iter()
8727 .map(|(k, v)| format!("- {k}: {v}"))
8728 .collect::<Vec<_>>()
8729 .join("\n");
8730 turns.push(Turn {
8731 who: format!("Judge {} (opening ranking)", j.judge),
8732 is_self: j.judge == self_idx + 1,
8733 body: format!(
8734 "Ranked {}{}{reasons}",
8735 j.ranking.iter().collect::<String>(),
8736 if reasons.is_empty() {
8737 ""
8738 } else {
8739 ", because:\n"
8740 }
8741 ),
8742 });
8743 }
8744 for t in self
8745 .state
8746 .deliberation
8747 .iter()
8748 .flat_map(|r| r.turns.iter())
8749 .chain(current)
8750 {
8751 turns.push(Turn {
8752 who: format!("Judge {}", t.judge),
8753 is_self: t.judge == self_idx + 1,
8754 body: t.body.clone(),
8755 });
8756 }
8757 turns
8758 }
8759}
8760
8761fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8763 agent::has_session(spec.kind, seat, sessions)
8764}
8765
8766fn next_untried_in_roster<'a>(
8787 roster: &'a [AgentSpec],
8788 start: usize,
8789 tried: &BTreeSet<String>,
8790) -> Option<&'a AgentSpec> {
8791 roster
8792 .get(start + 1..)?
8793 .iter()
8794 .find(|s| !tried.contains(&s.id))
8795}
8796
8797fn pick_successor<'a>(
8818 roster: &'a [AgentSpec],
8819 start: usize,
8820 tried: &BTreeSet<String>,
8821 carried: Option<&BTreeSet<String>>,
8822 others: &BTreeSet<String>,
8823) -> Option<&'a AgentSpec> {
8824 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8825 roster
8826 .get(start + 1..)
8827 .and_then(|tail| {
8828 tail.iter()
8829 .filter(free)
8830 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8831 })
8832 .or_else(|| {
8833 carried
8834 .is_some()
8835 .then(|| roster.iter().find(free))
8836 .flatten()
8837 })
8838 .or_else(|| match carried {
8839 Some(c) => next_for_seat(roster, start, tried, c),
8840 None => next_untried_in_roster(roster, start, tried),
8841 })
8842}
8843
8844fn next_for_seat<'a>(
8854 roster: &'a [AgentSpec],
8855 start: usize,
8856 round_tried: &BTreeSet<String>,
8857 carried_failed: &BTreeSet<String>,
8858) -> Option<&'a AgentSpec> {
8859 roster
8860 .get(start + 1..)?
8861 .iter()
8862 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8863 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8864}
8865
8866fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8872 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8873 return spec;
8874 };
8875 let ok = |id: &str| !h.failed.contains(id);
8876 if let Some(last) = h.last_ok.as_deref()
8877 && ok(last)
8878 && let Some(s) = roster.iter().find(|s| s.id == last)
8879 {
8880 return s.clone();
8881 }
8882 if ok(&spec.id) {
8883 return spec;
8884 }
8885 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8886 roster
8887 .get(from + 1..)
8888 .into_iter()
8889 .flatten()
8890 .chain(roster.iter())
8891 .find(|s| ok(&s.id))
8892 .cloned()
8893 .unwrap_or(spec)
8894}
8895
8896pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8900 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8901}
8902
8903const TIMED_OUT: &str = "timed out";
8906
8907impl FailClass {
8908 fn of(out: &AgentOutcome) -> Option<Self> {
8910 match out {
8911 AgentOutcome::Ok(_) => None,
8912 AgentOutcome::Quota(_) => Some(Self::Quota),
8913 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8914 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8915 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8916 AgentOutcome::ResourceBlocked(_) => None,
8919 }
8920 }
8921
8922 fn stem_word(&self) -> &'static str {
8924 match self {
8925 Self::Quota => "quota",
8926 _ => "handover",
8927 }
8928 }
8929}
8930
8931fn failure_signature(msg: &str) -> String {
8935 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8936 if let Some(code) = line
8939 .strip_prefix("process exited with no usable output (exit code ")
8940 .and_then(|s| s.strip_suffix(')'))
8941 .and_then(|s| s.parse::<i32>().ok())
8942 {
8943 line = format!("exited with some({code}) and no usable output");
8944 } else if matches!(
8945 line.as_str(),
8946 "process exited with no usable output (no exit code reported)"
8947 | "process exited with no usable output (exit code unavailable)"
8948 ) {
8949 line = "exited with none and no usable output".to_owned();
8950 }
8951 let mut out = Vec::new();
8952 for word in line.split_whitespace() {
8953 if word.contains('/') || word.contains('\\') {
8954 out.push("<path>".to_owned());
8955 continue;
8956 }
8957 let mut w = String::new();
8958 let mut in_digits = false;
8959 for c in word.chars() {
8960 if c.is_ascii_digit() {
8961 if !in_digits {
8962 w.push('#');
8963 }
8964 in_digits = true;
8965 } else {
8966 in_digits = false;
8967 w.push(c);
8968 }
8969 }
8970 out.push(w);
8971 }
8972 out.join(" ").chars().take(120).collect()
8973}
8974
8975fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8982 match cur {
8983 FailClass::Quota | FailClass::Timeout => true,
8984 FailClass::Other(_) => prev != Some(cur),
8985 }
8986}
8987
8988fn fail_reason(out: &AgentOutcome) -> String {
8990 match out {
8991 AgentOutcome::Ok(_) => String::new(),
8992 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8993 AgentOutcome::Dropped(o) => format!(
8994 "the CLI dropped the stream ({})",
8995 o.dropped
8996 .as_ref()
8997 .map(|d| d.why.as_str())
8998 .unwrap_or("it ended without delivering its answer")
8999 ),
9000 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9001 e.lines().next().unwrap_or("").chars().take(160).collect()
9002 }
9003 }
9004}
9005
9006pub(crate) fn record_handover(
9009 state: &mut RunState,
9010 node: &str,
9011 seat: &str,
9012 from: &str,
9013 to: &str,
9014 class: &FailClass,
9015 reason: &str,
9016) {
9017 let message = if *class == FailClass::Quota {
9018 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9019 } else {
9020 format!("{seat}: handed over {from} -> {to} ({reason})")
9021 };
9022 state.event(node, message);
9023 state.handovers.push(Handover {
9024 at: Timestamp::now(),
9025 node: node.to_owned(),
9026 seat: seat.to_owned(),
9027 from: from.to_owned(),
9028 to: to.to_owned(),
9029 reason: reason.to_owned(),
9030 });
9031}
9032
9033fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9047 commands.iter().any(|c| c.exit_code.is_none())
9048}
9049
9050fn verified_noop_claim(
9063 usable: bool,
9064 commands: &[agent::CommandEvidence],
9065 text: &str,
9066) -> Option<String> {
9067 (usable && !has_unconfirmed_command(commands))
9068 .then(|| verdict::verified_noop(text))
9069 .flatten()
9070}
9071
9072fn short(commit: &str) -> String {
9073 commit.chars().take(7).collect()
9074}
9075
9076fn make_executable(path: &Path) -> Result<()> {
9077 #[cfg(unix)]
9078 {
9079 use std::os::unix::fs::PermissionsExt as _;
9080 let mut perms = std::fs::metadata(path)?.permissions();
9081 perms.set_mode(0o755);
9082 std::fs::set_permissions(path, perms)?;
9083 }
9084 #[cfg(not(unix))]
9085 {
9086 let _ = path;
9087 }
9088 Ok(())
9089}
9090
9091struct WaveCtx<'a> {
9098 run: &'a str,
9101 node: &'a str,
9103 prompts: &'a Prompts,
9104 cache: Option<&'a Path>,
9106 round: Option<usize>,
9109 carry_seats: bool,
9113 pauses: Option<(&'a Pause, &'a Pause)>,
9116}
9117
9118fn repository_wait(
9120 state: &RunState,
9121 pauses: Option<(&Pause, &Pause)>,
9122) -> crate::repo_lock::WaitPolicy {
9123 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9124 let inherited = crate::repo_lock::WaitPolicy::current();
9125 crate::repo_lock::WaitPolicy::new(
9126 Duration::from_secs(state.config.graph.timeout_resource_wait),
9127 move || {
9128 inherited.cancelled()
9129 || pauses
9130 .as_ref()
9131 .is_some_and(|(a, b)| a.parked() || b.parked())
9132 },
9133 )
9134}
9135
9136fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9137 if error.is::<crate::repo_lock::ResourceBlocked>() {
9138 state.resource_block = Some(crate::run::ResourceBlock {
9139 node: "repository".into(),
9140 kind: if error.to_string().contains("disk pressure") {
9141 "disk pressure"
9142 } else {
9143 "repository coordination"
9144 }
9145 .into(),
9146 detail: error.to_string(),
9147 at: Timestamp::now(),
9148 });
9149 state.event("resource", error.to_string());
9150 }
9151}
9152
9153async fn coordinate_operation(
9154 state: &mut RunState,
9155 cwd: &Path,
9156 wait: &crate::repo_lock::WaitPolicy,
9157) -> Result<Option<crate::repo_lock::SeatLease>> {
9158 let result =
9159 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9160 .await;
9161 if let Err(e) = &result {
9162 record_repository_block(state, e);
9163 }
9164 result
9165}
9166
9167async fn run_one(
9169 job: SeatJob,
9170 sem: Arc<Semaphore>,
9171 ctx: &WaveCtx<'_>,
9172 state: &mut RunState,
9173 attempt: usize,
9174) -> (SeatState, AgentOutcome) {
9175 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9176 .await
9177 .pop()
9178 .expect("one job in, one result out");
9179 (seat, out)
9180}
9181
9182async fn wave(
9188 jobs: Vec<SeatJob>,
9189 sem: Arc<Semaphore>,
9190 ctx: &WaveCtx<'_>,
9191 state: &mut RunState,
9192 attempt: usize,
9193) -> Vec<(usize, SeatState, AgentOutcome)> {
9194 let WaveCtx {
9195 run,
9196 node,
9197 prompts,
9198 cache,
9199 round,
9200 carry_seats: _,
9201 pauses,
9202 } = *ctx;
9203 for job in &jobs {
9204 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9205 }
9206 if let Err(e) = state.save() {
9207 tracing::warn!("could not persist in-progress seats: {e:#}");
9212 }
9213 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9229 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9233 let cache_guard = if let Some(cache_dir) = cache {
9234 if jobs_had_a_writer {
9235 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9236 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9237 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9238 {
9239 Ok(Some(g)) => Some(g),
9240 Ok(None) => {
9243 state.resource_block = Some(crate::run::ResourceBlock {
9244 node: node.to_owned(),
9245 kind: "shared build cache".to_owned(),
9246 detail: "the run is parking".to_owned(),
9247 at: Timestamp::now(),
9248 });
9249 return block_wave(
9250 jobs,
9251 state,
9252 node,
9253 attempt,
9254 "parked while waiting for the shared build cache",
9255 );
9256 }
9257 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9260 Err(e) => {
9261 let why = format!("{e:#}");
9265 state.resource_block = Some(crate::run::ResourceBlock {
9266 node: node.to_owned(),
9267 kind: "shared build cache".to_owned(),
9268 detail: why.clone(),
9269 at: Timestamp::now(),
9270 });
9271 state.event(
9272 "resource",
9273 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9274 );
9275 return block_wave(jobs, state, node, attempt, &why);
9276 }
9277 }
9278 } else {
9279 None
9280 }
9281 } else {
9282 None
9283 };
9284 state.seats_spawning(node, attempt);
9287 if let Err(e) = state.save() {
9288 tracing::warn!("could not persist the start of a wave: {e:#}");
9289 }
9290 let slot_limit = resource_limit
9294 + jobs
9295 .iter()
9296 .map(|j| j.timeout)
9297 .max()
9298 .unwrap_or_default()
9299 .saturating_mul(jobs.len() as u32);
9300 let mut set = tokio::task::JoinSet::new();
9301 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9304 let overlay = prompts.overlay(node);
9305 let repository_wait = repository_wait(state, pauses);
9306 for (i, mut job) in jobs.into_iter().enumerate() {
9307 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9308 if cache.is_some() {
9309 job.prompt.push('\n');
9310 job.prompt
9311 .push_str(&prompt::build_cache_note(node, job.allow_write));
9312 }
9313 let sem = Arc::clone(&sem);
9314 let run = run.to_owned();
9315 let node = node.to_owned();
9316 let attachments = if node == "implement" {
9319 state.attachments.clone()
9320 } else {
9321 Vec::new()
9322 };
9323 let cache = cache
9334 .filter(|_| job.allow_write && cache_guard.is_some())
9335 .map(Path::to_path_buf);
9336 let spawned_tx = spawned_tx.clone();
9337 let repository_wait = repository_wait.clone();
9338 let coordination = state.config.coordination.clone();
9339 let fetched_base = state.fetched_base.clone();
9340 set.spawn(async move {
9341 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9345 Ok(p) => p,
9346 Err(_) => {
9347 return (
9348 i,
9349 job.seat,
9350 AgentOutcome::ResourceBlocked(format!(
9351 "no parallel slot became free within {}s",
9352 slot_limit.as_secs()
9353 )),
9354 false,
9355 );
9356 }
9357 };
9358 spawned_tx.send(job.seat.key.clone()).ok();
9359 let mut seat = job.seat;
9360 let out = repository_wait
9361 .scope(agent::invoke(
9362 &job.spec,
9363 &mut seat,
9364 &Invocation {
9365 scope: crate::agent::InvocationScope::Repository,
9366 coordination: coordination.as_ref(),
9367 fetched_base: fetched_base.as_ref(),
9368 cwd: &job.cwd,
9369 prompt: &job.prompt,
9370 timeout: job.timeout,
9371 allow_write: job.allow_write,
9372 unsandboxed: false,
9373 sessions: job.sessions,
9374 artifacts: &job.artifacts,
9375 stem: &job.stem,
9376 run: &run,
9377 node: &node,
9378 cache_dir: cache.as_deref(),
9379 attachments: &attachments,
9380 writable: &[],
9381 },
9382 ))
9383 .await;
9384 let interrupted = out
9385 .as_ref()
9386 .err()
9387 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9388 let out = match out {
9389 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9390 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9391 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9399 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9400 Ok(o) => AgentOutcome::Failed(format!(
9401 "process exited with no usable output ({})",
9402 crate::notices::exit_status(o.exit_code)
9403 )),
9404 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9405 AgentOutcome::ResourceBlocked(e.to_string())
9406 }
9407 Err(e) => AgentOutcome::Failed(e.to_string()),
9408 };
9409 (i, seat, out, interrupted)
9410 });
9411 }
9412 drop(spawned_tx);
9413 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9414 loop {
9415 let joined = tokio::select! {
9416 Some(key) = spawned_rx.recv() => {
9417 state.seat_spawned(&key);
9419 if let Err(e) = state.save() {
9420 tracing::warn!("could not persist a seat's start: {e:#}");
9421 }
9422 continue;
9423 }
9424 joined = set.join_next() => joined,
9425 };
9426 let Some(joined) = joined else {
9427 break;
9428 };
9429 let (i, seat, out, interrupted) = match joined {
9430 Ok(v) => v,
9431 Err(e) => {
9435 tracing::error!("agent task panicked: {e}");
9436 continue;
9437 }
9438 };
9439 state.seat_finished(&seat.key);
9440 if interrupted
9441 && node == "implement"
9442 && let Some(label) = state
9443 .candidates
9444 .iter()
9445 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9446 .map(|candidate| candidate.label)
9447 {
9448 state.candidate_preservation.insert(
9449 label,
9450 "interrupted implementation awaiting persistence".into(),
9451 );
9452 state.interrupted_implementations.insert(label);
9453 }
9454 if let AgentOutcome::ResourceBlocked(why) = &out
9455 && state.resource_block.is_none()
9456 {
9457 state.resource_block = Some(crate::run::ResourceBlock {
9458 node: node.to_owned(),
9459 kind: if why.contains("disk pressure") {
9460 "disk pressure"
9461 } else if why.contains("repository coordination") {
9462 "repository coordination"
9463 } else {
9464 "parallel slot"
9465 }
9466 .to_owned(),
9467 detail: why.clone(),
9468 at: Timestamp::now(),
9469 });
9470 let activity = if interrupted {
9471 "agent execution interrupted"
9472 } else {
9473 "no agent was started"
9474 };
9475 state.event("resource", format!("{node}: {activity} - {why}"));
9476 }
9477 record_jobs(state, node, round, &seat.key, &out);
9478 if let Err(e) = state.save() {
9479 tracing::warn!("could not persist a seat's completion: {e:#}");
9480 }
9481 if collected.len() <= i {
9482 collected.resize_with(i + 1, || None);
9483 }
9484 collected[i] = Some((i, seat, out));
9485 }
9486 if state
9492 .active
9493 .values()
9494 .any(|a| a.node == node && a.attempt == attempt)
9495 {
9496 state
9497 .active
9498 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9499 if let Err(e) = state.save() {
9500 tracing::warn!("could not persist the end of a wave: {e:#}");
9501 }
9502 }
9503 if let Some(cache_dir) = cache
9510 && jobs_had_a_writer
9511 {
9512 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9513 }
9514 if let Some(guard) = cache_guard {
9515 guard.release();
9516 }
9517 collected.into_iter().flatten().collect()
9518}
9519
9520fn record_jobs(
9531 state: &mut RunState,
9532 node: &str,
9533 round: Option<usize>,
9534 seat: &str,
9535 out: &AgentOutcome,
9536) {
9537 let commands: &[agent::CommandEvidence] = match out {
9538 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9539 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9540 };
9541 let checked_at = Timestamp::now();
9542 for c in commands {
9543 state.jobs.push(JobRecord {
9544 node: node.to_owned(),
9545 round,
9546 seat: seat.to_owned(),
9547 id: c.id.clone(),
9548 description: c.description.clone(),
9549 checked_at,
9550 status: match c.exit_code {
9551 Some(0) => JobStatus::Completed,
9552 Some(_) => JobStatus::Failed,
9553 None => JobStatus::Unknown,
9554 },
9555 exit_code: c.exit_code,
9556 result_summary: c.result_summary.clone(),
9557 source: c.source.clone(),
9558 });
9559 }
9560}
9561
9562fn round_is_clean(
9583 blocking: usize,
9584 e2e_ok: bool,
9585 answered: usize,
9586 expected: usize,
9587 quota_missing: usize,
9588 policy: IncompleteReviewPolicy,
9589) -> bool {
9590 if blocking != 0 || !e2e_ok {
9591 return false;
9592 }
9593 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9594 return true;
9595 }
9596 answered > 0 && expected - answered <= quota_missing
9597}
9598
9599pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9623 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9624 return Some(RunStatus::Gating);
9625 }
9626 let last = reviews.last()?;
9627 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9628 if reviews.len() < max_rounds && !stagnant {
9629 return None;
9630 }
9631 if last.incomplete() && last.blocking == 0 {
9632 return Some(RunStatus::Blocked);
9633 }
9634 if last.e2e_status() == E2eStatus::ResourceBlocked {
9635 return None;
9636 }
9637 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9638 RunStatus::Gating
9639 } else {
9640 RunStatus::Blocked
9641 })
9642}
9643
9644pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9659 if nudged {
9660 (full / 4).max(Duration::from_secs(120)).min(full)
9661 } else {
9662 full
9663 }
9664}
9665
9666#[allow(clippy::too_many_arguments)]
9695async fn ask_json_wave<T>(
9696 jobs: Vec<SeatJob>,
9697 sem: Arc<Semaphore>,
9698 retries: usize,
9699 roster: &[AgentSpec],
9700 ctx: &WaveCtx<'_>,
9701 losses: &mut Vec<QuotaLoss>,
9702 state: &mut RunState,
9703 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9704) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9705where
9706 T: serde::de::DeserializeOwned + Send + 'static,
9707{
9708 ask_wave_with(
9709 jobs,
9710 sem,
9711 retries,
9712 roster,
9713 ctx,
9714 losses,
9715 state,
9716 &|text: &str| {
9717 let v = verdict::extract_json::<T>(text)?;
9718 validate(&v)?;
9719 Ok(v)
9720 },
9721 )
9722 .await
9723}
9724
9725#[allow(clippy::too_many_arguments)]
9734async fn ask_wave_with<T>(
9735 jobs: Vec<SeatJob>,
9736 sem: Arc<Semaphore>,
9737 retries: usize,
9738 roster: &[AgentSpec],
9739 ctx: &WaveCtx<'_>,
9740 losses: &mut Vec<QuotaLoss>,
9741 state: &mut RunState,
9742 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9743) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9744where
9745 T: Send + 'static,
9746{
9747 let n = jobs.len();
9748 let originals: Vec<SeatJob> = jobs;
9749 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9750 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9751 let mut nudges: Vec<usize> = vec![0; n];
9758 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9762 let mut tried: Vec<BTreeSet<String>> = specs
9763 .iter()
9764 .map(|s| BTreeSet::from([s.id.clone()]))
9765 .collect();
9766 let starts: Vec<usize> = specs
9767 .iter()
9768 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9769 .collect();
9770 let carry = ctx.carry_seats && !roster.is_empty();
9771 let carried: Vec<BTreeSet<String>> = originals
9774 .iter()
9775 .map(|j| {
9776 state
9777 .seat_history
9778 .get(&j.seat.key)
9779 .filter(|_| carry)
9780 .map(|h| h.failed.clone())
9781 .unwrap_or_default()
9782 })
9783 .collect();
9784 let mut prev: Vec<Option<FailClass>> = originals
9785 .iter()
9786 .map(|j| {
9787 state
9788 .seat_history
9789 .get(&j.seat.key)
9790 .filter(|_| carry)
9791 .and_then(|h| h.last_fail.clone())
9792 })
9793 .collect();
9794 let next_agent =
9795 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9796 let others: BTreeSet<String> = specs
9797 .iter()
9798 .enumerate()
9799 .filter(|(j, _)| *j != i)
9800 .map(|(_, s)| s.id.clone())
9801 .collect();
9802 pick_successor(
9803 roster,
9804 starts[i],
9805 tried,
9806 carry.then(|| &carried[i]),
9807 &others,
9808 )
9809 .cloned()
9810 };
9811 let mut fresh: Vec<Option<String>> = vec![None; n];
9812 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9813 let mut pending: Vec<usize> = (0..n).collect();
9814
9815 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9820 for round in 0..max_rounds {
9821 if pending.is_empty() {
9822 break;
9823 }
9824 let mut batch = Vec::with_capacity(pending.len());
9825 let mut renudged: Vec<&str> = Vec::new();
9826 for &i in &pending {
9827 let src = &originals[i];
9828 let full: &str = match &src.handover {
9833 Some(h) if specs[i].id != src.spec.id => h,
9834 _ => &src.prompt,
9835 };
9836 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9839 (
9840 full.to_owned(),
9841 src.timeout,
9842 format!("{}-{word}-{}", src.stem, specs[i].id),
9843 )
9844 } else if nudges[i] == 0 {
9845 (full.to_owned(), src.timeout, src.stem.clone())
9846 } else {
9847 renudged.push(src.seat.key.as_str());
9848 let why = done[i]
9849 .as_ref()
9850 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9851 .unwrap_or_else(|| "no parsable answer".to_owned());
9852 let nudge = prompt::nudge(&why);
9853 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9854 let prompt = if nudged {
9855 nudge
9856 } else {
9857 format!("{full}\n\n---\n\n{nudge}")
9858 };
9859 (
9860 prompt,
9861 retry_budget(src.timeout, nudged),
9862 format!("{}-retry{}", src.stem, nudges[i]),
9863 )
9864 };
9865 batch.push(SeatJob {
9866 spec: specs[i].clone(),
9867 seat: seats[i].clone(),
9868 cwd: src.cwd.clone(),
9869 prompt,
9870 timeout,
9871 allow_write: src.allow_write,
9872 sessions: src.sessions,
9873 artifacts: src.artifacts.clone(),
9874 stem,
9875 handover: None,
9876 });
9877 }
9878
9879 if !renudged.is_empty() {
9880 state.event(
9881 ctx.node,
9882 format!("retry {round}: re-asking {}", renudged.join(", ")),
9883 );
9884 }
9885 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9886 let mut still = Vec::new();
9887 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9888 seats[i] = seat;
9889 if let AgentOutcome::ResourceBlocked(why) = &out {
9892 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9893 continue;
9894 }
9895 let class = FailClass::of(&out);
9896 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9899 && nudges[i] < retries
9900 && !roster.is_empty();
9901 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9902 let next = should_hand_over(prev[i].as_ref(), &cur)
9903 .then(|| next_agent(i, &tried[i], &specs))
9904 .flatten();
9905 if carry {
9906 let h = state
9907 .seat_history
9908 .entry(originals[i].seat.key.clone())
9909 .or_default();
9910 h.failed.insert(specs[i].id.clone());
9911 h.last_fail = Some(cur.clone());
9912 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9913 h.last_ok = None;
9914 }
9915 if let Err(e) = state.save() {
9918 tracing::warn!("could not persist a seat's failure history: {e:#}");
9919 }
9920 }
9921 if let Some(next) = next {
9922 record_handover(
9923 state,
9924 ctx.node,
9925 &originals[i].seat.key,
9926 &specs[i].id,
9927 &next.id,
9928 &cur,
9929 &fail_reason(&out),
9930 );
9931 tried[i].insert(next.id.clone());
9932 prev[i] = Some(cur.clone());
9933 seats[i] =
9934 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9935 specs[i] = next;
9936 fresh[i] = Some(cur.stem_word().to_owned());
9937 nudges[i] = 0;
9938 done[i] = Some(Err(anyhow::anyhow!(
9939 "handed over after: {}",
9940 fail_reason(&out)
9941 )));
9942 still.push(i);
9943 continue;
9944 }
9945 }
9946 if carry
9947 && !nudge_first
9948 && let Some(cur) = class.clone()
9949 {
9950 let h = state
9952 .seat_history
9953 .entry(originals[i].seat.key.clone())
9954 .or_default();
9955 h.failed.insert(specs[i].id.clone());
9956 h.last_fail = Some(cur);
9957 }
9958 let parsed = match out {
9959 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9960 AgentOutcome::Quota(o) => {
9961 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9962 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9963 }
9964 AgentOutcome::Dropped(o) => {
9968 let why = o
9969 .dropped
9970 .as_ref()
9971 .map(|d| d.why.as_str())
9972 .unwrap_or("the CLI ended the stream without delivering its answer");
9973 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9974 }
9975 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9976 Err(anyhow::anyhow!(e))
9977 }
9978 };
9979 let quota = class == Some(FailClass::Quota);
9980 let failed = parsed.is_err();
9981 done[i] = Some(parsed);
9982 if carry && !failed {
9983 let h = state
9984 .seat_history
9985 .entry(originals[i].seat.key.clone())
9986 .or_default();
9987 h.failed.remove(&specs[i].id);
9988 h.last_ok = Some(specs[i].id.clone());
9989 h.last_fail = None;
9990 }
9991 let agent_failure = class.is_some()
9999 && !nudge_first
10000 && !roster.is_empty()
10001 && next_agent(i, &tried[i], &specs).is_some();
10002 if failed && !quota && !agent_failure && nudges[i] < retries {
10003 nudges[i] += 1;
10004 still.push(i);
10005 }
10006 }
10007 pending = still;
10008 }
10009
10010 for (i, q) in last_quota.into_iter().enumerate() {
10013 if let Some(reset) = q {
10014 losses.push(QuotaLoss {
10015 seat: originals[i].seat.key.clone(),
10016 node: ctx.node.to_owned(),
10017 at: Timestamp::now(),
10018 reset,
10019 });
10020 }
10021 }
10022
10023 seats
10024 .into_iter()
10025 .zip(done)
10026 .zip(nudges)
10027 .map(|((seat, res), attempts)| {
10028 (
10029 seat,
10030 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10031 attempts,
10032 )
10033 })
10034 .collect()
10035}
10036
10037fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10039 results
10040 .iter()
10041 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10042}
10043
10044fn block_wave(
10048 jobs: Vec<SeatJob>,
10049 state: &mut RunState,
10050 node: &str,
10051 attempt: usize,
10052 why: &str,
10053) -> Vec<(usize, SeatState, AgentOutcome)> {
10054 state.resource_wait = None;
10055 state
10056 .active
10057 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10058 if let Err(e) = state.save() {
10059 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10060 }
10061 jobs.into_iter()
10062 .enumerate()
10063 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10064 .collect()
10065}
10066
10067async fn acquire_cache_lease(
10080 state: &mut RunState,
10081 cache_dir: &Path,
10082 owner: &crate::cache::Owner,
10083 budget: Duration,
10084 context: &str,
10085 cancel: &(dyn Fn() -> bool + Sync),
10086) -> Result<Option<crate::cache::Guard>> {
10087 let home = crate::run::home();
10088 let started = Instant::now();
10089 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10090 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10091 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10092 Err(e) => {
10093 state.event(
10094 "verify",
10095 format!("{context}: could not check the shared build cache: {e:#}"),
10096 );
10097 if let Err(e2) = state.save() {
10098 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10099 }
10100 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10101 }
10102 };
10103 state.event(
10104 "verify",
10105 format!(
10106 "{context}: waiting for the shared build cache at {} ({})",
10107 cache_dir.display(),
10108 busy.describe()
10109 ),
10110 );
10111 state.resource_wait = Some(crate::run::ResourceWait {
10114 node: context.to_owned(),
10115 kind: "shared build cache".to_owned(),
10116 holder: busy.describe(),
10117 since: Timestamp::now(),
10118 limit_secs: budget.as_secs(),
10119 });
10120 if let Err(e) = state.save() {
10121 tracing::warn!("could not persist a cache wait: {e:#}");
10122 }
10123 let remaining = budget.saturating_sub(started.elapsed());
10124 let waited = crate::cache::wait_for_unless(
10125 &home,
10126 cache_dir,
10127 owner,
10128 remaining,
10129 Duration::from_secs(5),
10130 cancel,
10131 )
10132 .await;
10133 state.resource_wait = None;
10134 match waited {
10135 Ok(g) => Ok(g),
10136 Err(e) => {
10137 state.event("verify", format!("{context}: {e:#}"));
10138 if let Err(e2) = state.save() {
10139 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10140 }
10141 Err(e)
10142 }
10143 }
10144}
10145
10146#[derive(Debug)]
10151struct CacheCheckError(String);
10152
10153impl std::fmt::Display for CacheCheckError {
10154 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10155 f.write_str(&self.0)
10156 }
10157}
10158
10159impl std::error::Error for CacheCheckError {}
10160
10161#[allow(clippy::too_many_arguments)]
10182async fn with_cache_lease<'s, F, Fut>(
10183 state: &'s mut RunState,
10184 cache_dir: Option<&Path>,
10185 node: &str,
10186 seat: &str,
10187 worktree: &Path,
10188 head: &str,
10189 budget: Duration,
10190 context: &str,
10191 wait: &crate::repo_lock::WaitPolicy,
10192 body: F,
10193) -> Result<(Vec<CommandOutcome>, bool)>
10194where
10195 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10196 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10197{
10198 let started = Instant::now();
10201 let home = crate::run::home();
10202 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10203 let guard = if let Some(cache_dir) = cache_dir {
10204 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10205 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10206 wait.cancelled()
10207 })
10208 .await
10209 .and_then(|g| {
10210 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10211 }) {
10212 Ok(g) => Some(g),
10213 Err(e) => {
10214 return Ok((
10215 vec![CommandOutcome {
10216 command: "(waiting for the shared build cache)".into(),
10217 code: None,
10218 output_tail: e.to_string(),
10219 duration_ms: started.elapsed().as_millis() as u64,
10220 resource_blocked: true,
10221 }],
10222 false,
10223 ));
10224 }
10225 }
10226 } else {
10227 None
10228 };
10229 let _coordination = coordinate_operation(state, worktree, wait).await?;
10230 if let Some(cache_dir) = cache_dir {
10231 let identity = crate::cache::Identity::new(worktree, head);
10232 if let Err(e) = crate::cache::ensure_fresh(
10233 &home,
10234 cache_dir,
10235 &identity,
10236 &state.config.verify.cargo_manifests(),
10237 ) {
10238 state.event(
10239 "verify",
10240 format!(
10241 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10242 worktree.display(),
10243 short(head)
10244 ),
10245 );
10246 if let Some(guard) = guard {
10247 guard.release();
10248 }
10249 return Ok((
10250 vec![CommandOutcome {
10251 command: "(confirming the shared build cache is fresh)".into(),
10252 code: None,
10253 output_tail: e.to_string(),
10254 duration_ms: started.elapsed().as_millis() as u64,
10255 resource_blocked: true,
10256 }],
10257 false,
10258 ));
10259 }
10260 }
10261 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10262 if !timed_out_pids.is_empty() {
10263 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10264 }
10265 drop(_coordination);
10268 if let Some(guard) = guard {
10269 guard.release();
10270 }
10271 Ok((outcomes, retried))
10272}
10273
10274async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10286 wait_for_pids_with(
10287 pids,
10288 crate::proc::pid_alive,
10289 LEASE_RELEASE_POLL,
10290 LEASE_RELEASE_MAX_WAIT,
10291 )
10292 .await;
10293}
10294
10295async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10301 pids: &[u32],
10302 alive: F,
10303 poll: Duration,
10304 max_wait: Duration,
10305) {
10306 let deadline = Instant::now() + max_wait;
10307 loop {
10308 if pids.iter().all(|&pid| !alive(pid)) {
10309 return;
10310 }
10311 if Instant::now() >= deadline {
10312 return;
10313 }
10314 tokio::time::sleep(poll).await;
10315 }
10316}
10317
10318fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10325 outcomes.iter().any(|o| o.resource_blocked)
10326}
10327
10328pub(crate) enum GateFix {
10330 Retry,
10332 Stop,
10335 Defer,
10338}
10339
10340fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10348 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10349 red.peek().is_some()
10350 && red.all(|o| {
10351 !o.resource_blocked
10352 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10353 && !o.output_tail.trim().is_empty()
10354 })
10355}
10356
10357fn e2e_outcome_label(o: &CommandOutcome) -> String {
10361 if o.ok() {
10362 return "pass".to_owned();
10363 }
10364 let reason = if o.build_failed() {
10365 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10366 } else {
10367 format!("FAIL ({:?})", o.code)
10368 };
10369 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10370}
10371
10372async fn run_e2e_with_retry(
10380 state: &mut RunState,
10381 shell: &[String],
10382 commands: &[String],
10383 worktree: &Path,
10384 timeout: Duration,
10385 context: &str,
10386) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10387 let (mut e2e, mut timed_out_pids) = run_commands(
10388 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10389 )
10390 .await;
10391 for o in &e2e {
10392 state.event(
10393 "verify",
10394 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10395 );
10396 }
10397 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10401 if verify_retried {
10402 state.event(
10403 "verify",
10404 format!(
10405 "{context}: verify could not build/link, not a test result — retrying once \
10406 before concluding"
10407 ),
10408 );
10409 let retried = run_commands(
10410 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10411 )
10412 .await;
10413 e2e = retried.0;
10414 timed_out_pids.extend(retried.1);
10417 for o in &e2e {
10418 state.event(
10419 "verify",
10420 format!(
10421 "{context}: retry `{}` -> {}",
10422 o.command,
10423 e2e_outcome_label(o)
10424 ),
10425 );
10426 }
10427 }
10428 (e2e, verify_retried, timed_out_pids)
10429}
10430
10431#[allow(clippy::too_many_arguments)]
10447async fn run_commands(
10448 state: &mut RunState,
10449 node: &str,
10450 task: &str,
10451 attempt: usize,
10452 shell: &[String],
10453 commands: &[String],
10454 cwd: &Path,
10455 timeout: Duration,
10456) -> (Vec<CommandOutcome>, Vec<u32>) {
10457 if commands.is_empty() {
10458 return (Vec::new(), Vec::new());
10463 }
10464 let mut out = Vec::new();
10465 let mut timed_out_pids = Vec::new();
10466 let total = commands.len();
10467 for (idx, command) in commands.iter().enumerate() {
10468 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10469 if let Err(e) = state.save() {
10470 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10471 }
10472 let started = Instant::now();
10473 let mut cmd = tokio::process::Command::new(&shell[0]);
10474 cmd.quiet();
10475 cmd.args(&shell[1..])
10476 .arg(command)
10477 .current_dir(cwd)
10478 .stdin(std::process::Stdio::null())
10479 .stdout(std::process::Stdio::piped())
10480 .stderr(std::process::Stdio::piped())
10481 .kill_on_drop(true);
10482 let mut outputs = Vec::new();
10483 let protected = (|| -> Result<()> {
10484 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10485 outputs.push(guard);
10486 }
10487 if let Some(cache) = state.config.cache_dir() {
10488 outputs.push(crate::retention::OutputLease::cache(
10489 &crate::run::home(),
10490 &cache,
10491 false,
10492 )?);
10493 }
10494 Ok(())
10495 })();
10496 if let Err(e) = protected {
10497 out.push(CommandOutcome {
10498 command: command.clone(),
10499 code: None,
10500 output_tail: format!("output ownership unavailable: {e:#}"),
10501 duration_ms: 0,
10502 resource_blocked: true,
10503 });
10504 continue;
10505 }
10506 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10507 let (code, body) = match spawned {
10508 Ok(mut running) => {
10509 let pid = running.child().id();
10510 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10511 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10512 let mut paths = vec![cwd.to_owned()];
10513 if let Some(cache) = state.config.cache_dir() {
10514 paths.push(cache);
10515 }
10516 let mut pressure = None;
10517 let status = tokio::select! {
10518 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10519 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10520 pressure = Some(reason); None
10521 }
10522 };
10523 let (code, failure) = match status {
10524 Some(Ok(Ok(status))) => (
10525 status.code(),
10526 status
10527 .code()
10528 .is_none()
10529 .then(|| format!("command terminated: {status}")),
10530 ),
10531 Some(Ok(Err(error))) => {
10532 (None, Some(format!("failed waiting for command: {error}")))
10533 }
10534 Some(Err(_)) => {
10535 if !running.coordinated()
10539 && let Some(pid) = pid
10540 {
10541 timed_out_pids.push(pid);
10542 }
10543 (
10544 None,
10545 Some(format!("timed out after {}s", timeout.as_secs())),
10546 )
10547 }
10548 None => (None, None),
10549 };
10550 running.finish().await;
10551 if let Some(reason) = pressure {
10552 state.resource_block = Some(crate::run::ResourceBlock {
10553 node: node.to_owned(),
10554 kind: "disk pressure".to_owned(),
10555 detail: reason.clone(),
10556 at: Timestamp::now(),
10557 });
10558 out.push(CommandOutcome {
10559 command: command.clone(),
10560 code: None,
10561 output_tail: reason,
10562 duration_ms: started.elapsed().as_millis() as u64,
10563 resource_blocked: true,
10564 });
10565 break;
10566 }
10567 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10568 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10569 if let Some(failure) = failure {
10570 body.push('\n');
10571 body.push_str(&failure);
10572 }
10573 (code, body)
10574 }
10575 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10576 };
10577 out.push(CommandOutcome {
10578 command: command.clone(),
10579 code,
10580 output_tail: tail(&body, OUTPUT_TAIL),
10581 duration_ms: started.elapsed().as_millis() as u64,
10582 resource_blocked: false,
10583 });
10584 }
10585 state.task_finished(task);
10586 if let Err(e) = state.save() {
10587 tracing::warn!("could not persist the end of {task}: {e:#}");
10588 }
10589 (out, timed_out_pids)
10590}
10591
10592fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10604 let repo = repo.display();
10605 match style {
10606 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10607 MergeStyle::Squash => {
10608 let subject = message
10611 .lines()
10612 .next()
10613 .unwrap_or(branch)
10614 .replace(['\\', '"', '$', '`'], "");
10615 format!(
10616 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10617 )
10618 }
10619 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10620 }
10621}
10622
10623const PR_TITLE_MAX: usize = 240;
10634
10635fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10646 let line = queue::first_line(text)?;
10647 let chars: Vec<char> = line.chars().collect();
10648 let (kept, mark) = if chars.len() <= max {
10649 (line.to_owned(), "")
10650 } else if let Some(end) = sentence_end(&chars, max) {
10651 (chars[..end].iter().collect::<String>(), "")
10652 } else {
10653 let room = max.saturating_sub(3);
10654 let cut = if chars[room].is_whitespace() {
10657 room
10658 } else {
10659 chars[..room]
10660 .iter()
10661 .rposition(|c| c.is_whitespace())
10662 .unwrap_or(room)
10663 };
10664 let head: String = chars[..cut].iter().collect();
10665 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10666 (head.to_owned(), "...")
10667 };
10668 if kept.is_empty() {
10669 return None;
10670 }
10671 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10672 return None;
10673 }
10674 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10675 return None;
10676 }
10677 Some(format!("{kept}{mark}"))
10678}
10679
10680fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10684 const MIN: usize = 20;
10685 for i in MIN..max.min(chars.len()) {
10686 if !matches!(chars[i], '.' | '!' | '?') {
10687 continue;
10688 }
10689 let Some(&next) = chars.get(i + 1) else {
10690 continue;
10691 };
10692 if !next.is_whitespace() {
10693 continue;
10694 }
10695 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10696 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10697 continue;
10698 }
10699 let word: String = chars[..i]
10700 .iter()
10701 .rev()
10702 .take_while(|c| !c.is_whitespace())
10703 .collect::<Vec<_>>()
10704 .into_iter()
10705 .rev()
10706 .collect();
10707 let word = word.to_ascii_lowercase();
10708 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10709 continue;
10710 }
10711 let end = chars[..i]
10712 .iter()
10713 .rposition(|c| !c.is_whitespace())
10714 .map_or(i, |p| p + 1);
10715 return Some(end);
10716 }
10717 None
10718}
10719
10720struct PrMessage {
10725 title: String,
10726 body: String,
10727}
10728
10729impl PrMessage {
10730 fn commit_message(&self) -> String {
10734 format!("{}\n\n{}", self.title, self.body)
10735 }
10736}
10737
10738fn title_marker(line: &str) -> Option<&str> {
10740 let line = line.trim();
10741 let head = line.get(..6)?;
10742 head.eq_ignore_ascii_case("title:")
10743 .then(|| line[6..].trim())
10744}
10745
10746fn summary_title(summary: &str) -> Option<String> {
10751 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10752 let raw = title_marker(first)?;
10753 if raw.is_empty() {
10754 return None;
10755 }
10756 let title = queue::title_from(raw, PR_TITLE_MAX);
10757 let lower = title.to_ascii_lowercase();
10758 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10759 return None;
10760 }
10761 Some(title)
10762}
10763
10764const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10766
10767pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10769
10770fn is_review_run(state: &RunState) -> bool {
10774 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10775}
10776
10777fn review_title(state: &RunState) -> Option<String> {
10782 english_subject(
10783 state.reviewed_commits.as_ref()?.first()?,
10784 state.config.graph.github_english(),
10785 )
10786}
10787
10788fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10791 let raw = raw.trim();
10792 if raw.is_empty() {
10793 return None;
10794 }
10795 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10796 return None;
10797 }
10798 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10799 return None;
10800 }
10801 let title = queue::title_from(raw, PR_TITLE_MAX);
10802 let lower = title.to_ascii_lowercase();
10803 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10804 return None;
10805 }
10806 Some(title)
10807}
10808
10809#[derive(Debug, Clone, PartialEq, Eq)]
10812struct BranchFacts {
10813 commits: Vec<(String, String)>,
10815 stat: String,
10817}
10818
10819const STAT_FILE_LINES: usize = 25;
10821const COMMITS_MAX_CHARS: usize = 20_000;
10823
10824async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10827 let commits = git::commit_log(repo, base, branch).await.ok()?;
10828 if commits.is_empty() {
10829 return None;
10830 }
10831 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10832 let lines: Vec<&str> = stat.lines().collect();
10833 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10834 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10835 let more = format!(" ... {omitted} more file(s)");
10836 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10837 kept.push(&more);
10838 kept.push(lines[lines.len() - 1]);
10839 kept.join("\n")
10840 } else {
10841 lines.join("\n")
10842 };
10843 Some(BranchFacts { commits, stat })
10844}
10845
10846fn markdown_safe(text: &str) -> String {
10849 text.replace("</details>", "</details>")
10850 .replace("\x60\x60\x60", "~~~")
10851}
10852
10853fn neutral_title(state: &RunState, winner: char) -> String {
10854 format!(
10855 "chore: land candidate {} of run {}",
10856 winner.to_ascii_uppercase(),
10857 state.id
10858 )
10859}
10860
10861pub(crate) async fn repair_landing_gate(
10864 state: &mut RunState,
10865 winner: &Candidate,
10866 outcomes: &[CommandOutcome],
10867) -> Result<GateFix> {
10868 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10869 return Ok(GateFix::Stop);
10870 }
10871 let mut runner = Runner {
10872 roles: state.config.resolve_roles()?,
10873 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10874 state: state.clone(),
10875 pause: Pause::new(),
10876 interrupt: Pause::new(),
10877 };
10878 let result = runner.gate_fix_round(winner, outcomes).await;
10879 *state = runner.state;
10880 result
10881}
10882
10883pub(crate) async fn check_gate_repair_e2e(
10886 state: &mut RunState,
10887 winner: &Candidate,
10888 wait: &crate::repo_lock::WaitPolicy,
10889) -> Result<GateFix> {
10890 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10891 return Ok(GateFix::Retry);
10892 }
10893 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10894 if state
10895 .gate_repair_e2e
10896 .as_ref()
10897 .is_some_and(|r| r.head == head && r.passed)
10898 {
10899 return Ok(GateFix::Retry);
10900 }
10901 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10902 head: head.clone(),
10903 passed: false,
10904 outcomes: Vec::new(),
10905 });
10906 state.save()?;
10907 let commands = state.config.verify.e2e.clone();
10908 let outcomes = if commands.is_empty() {
10909 Vec::new()
10910 } else {
10911 let shell = state.config.shell();
10912 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10913 let cache_dir = state.config.cache_dir();
10914 let (outcomes, _) = with_cache_lease(
10915 state,
10916 cache_dir.as_deref(),
10917 "e2e",
10918 "e2e",
10919 &winner.worktree,
10920 &head,
10921 timeout,
10922 "gate repair e2e",
10923 wait,
10924 |state, budget| async move {
10925 run_e2e_with_retry(
10926 state,
10927 &shell,
10928 &commands,
10929 &winner.worktree,
10930 budget,
10931 "gate repair e2e",
10932 )
10933 .await
10934 },
10935 )
10936 .await?;
10937 outcomes
10938 };
10939 let deferred = verify_inconclusive(&outcomes);
10940 let passed = outcomes.iter().all(CommandOutcome::ok)
10941 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10942 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10943 head,
10944 passed,
10945 outcomes,
10946 });
10947 state.save()?;
10948 if deferred {
10949 Ok(GateFix::Defer)
10950 } else if !passed {
10951 state.event(
10952 "gate",
10953 "verify.e2e failed after gate repair; nothing published",
10954 );
10955 Ok(GateFix::Stop)
10956 } else {
10957 Ok(GateFix::Retry)
10958 }
10959}
10960
10961pub(crate) async fn run_landing_gate(
10963 state: &mut RunState,
10964 winner: &Candidate,
10965 wait: &crate::repo_lock::WaitPolicy,
10966) -> Result<Vec<CommandOutcome>> {
10967 let shell = state.config.shell();
10968 let gate_commands = state.config.verify.gate.clone();
10969 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10970 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10971 Vec::new()
10972 } else {
10973 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10974 let cache_dir = state.config.cache_dir();
10975 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10976 let (outcomes, _) = with_cache_lease(
10977 state,
10978 cache_dir.as_deref(),
10979 "gate",
10980 "gate",
10981 &winner.worktree,
10982 &head,
10983 timeout,
10984 "final gate",
10985 wait,
10986 |state, budget| {
10987 let shell = shell.clone();
10988 let gate_commands = gate_commands.clone();
10989 let winner = winner.clone();
10990 async move {
10991 Runner::run_pre_gate(state, &winner).await;
10992 let (outcomes, timed_out_pids) = run_commands(
10993 state,
10994 "gate",
10995 "gate",
10996 0,
10997 &shell,
10998 &gate_commands,
10999 &winner.worktree,
11000 budget,
11001 )
11002 .await;
11003 (outcomes, false, timed_out_pids)
11004 }
11005 },
11006 )
11007 .await?;
11008 outcomes
11009 };
11010 if outcomes.is_empty() {
11011 state.event("gate", "no gate commands configured; not tested");
11016 }
11017 for o in &outcomes {
11018 state.event(
11019 "gate",
11020 format!(
11021 "`{}` -> {}",
11022 o.command,
11023 if o.ok() {
11024 "pass".to_owned()
11025 } else {
11026 format!(
11027 "FAIL ({:?})\n{}",
11028 o.code,
11029 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11030 )
11031 }
11032 ),
11033 );
11034 }
11035 Ok(outcomes)
11036}
11037
11038pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11044 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11045 ""
11046 } else {
11047 pr_title
11048 }
11049}
11050
11051pub fn landing_subject_source(state: &RunState) -> String {
11055 if is_review_run(state) {
11056 let winner = state.candidates.first().map_or('A', |c| c.label);
11057 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11058 }
11059 state.instruction.clone()
11060}
11061
11062fn summary_without_title(summary: &str) -> String {
11065 let mut lines = summary.trim().lines().peekable();
11066 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11067 lines.next();
11068 }
11069 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11070}
11071
11072#[cfg(test)]
11084fn pr_message(state: &RunState, winner: char) -> PrMessage {
11085 pr_message_with(state, winner, None)
11086}
11087
11088#[cfg(test)]
11091fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11092 let raw = pr_message_raw(
11093 state,
11094 winner,
11095 facts,
11096 crate::pr_outcome::FinalHead::Unreported,
11097 ChangeNote::NotAsked,
11098 );
11099 let id = crate::scrub::Identity::current();
11100 PrMessage {
11101 title: crate::scrub::scrub(&raw.title, &id),
11102 body: crate::scrub::scrub(&raw.body, &id),
11103 }
11104}
11105
11106#[derive(Debug, Clone, Copy)]
11108enum ChangeNote<'a> {
11109 NotAsked,
11111 Summary(&'a crate::run::ChangeSummary),
11113 Unavailable(&'a str),
11115}
11116
11117fn pr_message_raw(
11118 state: &RunState,
11119 winner: char,
11120 facts: Option<&BranchFacts>,
11121 head: crate::pr_outcome::FinalHead<'_>,
11122 change: ChangeNote<'_>,
11123) -> PrMessage {
11124 if !matches!(change, ChangeNote::NotAsked) {
11125 let (title, mut body) = match change {
11126 ChangeNote::Summary(c) => (
11127 c.title.clone(),
11128 format!("## Summary\n\n{}\n\n", c.body.trim()),
11129 ),
11130 ChangeNote::Unavailable(why) => (
11131 neutral_title(state, winner),
11132 format!(
11133 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11134 ),
11135 ),
11136 ChangeNote::NotAsked => unreachable!(),
11137 };
11138 body.push_str(&public_pr_evidence(state, head));
11139 body.push_str(&format!(
11140 "\n---\nmagi:run/{} magi:candidate-{}\n",
11141 state.id,
11142 winner.to_ascii_lowercase()
11143 ));
11144 return PrMessage { title, body };
11145 }
11146 pr_preview(state, winner, facts, head)
11147}
11148
11149fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11152 let mut body = crate::pr_outcome::review_rounds(state);
11153 body.push_str(&crate::pr_outcome::final_verification(state, head));
11154 let open = state.last_round_findings();
11155 if !open.is_empty() {
11156 body.push_str("## Open review findings\n\n");
11157 for f in open {
11158 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11159 }
11160 body.push('\n');
11161 }
11162 if let Some(c) = state.change_summary.as_ref()
11163 && head == crate::pr_outcome::FinalHead::At(&c.head)
11164 {
11165 body.push_str(&format!(
11166 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11167 c.base, c.head
11168 ));
11169 }
11170 body
11171}
11172
11173fn pr_preview(
11175 state: &RunState,
11176 winner: char,
11177 facts: Option<&BranchFacts>,
11178 head: crate::pr_outcome::FinalHead<'_>,
11179) -> PrMessage {
11180 let summary = state
11181 .candidates
11182 .iter()
11183 .find(|c| c.label == winner)
11184 .map(|c| c.summary.as_str())
11185 .unwrap_or_default();
11186 let review = is_review_run(state);
11189 let title = if review {
11190 facts
11191 .and_then(|f| {
11192 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11193 })
11194 .or_else(|| review_title(state))
11195 .or_else(|| {
11196 state
11197 .candidates
11198 .iter()
11199 .find(|c| c.label == winner)
11200 .filter(|c| !c.branch.starts_with("magi/"))
11201 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11202 })
11203 .unwrap_or_else(|| neutral_title(state, winner))
11204 } else {
11205 summary_title(summary).unwrap_or_else(|| {
11206 english_title(
11207 &state.instruction,
11208 PR_TITLE_MAX,
11209 state.config.graph.github_english(),
11210 )
11211 .unwrap_or_else(|| neutral_title(state, winner))
11212 })
11213 };
11214
11215 let mut body = String::new();
11216 let what = summary_without_title(summary);
11217 if !what.is_empty() {
11218 body.push_str("## Summary\n\n");
11219 body.push_str(&what);
11220 body.push_str("\n\n");
11221 }
11222
11223 body.push_str(&crate::pr_outcome::review_rounds(state));
11225 body.push_str(&crate::pr_outcome::final_verification(state, head));
11226 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11227
11228 let open = state.open_findings();
11229 if !open.is_empty() {
11230 body.push_str("## Open review findings\n\n");
11231 for f in &open {
11232 body.push_str(&format!(
11233 "- `{}` [{:?}] {}\n",
11234 f.id, f.severity, "Context retained in local review records."
11235 ));
11236 }
11237 body.push('\n');
11238 }
11239
11240 if let Some(fix) = fix
11241 && !fix.rejected.is_empty()
11242 {
11243 body.push_str("## Declined by the fixer\n\n");
11244 for r in &fix.rejected {
11245 body.push_str(&format!(
11246 "- `{}`: {}\n",
11247 r.id, "Reason retained in local review records."
11248 ));
11249 }
11250 body.push('\n');
11251 }
11252
11253 if review {
11254 body.push_str("## Commits under review\n\n");
11256 if let Some(facts) = facts {
11257 let mut left = COMMITS_MAX_CHARS;
11258 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11259 let mut entry = format!("- {}\n", markdown_safe(subject));
11260 for l in markdown_safe(text).lines() {
11261 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11262 }
11263 if left == 0 {
11264 body.push_str(&format!(
11265 "- ... {} more commit(s)\n",
11266 facts.commits.len() - i
11267 ));
11268 break;
11269 }
11270 if entry.len() > left {
11271 let mut end = left;
11274 while !entry.is_char_boundary(end) {
11275 end -= 1;
11276 }
11277 entry.truncate(end);
11278 entry.push_str("\n ... (truncated)\n");
11279 left = 0;
11280 } else {
11281 left -= entry.len();
11282 }
11283 body.push_str(&entry);
11284 }
11285 if !facts.stat.trim().is_empty() {
11286 body.push_str(&format!(
11287 "\n## Diff stat\n\n```\n{}\n```\n",
11288 markdown_safe(facts.stat.trim())
11289 ));
11290 }
11291 } else {
11292 match &state.reviewed_commits {
11293 Some(subjects) => {
11294 for s in subjects {
11295 body.push_str(&format!("- {}\n", s.trim()));
11296 }
11297 }
11298 None => {
11299 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11302 body.push_str(rest.trim());
11303 body.push('\n');
11304 }
11305 }
11306 }
11307 } else {
11308 let task = state.instruction.trim();
11309 let task = if task.is_empty() {
11310 "(empty task)"
11311 } else {
11312 task
11313 };
11314 body.push_str(&format!(
11315 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11316 task.replace("</details>", "</details>")
11317 ));
11318 }
11319
11320 body.push_str(&format!(
11321 "\n---\nmagi:run/{} magi:candidate-{}\n",
11322 state.id,
11323 winner.to_ascii_lowercase()
11324 ));
11325
11326 PrMessage { title, body }
11327}
11328
11329fn seeded_instruction(state: &RunState) -> String {
11333 match refs::describe(&state.seeds) {
11334 Some(facts) => format!(
11335 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11336 Candidates start from the unmerged branch named above, when there \
11337 is one, and carry any unmerged commit named by sha as a \
11338 cherry-pick. Check that this is what the task meant before \
11339 building on it.",
11340 state.instruction
11341 ),
11342 None => state.instruction.clone(),
11343 }
11344}
11345
11346async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11350 let base = &state.base_branch;
11351 let mut against = base.clone();
11352 if mode == MergeMode::Pr {
11353 let remote = &state.config.merge.remote;
11357 let tracking = format!("{remote}/{base}");
11358 let fetched = git::fetch(repo, remote, base).await;
11359 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11360 against = tracking;
11361 } else {
11362 return false;
11363 }
11364 }
11365 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11366}
11367
11368fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11371 let mut detail = format!(
11372 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11373 nothing to open a pull request for"
11374 );
11375 match refs::describe(&state.seeds) {
11376 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11377 None => detail.push_str(
11378 "\nThe task names no existing branch or commit; if it means to land work \
11379 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11380 ),
11381 }
11382 detail
11383}
11384
11385#[derive(Debug, PartialEq, Eq)]
11388enum PrPlan {
11389 Create,
11390 Adopt { url: String, title: String },
11391 Stop(String),
11392}
11393
11394fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11397 match found {
11398 Ok(land::OpenPr::None) => PrPlan::Create,
11399 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11400 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11401 "several open pull requests exist for this branch, not picking one: {}",
11402 urls.join(" ")
11403 )),
11404 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11405 }
11406}
11407
11408pub(crate) async fn gh_pr_create(
11410 cwd: &Path,
11411 base: &str,
11412 head: &str,
11413 title: &str,
11414 body: &str,
11415) -> Result<String> {
11416 let out = tokio::process::Command::new("gh")
11417 .args([
11418 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11419 ])
11420 .current_dir(cwd)
11421 .quiet()
11422 .stdin(std::process::Stdio::null())
11423 .output()
11424 .await
11425 .context("spawn gh")?;
11426 if out.status.success() {
11427 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11428 } else {
11429 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11430 }
11431}
11432
11433async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11435 match std::fs::symlink_metadata(path) {
11436 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11437 git::worktree_remove_clean(repo, path).await?;
11440 return Ok(());
11441 }
11442 Err(e) => return Err(e.into()),
11443 Ok(_) => {}
11444 }
11445 if !git::is_clean(path).await? {
11446 bail!(
11447 "worktree has uncommitted work; refusing removal: {}",
11448 path.display()
11449 );
11450 }
11451 if !git::worktree_remove_clean(repo, path).await? {
11452 bail!("worktree removal refused: {}", path.display());
11453 }
11454 Ok(())
11455}
11456
11457pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11466 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11467 bail!("refusing fold: {reason}");
11468 }
11469 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11470 bail!("candidate preservation incomplete; refusing cleanup");
11471 }
11472 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11476 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11477 let _driver = crate::driver::maintenance(home, &state.id)?;
11478 let mut output_guards = Vec::new();
11479 for entry in std::fs::read_dir(state.worktree_root())
11480 .into_iter()
11481 .flatten()
11482 .flatten()
11483 {
11484 if entry.path().join(".git").is_file() {
11485 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11486 }
11487 }
11488 let repo = state.repo.clone();
11489 let wait = repository_wait(state, None);
11492 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11493 Ok(lease) => lease,
11494 Err(e) => {
11495 state.save_under(home)?;
11496 return Err(e);
11497 }
11498 };
11499 let _source_outputs = if state.status == RunStatus::Merged
11503 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11504 {
11505 verify_merged_fold_sources(state, drop_winner, home).await?
11506 } else {
11507 Vec::new()
11508 };
11509 let root = state.worktree_root();
11510 let winner = state.tally.as_ref().map(|t| t.winner);
11511 let mut removed = Vec::new();
11512
11513 for i in 0..state.candidates.len() {
11514 let c = state.candidates[i].clone();
11515 let is_winner = Some(c.label) == winner;
11516 if is_winner && !drop_winner {
11517 continue;
11518 }
11519 let existed = c.worktree.exists();
11520 remove_preserved_worktree(&repo, &c.worktree).await?;
11523 if existed {
11524 removed.push(c.worktree.to_string_lossy().into_owned());
11525 }
11526 let handed_over = state.released_branches.contains(&c.branch);
11529 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11530 if !git::branch_delete(&repo, &c.branch).await? {
11531 bail!("candidate branch removal refused");
11532 }
11533 removed.push(c.branch.clone());
11534 }
11535 state.candidates[i].folded = true;
11536 }
11537
11538 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11539 let path = name.path();
11540 let keep = !drop_winner
11541 && winner.is_some_and(|w| {
11542 path.file_name()
11543 .is_some_and(|n| n == format!("cand-{w}").as_str())
11544 });
11545 if keep {
11546 continue;
11547 }
11548 remove_preserved_worktree(&repo, &path).await?;
11549 removed.push(path.to_string_lossy().into_owned());
11550 }
11551
11552 remove_if_empty(&root);
11561
11562 if state.enabled_worktree_config && drop_winner {
11563 git::release_worktree_config(&repo).await.ok();
11567 state.enabled_worktree_config = false;
11568 }
11569 state.save_under(home)?;
11570 Ok(removed)
11571}
11572
11573async fn verify_merged_fold_sources(
11574 state: &RunState,
11575 drop_winner: bool,
11576 home: &Path,
11577) -> Result<Vec<std::fs::File>> {
11578 let winner = state.tally.as_ref().map(|t| t.winner);
11579 let mut heads = BTreeSet::new();
11580 for candidate in &state.candidates {
11581 if Some(candidate.label) == winner && !drop_winner {
11582 continue;
11583 }
11584 match std::fs::symlink_metadata(&candidate.worktree) {
11585 Ok(_) => anyhow::ensure!(
11586 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11587 "candidate recovery source is outside the inventoried run root: {}",
11588 candidate.worktree.display()
11589 ),
11590 Err(error) if error.kind() == std::io::ErrorKind::NotFound && candidate.folded => {}
11591 Err(error) => {
11592 return Err(error).context(format!(
11593 "cannot inspect candidate recovery source: {}",
11594 candidate.worktree.display()
11595 ));
11596 }
11597 }
11598 if git::branch_exists(&state.repo, &candidate.branch).await? {
11599 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11600 heads.extend(
11601 git::git(
11602 &state.repo,
11603 &["reflog", "show", "--format=%H", &candidate.branch],
11604 )
11605 .await?
11606 .lines()
11607 .map(str::to_owned),
11608 );
11609 }
11610 }
11611 let mut cargo_guards = Vec::new();
11612 let entries = match std::fs::read_dir(state.worktree_root()) {
11613 Ok(entries) => Some(entries),
11614 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11615 Err(error) => return Err(error.into()),
11616 };
11617 for entry in entries.into_iter().flatten() {
11618 let path = entry?.path();
11619 let metadata = std::fs::symlink_metadata(&path)?;
11620 anyhow::ensure!(
11621 metadata.is_dir() && !metadata.file_type().is_symlink(),
11622 "non-directory or symlink recovery source retained: {}",
11623 path.display()
11624 );
11625 if !drop_winner
11626 && winner.is_some_and(|w| {
11627 path.file_name()
11628 .is_some_and(|name| name == format!("cand-{w}").as_str())
11629 })
11630 {
11631 continue;
11632 }
11633 anyhow::ensure!(
11634 path.join(".git").is_file(),
11635 "unregistered recovery source retained: {}",
11636 path.display()
11637 );
11638 anyhow::ensure!(
11639 git::git(
11640 &path,
11641 &[
11642 "status",
11643 "--porcelain",
11644 "--untracked-files=all",
11645 "--ignored",
11646 "--",
11647 ".",
11648 ":(exclude)target"
11649 ]
11650 )
11651 .await?
11652 .is_empty(),
11653 "source changes or ignored recovery files retained: {}",
11654 path.display()
11655 );
11656 match std::fs::symlink_metadata(path.join("target")) {
11657 Ok(_) => {
11658 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11659 }
11660 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11661 Err(error) => return Err(error.into()),
11662 }
11663 for checkpoint in [
11664 "rebase-merge",
11665 "rebase-apply",
11666 "sequencer",
11667 "CHERRY_PICK_HEAD",
11668 "MERGE_HEAD",
11669 "REBASE_HEAD",
11670 "REVERT_HEAD",
11671 "BISECT_LOG",
11672 "AUTO_MERGE",
11673 "MERGE_MSG",
11674 ] {
11675 let location = git::git(
11676 &path,
11677 &[
11678 "rev-parse",
11679 "--path-format=absolute",
11680 "--git-path",
11681 checkpoint,
11682 ],
11683 )
11684 .await?;
11685 anyhow::ensure!(
11686 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11687 if e.kind() == std::io::ErrorKind::NotFound),
11688 "Git recovery checkpoint retained: {checkpoint}"
11689 );
11690 }
11691 heads.insert(git::rev_parse(&path, "HEAD").await?);
11692 heads.extend(
11693 git::git(&path, &["reflog", "show", "--format=%H", "HEAD"])
11694 .await?
11695 .lines()
11696 .map(str::to_owned),
11697 );
11698 if let Ok(original) = git::rev_parse(&path, "ORIG_HEAD").await {
11699 heads.insert(original);
11700 }
11701 }
11702 if !heads.is_empty() {
11703 let reference = format!(
11707 "refs/remotes/{}/{}",
11708 state.config.merge.remote, state.base_branch
11709 );
11710 let base = git::rev_parse(&state.repo, &reference)
11711 .await
11712 .context("cannot prove merged sources against the fetched base")?;
11713 let mut squash_winner = None;
11714 let mut checked_squash = false;
11715 for head in heads {
11716 if git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, &base])
11717 .await?
11718 .ok()
11719 {
11720 continue;
11721 }
11722 if !checked_squash {
11723 squash_winner = proven_squash_winner(state, &base).await?;
11724 checked_squash = true;
11725 }
11726 let incorporated = if let Some(winner) = &squash_winner {
11727 git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, winner])
11728 .await?
11729 .ok()
11730 } else {
11731 false
11732 };
11733 anyhow::ensure!(
11734 incorporated,
11735 "recovery source {head} is not proven incorporated in {reference}; retaining the entire run"
11736 );
11737 }
11738 }
11739 Ok(cargo_guards)
11740}
11741
11742async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
11748 let Some(winner) = state.winner() else {
11749 return Ok(None);
11750 };
11751 let head = git::rev_parse(&state.repo, &winner.branch).await?;
11752 if !git::git_raw(
11753 &state.repo,
11754 &["merge-base", "--is-ancestor", &state.base_commit, &head],
11755 )
11756 .await?
11757 .ok()
11758 {
11759 return Ok(None);
11760 }
11761 let patch = fold_source_patch(&state.repo, &state.base_commit, &head).await?;
11762 if patch.is_empty() {
11763 return Ok(None);
11764 }
11765 let range = format!("{}..{base}", state.base_commit);
11766 let commits = git::git(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
11767 for commit in commits.lines() {
11768 let parent = format!("{commit}^");
11769 if fold_source_patch(&state.repo, &parent, commit).await? == patch {
11770 return Ok(Some(head));
11771 }
11772 }
11773 Ok(None)
11774}
11775
11776async fn fold_source_patch(repo: &Path, from: &str, to: &str) -> Result<String> {
11777 git::git(
11778 repo,
11779 &[
11780 "-c",
11781 "core.quotePath=true",
11782 "diff",
11783 "--binary",
11784 "--full-index",
11785 "--no-renames",
11786 "--no-ext-diff",
11787 "--no-textconv",
11788 "--no-color",
11789 "--ignore-submodules=none",
11790 "--submodule=short",
11791 from,
11792 to,
11793 "--",
11794 ],
11795 )
11796 .await
11797}
11798
11799fn remove_if_empty(dir: &Path) {
11810 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11811 std::fs::remove_dir(dir).ok();
11812 }
11813}
11814
11815pub fn worst_open(state: &RunState) -> Option<Severity> {
11817 state
11818 .reviews
11819 .last()?
11820 .reviews
11821 .iter()
11822 .flat_map(|r| r.findings.iter())
11823 .map(|f| f.severity)
11824 .max()
11825}
11826
11827async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11831 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11832 git::acquire_worktree_config(repo).await?;
11833 state.enabled_worktree_config = true;
11834 }
11835 Ok(())
11836}
11837
11838#[cfg(test)]
11839mod tests {
11840 #[tokio::test]
11841 async fn launcher_base_fetch_requires_remote_success() {
11842 let tmp = tempfile::tempdir().unwrap();
11843 let repo = tmp.path().join("repo");
11844 let remote = tmp.path().join("remote");
11845 std::fs::create_dir(&repo).unwrap();
11846 std::fs::create_dir(&remote).unwrap();
11847 git::git(&remote, &["init", "--bare", "-b", "main"])
11848 .await
11849 .unwrap();
11850 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11851 git::git(
11852 &repo,
11853 &[
11854 "-c",
11855 "user.name=Fixture",
11856 "-c",
11857 "user.email=fixture@example.invalid",
11858 "commit",
11859 "--allow-empty",
11860 "-m",
11861 "fixture",
11862 ],
11863 )
11864 .await
11865 .unwrap();
11866 git::git(
11867 &repo,
11868 &["remote", "add", "origin", remote.to_str().unwrap()],
11869 )
11870 .await
11871 .unwrap();
11872 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11873 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11874 assert_eq!(
11875 resolve_base(&repo, "main", "origin", None)
11876 .await
11877 .unwrap()
11878 .commit,
11879 head
11880 );
11881 #[cfg(unix)]
11882 {
11883 let policy: crate::config::Coordination = toml::from_str(
11884 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11885 ).unwrap();
11886 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11887 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11888 let held = crate::repo_lock::SeatLease::operation_with(
11889 &repo,
11890 Some(&policy),
11891 &crate::repo_lock::WaitPolicy::default(),
11892 )
11893 .await
11894 .unwrap();
11895 assert!(held.is_some());
11896 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11897 tokio::pin!(fetch);
11898 assert!(
11899 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11900 .await
11901 .is_err()
11902 );
11903 assert!(
11904 !repo.join(".git/FETCH_HEAD").exists(),
11905 "no fetch before admission"
11906 );
11907 drop(held);
11908 let fetched = fetch.await.unwrap();
11909 assert_eq!(fetched.commit, head);
11910 assert_eq!(fetched.remote, "origin");
11911 assert_eq!(fetched.branch, "main");
11912 }
11913 git::git(
11915 &repo,
11916 &[
11917 "remote",
11918 "set-url",
11919 "origin",
11920 tmp.path().join("missing").to_str().unwrap(),
11921 ],
11922 )
11923 .await
11924 .unwrap();
11925 let error = resolve_base(&repo, "main", "origin", None)
11926 .await
11927 .unwrap_err()
11928 .to_string();
11929 assert!(error.contains("cannot read origin/main"));
11930 assert!(error.contains("refusing to branch off"));
11931 }
11932
11933 #[test]
11934 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11935 let leaked = vec!["chore(deps): update a crate".to_owned()];
11936 let own = "fix(daemon): apply a chosen action";
11937 assert!(should_retitle("", own, &leaked));
11938 assert!(should_retitle(
11939 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11940 own,
11941 &leaked
11942 ));
11943 assert!(should_retitle(
11944 "chore: land candidate A of run 1",
11945 own,
11946 &leaked
11947 ));
11948 assert!(should_retitle(
11949 "magi: candidate A (uncommitted work)",
11950 own,
11951 &leaked
11952 ));
11953 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11954 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11955 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11956 }
11957
11958 #[test]
11959 fn pr_merge_plan_creates_adopts_or_stops() {
11960 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11961 assert_eq!(
11962 pr_merge_plan(Ok(land::OpenPr::One {
11963 url: "u".into(),
11964 title: "t".into()
11965 })),
11966 PrPlan::Adopt {
11967 url: "u".into(),
11968 title: "t".into()
11969 }
11970 );
11971 let PrPlan::Stop(many) =
11972 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11973 else {
11974 panic!("many must stop");
11975 };
11976 assert!(many.contains('a') && many.contains('b'));
11977 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11978 panic!("a failed lookup must stop");
11979 };
11980 assert!(err.contains("bad token"));
11981 }
11982
11983 use super::*;
11984 use crate::run::GateStatus;
11985 use std::collections::BTreeMap;
11986 use std::time::Duration;
11987
11988 fn conductor() -> AgentSpec {
11989 AgentSpec {
11990 id: "conductor".to_owned(),
11991 kind: crate::config::AgentKind::Command,
11992 model: None,
11993 command: vec!["true".to_owned()],
11994 extra_args: Vec::new(),
11995 env: BTreeMap::new(),
11996 prompt_delivery: None,
11997 }
11998 }
11999
12000 fn spec(id: &str) -> AgentSpec {
12001 AgentSpec {
12002 id: id.to_owned(),
12003 kind: crate::config::AgentKind::Command,
12004 model: None,
12005 command: vec!["true".to_owned()],
12006 extra_args: Vec::new(),
12007 env: BTreeMap::new(),
12008 prompt_delivery: None,
12009 }
12010 }
12011
12012 fn ids(xs: &[&str]) -> BTreeSet<String> {
12013 xs.iter().map(|s| (*s).to_owned()).collect()
12014 }
12015
12016 #[test]
12017 fn pick_successor_skips_an_agent_another_seat_holds() {
12018 let roster = [spec("a"), spec("b"), spec("c")];
12020 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12021 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12022 }
12023
12024 #[test]
12025 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12026 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12028 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12029 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12030 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12031 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12032 }
12033
12034 #[test]
12035 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12036 let roster = [spec("a"), spec("b")];
12037 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12038 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12039 let carried = ids(&["b"]);
12040 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12041 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12042 }
12043
12044 #[test]
12045 fn pick_successor_returns_none_without_any_untried_successor() {
12046 let roster = [spec("a"), spec("b")];
12047 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12048 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12049 let carried = ids(&["a"]);
12050 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12051 }
12052
12053 #[test]
12054 fn pick_successor_rescue_avoids_another_seats_occupant() {
12055 let roster = [spec("a"), spec("b"), spec("c")];
12057 let carried = ids(&["b", "c"]);
12058 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12059 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12060 }
12061
12062 #[test]
12063 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12064 let roster = [spec("a"), spec("b"), spec("c")];
12065 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12066 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12067 }
12068
12069 #[test]
12070 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12071 let roster = [spec("a"), spec("b"), spec("c")];
12072 let failed = ids(&["a", "b", "c"]);
12073 let mut tried = ids(&["b"]);
12075 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12076 assert_eq!(first.id, "a");
12077 tried.insert(first.id.clone());
12078 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12079 assert_eq!(second.id, "c");
12080 tried.insert(second.id.clone());
12081 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12082 }
12083
12084 #[test]
12085 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12086 let roster = [spec("a"), spec("b")];
12087 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12088 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12089 }
12090
12091 #[test]
12092 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12093 let roster = [spec("a"), spec("b"), spec("c")];
12094 let h = SeatHistory {
12095 failed: ids(&["a"]),
12096 last_ok: Some("b".to_owned()),
12097 last_fail: None,
12098 };
12099 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12100 let gone = SeatHistory {
12102 last_ok: Some("zzz".to_owned()),
12103 ..h.clone()
12104 };
12105 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12106 let failed = SeatHistory {
12107 failed: ids(&["a", "b"]),
12108 last_ok: Some("b".to_owned()),
12109 last_fail: None,
12110 };
12111 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12112 }
12113
12114 #[test]
12115 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12116 let first = SeatState::new("review-1", "alpha", 7);
12117 let next = handover_seat("review-1", "gamma", 7);
12118 assert_ne!(first.claude_session, next.claude_session);
12119 }
12120
12121 #[test]
12122 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12123 let mut state = state_with_summary("x", "y");
12124 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12125 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12126 assert_ne!(a.claude_session, b.claude_session);
12127 }
12128
12129 #[test]
12130 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12131 let roster = [spec("a"), spec("b")];
12132 let h = SeatHistory {
12133 failed: ids(&["a", "b"]),
12134 ..SeatHistory::default()
12135 };
12136 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12137 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12138 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12139 }
12140
12141 #[test]
12148 fn failure_signature_ignores_numbers_and_paths() {
12149 assert_eq!(
12150 failure_signature("exited with Some(2) and no usable output"),
12151 failure_signature("exited with Some(137) and no usable output")
12152 );
12153 assert_eq!(
12154 failure_signature("process exited with no usable output (exit code 2)"),
12155 failure_signature("process exited with no usable output (exit code 137)")
12156 );
12157 for (old, new) in [
12158 (
12159 "exited with Some(2) and no usable output",
12160 "process exited with no usable output (exit code 2)",
12161 ),
12162 (
12163 "exited with None and no usable output",
12164 "process exited with no usable output (exit code unavailable)",
12165 ),
12166 ] {
12167 let previous = FailClass::Other(failure_signature(old));
12168 let current = FailClass::Other(failure_signature(new));
12169 assert_eq!(previous, current, "resumption retains the failure class");
12170 assert!(!should_hand_over(Some(&previous), ¤t));
12171 }
12172 assert_eq!(
12173 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12174 failure_signature("cannot open /var/x.txt: denied")
12175 );
12176 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12177 }
12178
12179 #[test]
12180 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12181 let other = FailClass::Other("x".into());
12182 assert!(should_hand_over(None, &FailClass::Quota));
12183 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12184 assert!(should_hand_over(
12185 Some(&FailClass::Timeout),
12186 &FailClass::Timeout
12187 ));
12188 assert!(should_hand_over(None, &other));
12189 assert!(!should_hand_over(Some(&other), &other));
12190 assert!(should_hand_over(
12191 Some(&other),
12192 &FailClass::Other("y".into())
12193 ));
12194 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12196 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12197 }
12198
12199 #[test]
12200 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12201 let a = SeatState::new("judge-1", "alpha", 7);
12202 let b = handover_seat("judge-1", "beta", 7);
12203 assert_ne!(a.claude_session, b.claude_session);
12204 assert_eq!(b.turns, 0);
12205 }
12206
12207 #[test]
12208 fn a_timeout_is_classified_apart_from_other_failures() {
12209 assert_eq!(
12210 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12211 Some(FailClass::Timeout)
12212 );
12213 assert!(matches!(
12214 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12215 Some(FailClass::Other(_))
12216 ));
12217 }
12218
12219 #[test]
12220 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12221 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12222 let tried = BTreeSet::from(["beta".to_owned()]);
12223 let next = next_untried_in_roster(&roster, 1, &tried);
12226 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12227 }
12228
12229 #[test]
12230 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12231 let roster = vec![spec("alpha"), spec("beta")];
12232 let tried = BTreeSet::from(["beta".to_owned()]);
12233 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12237 }
12238
12239 #[test]
12240 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12241 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12242 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12243 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12247 }
12248
12249 #[test]
12250 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12251 let roster = vec![spec("a"), spec("a"), spec("b")];
12252 let tried = BTreeSet::from(["a".to_owned()]);
12253 let next = next_untried_in_roster(&roster, 0, &tried);
12254 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12255 }
12256
12257 #[test]
12258 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12259 let roster = vec![spec("a"), spec("b")];
12260 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12261 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12262 }
12263
12264 #[test]
12265 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12266 let dir = tempfile::tempdir().unwrap();
12267 let bay = dir.path().join("ffff");
12268
12269 remove_if_empty(&bay);
12271 assert!(!bay.exists());
12272
12273 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12276 remove_if_empty(&bay);
12277 assert!(bay.exists(), "non-empty directory must survive");
12278
12279 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12281 remove_if_empty(&bay);
12282 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12283 }
12284
12285 #[test]
12294 fn a_full_panel_that_found_nothing_is_clean() {
12295 assert!(round_is_clean(
12296 0,
12297 true,
12298 2,
12299 2,
12300 0,
12301 IncompleteReviewPolicy::Block
12302 ));
12303 }
12304
12305 #[test]
12306 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12307 assert!(!round_is_clean(
12308 0,
12309 true,
12310 1,
12311 2,
12312 0,
12313 IncompleteReviewPolicy::Block
12314 ));
12315 }
12316
12317 #[test]
12318 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12319 assert!(!round_is_clean(
12320 1,
12321 true,
12322 1,
12323 2,
12324 0,
12325 IncompleteReviewPolicy::Warn
12326 ));
12327 }
12328
12329 #[test]
12330 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12331 assert!(round_is_clean(
12332 0,
12333 true,
12334 1,
12335 2,
12336 0,
12337 IncompleteReviewPolicy::Warn
12338 ));
12339 }
12340
12341 #[test]
12342 fn a_full_panel_with_an_open_finding_is_not_clean() {
12343 assert!(!round_is_clean(
12344 1,
12345 true,
12346 2,
12347 2,
12348 0,
12349 IncompleteReviewPolicy::Block
12350 ));
12351 }
12352
12353 #[test]
12354 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12355 assert!(!round_is_clean(
12356 0,
12357 false,
12358 2,
12359 2,
12360 0,
12361 IncompleteReviewPolicy::Block
12362 ));
12363 }
12364
12365 #[test]
12372 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12373 assert!(round_is_clean(
12376 0,
12377 true,
12378 1,
12379 2,
12380 1,
12381 IncompleteReviewPolicy::Block
12382 ));
12383 }
12384
12385 #[test]
12386 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12387 assert!(!round_is_clean(
12390 0,
12391 true,
12392 1,
12393 2,
12394 0,
12395 IncompleteReviewPolicy::Block
12396 ));
12397 }
12398
12399 #[test]
12400 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12401 assert!(!round_is_clean(
12402 1,
12403 true,
12404 1,
12405 2,
12406 1,
12407 IncompleteReviewPolicy::Block
12408 ));
12409 assert!(!round_is_clean(
12410 0,
12411 false,
12412 1,
12413 2,
12414 1,
12415 IncompleteReviewPolicy::Block
12416 ));
12417 }
12418
12419 #[test]
12420 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12421 assert!(!round_is_clean(
12425 0,
12426 true,
12427 0,
12428 2,
12429 2,
12430 IncompleteReviewPolicy::Block
12431 ));
12432 }
12433
12434 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12435 CommandOutcome {
12436 command: "test".to_owned(),
12437 code,
12438 output_tail: String::new(),
12439 duration_ms: 0,
12440 resource_blocked,
12441 }
12442 }
12443
12444 #[test]
12445 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12446 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12447 assert!(
12448 !verify_inconclusive(&[outcome(Some(1), false)]),
12449 "an ordinary failure is still evidence about the patch"
12450 );
12451 assert!(verify_inconclusive(&[outcome(None, true)]));
12452 assert!(
12453 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12454 "one inconclusive outcome taints the whole batch"
12455 );
12456 assert!(!verify_inconclusive(&[]));
12457 }
12458
12459 #[tokio::test]
12460 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12461 let calls = std::sync::atomic::AtomicUsize::new(0);
12465 let started = Instant::now();
12466 wait_for_pids_with(
12467 &[123],
12468 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12469 Duration::from_millis(5),
12470 Duration::from_secs(5),
12471 )
12472 .await;
12473 assert!(
12474 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12475 "must keep checking rather than deciding on the first answer"
12476 );
12477 assert!(
12478 started.elapsed() < Duration::from_secs(1),
12479 "must return the moment it is confirmed dead, not wait out the ceiling"
12480 );
12481 }
12482
12483 #[tokio::test]
12484 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12485 let started = Instant::now();
12486 wait_for_pids_with(
12487 &[123],
12488 |_| true, Duration::from_millis(5),
12490 Duration::from_millis(30),
12491 )
12492 .await;
12493 let elapsed = started.elapsed();
12494 assert!(
12495 elapsed >= Duration::from_millis(30),
12496 "must not give up before its own ceiling: {elapsed:?}"
12497 );
12498 assert!(
12499 elapsed < Duration::from_secs(1),
12500 "must not wait past its own ceiling either: {elapsed:?}"
12501 );
12502 }
12503
12504 #[tokio::test]
12505 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12506 let started = Instant::now();
12507 wait_for_pids_with(
12508 &[],
12509 |_| true,
12510 Duration::from_secs(5),
12511 Duration::from_secs(5),
12512 )
12513 .await;
12514 assert!(
12515 started.elapsed() < Duration::from_millis(200),
12516 "an empty pid list has nothing to confirm"
12517 );
12518 }
12519
12520 fn review_round(
12526 clean: bool,
12527 blocking: usize,
12528 answered: usize,
12529 expected: usize,
12530 progressed: bool,
12531 e2e_ok: bool,
12532 ) -> ReviewRound {
12533 ReviewRound {
12534 round: 1,
12535 head: "h".to_owned(),
12536 verified_head: None,
12537 verified_at: None,
12538 reviews: Vec::new(),
12539 e2e: vec![CommandOutcome {
12540 command: "test".to_owned(),
12541 code: Some(if e2e_ok { 0 } else { 1 }),
12542 output_tail: String::new(),
12543 duration_ms: 0,
12544 resource_blocked: false,
12545 }],
12546 verify_retried: false,
12547 e2e_deferred: false,
12548 e2e_defer_reason: None,
12549 fix: None,
12550 blocking,
12551 answered,
12552 expected,
12553 clean,
12554 progressed,
12555 vote_split: false,
12556 reconsideration: Vec::new(),
12557 verdict: None,
12558 }
12559 }
12560
12561 #[test]
12562 fn review_conclusion_is_none_when_nothing_has_run() {
12563 assert_eq!(review_conclusion(&[], 3), None);
12564 }
12565
12566 #[test]
12567 fn review_conclusion_is_none_while_rounds_remain() {
12568 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12569 assert_eq!(review_conclusion(&rounds, 3), None);
12570 }
12571
12572 #[test]
12573 fn review_conclusion_is_gating_once_a_round_is_clean() {
12574 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12575 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12576 }
12577
12578 #[test]
12579 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12580 let rounds = vec![
12581 review_round(false, 1, 2, 2, true, true),
12582 review_round(false, 1, 2, 2, true, true),
12583 ];
12584 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12585 }
12586
12587 #[test]
12588 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12589 let rounds = vec![
12590 review_round(false, 1, 2, 2, true, true),
12591 review_round(false, 1, 2, 2, true, false),
12592 ];
12593 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12594 }
12595
12596 #[test]
12597 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12598 let mut blocked = review_round(false, 1, 2, 2, true, false);
12605 blocked.e2e[0].resource_blocked = true;
12606 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12607 assert_eq!(review_conclusion(&rounds, 2), None);
12608 }
12609
12610 #[test]
12611 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12612 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12614 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12615 }
12616
12617 #[test]
12618 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12619 let rounds = vec![
12620 review_round(false, 1, 2, 2, false, true),
12621 review_round(false, 1, 2, 2, false, true),
12622 ];
12623 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12624 }
12625
12626 fn secs(n: u64) -> Duration {
12627 Duration::from_secs(n)
12628 }
12629
12630 #[tokio::test]
12633 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12634 ask_test_home();
12635 let tmp = tempfile::tempdir().unwrap();
12636 let repo = tmp.path().join("repo");
12637 std::fs::create_dir_all(&repo).unwrap();
12638 init_repo(&repo);
12639 let g = |args: &[&str]| {
12640 let out = std::process::Command::new("git")
12641 .args(args)
12642 .current_dir(&repo)
12643 .quiet()
12644 .output()
12645 .unwrap();
12646 assert!(
12647 out.status.success(),
12648 "{}",
12649 String::from_utf8_lossy(&out.stderr)
12650 );
12651 };
12652 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12653 g(&["checkout", "-b", "historical"]);
12654 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12655 g(&["add", "."]);
12656 g(&["commit", "-m", "old alternative"]);
12657 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12658 g(&["checkout", "-b", "magi/task-selected", &base]);
12659 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12660 g(&["add", "."]);
12661 g(&["commit", "-m", "measurement infrastructure"]);
12662 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12663 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12664 g(&["add", "."]);
12665 g(&["commit", "-m", "dependent implementation"]);
12666 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12667 g(&["checkout", "main"]);
12668 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12669 g(&["add", "."]);
12670 g(&["commit", "-m", "unrelated base change"]);
12671 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12672 let origin = tmp.path().join("origin.git");
12673 g(&[
12674 "clone",
12675 "--bare",
12676 repo.to_str().unwrap(),
12677 origin.to_str().unwrap(),
12678 ]);
12679 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12680 g(&["fetch", "origin"]);
12681 let selection = crate::pr_recovery::Selection {
12682 url: "https://example.test/project/pull/1".into(),
12683 branch: "magi/task-selected".into(),
12684 head: head.clone(),
12685 source_head: None,
12686 source_run: None,
12687 source_proof: None,
12688 disposition: Some(crate::pr_recovery::Disposition::Review),
12689 instruction: "preserve measured behavior".into(),
12690 approved_at: jiff::Timestamp::now(),
12691 };
12692 let mut template = runner_at(RunStatus::Reviewing);
12693 template.state.repo = repo.clone();
12694 template.state.base_commit = current.clone();
12695 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12696 template.state.origin = Some(crate::run::Origin::queue("selected"));
12697 template.state.pr_head = Some(selection.branch.clone());
12698 template.state.pr_recovery = Some(selection.clone());
12699 let mut runner =
12700 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12701 .await
12702 .unwrap();
12703 let wt = runner.state.winner().unwrap().worktree.clone();
12704 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12705 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12706 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12707 assert!(runner.state.seeds.is_empty());
12708 assert_eq!(runner.state.instruction, "preserve measured behavior");
12709 runner.sync_to_base().await.unwrap();
12710 assert_eq!(
12711 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12712 "prerequisite"
12713 );
12714 assert_eq!(
12715 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12716 "measured implementation"
12717 );
12718 assert_eq!(
12719 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12720 "current main change"
12721 );
12722 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12723 assert_eq!(
12724 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12725 head
12726 );
12727
12728 let mut partial = runner_at(RunStatus::Reviewing);
12730 partial.state.repo = repo.clone();
12731 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12732 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12733 run: Some("exit 1".into()),
12734 ..Default::default()
12735 }];
12736 let selection = runner.state.pr_recovery.clone().unwrap();
12737 partial.state.pr_recovery = Some(selection.clone());
12738 partial.state.pr_head = Some(selection.branch.clone());
12739 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12740 let id = partial.state.id.clone();
12741 let error = match Runner::open_selected(
12742 &repo,
12743 partial.state,
12744 partial.roles,
12745 1,
12746 runner.state.base_commit.clone(),
12747 selection,
12748 )
12749 .await
12750 {
12751 Ok(_) => panic!("setup must fail"),
12752 Err(e) => e,
12753 };
12754 assert_eq!(
12755 error
12756 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12757 .unwrap()
12758 .run,
12759 id
12760 );
12761 let failed = RunState::load(&id).unwrap();
12762 assert_eq!(failed.status, RunStatus::Blocked);
12763 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12764 assert!(Runner::resume(&id).is_err());
12765 runner.state.status = RunStatus::Ready;
12766 runner.state.driver_exited = true;
12767 runner.state.save().unwrap();
12768 let mut retry = runner_at(RunStatus::Reviewing);
12769 retry.state.repo = repo.clone();
12770 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12771 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12772 reselected.approved_at = jiff::Timestamp::now();
12773 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12774 retry.state.pr_head = Some(reselected.branch.clone());
12775 retry.state.pr_recovery = Some(reselected.clone());
12776 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12777 .await
12778 .unwrap();
12779 let restarted = Runner::open_selected(
12780 &repo,
12781 retry.state,
12782 retry.roles,
12783 1,
12784 runner.state.base_commit.clone(),
12785 reselected,
12786 )
12787 .await
12788 .unwrap();
12789 assert_eq!(
12790 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12791 .await
12792 .unwrap(),
12793 head
12794 );
12795 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12796 assert_eq!(
12797 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12798 head
12799 );
12800 }
12801
12802 #[tokio::test]
12803 #[cfg(unix)]
12804 async fn fallback_and_interrupted_restoration_require_coordination() {
12805 use std::os::fd::AsRawFd as _;
12806 ask_test_home();
12807 let tmp = tempfile::tempdir().unwrap();
12808 let mut runner = registered_runner(tmp.path()).await;
12809 runner.state.config.graph.timeout_resource_wait = 1;
12810 let winner = runner.state.candidates[0].clone();
12811 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12812 let leftover = winner.worktree.join("leftover");
12813 std::fs::write(&leftover, "untrusted edits").unwrap();
12814 let next = tmp.path().join("next-fixer");
12815 let mut failed = spec("first");
12816 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12817 let mut successor = spec("second");
12818 successor.command = vec![
12819 "sh".into(),
12820 "-c".into(),
12821 format!("touch '{}'; echo done", next.display()),
12822 ];
12823 let attempts = vec![
12824 (failed.clone(), "first".into()),
12825 (successor.clone(), "second".into()),
12826 ];
12827 let lock_path = runner.state.repo.join(".git/updater.lock");
12828 let external = std::fs::File::create(&lock_path).unwrap();
12829 assert_eq!(
12830 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12831 0
12832 );
12833 let artifacts = tmp.path().join("artifacts");
12834 let build = |spec, seat| SeatJob {
12835 spec,
12836 seat,
12837 cwd: winner.worktree.clone(),
12838 prompt: "fixture".into(),
12839 timeout: Duration::from_secs(10),
12840 allow_write: true,
12841 sessions: false,
12842 artifacts: artifacts.clone(),
12843 stem: "fix".into(),
12844 handover: None,
12845 };
12846 let (_, _, out) = runner
12847 .ask_fixer_among(
12848 attempts.clone(),
12849 "graft",
12850 None,
12851 Some((&winner.worktree, &head)),
12852 build,
12853 )
12854 .await;
12855 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12856 assert!(leftover.exists(), "contention must prevent clean");
12857 assert!(!next.exists());
12858 assert!(runner.state.handovers.is_empty());
12859 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12861 from_run: "earlier".into(),
12862 generation: 1,
12863 mode: crate::recovery::Mode::VerifyRepair,
12864 reasons: Vec::new(),
12865 reviewed_head: None,
12866 rounds: Vec::new(),
12867 open_findings: Vec::new(),
12868 evidence: None,
12869 criteria_inherited: true,
12870 repair: crate::recovery::Repair::Started {
12871 before: head.clone(),
12872 },
12873 });
12874 assert!(
12875 runner
12876 .recovery_repair(&winner)
12877 .await
12878 .unwrap_err()
12879 .is::<crate::repo_lock::ResourceBlocked>()
12880 );
12881 assert!(matches!(
12882 runner.state.recovery.as_ref().unwrap().repair,
12883 crate::recovery::Repair::Started { .. }
12884 ));
12885 assert!(leftover.exists());
12886 drop(external);
12887 runner.recovery_repair(&winner).await.unwrap();
12888 assert!(!leftover.exists());
12889 assert!(matches!(
12890 runner.state.recovery.as_ref().unwrap().repair,
12891 crate::recovery::Repair::Done { .. }
12892 ));
12893 std::fs::remove_file(&lock_path).unwrap();
12895 std::fs::create_dir(&lock_path).unwrap();
12896 std::fs::write(&leftover, "untrusted edits").unwrap();
12897 let (_, _, out) = runner
12898 .ask_fixer_among(
12899 attempts,
12900 "graft",
12901 None,
12902 Some((&winner.worktree, &head)),
12903 build,
12904 )
12905 .await;
12906 assert!(matches!(out, AgentOutcome::Failed(_)));
12907 assert!(leftover.exists());
12908 assert!(!next.exists());
12909 assert_eq!(
12910 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12911 head
12912 );
12913 }
12914
12915 #[cfg(unix)]
12916 async fn registered_runner(root: &Path) -> Runner {
12917 ask_test_home();
12918 let repo = root.join("repo");
12919 std::fs::create_dir_all(&repo).unwrap();
12920 init_repo(&repo);
12921 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12922 let mut runner = runner_at(RunStatus::Prep);
12923 runner.state.repo = repo.clone();
12924 runner.state.config.coordination = crate::config::test_coordination();
12925 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12926 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12927 runner.state.config.graph.advise = false;
12928 runner.roles.implementers = vec![spec("implement")];
12929 runner.roles.judges = vec![spec("judge")];
12930 runner.prep().await.unwrap();
12931 runner
12932 }
12933
12934 #[tokio::test]
12935 #[cfg(unix)]
12936 async fn parked_implementer_persists_completed_work() {
12937 for registered in [false, true] {
12938 let tmp = tempfile::tempdir().unwrap();
12939 let mut runner = registered_runner(tmp.path()).await;
12940 if !registered {
12941 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12942 }
12943 let marker = tmp.path().join("started");
12944 let release = tmp.path().join("release");
12945 let mut implementer = spec("implement");
12946 implementer.command = vec![
12947 "sh".into(),
12948 "-c".into(),
12949 format!(
12950 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12951 marker.display(),
12952 release.display()
12953 ),
12954 ];
12955 runner.roles.implementers = vec![implementer.clone()];
12956 runner.state.config.agents = vec![implementer];
12957 let pause = runner.pause.clone();
12958 let (result, ()) = tokio::join!(runner.implement(), async {
12959 tokio::time::timeout(Duration::from_secs(10), async {
12960 while !marker.exists() {
12961 tokio::time::sleep(Duration::from_millis(10)).await;
12962 }
12963 })
12964 .await
12965 .unwrap();
12966 pause.park();
12967 std::fs::write(&release, "go").unwrap();
12968 });
12969 result.unwrap();
12970 let candidate = &runner.state.candidates[0];
12971 assert!(candidate.commits > 0, "{candidate:?}");
12972 assert!(candidate.summary.contains("saved-result"));
12973 assert_eq!(candidate.files, 1);
12974 let before = candidate.commits;
12975 runner.implement().await.unwrap();
12976 assert_eq!(runner.state.candidates[0].commits, before);
12977 }
12978 }
12979
12980 #[tokio::test]
12981 #[cfg(unix)]
12982 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12983 let tmp = tempfile::tempdir().unwrap();
12984 let mut runner = registered_runner(tmp.path()).await;
12985 let winner = runner.state.candidates[0].clone();
12986 let cache = tmp.path().join("cache");
12987 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12988 runner.state.config.graph.timeout_resource_wait = 0;
12989 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12990 let crate::cache::AcquireOutcome::Acquired(held) =
12991 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12992 else {
12993 panic!("cache must be free")
12994 };
12995 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12996 from_run: "earlier".into(),
12997 generation: 1,
12998 mode: crate::recovery::Mode::VerifyRepair,
12999 reasons: Vec::new(),
13000 reviewed_head: None,
13001 rounds: Vec::new(),
13002 open_findings: Vec::new(),
13003 evidence: None,
13004 criteria_inherited: true,
13005 repair: crate::recovery::Repair::NotStarted,
13006 });
13007 runner.recovery_repair(&winner).await.unwrap();
13008 assert!(matches!(
13009 runner.state.recovery.as_ref().unwrap().repair,
13010 crate::recovery::Repair::WaitingForRestore { .. }
13011 ));
13012 drop(held);
13013 runner.state.resource_block = None;
13014 runner.state.config.graph.timeout_resource_wait = 1;
13015 runner.recovery_repair(&winner).await.unwrap();
13016 assert!(matches!(
13017 runner.state.recovery.as_ref().unwrap().repair,
13018 crate::recovery::Repair::Done { .. }
13019 ));
13020 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13021 }
13022
13023 #[tokio::test]
13024 #[cfg(unix)]
13025 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13026 use std::os::fd::AsRawFd as _;
13027 for graft in [false, true] {
13028 let tmp = tempfile::tempdir().unwrap();
13029 let mut runner = registered_runner(tmp.path()).await;
13030 runner.state.config.graph.timeout_resource_wait = 1;
13031 let winner = runner.state.candidates[0].clone();
13032 let marker = tmp.path().join("fixer-started");
13033 let mut fixer = spec("fixer");
13034 fixer.command = vec![
13035 "sh".into(),
13036 "-c".into(),
13037 format!("echo started >> '{}'; echo done", marker.display()),
13038 ];
13039 runner.roles.fixer = Some(vec![fixer]);
13040 if graft {
13041 let mut loser = winner.clone();
13042 loser.label = 'B';
13043 loser.index = 1;
13044 runner.state.candidates.push(loser);
13045 runner.state.config.graph.graft_runner_up = true;
13046 runner.state.tally = Some(Tally {
13047 first_choice: BTreeMap::new(),
13048 borda: BTreeMap::new(),
13049 winner: winner.label,
13050 rankings: 1,
13051 unanimous_initial: true,
13052 deliberated: false,
13053 changed_votes: 0,
13054 unanimous_final: true,
13055 tie_break: None,
13056 judges: 1,
13057 present: 1,
13058 quorum: 1,
13059 met_quorum: true,
13060 uncontested: None,
13061 });
13062 runner.state.judgements.push(crate::run::Judgement {
13063 judge: 1,
13064 seat: "judge-1".into(),
13065 agent: "judge".into(),
13066 ranking: vec![winner.label, 'B'],
13067 reasons: BTreeMap::new(),
13068 confidence: None,
13069 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13070 order: vec![0, 1],
13071 failed: None,
13072 duration_ms: 0,
13073 });
13074 } else {
13075 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13076 from_run: "earlier".into(),
13077 generation: 1,
13078 mode: crate::recovery::Mode::VerifyRepair,
13079 reasons: Vec::new(),
13080 reviewed_head: None,
13081 rounds: Vec::new(),
13082 open_findings: Vec::new(),
13083 evidence: None,
13084 criteria_inherited: true,
13085 repair: crate::recovery::Repair::NotStarted,
13086 });
13087 }
13088 let reader = crate::repo_lock::SeatLease::for_checkout(
13089 &winner.worktree,
13090 false,
13091 crate::config::test_coordination().as_ref(),
13092 )
13093 .await
13094 .unwrap();
13095 let pause = runner.pause.clone();
13096 let (result, ()) = tokio::join!(
13097 async {
13098 if graft {
13099 runner.graft().await
13100 } else {
13101 runner.recovery_repair(&winner).await
13102 }
13103 },
13104 async {
13105 tokio::time::sleep(Duration::from_millis(100)).await;
13106 pause.park();
13107 }
13108 );
13109 result.unwrap();
13110 assert!(!marker.exists(), "a blocked seat must not spawn");
13111 runner.state = RunState::load(&runner.state.id).unwrap();
13113 if graft {
13114 assert_eq!(
13115 runner.state.graft.as_ref().unwrap().outcome,
13116 GraftOutcome::WaitingForRestore
13117 );
13118 } else {
13119 assert!(matches!(
13120 runner.state.recovery.as_ref().unwrap().repair,
13121 crate::recovery::Repair::WaitingForRestore { .. }
13122 ));
13123 }
13124 drop(reader);
13125 runner.pause = Pause::new();
13126 runner.state.resource_block = None;
13127 let external =
13129 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13130 assert_eq!(
13131 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13132 0
13133 );
13134 let leftover = winner.worktree.join("leftover");
13135 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13136 let result = if graft {
13137 runner.graft().await
13138 } else {
13139 runner.recovery_repair(&winner).await
13140 };
13141 assert!(
13142 result
13143 .unwrap_err()
13144 .is::<crate::repo_lock::ResourceBlocked>()
13145 );
13146 assert!(leftover.exists());
13147 assert!(!marker.exists());
13148 drop(external);
13149 runner.state.resource_block = None;
13150 let result = if graft {
13151 runner.graft().await
13152 } else {
13153 runner.recovery_repair(&winner).await
13154 };
13155 result.unwrap();
13156 assert!(!leftover.exists(), "restore must precede retry");
13157 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13158 if graft {
13159 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13160 } else {
13161 assert!(matches!(
13162 runner.state.recovery.as_ref().unwrap().repair,
13163 crate::recovery::Repair::Done { .. }
13164 ));
13165 }
13166 }
13167 }
13168
13169 #[tokio::test]
13170 #[cfg(unix)]
13171 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13172 use std::os::fd::AsRawFd as _;
13173 let tmp = tempfile::tempdir().unwrap();
13174 let mut runner = registered_runner(tmp.path()).await;
13175 runner.state.config.graph.timeout_resource_wait = 1;
13176 let winner = runner.state.candidates[0].clone();
13177 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13178 runner.state.tally = Some(Tally {
13179 first_choice: BTreeMap::new(),
13180 borda: BTreeMap::new(),
13181 winner: winner.label,
13182 rankings: 0,
13183 unanimous_initial: false,
13184 deliberated: false,
13185 changed_votes: 0,
13186 unanimous_final: false,
13187 tie_break: None,
13188 judges: 0,
13189 present: 0,
13190 quorum: 0,
13191 met_quorum: true,
13192 uncontested: Some("fixture".into()),
13193 });
13194 runner.state.graft = Some(GraftRecord {
13195 pre: pre.clone(),
13196 post: None,
13197 lines: None,
13198 limit: None,
13199 agent: None,
13200 outcome: GraftOutcome::Running,
13201 requested: Vec::new(),
13202 adoption: None,
13203 adoption_error: None,
13204 });
13205 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13206 git::commit_all(&winner.worktree, "unfinished graft")
13207 .await
13208 .unwrap();
13209 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13210 let leftover = winner.worktree.join("leftover");
13211 std::fs::write(&leftover, "untrusted edits").unwrap();
13212 let path = runner.state.repo.join(".git/updater.lock");
13213 let external = std::fs::File::create(&path).unwrap();
13214 assert_eq!(
13215 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13216 0
13217 );
13218 assert!(
13219 runner
13220 .graft()
13221 .await
13222 .unwrap_err()
13223 .is::<crate::repo_lock::ResourceBlocked>()
13224 );
13225 assert!(matches!(
13226 runner.state.graft.as_ref().unwrap().outcome,
13227 GraftOutcome::Running
13228 ));
13229 assert_eq!(
13230 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13231 unfinished
13232 );
13233 assert!(leftover.exists());
13234 drop(external);
13235 std::fs::remove_file(&path).unwrap();
13236 std::fs::create_dir(&path).unwrap();
13237 assert!(runner.graft().await.is_err());
13238 assert!(matches!(
13239 runner.state.graft.as_ref().unwrap().outcome,
13240 GraftOutcome::Running
13241 ));
13242 assert_eq!(
13243 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13244 unfinished
13245 );
13246 assert!(leftover.exists());
13247 std::fs::remove_dir(&path).unwrap();
13248 runner.graft().await.unwrap();
13249 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13250 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13251 assert!(!leftover.exists());
13252 }
13253
13254 #[tokio::test]
13255 #[cfg(unix)]
13256 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13257 use std::os::fd::AsRawFd as _;
13258 let tmp = tempfile::tempdir().unwrap();
13259 let mut runner = registered_runner(tmp.path()).await;
13260 runner.state.prep_complete = Some(false);
13261 runner.state.config.graph.timeout_resource_wait = 1;
13262 runner.state.save().unwrap();
13263 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13264 assert_eq!(
13265 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13266 0
13267 );
13268 runner.execute().await.unwrap();
13269 let disk = RunState::load(&runner.state.id).unwrap();
13270 assert!(disk.driver_exited);
13271 assert_eq!(disk.driver_pid, None);
13272 assert_eq!(runner.state.status, RunStatus::Blocked);
13273 assert_eq!(
13274 runner.state.resource_block.as_ref().unwrap().kind,
13275 "repository coordination"
13276 );
13277 assert!(runner.state.prep_failure.is_none());
13278 assert!(runner.state.seats.is_empty());
13279 runner.pause.park();
13280 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13283 .await
13284 .unwrap()
13285 .unwrap();
13286 assert!(runner.state.parked);
13287 assert!(runner.state.prep_failure.is_none());
13288 let disk = RunState::load(&runner.state.id).unwrap();
13289 assert!(disk.driver_exited);
13290 assert_eq!(disk.driver_pid, None);
13291 }
13292
13293 #[tokio::test]
13294 #[cfg(unix)]
13295 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13296 use std::os::fd::AsRawFd as _;
13297 let tmp = tempfile::tempdir().unwrap();
13298 let mut runner = registered_runner(tmp.path()).await;
13299 runner.state.prep_complete = Some(false);
13300 runner.state.config.graph.timeout_resource_wait = 30;
13301 runner.state.save().unwrap();
13302 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13303 assert_eq!(
13304 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13305 0
13306 );
13307 assert!(
13308 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13309 .await
13310 .is_err()
13311 );
13312 let disk = RunState::load(&runner.state.id).unwrap();
13313 assert!(disk.driver_exited);
13314 assert_eq!(disk.driver_pid, None);
13315 assert!(disk.status.resumable());
13316 assert!(runner.state.driver_exited);
13317 assert_eq!(runner.state.driver_pid, None);
13318 runner.state.save().unwrap();
13319 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13320 }
13321
13322 #[tokio::test]
13323 #[cfg(unix)]
13324 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13325 let tmp = tempfile::tempdir().unwrap();
13326 let mut runner = registered_runner(tmp.path()).await;
13327 let winner = runner.state.candidates[0].clone();
13328 let cache = tmp.path().join("cache");
13329 let started = tmp.path().join("pre-gate-started");
13330 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13331 runner.state.config.verify.gate =
13332 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13333 let home = crate::run::home();
13334 let owner =
13335 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13336 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13337 crate::cache::AcquireOutcome::Acquired(g) => g,
13338 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13339 };
13340 let mut gate = Box::pin(runner.run_gate(&winner));
13341 assert!(
13342 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13343 .await
13344 .is_err()
13345 );
13346 assert!(!started.exists(), "pre-gate must wait for the cache too");
13347 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13351 let editor = crate::repo_lock::SeatLease::checkout_with(
13352 &winner.worktree,
13353 true,
13354 crate::config::test_coordination().as_ref(),
13355 &wait,
13356 )
13357 .await
13358 .unwrap();
13359 drop(editor);
13360 guard.release();
13361 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13362 .await
13363 .unwrap()
13364 .unwrap();
13365 drop(gate);
13366 assert!(outcomes.iter().all(CommandOutcome::ok));
13367 assert!(started.exists());
13368 }
13369
13370 #[tokio::test]
13371 #[cfg(unix)]
13372 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13373 use std::os::fd::AsRawFd as _;
13374
13375 ask_test_home();
13376 let tmp = tempfile::tempdir().unwrap();
13377 let repo = tmp.path().join("repo");
13378 std::fs::create_dir_all(&repo).unwrap();
13379 init_repo(&repo);
13380 let common = repo.join(".git");
13381 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13382 let mut runner = runner_at(RunStatus::Prep);
13383 runner.state.repo = repo.clone();
13384 runner.state.config.coordination = crate::config::test_coordination();
13385 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13386 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13387 runner.state.config.graph.advise = false;
13388 runner.roles.implementers = vec![spec("implement")];
13389 runner.roles.judges = vec![spec("judge")];
13390 runner.prep().await.unwrap();
13391 let winner = runner.state.candidates[0].clone();
13392 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13395 let completed = tmp.path().join("seat-completed");
13396 runner.roles.implementers[0].command = vec![
13397 "sh".into(),
13398 "-c".into(),
13399 format!("touch '{}'; echo completed", completed.display()),
13400 ];
13401 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13402 assert_eq!(
13403 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13404 0
13405 );
13406 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13407 let mut implement = Box::pin(runner.implement());
13408 tokio::select! {
13409 result = &mut implement => panic!("rescue must wait: {result:?}"),
13410 result = tokio::time::timeout(Duration::from_secs(10), async {
13411 while !completed.exists() {
13412 tokio::time::sleep(Duration::from_millis(20)).await;
13413 }
13414 }) => result.unwrap(),
13415 }
13416 assert!(
13417 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13418 .await
13419 .is_err()
13420 );
13421 assert_eq!(
13422 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13423 before
13424 );
13425 assert_eq!(
13426 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13427 0
13428 );
13429 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13430 .await
13431 .unwrap()
13432 .unwrap();
13433 drop(implement);
13434 assert_ne!(
13435 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13436 before
13437 );
13438 assert!(git::is_clean(&winner.worktree).await.unwrap());
13439
13440 let started = tmp.path().join("pre-gate-started");
13441 let proceed = tmp.path().join("pre-gate-proceed");
13442 let checked = tmp.path().join("gate-ran");
13443 runner.state.config.verify.pre_gate = vec![format!(
13444 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13445 started.display(),
13446 proceed.display()
13447 )];
13448 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13449 assert_eq!(
13450 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13451 0
13452 );
13453 let mut gate = Box::pin(runner.run_gate(&winner));
13454 assert!(
13455 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13456 .await
13457 .is_err()
13458 );
13459 assert!(!started.exists());
13460 assert!(!checked.exists());
13461 assert!(git::is_clean(&winner.worktree).await.unwrap());
13462 assert_eq!(
13463 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13464 0
13465 );
13466 tokio::select! {
13467 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13468 result = tokio::time::timeout(Duration::from_secs(10), async {
13469 while !started.exists() {
13470 tokio::time::sleep(Duration::from_millis(20)).await;
13471 }
13472 }) => result.unwrap(),
13473 }
13474 assert_eq!(
13475 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13476 -1,
13477 "mechanical edits must hold the real lock"
13478 );
13479 std::fs::write(proceed, "ready").unwrap();
13480 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13481 .await
13482 .unwrap()
13483 .unwrap();
13484 drop(gate);
13485 assert!(outcomes.iter().all(CommandOutcome::ok));
13486 assert!(checked.exists());
13487 assert!(runner.state.pre_gate_commit.is_some());
13488 assert!(git::is_clean(&winner.worktree).await.unwrap());
13489 assert_eq!(
13490 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13491 0,
13492 "gate completion must release coordination"
13493 );
13494 drop(outsider);
13495 std::fs::remove_file(common.join("updater.lock")).unwrap();
13496 std::fs::create_dir(common.join("updater.lock")).unwrap();
13497 std::fs::remove_file(&started).unwrap();
13498 std::fs::remove_file(&checked).unwrap();
13499 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13500 let error = runner.run_gate(&winner).await.unwrap_err();
13501 assert!(
13502 error
13503 .to_string()
13504 .contains("acquire mandatory repository lock")
13505 );
13506 assert!(
13507 !started.exists(),
13508 "failed acquisition must prevent pre-gate"
13509 );
13510 assert!(!checked.exists(), "failed acquisition must prevent gate");
13511 assert!(git::is_clean(&winner.worktree).await.unwrap());
13512 assert_eq!(
13513 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13514 head
13515 );
13516 }
13517
13518 #[tokio::test]
13519 #[cfg(unix)]
13520 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13521 use std::os::fd::AsRawFd as _;
13522 use std::time::Duration;
13523
13524 ask_test_home();
13525 let tmp = tempfile::tempdir().unwrap();
13526 let repo = tmp.path().join("repo");
13527 std::fs::create_dir_all(&repo).unwrap();
13528 init_repo(&repo);
13529 let common = repo.join(".git");
13530 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13531 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13532 assert_eq!(
13533 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13534 0
13535 );
13536 let mut runner = runner_at(RunStatus::Prep);
13537 runner.state.repo = repo.clone();
13538 runner.state.config.coordination = crate::config::test_coordination();
13539 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13540 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13541 runner.state.config.graph.advise = true;
13542 runner.state.config.graph.advisors = 1;
13543 runner.roles.implementers = vec![spec("implement")];
13544 runner.roles.judges = vec![spec("judge")];
13545 let root = runner.state.worktree_root();
13546 let started = tmp.path().join("setup-started");
13547 let proceed = tmp.path().join("setup-proceed");
13548 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13549 run: Some(format!(
13550 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13551 started.display(),
13552 proceed.display()
13553 )),
13554 ..Default::default()
13555 }];
13556 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13557 let config = git::git(&repo, &["config", "--local", "--list"])
13558 .await
13559 .unwrap();
13560 let mut prep = Box::pin(runner.prep());
13561 assert!(
13562 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13563 .await
13564 .is_err()
13565 );
13566 assert!(
13567 !root.exists(),
13568 "contention must prevent even checkout creation"
13569 );
13570 assert!(!started.exists(), "contention must prevent setup");
13571 assert_eq!(
13572 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13573 branches
13574 );
13575 assert_eq!(
13576 git::git(&repo, &["config", "--local", "--list"])
13577 .await
13578 .unwrap(),
13579 config
13580 );
13581 drop(lock);
13582 tokio::select! {
13583 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13584 result = tokio::time::timeout(Duration::from_secs(10), async {
13585 while !started.exists() {
13586 tokio::time::sleep(Duration::from_millis(20)).await;
13587 }
13588 }) => result.unwrap(),
13589 }
13590 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13591 assert_eq!(
13592 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13593 -1
13594 );
13595 assert!(
13596 tokio::time::timeout(
13597 Duration::from_millis(100),
13598 crate::repo_lock::SeatLease::for_checkout(
13599 &repo,
13600 true,
13601 crate::config::test_coordination().as_ref()
13602 )
13603 )
13604 .await
13605 .is_err(),
13606 "preparation must also exclude local editors"
13607 );
13608 std::fs::write(proceed, "ready").unwrap();
13609 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13610 .await
13611 .unwrap()
13612 .unwrap();
13613 drop(prep);
13614 assert_eq!(runner.state.prep_complete, Some(true));
13615 assert!(root.join("judge-1").exists());
13616 assert!(root.join("advisor-1").exists());
13617 assert_eq!(
13618 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13619 0
13620 );
13621 }
13622
13623 #[tokio::test]
13624 #[cfg(unix)]
13625 async fn preparation_lock_failure_prevents_repository_changes() {
13626 ask_test_home();
13627 let tmp = tempfile::tempdir().unwrap();
13628 let repo = tmp.path().join("repo");
13629 std::fs::create_dir_all(&repo).unwrap();
13630 init_repo(&repo);
13631 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13632 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13635 let mut runner = runner_at(RunStatus::Prep);
13636 runner.state.repo = repo.clone();
13637 runner.state.config.coordination = crate::config::test_coordination();
13638 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13639 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13640 runner.roles.implementers = vec![spec("implement")];
13641 let error = runner.prep().await.unwrap_err();
13642 assert!(
13643 error
13644 .to_string()
13645 .contains("acquire mandatory repository lock")
13646 );
13647 assert!(!runner.state.worktree_root().exists());
13648 assert!(runner.state.candidates.is_empty());
13649 assert!(!repo.join(".git/config.worktree").exists());
13650 assert_eq!(
13651 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13652 "* main"
13653 );
13654 }
13655
13656 #[tokio::test]
13657 async fn preparation_records_creation_intent_before_git_can_fail() {
13658 ask_test_home();
13659 let tmp = tempfile::tempdir().unwrap();
13660 let repo = tmp.path().join("repo");
13661 std::fs::create_dir_all(&repo).unwrap();
13662 init_repo(&repo);
13663 let mut runner = runner_at(RunStatus::Prep);
13664 runner.state.repo = repo.clone();
13665 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13666 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13667 runner.state.config.blind.commit_msg_hook = false;
13668 runner.roles.implementers = vec![spec("one")];
13669 let label = blind::assign_labels(1, runner.state.seed)[0];
13670 let branch = runner.state.branch_for(label);
13671 git::git(&repo, &["branch", &branch]).await.unwrap();
13674 assert!(runner.prep().await.is_err());
13675 let saved = RunState::load(&runner.state.id).unwrap();
13676 assert!(saved.candidates.is_empty());
13677 let event = saved
13678 .events
13679 .iter()
13680 .find(|e| e.node == "prep-intent")
13681 .unwrap();
13682 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13683 assert_eq!(intent["branch"], branch);
13684 assert_eq!(intent["start"], runner.state.base_commit);
13685 assert_eq!(
13686 intent["worktree"],
13687 runner
13688 .state
13689 .worktree_root()
13690 .join(format!("cand-{label}"))
13691 .to_string_lossy()
13692 .as_ref()
13693 );
13694 assert_eq!(intent["picks"], serde_json::json!([]));
13695 }
13696
13697 #[tokio::test]
13698 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13699 ask_test_home();
13700 let tmp = tempfile::tempdir().unwrap();
13701 let repo = tmp.path().join("repo");
13702 std::fs::create_dir_all(&repo).unwrap();
13703 init_repo(&repo);
13704 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13705 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13706 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13707 git::git(&repo, &["add", "."]).await.unwrap();
13708 git::git(&repo, &["commit", "-m", "required seed"])
13709 .await
13710 .unwrap();
13711 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13712 git::git(&repo, &["checkout", "main"]).await.unwrap();
13713 let mut runner = runner_at(RunStatus::Prep);
13714 runner.state.repo = repo.clone();
13715 runner.state.base_commit = base.clone();
13716 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13717 runner.state.config.graph.advise = false;
13718 runner.state.config.blind.commit_msg_hook = false;
13719 runner.state.seeds = vec![refs::Seed {
13720 token: seed.clone(),
13721 sha: seed.clone(),
13722 kind: refs::SeedKind::Unmerged,
13723 branch: false,
13724 detail: String::new(),
13725 }];
13726 runner.roles.implementers = vec![spec("one"), spec("two")];
13727 runner.roles.judges = vec![spec("judge")];
13728 let label = blind::assign_labels(2, runner.state.seed)[0];
13729 let branch = runner.state.branch_for(label);
13730 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13731 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13732 .await
13733 .unwrap();
13734 runner.state.candidates.push(Candidate {
13735 index: 0,
13736 label,
13737 agent: "one".into(),
13738 branch,
13739 worktree,
13740 summary: String::new(),
13741 stat: String::new(),
13742 files: 0,
13743 commits: 0,
13744 empty: false,
13745 failed: None,
13746 verified_noop: None,
13747 duration_ms: 0,
13748 folded: false,
13749 });
13750 runner.state.save().unwrap();
13751 runner.state = RunState::load(&runner.state.id).unwrap();
13753 assert_eq!(runner.state.prep_complete, Some(false));
13754 runner.prep().await.unwrap();
13755 assert_eq!(runner.state.prep_complete, Some(true));
13756 assert_eq!(runner.state.candidates.len(), 2);
13757 for candidate in &runner.state.candidates {
13758 assert_eq!(
13759 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13760 git::tree_of(&repo, &seed).await.unwrap()
13761 );
13762 assert_eq!(
13763 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13764 "required commit"
13765 );
13766 }
13767 assert!(runner.state.worktree_root().join("judge-1").exists());
13768 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13769 runner.prep().await.unwrap();
13770 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13771
13772 runner.state.prep_complete = Some(false);
13773 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13774 run: Some(format!(
13775 "test -f '{}'",
13776 tmp.path().join("dependency-ready").display()
13777 )),
13778 ..Default::default()
13779 }];
13780 assert!(runner.prep().await.is_err());
13781 assert!(runner.state.prep_failure.is_some());
13782 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13783 runner.state = RunState::load(&runner.state.id).unwrap();
13784 assert!(
13785 runner.prep().await.is_err(),
13786 "correcting the cause alone must not auto-retry"
13787 );
13788 let failed_at = runner.state.events.last().unwrap().at;
13789 assert!(
13790 runner.retry_preparation(failed_at).is_err(),
13791 "old approvals cannot unlock the hold"
13792 );
13793 let mut task = crate::queue::Task::new(
13794 "retry".into(),
13795 "retry setup".into(),
13796 repo.clone(),
13797 crate::queue::Source::Human,
13798 );
13799 task.start(runner.state.id.clone());
13800 task.hold_machine(Some("setup failed".into()));
13801 task.release_operator("cli", None);
13802 runner
13803 .retry_preparation(task.resumption.as_ref().unwrap().at)
13804 .unwrap();
13805 runner.prep().await.unwrap();
13806 assert!(runner.state.prep_failure.is_none());
13807 assert_eq!(runner.state.prep_complete, Some(true));
13808 assert_eq!(runner.state.candidates.len(), 2);
13809 assert!(runner.state.worktree_root().join("judge-1").exists());
13810 for candidate in &runner.state.candidates {
13811 assert_eq!(
13812 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13813 git::tree_of(&repo, &seed).await.unwrap()
13814 );
13815 }
13816 }
13817
13818 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13819 let mut heads = Vec::new();
13820 for candidate in candidates {
13821 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13822 }
13823 heads
13824 }
13825
13826 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13827 ask_test_home();
13828 let tmp = tempfile::tempdir().unwrap();
13829 let repo = tmp.path().join("repo");
13830 std::fs::create_dir(&repo).unwrap();
13831 init_repo(&repo);
13832 let mut runner = runner_at(RunStatus::Reviewing);
13833 runner.state.repo = repo.clone();
13834 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13835 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13836 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13837 let branch = format!("candidate-{label}");
13838 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13839 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13840 .await
13841 .unwrap();
13842 if label == 'B' {
13843 std::fs::write(worktree.join("change"), "winner").unwrap();
13844 git::rescue_commit(&worktree, "candidate change")
13845 .await
13846 .unwrap();
13847 }
13848 runner.state.candidates.push(Candidate {
13849 index,
13850 label,
13851 agent: "alpha".into(),
13852 branch,
13853 worktree,
13854 summary: "completed reply".into(),
13855 stat: String::new(),
13856 files: usize::from(label == 'B'),
13857 commits: usize::from(label == 'B'),
13858 empty: label == 'A',
13859 failed: None,
13860 verified_noop: None,
13861 duration_ms: 1,
13862 folded: false,
13863 });
13864 }
13865 runner.state.judge_skipped = true;
13866 runner.state.tally = Some(Tally {
13867 first_choice: BTreeMap::new(),
13868 borda: BTreeMap::new(),
13869 winner: 'B',
13870 rankings: 0,
13871 unanimous_initial: false,
13872 deliberated: false,
13873 changed_votes: 0,
13874 unanimous_final: false,
13875 tie_break: None,
13876 judges: 0,
13877 present: 0,
13878 quorum: 0,
13879 met_quorum: true,
13880 uncontested: Some("only B produced a change".into()),
13881 });
13882 runner.state.save().unwrap();
13883 (tmp, runner)
13884 }
13885
13886 #[tokio::test]
13887 #[cfg(unix)]
13888 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13889 for (solo, failed_checkpoint) in
13890 [(true, false), (false, false), (true, true), (false, true)]
13891 {
13892 let (_tmp, mut runner) = preservation_resume_fixture().await;
13893 if solo {
13894 runner.state.candidates.retain(|c| c.label == 'A');
13895 }
13896 runner.state.candidates[0].empty = false;
13897 runner.state.candidates[0].summary.clear();
13898 runner.state.tally = None;
13899 runner.state.judge_skipped = false;
13900 runner.state.status = RunStatus::Implementing;
13901 runner.state.config.disk.min_free_bytes = u64::MAX;
13902 let mut implementer = spec("alpha");
13903 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13904 runner.roles.implementers = vec![implementer];
13905 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13906 if failed_checkpoint {
13907 std::fs::create_dir_all(&patch).unwrap();
13908 }
13909 runner.state.save().unwrap();
13910 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13911 runner.state = RunState::load(&runner.state.id).unwrap();
13912 assert_eq!(runner.state.status, RunStatus::Blocked);
13913 assert!(runner.state.interrupted_implementations.contains(&'A'));
13914 assert_eq!(runner.state.candidates[0].commits, 0);
13915 assert!(!runner.state.candidates[0].empty);
13916 assert!(runner.state.candidates[0].failed.is_none());
13917 if failed_checkpoint {
13918 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13919 std::fs::remove_dir(patch).unwrap();
13920 }
13921 runner.state.resource_block = None;
13922 runner.state.config.disk.min_free_bytes = 0;
13923 runner.roles.implementers[0].command = vec![
13924 "sh".into(),
13925 "-c".into(),
13926 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13927 ];
13928 runner.implement().await.unwrap();
13929 assert!(runner.state.interrupted_implementations.is_empty());
13930 assert!(runner.state.candidate_preservation.is_empty());
13931 assert_eq!(runner.state.candidates[0].commits, 1);
13932 assert!(!runner.state.candidates[0].empty);
13933 assert!(
13934 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13935 .unwrap()
13936 .contains("retried empty interrupted seat")
13937 );
13938 if !solo {
13939 assert_eq!(runner.state.candidates[1].commits, 1);
13940 }
13941 }
13942 }
13943
13944 #[tokio::test]
13945 #[cfg(unix)]
13946 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13947 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13948 let (_tmp, mut runner) = preservation_resume_fixture().await;
13949 runner.state.candidates.retain(|c| c.label == 'A');
13950 runner.state.candidates[0].empty = false;
13951 runner.state.candidates[0].summary.clear();
13952 runner.state.tally = None;
13953 runner.state.judge_skipped = false;
13954 runner.state.status = RunStatus::Implementing;
13955 runner.state.config.disk.min_free_bytes = u64::MAX;
13958 let wt = runner.state.candidates[0].worktree.clone();
13959 std::fs::write(
13960 wt.join("interrupted-source"),
13961 "retain implementation work\n",
13962 )
13963 .unwrap();
13964 if committed {
13965 git::rescue_commit(&wt, "implementation checkpoint")
13966 .await
13967 .unwrap();
13968 }
13969 let lock = git::git(
13970 &wt,
13971 &[
13972 "rev-parse",
13973 "--path-format=absolute",
13974 "--git-path",
13975 "index.lock",
13976 ],
13977 )
13978 .await
13979 .unwrap();
13980 if block_rescue {
13981 std::fs::write(&lock, "fixture-owned lock").unwrap();
13982 }
13983 let mut implementer = spec("alpha");
13984 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13985 runner.roles.implementers = vec![implementer];
13986 runner.state.save().unwrap();
13987 let result = runner.implement().await;
13988 assert_eq!(result.is_err(), block_rescue);
13989 assert_eq!(
13990 runner.state.resource_block.as_ref().unwrap().kind,
13991 "disk pressure"
13992 );
13993 assert!(
13994 runner
13995 .state
13996 .events
13997 .iter()
13998 .any(|e| e.message.contains("agent execution interrupted"))
13999 );
14000 assert!(
14001 runner
14002 .state
14003 .dir()
14004 .join("artifacts/candidate-A-recovery/staged.patch")
14005 .exists()
14006 );
14007 if block_rescue {
14008 let saved = RunState::load(&runner.state.id).unwrap();
14009 assert!(saved.candidate_preservation.contains_key(&'A'));
14010 assert!(!saved.candidates[0].empty);
14011 assert_eq!(
14012 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14013 "retain implementation work\n"
14014 );
14015 std::fs::remove_file(lock).unwrap();
14016 runner.state = saved;
14017 } else {
14018 assert_eq!(runner.state.status, RunStatus::Blocked);
14019 assert_eq!(runner.state.candidates[0].commits, 1);
14020 }
14021 runner.roles.implementers.clear();
14024 runner.state.config.disk.min_free_bytes = 0;
14025 runner.state.resource_block = None;
14026 runner.implement().await.unwrap();
14027 assert!(runner.state.candidate_preservation.is_empty());
14028 assert_eq!(runner.state.candidates[0].commits, 1);
14029 assert!(!runner.state.candidates[0].empty);
14030 assert!(git::is_clean(&wt).await.unwrap());
14031 assert!(
14032 crate::run::read_artifact(&runner.state, "cand-A.patch")
14033 .unwrap()
14034 .contains("retain implementation work")
14035 );
14036 }
14037 }
14038
14039 #[tokio::test]
14040 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14041 let (_tmp, mut runner) = preservation_resume_fixture().await;
14042 runner.implement().await.unwrap();
14044 assert!(runner.state.candidate_preservation.is_empty());
14045 assert!(
14046 !runner
14047 .state
14048 .dir()
14049 .join("artifacts/candidate-A-recovery")
14050 .exists()
14051 );
14052 runner.fold_losers().await.unwrap();
14053 assert!(!runner.state.candidates[0].worktree.exists());
14054 for schema in [14, crate::run::SCHEMA] {
14055 runner.state.schema = schema;
14056 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14057 runner.state.save().unwrap();
14058 runner.state = RunState::load(&runner.state.id).unwrap();
14059 runner.implement().await.unwrap();
14060 assert!(runner.state.candidate_preservation.is_empty());
14061 assert!(runner.state.candidates[0].folded);
14062 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14063 }
14064 }
14065
14066 #[tokio::test]
14067 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14068 let (_tmp, mut runner) = preservation_resume_fixture().await;
14069 let wt = runner.state.candidates[0].worktree.clone();
14070 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14071 runner.state.schema = 14;
14072 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14073 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14074 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14075 for _ in 0..2 {
14076 assert!(runner.implement().await.is_err());
14077 assert_eq!(runner.state.status, RunStatus::Blocked);
14078 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14079 assert!(
14080 runner.state.candidates[0].empty,
14081 "old verdict must remain unusable"
14082 );
14083 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14084 assert!(runner.state.judge_skipped);
14085 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14086 assert!(
14087 crate::run::read_artifact(&runner.state, "cand-A.patch")
14088 .unwrap()
14089 .contains("owned edits")
14090 );
14091 assert!(wt.exists());
14092 assert_eq!(
14093 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14094 .await
14095 .unwrap(),
14096 1
14097 );
14098 runner.state = RunState::load(&runner.state.id).unwrap();
14099 }
14100 }
14101
14102 #[tokio::test]
14103 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14104 let (_tmp, mut runner) = preservation_resume_fixture().await;
14105 let wt = runner.state.candidates[0].worktree.clone();
14106 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14107 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14108 let index_lock = PathBuf::from(
14109 git::git(
14110 &wt,
14111 &[
14112 "rev-parse",
14113 "--path-format=absolute",
14114 "--git-path",
14115 "index.lock",
14116 ],
14117 )
14118 .await
14119 .unwrap(),
14120 );
14121 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14122 runner.state.schema = 14;
14123 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14124 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14125 assert!(runner.implement().await.is_err());
14126 assert!(runner.after_implement().is_err());
14127 assert!(runner.fold_losers().await.is_err());
14128 assert_eq!(
14129 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14130 "failed seat edits"
14131 );
14132 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14133 runner.state = RunState::load(&runner.state.id).unwrap();
14134 std::fs::remove_file(index_lock).unwrap();
14135 assert!(runner.implement().await.is_err());
14137 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14138 assert!(runner.state.candidates[0].empty);
14139 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14140 assert!(
14141 crate::run::read_artifact(&runner.state, "cand-A.patch")
14142 .unwrap()
14143 .contains("failed seat edits")
14144 );
14145 assert!(wt.exists());
14146 }
14147
14148 #[tokio::test]
14149 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14150 let (_tmp, mut runner) = preservation_resume_fixture().await;
14151 let repo = runner.state.repo.clone();
14152 let common = git::git(
14153 &repo,
14154 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14155 )
14156 .await
14157 .unwrap();
14158 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14159 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14160 assert!(runner.fold_losers().await.is_err());
14161 assert!(!runner.state.candidates[0].worktree.exists());
14162 assert!(!runner.state.candidates[0].folded);
14163 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14164 runner.state = RunState::load(&runner.state.id).unwrap();
14165 std::fs::remove_file(branch_lock).unwrap();
14166 let judge = runner.state.worktree_root().join("judge-1");
14167 git::worktree_add_detached(&repo, &judge, "main")
14168 .await
14169 .unwrap();
14170 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14171 runner.roles.judges = vec![spec("judge")];
14172 assert!(runner.fold_losers().await.is_err());
14173 runner.state = RunState::load(&runner.state.id).unwrap();
14174 assert!(
14175 runner.state.candidates[0].folded,
14176 "candidate progress saved before judge failure"
14177 );
14178 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14179 assert!(judge.join("uncommitted").exists());
14180 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14181 runner.fold_losers().await.unwrap();
14182 assert!(!judge.exists());
14183 runner.fold_losers().await.unwrap();
14184 }
14185
14186 #[tokio::test]
14187 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14188 let (_tmp, mut runner) = preservation_resume_fixture().await;
14189 runner.state.candidates.retain(|c| c.label == 'B');
14190 runner.state.tally = None;
14191 runner.state.judge_skipped = false;
14192 runner.state.prep_complete = Some(true);
14193 runner.state.config.graph.advise = false;
14194 runner.state.config.graph.selected_implementer = Some("alpha".into());
14195 let wt = runner.state.candidates[0].worktree.clone();
14196 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14197 runner
14198 .state
14199 .candidate_preservation
14200 .insert('B', "awaiting persistence".into());
14201 let index_lock = git::git(
14202 &wt,
14203 &[
14204 "rev-parse",
14205 "--path-format=absolute",
14206 "--git-path",
14207 "index.lock",
14208 ],
14209 )
14210 .await
14211 .unwrap();
14212 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14213 runner.state.save().unwrap();
14214 assert!(runner.execute_graph().await.is_err());
14215 assert_eq!(runner.state.status, RunStatus::Blocked);
14216 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14217 std::fs::remove_file(index_lock).unwrap();
14218 for _ in 0..2 {
14221 runner.state = RunState::load(&runner.state.id).unwrap();
14222 runner.pause.park();
14223 runner.execute_graph().await.unwrap();
14224 assert_eq!(runner.state.status, RunStatus::Implementing);
14225 assert!(runner.state.candidate_preservation.is_empty());
14226 assert_eq!(runner.state.candidates[0].commits, 2);
14227 assert!(runner.state.tally.is_none());
14228 }
14229 runner.implement().await.unwrap();
14230 assert_eq!(runner.state.status, RunStatus::Judging);
14231 runner.state.status = RunStatus::Blocked;
14233 runner.state.candidates[0].failed = Some("implementation failed".into());
14234 runner.state.save().unwrap();
14235 let error = runner.execute_graph().await.unwrap_err();
14236 assert!(
14237 error
14238 .to_string()
14239 .contains("selected implementation is held")
14240 );
14241 }
14242
14243 #[tokio::test]
14244 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14245 let (_tmp, mut runner) = preservation_resume_fixture().await;
14246 let repo = runner.state.repo.clone();
14247 let candidate = runner.state.candidates[0].clone();
14248 let registered = git::worktree_holding(&repo, &candidate.branch)
14251 .await
14252 .unwrap()
14253 .unwrap();
14254 assert_eq!(
14255 registered.canonicalize().unwrap(),
14256 candidate.worktree.canonicalize().unwrap()
14257 );
14258 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14259 assert_eq!(
14260 git::worktree_holding(&repo, &candidate.branch)
14261 .await
14262 .unwrap(),
14263 Some(registered)
14264 );
14265 assert!(
14266 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14267 "the stale registration initially keeps the branch checked out"
14268 );
14269 runner.fold_losers().await.unwrap();
14270 assert!(runner.state.candidates[0].folded);
14271 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14272 assert_eq!(
14273 git::worktree_holding(&repo, &candidate.branch)
14274 .await
14275 .unwrap(),
14276 None
14277 );
14278 runner.state = RunState::load(&runner.state.id).unwrap();
14279 runner.fold_losers().await.unwrap();
14280 }
14281
14282 #[tokio::test]
14283 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14284 let (_tmp, mut runner) = preservation_resume_fixture().await;
14285 let repo = runner.state.repo.clone();
14286 let candidate = runner.state.candidates[0].clone();
14287 let winner = runner.state.candidates[1].clone();
14288 let registered = git::worktree_holding(&repo, &candidate.branch)
14289 .await
14290 .unwrap()
14291 .unwrap();
14292 assert_eq!(
14293 registered.canonicalize().unwrap(),
14294 candidate.worktree.canonicalize().unwrap()
14295 );
14296 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14297 assert_eq!(
14298 git::worktree_holding(&repo, &candidate.branch)
14299 .await
14300 .unwrap(),
14301 Some(registered)
14302 );
14303 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14304
14305 for _ in 0..2 {
14306 fold_run(&mut runner.state, false, &crate::run::home())
14307 .await
14308 .unwrap();
14309 runner.state = RunState::load(&runner.state.id).unwrap();
14310 assert!(runner.state.candidates[0].folded);
14311 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14312 assert_eq!(
14313 git::worktree_holding(&repo, &candidate.branch)
14314 .await
14315 .unwrap(),
14316 None
14317 );
14318 assert!(!runner.state.candidates[1].folded);
14319 assert!(winner.worktree.exists());
14320 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14321 }
14322 }
14323
14324 #[tokio::test]
14325 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14326 ask_test_home();
14327 let tmp = tempfile::tempdir().unwrap();
14328 let repo = tmp.path().join("repo");
14329 std::fs::create_dir(&repo).unwrap();
14330 init_repo(&repo);
14331 let mut runner = runner_at(RunStatus::Implementing);
14332 runner.state.repo = repo.clone();
14333 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14334 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14335 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14336 let branch = format!("candidate-{label}");
14337 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14338 git::worktree_add_branch(&repo, &wt, &branch, "main")
14339 .await
14340 .unwrap();
14341 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14342 git::git(&wt, &["add", "README.md"]).await.unwrap();
14343 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14344 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14345 runner.state.candidates.push(Candidate {
14346 index,
14347 label,
14348 agent: "alpha".to_owned(),
14349 branch,
14350 worktree: wt,
14351 summary: "completed implementation".to_owned(),
14352 stat: String::new(),
14353 files: 0,
14354 commits: 0,
14355 empty: false,
14356 failed: None,
14357 verified_noop: None,
14358 duration_ms: 42,
14359 folded: false,
14360 });
14361 runner
14362 .state
14363 .candidate_preservation
14364 .insert(label, "awaiting persistence".to_owned());
14365 }
14366 let wt = runner.state.candidates[0].worktree.clone();
14367 let index_lock = git::git(
14368 &wt,
14369 &[
14370 "rev-parse",
14371 "--path-format=absolute",
14372 "--git-path",
14373 "index.lock",
14374 ],
14375 )
14376 .await
14377 .unwrap();
14378 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14379 runner.state.save().unwrap();
14380 assert!(runner.persist_candidates().await.is_err());
14381 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14382 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14383 assert_eq!(
14384 runner.state.candidates[1].commits, 1,
14385 "other results are still collected"
14386 );
14387 assert!(
14388 !runner.state.candidates[0].empty,
14389 "failed rescue is not no change"
14390 );
14391 assert_eq!(
14392 runner.state.candidates[0].summary,
14393 "completed implementation"
14394 );
14395 assert!(runner.after_implement().is_err());
14396 let home = tmp.path().join("fold-home");
14397 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14398 assert!(wt.join("new-file").exists());
14399 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14400 assert!(recovery.join("staged.patch").exists());
14401 assert!(recovery.join("unstaged.patch").exists());
14402 assert_eq!(
14403 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14404 "untracked edits\n"
14405 );
14406 runner.state.begin_pass();
14407 runner.state.end_pass_parked(ParkCause::Upgrade {
14408 from: "1.0".into(),
14409 to: Some("1.1".into()),
14410 });
14411 runner.state.save().unwrap();
14412 runner.state = RunState::load(&runner.state.id).unwrap();
14413 assert!(matches!(
14414 runner.state.passes.last().unwrap().end,
14415 Some(crate::run::PassEnd::Parked {
14416 cause: ParkCause::Upgrade { .. }
14417 })
14418 ));
14419 std::fs::remove_file(index_lock).unwrap();
14420 runner.implement().await.unwrap();
14422 assert!(runner.state.candidate_preservation.is_empty());
14423 assert!(
14424 runner
14425 .state
14426 .candidates
14427 .iter()
14428 .all(|c| c.commits == 1 && !c.empty)
14429 );
14430 assert!(git::is_clean(&wt).await.unwrap());
14431 assert!(
14432 crate::run::read_artifact(&runner.state, "cand-A.patch")
14433 .unwrap()
14434 .contains("later tracked edits")
14435 );
14436 }
14437
14438 #[tokio::test]
14439 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14440 ask_test_home();
14441 let tmp = tempfile::tempdir().unwrap();
14442 let repo = tmp.path().join("repo");
14443 std::fs::create_dir(&repo).unwrap();
14444 init_repo(&repo);
14445 let wt = tmp.path().join("candidate");
14446 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14447 .await
14448 .unwrap();
14449 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14450 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14451 assert!(wt.join("uncommitted").exists());
14452 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14453 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14454 let mut runner = runner_at(RunStatus::Implementing);
14455 runner.state.repo = repo.clone();
14456 runner.state.base_commit = "missing-base".to_owned();
14457 runner.state.candidates.push(Candidate {
14458 index: 0,
14459 label: 'A',
14460 agent: "alpha".into(),
14461 branch: "candidate".into(),
14462 worktree: wt.clone(),
14463 summary: "completed".into(),
14464 stat: String::new(),
14465 files: 0,
14466 commits: 0,
14467 empty: false,
14468 failed: None,
14469 verified_noop: None,
14470 duration_ms: 1,
14471 folded: false,
14472 });
14473 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14474 assert!(
14475 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14476 .await
14477 .is_err()
14478 );
14479 assert!(!runner.state.candidates[0].folded);
14480 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14481 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14482 runner
14483 .state
14484 .candidate_preservation
14485 .insert('A', "pending".into());
14486 assert!(runner.persist_candidates().await.is_err());
14487 assert!(!runner.state.candidates[0].empty);
14488 assert!(!runner.state.candidates[0].folded);
14489 assert!(wt.exists());
14490 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14491 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14492 std::fs::create_dir_all(&patch_path).unwrap();
14493 assert!(
14494 runner.persist_candidates().await.is_err(),
14495 "artifact failure stays pending"
14496 );
14497 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14498 assert!(!runner.state.candidates[0].empty);
14499 std::fs::remove_dir(patch_path).unwrap();
14500 runner.persist_candidates().await.unwrap();
14501 assert!(
14502 runner.state.candidates[0].empty,
14503 "only successful inspection confirms no change"
14504 );
14505 }
14506
14507 fn init_repo(dir: &Path) {
14508 let run = |args: &[&str]| {
14509 let out = std::process::Command::new("git")
14510 .args(args)
14511 .current_dir(dir)
14512 .quiet()
14513 .output()
14514 .expect("spawn git");
14515 assert!(
14516 out.status.success(),
14517 "git {args:?} failed: {}",
14518 String::from_utf8_lossy(&out.stderr)
14519 );
14520 };
14521 run(&["init", "-b", "main"]);
14522 run(&["config", "user.name", "magi test"]);
14523 run(&["config", "user.email", "magi@example.com"]);
14524 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14525 run(&["add", "-A"]);
14526 run(&["commit", "-m", "init"]);
14527 }
14528
14529 fn ask_test_home() {
14537 crate::run::pin_test_home();
14538 }
14539
14540 fn runner_at(status: RunStatus) -> Runner {
14543 let mut state = RunState::new(
14544 PathBuf::from("/nonexistent/repo"),
14545 "main".to_owned(),
14546 "deadbeef".to_owned(),
14547 "task".to_owned(),
14548 Config::default(),
14549 );
14550 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14553 state.id = format!(
14554 "20000101-000000-{:04x}",
14555 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14556 );
14557 state.status = status;
14558 Runner {
14559 state,
14560 roles: ResolvedRoles {
14561 implementers: Vec::new(),
14562 judges: Vec::new(),
14563 reviewers: Vec::new(),
14564 fixer: None,
14565 conductor: conductor(),
14566 implementer_roster: Vec::new(),
14567 judge_roster: Vec::new(),
14568 reviewer_roster: Vec::new(),
14569 },
14570 sem: Arc::new(Semaphore::new(1)),
14571 pause: Pause::new(),
14572 interrupt: Pause::new(),
14573 }
14574 }
14575
14576 #[test]
14577 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14578 crate::run::pin_test_home();
14579 let mut runner = runner_at(RunStatus::Gating);
14580 runner.state.save().unwrap();
14581 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14584 assert!(runner.claim_driver().is_err());
14585 assert!(runner.state.driver_exited);
14586 assert_eq!(runner.state.driver_pid, None);
14587 let disk = RunState::load(&runner.state.id).unwrap();
14588 assert!(disk.driver_exited);
14589 assert_eq!(disk.driver_pid, None);
14590 assert_eq!(disk.status, RunStatus::Gating);
14591 }
14592
14593 #[tokio::test]
14594 async fn an_early_graph_error_releases_its_driver() {
14595 crate::run::pin_test_home();
14596 let mut runner = runner_at(RunStatus::Blocked);
14597 runner.state.config.graph.selected_implementer = Some("selected".into());
14598 runner.state.save().unwrap();
14599 assert!(runner.execute().await.is_err());
14600 let disk = RunState::load(&runner.state.id).unwrap();
14601 assert!(disk.driver_exited);
14602 assert_eq!(disk.driver_pid, None);
14603 assert_eq!(disk.status, RunStatus::Blocked);
14604 }
14605
14606 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14607 crate::run::MergeRepair {
14608 question: "repair-question".into(),
14609 head: "old-head".into(),
14610 pr: "https://github.com/o/r/pull/42".into(),
14611 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14612 gate: vec![outcome(Some(0), false)],
14613 gate_head: Some("old-head".into()),
14614 gate_fixes: Vec::new(),
14615 pre_gate: Vec::new(),
14616 phase,
14617 failure: None,
14618 }
14619 }
14620
14621 #[tokio::test]
14622 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14623 crate::run::pin_test_home();
14624 let mut runner = runner_at(RunStatus::Landing);
14625 runner.state.merge_repairs.push(merge_repair_request(
14626 crate::run::MergeRepairPhase::Requested,
14627 ));
14628 runner.state.save().unwrap();
14629 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14630 assert!(
14631 runner
14632 .execute()
14633 .await
14634 .unwrap_err()
14635 .to_string()
14636 .contains("already running")
14637 );
14638 assert_eq!(
14639 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14640 crate::run::MergeRepairPhase::Requested
14641 );
14642 drop(claim);
14643 let mut disk = runner.state.clone();
14644 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14645 disk.save().unwrap();
14646 assert!(
14647 runner
14648 .execute()
14649 .await
14650 .unwrap_err()
14651 .to_string()
14652 .contains("decision changed")
14653 );
14654 assert_eq!(
14655 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14656 crate::run::MergeRepairPhase::Complete
14657 );
14658 }
14659
14660 #[test]
14661 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14662 crate::run::pin_test_home();
14663 let mut runner = runner_at(RunStatus::Landing);
14664 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14665 runner.state.reviews = request.reviews.clone();
14666 runner.state.gate = request.gate.clone();
14667 runner.state.gate_ran = true;
14668 runner.state.land_approval = Some(crate::run::LandApproval {
14669 question: "old".into(),
14670 head: "old-head".into(),
14671 });
14672 runner.state.merge_repairs.push(request);
14673 runner.begin_merge_repair_review(0).unwrap();
14674 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14675 assert!(!runner.state.gate_status().ok());
14676 assert!(runner.state.land_approval.is_none());
14677 let disk = RunState::load(&runner.state.id).unwrap();
14678 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14679 assert!(disk.merge_repairs[0].gate[0].ok());
14680 assert_eq!(
14681 disk.merge_repairs[0].phase,
14682 crate::run::MergeRepairPhase::Reviewing
14683 );
14684 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14685 assert!(disk.reviews.is_empty());
14686 }
14687
14688 #[tokio::test]
14689 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14690 crate::run::pin_test_home();
14691 for phase in [
14692 crate::run::MergeRepairPhase::Fixing,
14693 crate::run::MergeRepairPhase::Requested,
14694 ] {
14695 let mut runner = runner_at(RunStatus::Landing);
14696 runner.state.config.graph.land_rounds = 0;
14697 runner.state.merge_repairs.push(merge_repair_request(phase));
14698 runner.state.save().unwrap();
14699 assert!(!runner.repair_merge_review().await.unwrap());
14700 assert_eq!(runner.state.status, RunStatus::Blocked);
14701 assert_eq!(
14702 runner.state.merge_repairs[0].phase,
14703 crate::run::MergeRepairPhase::Failed
14704 );
14705 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14706 assert!(
14707 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14708 "interrupted"
14709 } else {
14710 "budget exhausted"
14711 })
14712 );
14713 assert!(runner.state.merge.is_none());
14714 }
14715 }
14716
14717 #[test]
14721 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14722 crate::run::pin_test_home();
14723 let mut runner = runner_at(RunStatus::Implementing);
14724 let interrupt = Pause::new();
14725 runner.watch_interrupt(interrupt.clone());
14726
14727 interrupt.park_because("task a1b2 asked to run first");
14728
14729 assert!(runner.park_here().expect("park_here"));
14730 assert!(runner.state.parked);
14731 let last = runner.state.events.last().expect("a park event");
14732 assert_eq!(last.node, "park");
14733 assert!(
14734 last.message.contains("task a1b2 asked to run first"),
14735 "expected the interrupt reason in {:?}",
14736 last.message
14737 );
14738 }
14739
14740 #[test]
14743 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14744 crate::run::pin_test_home();
14745 let mut runner = runner_at(RunStatus::Implementing);
14746 runner.state.begin_pass();
14747 let shutdown = Pause::new();
14748 runner.on_pause(shutdown.clone());
14749 let upgrade = ParkCause::Upgrade {
14750 from: "0.121.0".to_owned(),
14751 to: Some("0.121.1".to_owned()),
14752 };
14753 shutdown.park_for(upgrade.clone());
14754 shutdown.park_for(ParkCause::Operator);
14755
14756 assert!(runner.park_here().expect("park_here"));
14757 assert_eq!(
14758 runner.state.passes[0].end,
14759 Some(crate::run::PassEnd::Parked { cause: upgrade })
14760 );
14761 let last = runner.state.events.last().expect("a park event");
14762 assert!(last.message.contains("0.121.0"), "{}", last.message);
14763 }
14764
14765 #[test]
14773 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14774 crate::run::pin_test_home();
14775 let mut runner = runner_at(RunStatus::Implementing);
14776 let shutdown = Pause::new();
14777 runner.on_pause(shutdown.clone());
14778 let interrupt = Pause::new();
14779 runner.watch_interrupt(interrupt.clone());
14780
14781 assert!(!runner.park_here().expect("park_here"));
14783 assert!(!runner.state.parked);
14784
14785 interrupt.park_because("test");
14787 assert!(!shutdown.parked());
14788 assert!(runner.park_here().expect("park_here"));
14789 }
14790
14791 #[tokio::test]
14805 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14806 crate::run::pin_test_home();
14807 let mut runner = runner_at(RunStatus::Implementing);
14808 let interrupt = Pause::new();
14809 runner.watch_interrupt(interrupt.clone());
14810
14811 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14812 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14813
14814 let node = async move {
14818 started_tx.send(()).expect("send started");
14819 finish_rx.await.expect("recv finish");
14820 "node finished"
14821 };
14822
14823 let interrupter = async move {
14824 started_rx.await.expect("recv started");
14825 interrupt.park_because("higher-priority task waiting");
14827 tokio::task::yield_now().await;
14831 finish_tx.send(()).expect("send finish");
14832 };
14833
14834 let (node_result, ()) = tokio::join!(node, interrupter);
14835 assert_eq!(
14836 node_result, "node finished",
14837 "the in-flight call ran to completion"
14838 );
14839
14840 assert!(runner.park_here().expect("park_here"));
14843 assert!(runner.state.parked);
14844 }
14845
14846 #[test]
14852 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14853 crate::run::pin_test_home();
14854 let mut runner = runner_at(RunStatus::Judging);
14855 runner.state.config.agents = vec![conductor()];
14859 runner.state.candidates = vec![Candidate {
14860 index: 0,
14861 label: 'A',
14862 agent: "alpha".to_owned(),
14863 branch: "magi/x/A".to_owned(),
14864 worktree: PathBuf::from("/nonexistent/worktree"),
14865 summary: "did the thing".to_owned(),
14866 stat: "1 file changed".to_owned(),
14867 files: 1,
14868 commits: 1,
14869 empty: false,
14870 failed: None,
14871 verified_noop: None,
14872 duration_ms: 1234,
14873 folded: false,
14874 }];
14875 let run_id = runner.state.id.clone();
14876
14877 let interrupt = Pause::new();
14878 runner.watch_interrupt(interrupt.clone());
14879 interrupt.park_because("task c3d4 asked to run first");
14880 assert!(runner.park_here().expect("park_here"));
14881
14882 let resumed = Runner::resume(&run_id).expect("resume");
14883 assert_eq!(resumed.state.candidates.len(), 1);
14884 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14885 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14886 assert_eq!(resumed.state.status, runner.state.status);
14887 assert!(
14888 resumed.state.parked,
14889 "still parked until `execute` actually walks the graph again"
14890 );
14891 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14892 }
14893
14894 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14896 let mut q = ask::Question::new(
14897 run.to_owned(),
14898 "implement".to_owned(),
14899 "impl-A".to_owned(),
14900 "Which storage backend should the cache use?".to_owned(),
14901 String::new(),
14902 vec!["SQLite".to_owned(), "Redis".to_owned()],
14903 );
14904 store.put(&mut q).unwrap();
14905 q
14906 }
14907
14908 #[test]
14909 fn a_failed_runs_open_question_is_abandoned() {
14910 ask_test_home();
14911 let store = ask::Questions::open();
14912 let mut runner = runner_at(RunStatus::Failed);
14913 let run = runner.state.id.clone();
14914 let q = ask_open_question(&store, &run);
14915
14916 runner.settle_questions();
14917
14918 let back = store.get(&q.id).unwrap();
14919 assert!(
14920 !back.status.open(),
14921 "the seat that asked died with the run; nobody is left to read an answer"
14922 );
14923 assert!(
14924 back.detail.contains(&run) && back.detail.contains("failed"),
14925 "the reason names what the run became, not just that it is gone: {}",
14926 back.detail
14927 );
14928 }
14929
14930 #[test]
14931 fn a_merged_runs_open_question_is_abandoned_too() {
14932 ask_test_home();
14933 let store = ask::Questions::open();
14934 for status in [RunStatus::Merged, RunStatus::Ready] {
14937 let mut runner = runner_at(status);
14938 let run = runner.state.id.clone();
14939 let q = ask_open_question(&store, &run);
14940
14941 runner.settle_questions();
14942
14943 let back = store.get(&q.id).unwrap();
14944 assert!(
14945 !back.status.open(),
14946 "{status:?} run's question must not outlive the run"
14947 );
14948 }
14949 }
14950
14951 #[test]
14952 fn a_still_resumable_runs_open_question_is_left_alone() {
14953 ask_test_home();
14954 let store = ask::Questions::open();
14955 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14961 let mut runner = runner_at(status);
14962 let run = runner.state.id.clone();
14963 let q = ask_open_question(&store, &run);
14964
14965 runner.settle_questions();
14966
14967 let back = store.get(&q.id).unwrap();
14968 assert!(
14969 back.status.open(),
14970 "{status:?} is still alive; the question must still be waiting"
14971 );
14972 }
14973 }
14974
14975 #[test]
14976 fn settle_questions_never_touches_an_already_answered_question() {
14977 ask_test_home();
14978 let store = ask::Questions::open();
14979 let mut runner = runner_at(RunStatus::Failed);
14980 let run = runner.state.id.clone();
14981 let mut q = ask_open_question(&store, &run);
14982 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14983 .unwrap();
14984 store.put(&mut q).unwrap();
14985
14986 runner.settle_questions();
14991 runner.settle_questions();
14992
14993 let back = store.get(&q.id).unwrap();
14994 assert_eq!(
14995 back.status,
14996 ask::QuestionStatus::Answered,
14997 "a real answer is a decision on record, never overwritten by a sweep"
14998 );
14999 }
15000
15001 #[tokio::test]
15012 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15013 crate::run::pin_test_home();
15014 let tmp = tempfile::tempdir().expect("tempdir");
15015 let repo = tmp.path().join("repo");
15016 std::fs::create_dir_all(&repo).unwrap();
15017 init_repo(&repo);
15018
15019 let mut config = Config::default();
15020 config.graph.worktree_root = Some(tmp.path().join("wt"));
15021
15022 let mut state = RunState::new(
15023 repo.clone(),
15024 "main".to_owned(),
15025 "deadbeef".to_owned(),
15026 "task".to_owned(),
15027 config,
15028 );
15029 let root = state.worktree_root();
15030 let wt_a = root.join("cand-A");
15031 let wt_b = root.join("cand-B");
15032 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15033 .await
15034 .expect("worktree A");
15035 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15036 .await
15037 .expect("worktree B");
15038
15039 state.candidates = vec![
15040 Candidate {
15041 index: 0,
15042 label: 'A',
15043 agent: "alpha".to_owned(),
15044 branch: "magi/x/A".to_owned(),
15045 worktree: wt_a.clone(),
15046 summary: String::new(),
15047 stat: String::new(),
15048 files: 0,
15049 commits: 0,
15050 empty: false,
15051 failed: None,
15052 verified_noop: None,
15053 duration_ms: 0,
15054 folded: false,
15055 },
15056 Candidate {
15057 index: 1,
15058 label: 'B',
15059 agent: "beta".to_owned(),
15060 branch: "magi/x/B".to_owned(),
15061 worktree: wt_b.clone(),
15062 summary: String::new(),
15063 stat: String::new(),
15064 files: 0,
15065 commits: 0,
15066 empty: false,
15067 failed: None,
15068 verified_noop: None,
15069 duration_ms: 0,
15070 folded: false,
15071 },
15072 ];
15073 state.tally = Some(Tally {
15074 first_choice: BTreeMap::from([('A', 1)]),
15075 borda: BTreeMap::new(),
15076 winner: 'A',
15077 rankings: 1,
15078 unanimous_initial: true,
15079 deliberated: false,
15080 changed_votes: 0,
15081 unanimous_final: true,
15082 tie_break: None,
15083 judges: 1,
15084 present: 1,
15085 quorum: 1,
15086 met_quorum: true,
15087 uncontested: None,
15088 });
15089 state.status = RunStatus::Ready;
15090
15091 fold_run(&mut state, false, &crate::run::home())
15092 .await
15093 .expect("fold_run");
15094
15095 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15096 assert!(
15097 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15098 "the unmerged winner's branch survives"
15099 );
15100 assert!(
15101 !state.candidates[0].folded,
15102 "the winner is not marked folded"
15103 );
15104
15105 assert!(!wt_b.exists(), "the loser's worktree is removed");
15106 assert!(
15107 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15108 "the loser's branch is removed"
15109 );
15110 assert!(state.candidates[1].folded, "the loser is marked folded");
15111 }
15112
15113 #[tokio::test]
15114 #[cfg(unix)]
15115 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15116 use std::os::fd::AsRawFd as _;
15117 use std::time::Duration;
15118
15119 let tmp = tempfile::tempdir().unwrap();
15120 let repo = tmp.path().join("repo");
15121 std::fs::create_dir_all(&repo).unwrap();
15122 init_repo(&repo);
15123 let common = repo.join(".git");
15124 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15125 let home = tmp.path().join("home");
15126 let mut config = Config::default();
15127 config.coordination = crate::config::test_coordination();
15128 config.graph.worktree_root = Some(tmp.path().join("wt"));
15129 config.graph.timeout_resource_wait = 1;
15130 let mut state = RunState::new(
15131 repo.clone(),
15132 "main".into(),
15133 "deadbeef".into(),
15134 "task".into(),
15135 config,
15136 );
15137 let root = state.worktree_root();
15138 let candidate = root.join("cand-A");
15139 let reviewer = root.join("review-1");
15140 let branch = state.branch_for('A');
15141 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15142 .await
15143 .unwrap();
15144 git::worktree_add_detached(&repo, &reviewer, "main")
15145 .await
15146 .unwrap();
15147 git::acquire_worktree_config(&repo).await.unwrap();
15148 state.enabled_worktree_config = true;
15149 state.status = RunStatus::Ready;
15150 state.candidates.push(Candidate {
15151 index: 0,
15152 label: 'A',
15153 agent: "implement".into(),
15154 branch: branch.clone(),
15155 worktree: candidate.clone(),
15156 summary: String::new(),
15157 stat: String::new(),
15158 files: 0,
15159 commits: 0,
15160 empty: false,
15161 failed: None,
15162 verified_noop: None,
15163 duration_ms: 0,
15164 folded: false,
15165 });
15166 state.save_under(&home).unwrap();
15167 let config_before = std::fs::read(common.join("config")).unwrap();
15168 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15169 .await
15170 .unwrap();
15171 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15174 assert_eq!(
15175 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15176 0
15177 );
15178 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15179 assert!(
15180 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15181 .await
15182 .is_err()
15183 );
15184 assert!(candidate.exists());
15185 assert!(reviewer.exists());
15186 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15187 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15188 assert_eq!(
15189 git::git(&repo, &["worktree", "list", "--porcelain"])
15190 .await
15191 .unwrap(),
15192 worktrees_before
15193 );
15194 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15195 .await
15196 .unwrap()
15197 .unwrap_err();
15198 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15199 drop(fold);
15200 assert!(candidate.exists() && reviewer.exists());
15201 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15202 assert_eq!(
15203 state.resource_block.as_ref().unwrap().kind,
15204 "repository coordination"
15205 );
15206 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15207 drop(external);
15208 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15209 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15210 .await
15211 .unwrap()
15212 .unwrap();
15213 drop(fold);
15214 assert!(removed.contains(&branch));
15215 assert!(!root.exists());
15216 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15217 assert!(state.candidates[0].folded);
15218 assert!(!state.enabled_worktree_config);
15219 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15220 let enabled = git::git_raw(
15221 &repo,
15222 &["config", "--local", "--get", "extensions.worktreeConfig"],
15223 )
15224 .await
15225 .unwrap();
15226 assert!(!enabled.ok(), "the last config reference was released");
15227 }
15228
15229 #[tokio::test]
15232 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15233 let tmp = tempfile::tempdir().expect("tempdir");
15234 let repo = tmp.path().join("repo");
15235 std::fs::create_dir_all(&repo).unwrap();
15236 init_repo(&repo);
15237 let home = tmp.path().join("home");
15238
15239 let mut config = Config::default();
15240 config.graph.worktree_root = Some(tmp.path().join("wt"));
15241 let mut state = RunState::new(
15242 repo.clone(),
15243 "main".to_owned(),
15244 "deadbeef".to_owned(),
15245 "task".to_owned(),
15246 config,
15247 );
15248 git::git(&repo, &["branch", "magi/x/A", "main"])
15250 .await
15251 .expect("branch");
15252 state.candidates = vec![Candidate {
15253 index: 0,
15254 label: 'A',
15255 agent: "alpha".to_owned(),
15256 branch: "magi/x/A".to_owned(),
15257 worktree: state.worktree_root().join("cand-A"),
15258 summary: String::new(),
15259 stat: String::new(),
15260 files: 0,
15261 commits: 0,
15262 empty: false,
15263 failed: None,
15264 verified_noop: None,
15265 duration_ms: 0,
15266 folded: true,
15267 }];
15268 state.released_to = Some("20260901-000000-new1".to_owned());
15269 state.released_branches = vec!["magi/x/A".to_owned()];
15270
15271 fold_run(&mut state, true, &home).await.expect("fold_run");
15272
15273 assert!(
15274 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15275 "the handed-over branch survives a fold"
15276 );
15277 }
15278
15279 #[tokio::test]
15283 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15284 let tmp = tempfile::tempdir().expect("tempdir");
15285 let repo = tmp.path().join("repo");
15286 std::fs::create_dir_all(&repo).unwrap();
15287 init_repo(&repo);
15288 let run = |args: &[&str]| {
15289 let out = std::process::Command::new("git")
15290 .quiet()
15291 .args(args)
15292 .current_dir(&repo)
15293 .output()
15294 .expect("spawn git");
15295 assert!(out.status.success(), "git {args:?}");
15296 };
15297 run(&["branch", "magi/x/A"]);
15298 run(&["checkout", "-q", "-b", "magi/x/B"]);
15299 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15300 run(&["add", "-A"]);
15301 run(&["commit", "-q", "-m", "work"]);
15302 run(&["checkout", "-q", "main"]);
15303
15304 let probe = RunState::new(
15308 repo.clone(),
15309 "main".to_owned(),
15310 "deadbeef".to_owned(),
15311 "task".to_owned(),
15312 Config::default(),
15313 );
15314 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15315 let bare = tmp.path().join("origin.git");
15316 let out = std::process::Command::new("git")
15317 .quiet()
15318 .args(["init", "-q", "--bare"])
15319 .arg(&bare)
15320 .output()
15321 .expect("spawn git");
15322 assert!(out.status.success(), "git init --bare");
15323 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15324 run(&["push", "-q", "origin", "main"]);
15325
15326 let mut state = RunState::new(
15327 repo.clone(),
15328 "main".to_owned(),
15329 "deadbeef".to_owned(),
15330 "task".to_owned(),
15331 Config::default(),
15332 );
15333 state.seeds = vec![refs::Seed {
15334 token: "magi/27b2/A".to_owned(),
15335 kind: refs::SeedKind::Unresolved,
15336 sha: String::new(),
15337 branch: true,
15338 detail: "no branch or commit named magi/27b2/A".to_owned(),
15339 }];
15340
15341 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15342 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15343 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15344 let detail = empty_candidate_detail(&state, "main");
15345 assert!(detail.starts_with("empty candidate"), "{detail}");
15346 assert!(detail.contains("magi/27b2/A"), "{detail}");
15347 }
15348
15349 #[tokio::test]
15358 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15359 let tmp = tempfile::tempdir().expect("tempdir");
15360 let repo = tmp.path().join("repo");
15361 std::fs::create_dir_all(&repo).unwrap();
15362 init_repo(&repo);
15363
15364 let mut config = Config::default();
15365 config.merge.mode = MergeMode::Local;
15366
15367 let mut state = RunState::new(
15368 repo.clone(),
15369 "main".to_owned(),
15370 "deadbeef".to_owned(),
15371 "task".to_owned(),
15372 config,
15373 );
15374 state.candidates = vec![Candidate {
15375 index: 0,
15376 label: 'A',
15377 agent: "alpha".to_owned(),
15378 branch: "does-not-exist".to_owned(),
15379 worktree: repo.clone(),
15380 summary: String::new(),
15381 stat: String::new(),
15382 files: 0,
15383 commits: 0,
15384 empty: false,
15385 failed: None,
15386 verified_noop: None,
15387 duration_ms: 0,
15388 folded: false,
15389 }];
15390 state.tally = Some(Tally {
15391 first_choice: BTreeMap::from([('A', 1)]),
15392 borda: BTreeMap::new(),
15393 winner: 'A',
15394 rankings: 1,
15395 unanimous_initial: true,
15396 deliberated: false,
15397 changed_votes: 0,
15398 unanimous_final: true,
15399 tie_break: None,
15400 judges: 0,
15401 present: 0,
15402 quorum: 0,
15403 met_quorum: true,
15404 uncontested: Some("only candidate A produced a change".to_owned()),
15405 });
15406 state.reviews = vec![ReviewRound {
15407 round: 1,
15408 head: "deadbeef".to_owned(),
15409 verified_head: None,
15410 verified_at: None,
15411 reviews: Vec::new(),
15412 e2e: Vec::new(),
15413 fix: None,
15414 blocking: 0,
15415 answered: 0,
15416 expected: 0,
15417 clean: true,
15418 verify_retried: false,
15419 e2e_deferred: false,
15420 e2e_defer_reason: None,
15421 progressed: false,
15422 vote_split: false,
15423 reconsideration: Vec::new(),
15424 verdict: None,
15425 }];
15426 state.gate = vec![CommandOutcome {
15427 command: "test".to_owned(),
15428 code: Some(0),
15429 output_tail: String::new(),
15430 duration_ms: 0,
15431 resource_blocked: false,
15432 }];
15433 state.gate_ran = true;
15434 state.status = RunStatus::Ready;
15439 state.merge = Some(MergeOutcome {
15440 mode: MergeMode::Local,
15441 ok: false,
15442 detail: "already concluded".to_owned(),
15443 empty: false,
15444 });
15445
15446 let mut runner = Runner {
15447 state,
15448 roles: ResolvedRoles {
15449 implementers: Vec::new(),
15450 judges: Vec::new(),
15451 reviewers: Vec::new(),
15452 fixer: None,
15453 conductor: conductor(),
15454 implementer_roster: Vec::new(),
15455 judge_roster: Vec::new(),
15456 reviewer_roster: Vec::new(),
15457 },
15458 sem: Arc::new(Semaphore::new(1)),
15459 pause: Pause::new(),
15460 interrupt: Pause::new(),
15461 };
15462
15463 runner.merge().await.expect("merge");
15464
15465 assert_eq!(
15466 runner.state.status,
15467 RunStatus::Ready,
15468 "a concluded run's status must not change on reentry"
15469 );
15470 assert_eq!(
15471 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15472 Some("already concluded"),
15473 "merge must not run again once the node already recorded an outcome"
15474 );
15475 }
15476
15477 #[tokio::test]
15486 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15487 let tmp = tempfile::tempdir().expect("tempdir");
15488 let repo = tmp.path().join("repo");
15489 std::fs::create_dir_all(&repo).unwrap();
15490 init_repo(&repo);
15491
15492 let mut config = Config::default();
15493 config.merge.mode = MergeMode::Local;
15494
15495 let mut state = RunState::new(
15496 repo.clone(),
15497 "main".to_owned(),
15498 "deadbeef".to_owned(),
15499 "task".to_owned(),
15500 config,
15501 );
15502 state.candidates = vec![Candidate {
15503 index: 0,
15504 label: 'A',
15505 agent: "alpha".to_owned(),
15506 branch: "does-not-exist".to_owned(),
15507 worktree: repo.clone(),
15508 summary: String::new(),
15509 stat: String::new(),
15510 files: 0,
15511 commits: 0,
15512 empty: false,
15513 failed: None,
15514 verified_noop: None,
15515 duration_ms: 0,
15516 folded: false,
15517 }];
15518 state.tally = Some(Tally {
15519 first_choice: BTreeMap::from([('A', 1)]),
15520 borda: BTreeMap::new(),
15521 winner: 'A',
15522 rankings: 1,
15523 unanimous_initial: true,
15524 deliberated: false,
15525 changed_votes: 0,
15526 unanimous_final: true,
15527 tie_break: None,
15528 judges: 0,
15529 present: 0,
15530 quorum: 0,
15531 met_quorum: true,
15532 uncontested: Some("only candidate A produced a change".to_owned()),
15533 });
15534 state.reviews = vec![ReviewRound {
15535 round: 1,
15536 head: "deadbeef".to_owned(),
15537 verified_head: None,
15538 verified_at: None,
15539 reviews: Vec::new(),
15540 e2e: Vec::new(),
15541 fix: None,
15542 blocking: 0,
15543 answered: 0,
15544 expected: 0,
15545 clean: true,
15546 verify_retried: false,
15547 e2e_deferred: false,
15548 e2e_defer_reason: None,
15549 progressed: false,
15550 vote_split: false,
15551 reconsideration: Vec::new(),
15552 verdict: None,
15553 }];
15554 state.gate = Vec::new();
15556 state.gate_ran = false;
15557 state.status = RunStatus::Gating;
15558
15559 let mut runner = Runner {
15560 state,
15561 roles: ResolvedRoles {
15562 implementers: Vec::new(),
15563 judges: Vec::new(),
15564 reviewers: Vec::new(),
15565 fixer: None,
15566 conductor: conductor(),
15567 implementer_roster: Vec::new(),
15568 judge_roster: Vec::new(),
15569 reviewer_roster: Vec::new(),
15570 },
15571 sem: Arc::new(Semaphore::new(1)),
15572 pause: Pause::new(),
15573 interrupt: Pause::new(),
15574 };
15575
15576 runner.merge().await.expect("merge");
15577
15578 assert!(
15579 runner.state.merge.is_none(),
15580 "an empty gate must never be read as a passing one: {:?}",
15581 runner.state.merge
15582 );
15583 }
15584
15585 #[tokio::test]
15592 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15593 let tmp = tempfile::tempdir().expect("tempdir");
15594 let repo = tmp.path().join("repo");
15595 std::fs::create_dir_all(&repo).unwrap();
15596 init_repo(&repo);
15597
15598 let config = Config::default();
15600
15601 let mut state = RunState::new(
15602 repo.clone(),
15603 "main".to_owned(),
15604 "deadbeef".to_owned(),
15605 "task".to_owned(),
15606 config,
15607 );
15608 state.candidates = vec![Candidate {
15609 index: 0,
15610 label: 'A',
15611 agent: "alpha".to_owned(),
15612 branch: "does-not-exist".to_owned(),
15613 worktree: repo.clone(),
15614 summary: String::new(),
15615 stat: String::new(),
15616 files: 0,
15617 commits: 0,
15618 empty: false,
15619 failed: None,
15620 verified_noop: None,
15621 duration_ms: 0,
15622 folded: false,
15623 }];
15624 state.tally = Some(Tally {
15625 first_choice: BTreeMap::from([('A', 1)]),
15626 borda: BTreeMap::new(),
15627 winner: 'A',
15628 rankings: 1,
15629 unanimous_initial: true,
15630 deliberated: false,
15631 changed_votes: 0,
15632 unanimous_final: true,
15633 tie_break: None,
15634 judges: 0,
15635 present: 0,
15636 quorum: 0,
15637 met_quorum: true,
15638 uncontested: Some("only candidate A produced a change".to_owned()),
15639 });
15640 state.reviews = vec![ReviewRound {
15641 round: 1,
15642 head: "deadbeef".to_owned(),
15643 verified_head: None,
15644 verified_at: None,
15645 reviews: Vec::new(),
15646 e2e: Vec::new(),
15647 fix: None,
15648 blocking: 0,
15649 answered: 0,
15650 expected: 0,
15651 clean: true,
15652 verify_retried: false,
15653 e2e_deferred: false,
15654 e2e_defer_reason: None,
15655 progressed: false,
15656 vote_split: false,
15657 reconsideration: Vec::new(),
15658 verdict: None,
15659 }];
15660
15661 let mut runner = Runner {
15662 state,
15663 roles: ResolvedRoles {
15664 implementers: Vec::new(),
15665 judges: Vec::new(),
15666 reviewers: Vec::new(),
15667 fixer: None,
15668 conductor: conductor(),
15669 implementer_roster: Vec::new(),
15670 judge_roster: Vec::new(),
15671 reviewer_roster: Vec::new(),
15672 },
15673 sem: Arc::new(Semaphore::new(1)),
15674 pause: Pause::new(),
15675 interrupt: Pause::new(),
15676 };
15677
15678 runner.gate().await.expect("gate");
15679 assert!(
15680 runner.state.gate_ran,
15681 "zero configured commands is still a real attempt, not an unrun gate"
15682 );
15683 assert!(runner.state.gate.is_empty());
15684 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15685 assert_ne!(
15686 runner.state.status,
15687 RunStatus::Blocked,
15688 "a gate with nothing to check must not read as failed"
15689 );
15690
15691 runner.merge().await.expect("merge");
15692 assert_eq!(
15693 runner.state.status,
15694 RunStatus::Ready,
15695 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15696 );
15697 }
15698
15699 #[tokio::test]
15710 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15711 crate::run::pin_test_home();
15712 let home = crate::run::home();
15713
15714 let tmp = tempfile::tempdir().expect("tempdir");
15715 let repo = tmp.path().join("repo");
15716 std::fs::create_dir_all(&repo).unwrap();
15717 init_repo(&repo);
15718 let cache_dir = tmp.path().join("target");
15721
15722 let mut config = Config::default();
15723 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15724 config.graph.timeout_verify = Some(2);
15727 config.graph.timeout_resource_wait = 2;
15728
15729 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15730 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15731 .expect("no io error acquiring directly")
15732 {
15733 crate::cache::AcquireOutcome::Acquired(g) => g,
15734 crate::cache::AcquireOutcome::Busy(b) => {
15735 panic!("expected the direct acquire to win the lease first: {b:?}")
15736 }
15737 };
15738
15739 let mut state = RunState::new(
15740 repo.clone(),
15741 "main".to_owned(),
15742 "deadbeef".to_owned(),
15743 "task".to_owned(),
15744 config,
15745 );
15746 state.candidates = vec![Candidate {
15747 index: 0,
15748 label: 'A',
15749 agent: "alpha".to_owned(),
15750 branch: "does-not-exist".to_owned(),
15751 worktree: repo.clone(),
15752 summary: String::new(),
15753 stat: String::new(),
15754 files: 0,
15755 commits: 0,
15756 empty: false,
15757 failed: None,
15758 verified_noop: None,
15759 duration_ms: 0,
15760 folded: false,
15761 }];
15762 state.tally = Some(Tally {
15763 first_choice: BTreeMap::from([('A', 1)]),
15764 borda: BTreeMap::new(),
15765 winner: 'A',
15766 rankings: 1,
15767 unanimous_initial: true,
15768 deliberated: false,
15769 changed_votes: 0,
15770 unanimous_final: true,
15771 tie_break: None,
15772 judges: 0,
15773 present: 0,
15774 quorum: 0,
15775 met_quorum: true,
15776 uncontested: Some("only candidate A produced a change".to_owned()),
15777 });
15778 state.reviews = vec![ReviewRound {
15779 round: 1,
15780 head: "deadbeef".to_owned(),
15781 verified_head: None,
15782 verified_at: None,
15783 reviews: Vec::new(),
15784 e2e: Vec::new(),
15785 fix: None,
15786 blocking: 0,
15787 answered: 0,
15788 expected: 0,
15789 clean: true,
15790 verify_retried: false,
15791 e2e_deferred: false,
15792 e2e_defer_reason: None,
15793 progressed: false,
15794 vote_split: false,
15795 reconsideration: Vec::new(),
15796 verdict: None,
15797 }];
15798
15799 let mut runner = Runner {
15800 state,
15801 roles: ResolvedRoles {
15802 implementers: Vec::new(),
15803 judges: Vec::new(),
15804 reviewers: Vec::new(),
15805 fixer: None,
15806 conductor: conductor(),
15807 implementer_roster: Vec::new(),
15808 judge_roster: Vec::new(),
15809 reviewer_roster: Vec::new(),
15810 },
15811 sem: Arc::new(Semaphore::new(1)),
15812 pause: Pause::new(),
15813 interrupt: Pause::new(),
15814 };
15815
15816 let started = std::time::Instant::now();
15817 runner.gate().await.expect("gate");
15818 assert!(
15819 started.elapsed() < Duration::from_secs(1),
15820 "a gate with nothing to run must never wait on a lease it never needed"
15821 );
15822 assert!(
15823 runner.state.gate_ran,
15824 "zero commands is still a real, immediate attempt"
15825 );
15826 assert!(runner.state.gate.is_empty());
15827 assert_ne!(
15828 runner.state.status,
15829 RunStatus::Blocked,
15830 "must not read as resource-blocked on a lease it never asked for"
15831 );
15832 }
15833
15834 #[cfg(unix)]
15835 #[tokio::test]
15836 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15837 crate::run::pin_test_home();
15838 let tmp = tempfile::tempdir().unwrap();
15839 let mut config = Config::default();
15840 config.disk.min_free_bytes = 0;
15841 let mut state = RunState::new(
15842 tmp.path().to_owned(),
15843 "main".into(),
15844 "base".into(),
15845 "verification evidence".into(),
15846 config,
15847 );
15848 let (outcomes, pids) = run_commands(
15849 &mut state,
15850 "gate",
15851 "gate",
15852 1,
15853 &["sh".into(), "-c".into()],
15854 &[
15855 "printf before-timeout; exec sleep 30".into(),
15856 "kill -TERM $$".into(),
15857 "exit 7".into(),
15858 "printf success".into(),
15859 ],
15860 tmp.path(),
15861 Duration::from_secs(1),
15862 )
15863 .await;
15864 wait_for_timed_out_children_to_die(&pids).await;
15865 assert_eq!(
15866 pids.len(),
15867 1,
15868 "unleased timeout retains its death-check PID"
15869 );
15870 assert_eq!(outcomes.len(), 4);
15871 assert_eq!(outcomes[0].code, None);
15872 assert!(outcomes[0].output_tail.contains("before-timeout"));
15873 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15874 assert_eq!(outcomes[1].code, None);
15875 assert!(outcomes[1].output_tail.contains("command terminated:"));
15876 assert!(!outcomes[1].output_tail.contains("timed out"));
15877 assert_eq!(outcomes[2].code, Some(7));
15878 assert!(!outcomes[2].output_tail.contains("timed out"));
15879 assert_eq!(outcomes[3].code, Some(0));
15880 assert_eq!(outcomes[3].output_tail, "success");
15881 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15882 }
15883
15884 #[tokio::test]
15894 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15895 crate::run::pin_test_home();
15896
15897 let tmp = tempfile::tempdir().expect("tempdir");
15898 let repo = tmp.path().join("repo");
15899 std::fs::create_dir_all(&repo).unwrap();
15900 init_repo(&repo);
15901
15902 let mut config = Config::default();
15903 config.verify.gate = vec![
15904 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15905 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15906 .to_owned(),
15907 ];
15908
15909 let mut state = RunState::new(
15910 repo.clone(),
15911 "main".to_owned(),
15912 "deadbeef".to_owned(),
15913 "task".to_owned(),
15914 config,
15915 );
15916 let run_id = state.id.clone();
15917 state.candidates = vec![Candidate {
15918 index: 0,
15919 label: 'A',
15920 agent: "alpha".to_owned(),
15921 branch: "does-not-exist".to_owned(),
15922 worktree: repo.clone(),
15923 summary: String::new(),
15924 stat: String::new(),
15925 files: 0,
15926 commits: 0,
15927 empty: false,
15928 failed: None,
15929 verified_noop: None,
15930 duration_ms: 0,
15931 folded: false,
15932 }];
15933 state.tally = Some(Tally {
15934 first_choice: BTreeMap::from([('A', 1)]),
15935 borda: BTreeMap::new(),
15936 winner: 'A',
15937 rankings: 1,
15938 unanimous_initial: true,
15939 deliberated: false,
15940 changed_votes: 0,
15941 unanimous_final: true,
15942 tie_break: None,
15943 judges: 0,
15944 present: 0,
15945 quorum: 0,
15946 met_quorum: true,
15947 uncontested: Some("only candidate A produced a change".to_owned()),
15948 });
15949 state.reviews = vec![ReviewRound {
15950 round: 1,
15951 head: "deadbeef".to_owned(),
15952 verified_head: None,
15953 verified_at: None,
15954 reviews: Vec::new(),
15955 e2e: Vec::new(),
15956 fix: None,
15957 blocking: 0,
15958 answered: 0,
15959 expected: 0,
15960 clean: true,
15961 verify_retried: false,
15962 e2e_deferred: false,
15963 e2e_defer_reason: None,
15964 progressed: false,
15965 vote_split: false,
15966 reconsideration: Vec::new(),
15967 verdict: None,
15968 }];
15969
15970 let mut runner = Runner {
15971 state,
15972 roles: ResolvedRoles {
15973 implementers: Vec::new(),
15974 judges: Vec::new(),
15975 reviewers: Vec::new(),
15976 fixer: None,
15977 conductor: conductor(),
15978 implementer_roster: Vec::new(),
15979 judge_roster: Vec::new(),
15980 reviewer_roster: Vec::new(),
15981 },
15982 sem: Arc::new(Semaphore::new(1)),
15983 pause: Pause::new(),
15984 interrupt: Pause::new(),
15985 };
15986
15987 let started_marker = repo.join("started.marker");
15988 let release_marker = repo.join("release.marker");
15989 let poller = tokio::spawn(async move {
15990 for _ in 0..100 {
15995 if started_marker.exists()
15996 && let Ok(s) = crate::run::RunState::load(&run_id)
15997 && let Some(a) = s.active.get("gate")
15998 {
15999 std::fs::write(&release_marker, b"go").expect("release marker");
16000 return Some(a.clone());
16001 }
16002 tokio::time::sleep(Duration::from_millis(50)).await;
16003 }
16004 None
16005 });
16006
16007 runner.gate().await.expect("gate");
16008 let captured = poller.await.expect("poller task");
16009 let captured = captured.expect(
16010 "the poller never saw a `gate` task entry in run.json while the command was \
16011 still blocked on its own release marker",
16012 );
16013
16014 assert_eq!(captured.task.as_deref(), Some("gate"));
16015 assert_eq!(captured.node, "gate");
16016 assert_eq!(captured.index, Some(1));
16017 assert_eq!(captured.total, Some(1));
16018 assert!(
16019 captured
16020 .command
16021 .as_deref()
16022 .is_some_and(|c| c.contains("started.marker")),
16023 "{captured:?}"
16024 );
16025
16026 assert!(
16027 runner.state.active.is_empty(),
16028 "the entry must be cleared once the command actually finished: {:?}",
16029 runner.state.active
16030 );
16031 assert!(runner.state.gate_ran);
16032 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16033 }
16034
16035 #[tokio::test]
16038 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16039 use crate::verdict::{Finding, ReviewVote, Severity};
16040 crate::run::pin_test_home();
16041 let tmp = tempfile::tempdir().expect("tempdir");
16042 let repo = tmp.path().join("repo");
16043 std::fs::create_dir_all(&repo).unwrap();
16044 init_repo(&repo);
16045
16046 for (severity, vote, expect) in [
16047 (Severity::Major, ReviewVote::Reject, true),
16048 (Severity::Minor, ReviewVote::Reject, false),
16049 (Severity::Major, ReviewVote::Approve, false),
16050 ] {
16051 let mut round = review_round(false, 1, 1, 1, false, true);
16052 round.reviews = vec![ReviewRecord {
16053 reviewer: 1,
16054 agent: "alpha".to_owned(),
16055 summary: String::new(),
16056 findings: vec![Finding {
16057 id: "R1-1-1".to_owned(),
16058 severity,
16059 file: None,
16060 line: None,
16061 title: "t".to_owned(),
16062 detail: String::new(),
16063 }],
16064 vote: Some(vote),
16065 failed: None,
16066 duration_ms: 0,
16067 attempts: 0,
16068 }];
16069 let mut state = RunState::new(
16070 repo.clone(),
16071 "main".to_owned(),
16072 "deadbeef".to_owned(),
16073 "task".to_owned(),
16074 Config::default(),
16075 );
16076 state.reviews = vec![round];
16077 let mut runner = Runner {
16078 state,
16079 roles: ResolvedRoles {
16080 implementers: Vec::new(),
16081 judges: Vec::new(),
16082 reviewers: Vec::new(),
16083 fixer: None,
16084 conductor: conductor(),
16085 implementer_roster: Vec::new(),
16086 judge_roster: Vec::new(),
16087 reviewer_roster: Vec::new(),
16088 },
16089 sem: Arc::new(Semaphore::new(1)),
16090 pause: Pause::new(),
16091 interrupt: Pause::new(),
16092 };
16093 let shell = runner.state.config.shell();
16094 runner
16095 .stop_reviewing("round budget spent", &shell, &repo)
16096 .await
16097 .expect("stop_reviewing");
16098 assert_eq!(runner.state.status, RunStatus::Gating);
16099 assert_eq!(
16100 runner.state.contested_handoff.is_some(),
16101 expect,
16102 "{severity:?} + {vote:?}"
16103 );
16104 }
16105 }
16106
16107 #[tokio::test]
16120 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16121 crate::run::pin_test_home();
16122 let home = crate::run::home();
16123
16124 let tmp = tempfile::tempdir().expect("tempdir");
16125 let repo = tmp.path().join("repo");
16126 std::fs::create_dir_all(&repo).unwrap();
16127 init_repo(&repo);
16128 let head = crate::git::rev_parse(&repo, "HEAD")
16129 .await
16130 .expect("rev-parse");
16131 let cache_dir = tmp.path().join("target");
16134
16135 let mut config = Config::default();
16136 config.verify.e2e = vec![format!(
16137 "CARGO_TARGET_DIR='{}' test -f README.md",
16138 cache_dir.display()
16139 )];
16140 config.graph.review_rounds = 1;
16141 config.graph.timeout_verify = Some(2);
16144 config.graph.timeout_resource_wait = 2;
16145
16146 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16147 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16148 .expect("no io error acquiring directly")
16149 {
16150 crate::cache::AcquireOutcome::Acquired(g) => g,
16151 crate::cache::AcquireOutcome::Busy(b) => {
16152 panic!("expected the direct acquire to win the lease first: {b:?}")
16153 }
16154 };
16155
16156 let mut state = RunState::new(
16157 repo.clone(),
16158 "main".to_owned(),
16159 head.clone(),
16160 "task".to_owned(),
16161 config,
16162 );
16163 state.candidates = vec![Candidate {
16164 index: 0,
16165 label: 'A',
16166 agent: "alpha".to_owned(),
16167 branch: "does-not-exist".to_owned(),
16168 worktree: repo.clone(),
16169 summary: String::new(),
16170 stat: String::new(),
16171 files: 0,
16172 commits: 0,
16173 empty: false,
16174 failed: None,
16175 verified_noop: None,
16176 duration_ms: 0,
16177 folded: false,
16178 }];
16179 state.tally = Some(Tally {
16180 first_choice: BTreeMap::from([('A', 1)]),
16181 borda: BTreeMap::new(),
16182 winner: 'A',
16183 rankings: 1,
16184 unanimous_initial: true,
16185 deliberated: false,
16186 changed_votes: 0,
16187 unanimous_final: true,
16188 tie_break: None,
16189 judges: 0,
16190 present: 0,
16191 quorum: 0,
16192 met_quorum: true,
16193 uncontested: Some("only candidate A produced a change".to_owned()),
16194 });
16195 state.reviews = vec![ReviewRound {
16199 round: 1,
16200 head: head.clone(),
16201 verified_head: None,
16202 verified_at: None,
16203 reviews: Vec::new(),
16204 e2e: Vec::new(),
16205 fix: None,
16206 blocking: 1,
16207 answered: 1,
16208 expected: 1,
16209 clean: false,
16210 verify_retried: false,
16211 e2e_deferred: true,
16212 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16213 progressed: false,
16214 vote_split: false,
16215 reconsideration: Vec::new(),
16216 verdict: None,
16217 }];
16218
16219 let mut runner = Runner {
16220 state,
16221 roles: ResolvedRoles {
16222 implementers: Vec::new(),
16223 judges: Vec::new(),
16224 reviewers: Vec::new(),
16225 fixer: None,
16226 conductor: conductor(),
16227 implementer_roster: Vec::new(),
16228 judge_roster: Vec::new(),
16229 reviewer_roster: Vec::new(),
16230 },
16231 sem: Arc::new(Semaphore::new(1)),
16232 pause: Pause::new(),
16233 interrupt: Pause::new(),
16234 };
16235
16236 let shell = runner.state.config.shell();
16237 runner
16238 .stop_reviewing("round budget spent", &shell, &repo)
16239 .await
16240 .expect("stop_reviewing");
16241
16242 let last = runner.state.reviews.last().expect("round record");
16243 assert_eq!(
16244 last.e2e_status(),
16245 E2eStatus::ResourceBlocked,
16246 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16247 failed: {last:?}"
16248 );
16249 assert_eq!(
16250 last.verified_head.as_deref(),
16251 Some(head.as_str()),
16252 "which commit this attempt targeted is known even though nothing finished checking \
16253 it"
16254 );
16255 let first_attempt_at = last
16256 .verified_at
16257 .expect("when this attempt ran is known too");
16258 assert_ne!(
16259 runner.state.status,
16260 RunStatus::Blocked,
16261 "contention is evidence about the machine, not the patch — it must not settle the \
16262 run as blocked: {:?}",
16263 runner.state.status
16264 );
16265 assert!(
16266 !runner
16267 .state
16268 .events
16269 .iter()
16270 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16271 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16272 runner.state.events
16273 );
16274
16275 runner
16280 .stop_reviewing("round budget spent", &shell, &repo)
16281 .await
16282 .expect("stop_reviewing retry");
16283 assert_eq!(
16284 runner.state.reviews.len(),
16285 1,
16286 "no new round was started: {:?}",
16287 runner.state.reviews
16288 );
16289 let last = runner.state.reviews.last().expect("round record");
16290 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16291 assert!(
16292 last.verified_at.expect("still known") > first_attempt_at,
16293 "a second reentry must be a fresh attempt, not a stale copy of the first"
16294 );
16295 assert_ne!(runner.state.status, RunStatus::Blocked);
16296
16297 held.release();
16298 }
16299
16300 #[tokio::test]
16312 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16313 crate::run::pin_test_home();
16314 let home = crate::run::home();
16315
16316 let tmp = tempfile::tempdir().expect("tempdir");
16317 let repo = tmp.path().join("repo");
16318 std::fs::create_dir_all(&repo).unwrap();
16319 init_repo(&repo);
16320 let head = crate::git::rev_parse(&repo, "HEAD")
16321 .await
16322 .expect("rev-parse");
16323 let cache_dir = tmp.path().join("target");
16324
16325 let mut config = Config::default();
16326 config.verify.e2e = vec![format!(
16327 "CARGO_TARGET_DIR='{}' test -f README.md",
16328 cache_dir.display()
16329 )];
16330 config.graph.review_rounds = 1;
16331 config.graph.timeout_verify = Some(2);
16332 config.graph.timeout_resource_wait = 2;
16333
16334 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16335 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16336 .expect("no io error acquiring directly")
16337 {
16338 crate::cache::AcquireOutcome::Acquired(g) => g,
16339 crate::cache::AcquireOutcome::Busy(b) => {
16340 panic!("expected the direct acquire to win the lease first: {b:?}")
16341 }
16342 };
16343
16344 let mut state = RunState::new(
16345 repo.clone(),
16346 "main".to_owned(),
16347 head.clone(),
16348 "task".to_owned(),
16349 config,
16350 );
16351 state.candidates = vec![Candidate {
16352 index: 0,
16353 label: 'A',
16354 agent: "alpha".to_owned(),
16355 branch: "does-not-exist".to_owned(),
16356 worktree: repo.clone(),
16357 summary: String::new(),
16358 stat: String::new(),
16359 files: 0,
16360 commits: 0,
16361 empty: false,
16362 failed: None,
16363 verified_noop: None,
16364 duration_ms: 0,
16365 folded: false,
16366 }];
16367 state.tally = Some(Tally {
16368 first_choice: BTreeMap::from([('A', 1)]),
16369 borda: BTreeMap::new(),
16370 winner: 'A',
16371 rankings: 1,
16372 unanimous_initial: true,
16373 deliberated: false,
16374 changed_votes: 0,
16375 unanimous_final: true,
16376 tie_break: None,
16377 judges: 0,
16378 present: 0,
16379 quorum: 0,
16380 met_quorum: true,
16381 uncontested: Some("only candidate A produced a change".to_owned()),
16382 });
16383 state.reviews = vec![ReviewRound {
16387 round: 1,
16388 head: head.clone(),
16389 verified_head: Some(head.clone()),
16390 verified_at: Some(jiff::Timestamp::now()),
16391 reviews: Vec::new(),
16392 e2e: vec![CommandOutcome {
16393 command: format!(
16394 "CARGO_TARGET_DIR='{}' test -f README.md",
16395 cache_dir.display()
16396 ),
16397 code: None,
16398 output_tail: "waiting for the shared build cache".to_owned(),
16399 duration_ms: 0,
16400 resource_blocked: true,
16401 }],
16402 fix: None,
16403 blocking: 1,
16404 answered: 1,
16405 expected: 1,
16406 clean: false,
16407 verify_retried: false,
16408 e2e_deferred: false,
16409 e2e_defer_reason: None,
16410 progressed: false,
16411 vote_split: false,
16412 reconsideration: Vec::new(),
16413 verdict: None,
16414 }];
16415
16416 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16417 let mut runner = Runner {
16418 state,
16419 roles: ResolvedRoles {
16420 implementers: Vec::new(),
16421 judges: Vec::new(),
16422 reviewers: Vec::new(),
16423 fixer: None,
16424 conductor: conductor(),
16425 implementer_roster: Vec::new(),
16426 judge_roster: Vec::new(),
16427 reviewer_roster: Vec::new(),
16428 },
16429 sem: Arc::new(Semaphore::new(1)),
16430 pause: Pause::new(),
16431 interrupt: Pause::new(),
16432 };
16433
16434 runner.review_loop().await.expect("review_loop");
16439
16440 assert_eq!(
16441 runner.state.reviews.len(),
16442 1,
16443 "no new round was started on top of the unresolved one: {:?}",
16444 runner.state.reviews
16445 );
16446 let last = &runner.state.reviews[0];
16447 assert_eq!(
16448 last.e2e_status(),
16449 E2eStatus::ResourceBlocked,
16450 "still contended: {last:?}"
16451 );
16452 assert!(
16453 last.verified_at.expect("still known") > first_attempt_at,
16454 "review_loop must have actually retried the check, not left it exactly as found"
16455 );
16456 assert_ne!(
16457 runner.state.status,
16458 RunStatus::Blocked,
16459 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16460 runner.state.status
16461 );
16462
16463 held.release();
16464 }
16465
16466 #[tokio::test]
16467 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16468 crate::run::pin_test_home();
16469 for status in [RunStatus::Blocked, RunStatus::Gating] {
16470 let mut runner = runner_at(status);
16471 runner.state.config.graph.land = true;
16472 runner.state.gate_ran = true;
16473 runner.state.merge = Some(MergeOutcome {
16474 mode: MergeMode::Pr,
16475 ok: false,
16476 detail: "the fixer produced no commit".into(),
16477 empty: false,
16478 });
16479 let url = "https://example.invalid/o/r/pull/1";
16480 runner.state.pr = Some(crate::run::PrRecord {
16481 url: url.into(),
16482 number: 1,
16483 state: "open".into(),
16484 checks: "red".into(),
16485 round: 1,
16486 rounds: 1,
16487 red_at_merge: Vec::new(),
16488 });
16489 runner.state.save().unwrap();
16490 runner.execute_graph().await.unwrap();
16494 assert_eq!(runner.state.status, RunStatus::Blocked);
16495 assert!(runner.state.land_sync_pending);
16496 assert!(runner.state.events.iter().any(|event|
16497 event.node == "land" && event.message == format!("watching {url}")));
16498 assert_eq!(
16499 runner.state.merge.as_ref().unwrap().detail,
16500 "the fixer produced no commit"
16501 );
16502 }
16503 }
16504
16505 #[tokio::test]
16506 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16507 crate::run::pin_test_home();
16508 let tmp = tempfile::tempdir().expect("tempdir");
16509 let repo = tmp.path().join("repo");
16510 std::fs::create_dir_all(&repo).unwrap();
16511 init_repo(&repo);
16512
16513 let mut config = Config::default();
16514 config.merge.mode = MergeMode::Pr;
16515 config.graph.land = true;
16516 config.graph.land_approval = false;
16517
16518 let mut state = RunState::new(
16519 repo.clone(),
16520 "main".to_owned(),
16521 "deadbeef".to_owned(),
16522 "task".to_owned(),
16523 config,
16524 );
16525 state.candidates = vec![Candidate {
16526 index: 0,
16527 label: 'A',
16528 agent: "alpha".to_owned(),
16529 branch: "does-not-exist".to_owned(),
16530 worktree: repo.clone(),
16531 summary: String::new(),
16532 stat: String::new(),
16533 files: 0,
16534 commits: 0,
16535 empty: false,
16536 failed: None,
16537 verified_noop: None,
16538 duration_ms: 0,
16539 folded: false,
16540 }];
16541 state.tally = Some(Tally {
16542 first_choice: BTreeMap::from([('A', 1)]),
16543 borda: BTreeMap::new(),
16544 winner: 'A',
16545 rankings: 1,
16546 unanimous_initial: true,
16547 deliberated: false,
16548 changed_votes: 0,
16549 unanimous_final: true,
16550 tie_break: None,
16551 judges: 0,
16552 present: 0,
16553 quorum: 0,
16554 met_quorum: true,
16555 uncontested: Some("only candidate A produced a change".to_owned()),
16556 });
16557 state.reviews = vec![ReviewRound {
16558 round: 1,
16559 head: "deadbeef".to_owned(),
16560 verified_head: None,
16561 verified_at: None,
16562 reviews: Vec::new(),
16563 e2e: Vec::new(),
16564 fix: None,
16565 blocking: 0,
16566 answered: 0,
16567 expected: 0,
16568 clean: true,
16569 verify_retried: false,
16570 e2e_deferred: false,
16571 e2e_defer_reason: None,
16572 progressed: false,
16573 vote_split: false,
16574 reconsideration: Vec::new(),
16575 verdict: None,
16576 }];
16577 state.gate = vec![CommandOutcome {
16578 command: "test".to_owned(),
16579 code: Some(0),
16580 output_tail: String::new(),
16581 duration_ms: 0,
16582 resource_blocked: false,
16583 }];
16584 state.gate_ran = true;
16585 state.status = RunStatus::Landing;
16589 state.merge = Some(MergeOutcome {
16590 mode: MergeMode::Pr,
16591 ok: true,
16592 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16593 empty: false,
16594 });
16595
16596 ask_test_home();
16600 state.save().expect("save landing run");
16603 let store = ask::Questions::open();
16604 let q = ask_open_question(&store, &state.id);
16605
16606 let mut runner = Runner {
16607 state,
16608 roles: ResolvedRoles {
16609 implementers: Vec::new(),
16610 judges: Vec::new(),
16611 reviewers: Vec::new(),
16612 fixer: None,
16613 conductor: conductor(),
16614 implementer_roster: Vec::new(),
16615 judge_roster: Vec::new(),
16616 reviewer_roster: Vec::new(),
16617 },
16618 sem: Arc::new(Semaphore::new(1)),
16619 pause: Pause::new(),
16620 interrupt: Pause::new(),
16621 };
16622
16623 runner.execute().await.expect("execute");
16628
16629 assert_eq!(
16630 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16631 Some("https://example.invalid/x/y/pull/1"),
16632 "reentry must not push again or open a second pull request over the \
16633 one `land` is already watching"
16634 );
16635 assert_ne!(
16636 runner.state.status,
16637 RunStatus::Landing,
16638 "land could not actually reach the fake pull request, so it must \
16639 have given up rather than left the run silently parked forever"
16640 );
16641 assert_eq!(runner.state.status, RunStatus::Blocked);
16645 assert!(
16646 store.get(&q.id).unwrap().status.open(),
16647 "Blocked is still alive; settle_questions must have been a no-op here"
16648 );
16649 }
16650
16651 fn state_with_round(round: ReviewRound) -> RunState {
16652 let mut s = RunState::new(
16653 PathBuf::from("/repo"),
16654 "main".to_owned(),
16655 "abc1234".to_owned(),
16656 "add retries".to_owned(),
16657 Config::default(),
16658 );
16659 s.reviews = vec![round];
16660 s
16661 }
16662
16663 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16664 crate::verdict::Finding {
16665 id: id.to_owned(),
16666 severity,
16667 file: None,
16668 line: None,
16669 title: title.to_owned(),
16670 detail: String::new(),
16671 }
16672 }
16673
16674 #[test]
16675 fn pr_body_names_open_findings_and_declined_ones() {
16676 let round = ReviewRound {
16677 round: 2,
16678 head: "deadbee".to_owned(),
16679 verified_head: None,
16680 verified_at: None,
16681 reviews: vec![ReviewRecord {
16682 attempts: 0,
16683 reviewer: 1,
16684 agent: "alpha".to_owned(),
16685 summary: String::new(),
16686 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16687 vote: None,
16688 failed: None,
16689 duration_ms: 0,
16690 }],
16691 e2e: vec![CommandOutcome {
16692 command: "cargo test".to_owned(),
16693 code: Some(0),
16694 output_tail: String::new(),
16695 duration_ms: 0,
16696 resource_blocked: false,
16697 }],
16698 verify_retried: false,
16699 e2e_deferred: false,
16700 e2e_defer_reason: None,
16701 fix: Some(FixRecord {
16702 agent: "alpha".to_owned(),
16703 addressed: Vec::new(),
16704 rejected: vec![crate::verdict::Rejection {
16705 id: "R1-1-1".to_owned(),
16706 why: "not reachable from any caller".to_owned(),
16707 }],
16708 notes: String::new(),
16709 committed: true,
16710 failed: None,
16711 duration_ms: 0,
16712 continuation: None,
16713 }),
16714 blocking: 0,
16715 answered: 1,
16716 expected: 1,
16717 clean: false,
16718 progressed: true,
16719 vote_split: false,
16720 reconsideration: Vec::new(),
16721 verdict: None,
16722 };
16723 let state = state_with_round(round);
16724 let body = pr_message(&state, 'A').body;
16725
16726 assert!(body.contains("add retries"), "the task must still be there");
16727 assert!(body.contains("R2-1-1"), "{body}");
16728 assert!(!body.contains("unused import"), "{body}");
16729 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16730 assert!(
16731 !body.contains("not reachable from any caller"),
16732 "the reason it was declined: {body}"
16733 );
16734 }
16735
16736 #[test]
16737 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16738 let round = ReviewRound {
16739 round: 1,
16740 head: "deadbee".to_owned(),
16741 verified_head: None,
16742 verified_at: None,
16743 reviews: vec![ReviewRecord {
16744 attempts: 0,
16745 reviewer: 1,
16746 agent: "alpha".to_owned(),
16747 summary: String::new(),
16748 findings: Vec::new(),
16749 vote: None,
16750 failed: None,
16751 duration_ms: 0,
16752 }],
16753 e2e: Vec::new(),
16754 verify_retried: false,
16755 e2e_deferred: false,
16756 e2e_defer_reason: None,
16757 fix: None,
16758 blocking: 0,
16759 answered: 1,
16760 expected: 1,
16761 clean: true,
16762 progressed: false,
16763 vote_split: false,
16764 reconsideration: Vec::new(),
16765 verdict: None,
16766 };
16767 let state = state_with_round(round);
16768 let body = pr_message(&state, 'A').body;
16769 assert!(!body.contains("Open review findings"), "{body}");
16770 assert!(!body.contains("Declined"), "{body}");
16771 }
16772
16773 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16774 let mut state = RunState::new(
16775 PathBuf::from("/repo"),
16776 "main".to_owned(),
16777 "abc1234".to_owned(),
16778 instruction.to_owned(),
16779 Config::default(),
16780 );
16781 state.candidates.push(Candidate {
16782 index: 0,
16783 label: 'A',
16784 agent: "alpha".to_owned(),
16785 branch: "magi/x/A".to_owned(),
16786 worktree: PathBuf::from("/wt"),
16787 summary: summary.to_owned(),
16788 stat: String::new(),
16789 files: 1,
16790 commits: 1,
16791 empty: false,
16792 failed: None,
16793 verified_noop: None,
16794 folded: false,
16795 duration_ms: 0,
16796 });
16797 state
16798 }
16799
16800 fn review_state(subjects: &[&str]) -> RunState {
16801 let mut state = state_with_summary(
16802 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16803 "",
16804 );
16805 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16806 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16807 state
16808 }
16809
16810 #[tokio::test]
16813 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16814 ask_test_home();
16815 let tmp = tempfile::tempdir().unwrap();
16816 let repo = tmp.path().join("repo");
16817 std::fs::create_dir_all(&repo).unwrap();
16818 init_repo(&repo);
16819 let origin = tmp.path().join("origin.git");
16820 let g = |dir: &Path, args: &[&str]| {
16821 let out = std::process::Command::new("git")
16822 .args(args)
16823 .current_dir(dir)
16824 .quiet()
16825 .output()
16826 .expect("spawn git");
16827 assert!(
16828 out.status.success(),
16829 "git {args:?}: {}",
16830 String::from_utf8_lossy(&out.stderr)
16831 );
16832 };
16833 g(
16834 tmp.path(),
16835 &[
16836 "clone",
16837 "--bare",
16838 "-q",
16839 repo.to_str().unwrap(),
16840 origin.to_str().unwrap(),
16841 ],
16842 );
16843 g(
16844 &repo,
16845 &["remote", "add", "origin", origin.to_str().unwrap()],
16846 );
16847 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16848
16849 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16851 g(&repo, &["add", "-A"]);
16852 g(
16853 &repo,
16854 &["commit", "-q", "-m", "chore(deps): update a crate"],
16855 );
16856 g(&repo, &["push", "-q", "origin", "main"]);
16857 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16858 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16859 g(&repo, &["add", "-A"]);
16860 g(
16861 &repo,
16862 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16863 );
16864 g(&repo, &["checkout", "-q", "main"]);
16865
16866 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16867 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16868 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16870 assert_eq!(fallback, c1);
16871
16872 let mut state = review_state(&[
16873 "chore(deps): update a crate",
16874 "fix(daemon): apply a chosen action",
16875 ]);
16876 state.repo = repo.clone();
16877 state.base_branch = "main".to_owned();
16878 state.base_commit = c1;
16879 refresh_reviewed_commits(&mut state, "feat/own").await;
16880 assert_eq!(
16881 state.reviewed_commits,
16882 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16883 );
16884 assert_eq!(
16885 review_title(&state).as_deref(),
16886 Some("fix(daemon): apply a chosen action")
16887 );
16888 assert_eq!(
16889 leaked_subjects(&state, "feat/own").await,
16890 Some(vec!["chore(deps): update a crate".to_owned()])
16891 );
16892 state.config.merge.remote = "nowhere".to_owned();
16895 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16896 }
16897
16898 struct SummaryFixture {
16902 _tmp: tempfile::TempDir,
16903 repo: PathBuf,
16904 reply: PathBuf,
16905 runner: Runner,
16906 }
16907
16908 fn summary_fixture() -> SummaryFixture {
16909 ask_test_home();
16910 let tmp = tempfile::tempdir().unwrap();
16911 let repo = tmp.path().join("repo");
16912 std::fs::create_dir_all(&repo).unwrap();
16913 init_repo(&repo);
16914 let origin = tmp.path().join("origin.git");
16915 let g = |args: &[&str]| {
16916 let out = std::process::Command::new("git")
16917 .args(args)
16918 .current_dir(&repo)
16919 .quiet()
16920 .output()
16921 .expect("spawn git");
16922 assert!(
16923 out.status.success(),
16924 "git {args:?}: {}",
16925 String::from_utf8_lossy(&out.stderr)
16926 );
16927 };
16928 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16929 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16930 g(&["push", "-q", "origin", "main"]);
16931 g(&["checkout", "-q", "-b", "feat/own"]);
16932 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16933 g(&["add", "-A"]);
16934 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16935 let reply = tmp.path().join("reply.txt");
16936 std::fs::write(
16937 &reply,
16938 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16939 )
16940 .unwrap();
16941 let mut runner = runner_at(RunStatus::Gating);
16942 let mut state = review_state(&["feat(own): add the own file"]);
16943 state.repo = repo.clone();
16944 state.base_branch = "main".to_owned();
16945 state.candidates[0].branch = "feat/own".to_owned();
16946 state.candidates[0].worktree = repo.clone();
16947 let mut author = spec("alpha");
16948 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16949 author
16950 .env
16951 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16952 state.config.agents = vec![author];
16953 runner.state = state;
16954 SummaryFixture {
16955 _tmp: tmp,
16956 repo,
16957 reply,
16958 runner,
16959 }
16960 }
16961
16962 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16963 let winner = f.runner.state.candidates[0].clone();
16964 f.runner
16965 .guarded_pr_message(&winner, None, true)
16966 .await
16967 .unwrap()
16968 .expect("an English description never parks")
16969 }
16970
16971 fn clean_round_with_minor_findings() -> ReviewRound {
16972 let mut round = passing_round(2, None);
16973 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16974 round.reviews = vec![ReviewRecord {
16975 attempts: 0,
16976 reviewer: 1,
16977 agent: "alpha".into(),
16978 summary: "内部レビュー".into(),
16979 findings: vec![
16980 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16981 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16982 ],
16983 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16984 failed: None,
16985 duration_ms: 0,
16986 }];
16987 round
16988 }
16989
16990 #[tokio::test]
16991 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16992 for review_only in [false, true] {
16993 let mut f = summary_fixture();
16994 if !review_only {
16995 f.runner.state.candidates[0].agent = "alpha".into();
16996 }
16997 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16998 assert!(f.runner.state.reviews[0].clean);
16999 assert!(f.runner.state.reviews[0].fix.is_none());
17000 assert!(f.runner.state.open_findings().is_empty());
17001 assert_eq!(
17002 crate::pr_outcome::required_finding_ids(&f.runner.state),
17003 ["R2-1-1", "R2-1-2"]
17004 );
17005 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17006 let evidence = crate::pr_outcome::summary_evidence(
17007 &f.runner.state,
17008 crate::pr_outcome::FinalHead::At(&head),
17009 );
17010 assert!(
17011 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17012 );
17013 let omitted = posted(&mut f).await;
17015 assert!(omitted.body.contains("No description"));
17016 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17017 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17018 assert!(f.runner.state.change_summary.is_none());
17019 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17020 let described = posted(&mut f).await;
17021 assert!(described.body.contains("improve documentation for callers"));
17022 assert!(described.body.contains("remaining edge case"));
17023 assert!(described.body.contains("## Open review findings"));
17024 assert!(!described.body.contains("未修正"));
17025 }
17026 }
17027
17028 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17029 let mut f = summary_fixture();
17030 if !review_only {
17031 f.runner.state.candidates[0].agent = "alpha".into();
17032 }
17033 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17034 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17035 fixed
17036 .fix
17037 .as_mut()
17038 .unwrap()
17039 .rejected
17040 .push(crate::verdict::Rejection {
17041 id: "R1-1-1".into(),
17042 why: "呼び出されないコードのため辞退".into(),
17043 });
17044 let mut clean = clean_round_with_minor_findings();
17045 clean.verified_head = Some(head.clone());
17046 f.runner.state.reviews = vec![fixed, clean];
17047 ok_gate(&mut f.runner.state, Some(&head));
17048 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17049 f.runner.state.config.agents.push(scripted("beta", reply));
17050 f.runner.state.config.roles.language_judge =
17051 Some(crate::config::AgentChoice::One("beta".into()));
17052 f
17053 }
17054
17055 #[tokio::test]
17056 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17057 for review_only in [false, true] {
17058 let mut f = rewrite_fixture_with_findings(review_only,
17059 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17060 let m = posted(&mut f).await;
17061 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17062 let expected =
17063 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17064 assert!(m.body.contains(&expected));
17065 assert!(m.body.contains("## Review rounds"));
17066 assert!(
17067 m.body
17068 .contains("Gate (`verify.gate`): passed on the final commit")
17069 );
17070 assert!(
17071 m.body
17072 .contains("Tests (`verify.e2e`): passed on the final commit")
17073 );
17074 let c = f.runner.state.change_summary.as_ref().unwrap();
17075 assert!(
17076 m.body
17077 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17078 );
17079 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17080 assert!(crate::pr_outcome::covers_required_findings(
17081 &f.runner.state,
17082 &m.body
17083 ));
17084 assert!(m.body.contains("documentation remains incomplete"));
17085 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17086 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17087 assert!(f.runner.state.github_text.is_none());
17088 }
17089 }
17090
17091 #[tokio::test]
17092 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17093 for review_only in [false, true] {
17094 for reply in [
17095 "TITLE: fix: public description\nA short English description.",
17096 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17097 ] {
17098 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17099 let winner = f.runner.state.candidates[0].clone();
17100 let posted = f
17101 .runner
17102 .guarded_pr_message(&winner, None, true)
17103 .await
17104 .unwrap();
17105 assert!(posted.is_none());
17106 assert!(f.runner.state.parked);
17107 let withheld = f.runner.state.github_text.as_ref().unwrap();
17108 assert!(
17109 withheld.rewrite.is_none(),
17110 "an incomplete rewrite must not be selectable"
17111 );
17112 let q = ask::Questions::open()
17113 .get(withheld.question.as_ref().unwrap())
17114 .unwrap();
17115 assert!(
17116 !q.choices
17117 .contains(&crate::github_text::USE_REWRITE.to_owned())
17118 );
17119 assert!(
17120 q.detail.contains("R1-1-1")
17121 && q.detail.contains("R2-1-1")
17122 && q.detail.contains("R2-1-2")
17123 );
17124 }
17125 }
17126 }
17127
17128 #[tokio::test]
17129 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17130 for review_only in [false, true] {
17131 let mut f = summary_fixture();
17132 if !review_only {
17133 f.runner.state.candidates[0].agent = "alpha".into();
17134 }
17135 f.runner.state.instruction = "内部タスクの説明".into();
17136 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17137 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17138 f.runner.state.reviews = vec![
17139 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17140 round_with_notes(2, Some("二回目の内部修正報告です。")),
17141 passing_round(3, Some(&head)),
17142 ];
17143 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17144 ok_gate(&mut f.runner.state, Some(&head));
17145 let captured = f.repo.join("public-prompt.txt");
17146 let author = &mut f.runner.state.config.agents[0];
17147 author.command = vec![
17148 "sh".into(),
17149 "-c".into(),
17150 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17151 ];
17152 author.env.insert(
17153 "PROMPT_FILE".into(),
17154 captured.to_string_lossy().into_owned(),
17155 );
17156 let m = posted(&mut f).await;
17157 assert!(m.body.contains("Adds `own.txt`"));
17158 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17159 assert!(m.body.contains("R1-1-1"));
17160 assert!(
17161 m.body
17162 .contains("Gate (`verify.gate`): passed on the final commit")
17163 );
17164 assert!(
17165 m.body
17166 .contains("Tests (`verify.e2e`): passed on the final commit")
17167 );
17168 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17169 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17170 let prompt = std::fs::read_to_string(captured).unwrap();
17171 assert!(prompt.contains(&format!(
17172 "git diff {} {head}",
17173 f.runner.state.change_summary.as_ref().unwrap().base
17174 )));
17175 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17176 assert_eq!(
17177 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17178 "二回目の内部修正報告です。"
17179 );
17180 }
17181 }
17182
17183 #[tokio::test]
17184 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17185 for review_only in [false, true] {
17186 let mut f = summary_fixture();
17187 if !review_only {
17188 f.runner.state.candidates[0].agent = "alpha".into();
17189 }
17190 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17191 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17192 std::fs::write(&f.reply, "no usable description").unwrap();
17193 let failed = posted(&mut f).await;
17194 assert!(
17195 failed
17196 .body
17197 .contains("No description of what this change does could be generated")
17198 );
17199 assert!(failed.body.is_ascii());
17200 assert!(f.runner.state.github_text.is_none());
17201 assert!(f.runner.state.change_summary.is_none());
17202 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17203 let evidence = crate::pr_outcome::summary_evidence(
17204 &f.runner.state,
17205 crate::pr_outcome::FinalHead::At(&head),
17206 );
17207 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17208 assert!(
17209 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17210 .exists()
17211 );
17212 f.runner.state =
17213 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17214 std::fs::write(
17215 &f.reply,
17216 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17217 )
17218 .unwrap();
17219 let retried = posted(&mut f).await;
17220 assert!(retried.body.contains("Adds the committed own file."));
17221 assert_eq!(
17222 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17223 "日本語の修正報告"
17224 );
17225 }
17226 }
17227
17228 #[tokio::test]
17229 async fn same_head_verification_and_findings_invalidate_public_prose() {
17230 let mut f = summary_fixture();
17231 posted(&mut f).await;
17232 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17233 let original = f
17234 .runner
17235 .state
17236 .change_summary
17237 .as_ref()
17238 .unwrap()
17239 .evidence
17240 .clone();
17241 ok_gate(&mut f.runner.state, Some(&head));
17242 std::fs::write(
17243 &f.reply,
17244 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17245 )
17246 .unwrap();
17247 let verified = posted(&mut f).await;
17248 assert!(verified.body.contains("now verified"));
17249 assert_ne!(
17250 f.runner.state.change_summary.as_ref().unwrap().evidence,
17251 original
17252 );
17253 let mut round = round_with_notes(1, Some("内部報告"));
17254 round.reviews.push(ReviewRecord {
17255 attempts: 0,
17256 reviewer: 1,
17257 agent: "alpha".into(),
17258 summary: "内部レビュー".into(),
17259 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17260 vote: None,
17261 failed: None,
17262 duration_ms: 0,
17263 });
17264 round
17265 .fix
17266 .as_mut()
17267 .unwrap()
17268 .rejected
17269 .push(crate::verdict::Rejection {
17270 id: "R1-1-1".into(),
17271 why: "呼び出し元から到達できない".into(),
17272 });
17273 f.runner.state.reviews = vec![round];
17274 let omitted = posted(&mut f).await;
17276 assert!(omitted.body.contains("No description"));
17277 assert!(omitted.body.contains("R1-1-1"));
17278 assert!(!omitted.body.contains("到達"));
17279 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17280 let described = posted(&mut f).await;
17281 assert!(described.body.contains("unreachable from callers"));
17282 assert!(described.body.contains("R1-1-1"));
17283 assert!(!described.body.contains("到達"));
17284 let evidence = f
17285 .runner
17286 .state
17287 .change_summary
17288 .as_ref()
17289 .unwrap()
17290 .evidence
17291 .clone();
17292 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17293 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17294 assert!(posted(&mut f).await.body.contains("rationale updated"));
17295 assert_ne!(
17296 f.runner.state.change_summary.as_ref().unwrap().evidence,
17297 evidence
17298 );
17299 f.runner
17301 .state
17302 .change_summary
17303 .as_mut()
17304 .unwrap()
17305 .evidence
17306 .clear();
17307 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17308 assert!(posted(&mut f).await.body.contains("current specification"));
17309 }
17310
17311 #[tokio::test]
17312 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17313 let mut f = summary_fixture();
17314 let m = posted(&mut f).await;
17315 assert_eq!(m.title, "feat(own): add the own file");
17316 assert!(
17317 m.body
17318 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17319 "{}",
17320 m.body
17321 );
17322 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17323 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17324 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17325 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17326 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17327 let kept = f.runner.state.change_summary.as_ref().unwrap();
17328 assert_eq!(
17329 (kept.head.as_str(), kept.base.as_str()),
17330 (head.as_str(), base.as_str())
17331 );
17332 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17333 }
17334
17335 #[tokio::test]
17336 async fn review_only_summary_receives_the_configured_github_language() {
17337 let mut f = summary_fixture();
17338 f.runner.state.config.graph.github_language = "ja".into();
17339 let captured = f.repo.join("summary-prompt.txt");
17340 let author = &mut f.runner.state.config.agents[0];
17341 author.command = vec![
17342 "sh".into(),
17343 "-c".into(),
17344 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17345 ];
17346 author.env.insert(
17347 "PROMPT_FILE".into(),
17348 captured.to_string_lossy().into_owned(),
17349 );
17350 std::fs::write(
17351 &f.reply,
17352 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17353 )
17354 .unwrap();
17355 let m = posted(&mut f).await;
17356 let prompt = std::fs::read_to_string(captured).unwrap();
17357 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17358 assert!(!prompt.contains("in English"));
17359 assert_eq!(m.title, "feat(own): ファイルを追加");
17360 assert!(m.body.contains("own.txt を追加します。"));
17361 }
17362
17363 #[tokio::test]
17364 async fn review_fixes_come_after_the_whole_change_summary() {
17365 let mut f = summary_fixture();
17366 f.runner.state.reviews = vec![
17367 round_with_notes(1, Some("handled the PATH case")),
17368 round_with_notes(2, None),
17369 ];
17370 let m = posted(&mut f).await;
17371 let summary = m.body.find("## Summary").expect("summary");
17372 let fixes = m.body.find("## Review rounds").expect("review rounds");
17373 assert!(summary < fixes, "{}", m.body);
17374 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17375 assert!(m.body.contains("## Final verification"), "{}", m.body);
17376 }
17377
17378 #[tokio::test]
17379 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17380 let mut f = summary_fixture();
17381 let first = posted(&mut f).await;
17382 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17383 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17385 let again = posted(&mut f).await;
17386 assert_eq!((again.title, again.body), (first.title, first.body));
17387
17388 let g = |args: &[&str]| {
17389 let out = std::process::Command::new("git")
17390 .args(args)
17391 .current_dir(&f.repo)
17392 .quiet()
17393 .output()
17394 .expect("spawn git");
17395 assert!(out.status.success(), "git {args:?}");
17396 };
17397 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17398 g(&["add", "-A"]);
17399 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17400 let second = posted(&mut f).await;
17401 let kept = f.runner.state.change_summary.clone().unwrap();
17402 assert_ne!(kept.head, first_head);
17403 assert!(second.body.contains("Not read yet."), "{}", second.body);
17404 assert!(!second.body.contains("so the first summary is read"));
17405 assert_eq!(second.title, "feat(own): other");
17406 }
17407
17408 #[tokio::test]
17409 async fn an_unavailable_summary_is_stated_and_never_invented() {
17410 let mut f = summary_fixture();
17411 std::fs::write(&f.reply, "no title line here").unwrap();
17412 let m = posted(&mut f).await;
17413 assert!(
17414 m.body
17415 .contains("No description of what this change does could be generated"),
17416 "{}",
17417 m.body
17418 );
17419 assert!(m.body.contains("internal reports remain in local records"));
17420 assert!(f.runner.state.change_summary.is_none());
17421 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17422 let winner = f.runner.state.candidates[0].clone();
17424 let preview = f
17425 .runner
17426 .guarded_pr_message(&winner, None, false)
17427 .await
17428 .unwrap()
17429 .unwrap();
17430 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17431 }
17432
17433 #[test]
17434 fn pr_message_review_single_commit_uses_its_subject() {
17435 let state = review_state(&["feat(nats): per-role user"]);
17436 let m = pr_message(&state, 'A');
17437 assert_eq!(m.title, "feat(nats): per-role user");
17438 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17439 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17440 }
17441
17442 #[test]
17443 fn pr_message_review_multi_commit_takes_the_oldest() {
17444 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17445 let m = pr_message(&state, 'A');
17446 assert_eq!(m.title, "feat: the change");
17447 for s in ["feat: the change", "fix: typo", "fix: again"] {
17448 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17449 }
17450 }
17451
17452 #[test]
17453 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17454 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17455 let mut state = review_state(&[first, "fix: later fixup"]);
17456 state.candidates[0].branch = "機能/ログイン".to_owned();
17457 let m = pr_message(&state, 'A');
17458 assert!(
17459 m.title.starts_with("chore: land candidate A of run"),
17460 "{}",
17461 m.title
17462 );
17463 }
17464 }
17465
17466 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17467 BranchFacts {
17468 commits: commits
17469 .iter()
17470 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17471 .collect(),
17472 stat: stat.to_owned(),
17473 }
17474 }
17475
17476 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17477 let mut r = review_round(true, 0, 1, 1, true, true);
17478 r.round = round;
17479 r.fix = notes.map(|n| FixRecord {
17480 agent: "fixer".to_owned(),
17481 addressed: Vec::new(),
17482 rejected: Vec::new(),
17483 notes: n.to_owned(),
17484 committed: true,
17485 failed: None,
17486 duration_ms: 0,
17487 continuation: None,
17488 });
17489 r
17490 }
17491
17492 #[test]
17493 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17494 let state = review_state(&["ignored"]);
17495 let f = facts(
17496 &[
17497 (
17498 "fix(login): resolve PATH on macOS",
17499 "Login shells skip rc files.",
17500 ),
17501 ("fix: address review", ""),
17502 ],
17503 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17504 );
17505 let m = pr_message_with(&state, 'A', Some(&f));
17506 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17507 assert!(
17508 m.body
17509 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17510 "{}",
17511 m.body
17512 );
17513 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17514 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17515 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17516 for banned in [
17517 "Review the work already",
17518 "no task statement",
17519 "Original task",
17520 ] {
17521 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17522 }
17523 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17524 }
17525
17526 #[test]
17527 fn pr_message_review_truncates_a_huge_first_commit_body() {
17528 let state = review_state(&["ignored"]);
17529 let f = facts(
17530 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17531 "s",
17532 );
17533 let m = pr_message_with(&state, 'A', Some(&f));
17534 assert!(m.body.len() < 30_000, "{}", m.body.len());
17535 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17536 assert!(
17537 m.body.contains("- ... 1 more commit(s)"),
17538 "{}",
17539 m.body.len()
17540 );
17541 assert!(m.body.ends_with("magi:candidate-a\n"));
17542 }
17543
17544 #[test]
17545 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17546 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17547 assert_eq!(m.title, "feat: x");
17548 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17549 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17550 assert!(!m.body.contains("no task statement"), "{}", m.body);
17551 }
17552
17553 #[test]
17554 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17555 let mut state = review_state(&["日本語の件名"]);
17556 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17557 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17558 state.candidates[0].branch = "機能/ログイン".to_owned();
17559 assert!(
17560 pr_message(&state, 'A')
17561 .title
17562 .starts_with("chore: land candidate A")
17563 );
17564 }
17565
17566 #[test]
17567 fn pr_message_review_fixes_survive_a_clean_final_round() {
17568 let mut state = review_state(&["feat: x"]);
17569 state.reviews = vec![
17570 round_with_notes(1, Some("handled the PATH case")),
17571 round_with_notes(2, None),
17572 ];
17573 let body = pr_message(&state, 'A').body;
17574 assert!(
17575 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17576 "{body}"
17577 );
17578 assert!(body.contains("### Round 1"), "{body}");
17579
17580 state.reviews = vec![
17581 round_with_notes(1, Some("first fix")),
17582 round_with_notes(2, Some("")),
17583 round_with_notes(3, Some("second fix")),
17584 round_with_notes(4, None),
17585 ];
17586 let body = pr_message(&state, 'A').body;
17587 assert!(
17588 body.contains("### Round 1") && !body.contains("first fix"),
17589 "{body}"
17590 );
17591 assert!(
17592 body.contains("### Round 3") && !body.contains("second fix"),
17593 "{body}"
17594 );
17595 assert!(body.contains("### Round 2"), "{body}");
17596 }
17597
17598 const FINAL: &str = "abcdef1234567890";
17599
17600 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17601 let mut r = round_with_notes(round, None);
17602 r.verified_head = verified.map(str::to_owned);
17603 r
17604 }
17605
17606 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17607 state.gate = vec![CommandOutcome {
17608 command: "gate".to_owned(),
17609 code: Some(0),
17610 output_tail: String::new(),
17611 duration_ms: 0,
17612 resource_blocked: false,
17613 }];
17614 state.gate_ran = true;
17615 state.gate_head = head.map(str::to_owned);
17616 }
17617
17618 fn body_at(state: &RunState) -> String {
17619 pr_message_raw(
17620 state,
17621 'A',
17622 None,
17623 crate::pr_outcome::FinalHead::At(FINAL),
17624 ChangeNote::NotAsked,
17625 )
17626 .body
17627 }
17628
17629 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17630
17631 #[test]
17632 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17633 let mut state = review_state(&["feat: x"]);
17634 state.reviews = vec![
17635 round_with_notes(1, Some(SANDBOXED)),
17636 passing_round(2, Some(FINAL)),
17637 ];
17638 ok_gate(&mut state, Some(FINAL));
17639 let body = body_at(&state);
17640 assert!(!body.contains("Handled the PATH case."), "{body}");
17641 assert!(!body.contains("could not commit"), "{body}");
17642 assert!(!body.contains("could not run"), "{body}");
17643 assert!(
17644 body.contains("The fix is committed on the branch"),
17645 "{body}"
17646 );
17647 assert!(!body.contains("sandbox"), "{body}");
17648 assert!(
17649 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17650 "{body}"
17651 );
17652 assert!(
17653 body.contains("Gate (`verify.gate`): passed on the final commit"),
17654 "{body}"
17655 );
17656 assert!(
17658 state.reviews[0]
17659 .fix
17660 .as_ref()
17661 .unwrap()
17662 .notes
17663 .contains("could not commit")
17664 );
17665 }
17666
17667 #[test]
17668 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17669 let mut state = review_state(&["feat: x"]);
17670 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17671 r1.fix.as_mut().unwrap().committed = false;
17672 let mut r2 = passing_round(2, Some(FINAL));
17673 r2.e2e[0].code = Some(1);
17674 state.reviews = vec![r1, r2];
17675 let body = body_at(&state);
17676 assert!(!body.contains("could not commit"), "{body}");
17677 assert!(!body.contains("could not run"), "{body}");
17678 assert!(body.contains("This round produced no commit"), "{body}");
17679 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17680 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17681 assert!(!body.contains("passed"), "{body}");
17682 }
17683
17684 #[test]
17685 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17686 let mut state = review_state(&["feat: x"]);
17687 state.reviews = vec![
17688 round_with_notes(1, Some(SANDBOXED)),
17689 passing_round(2, Some("0123456789abcdef")),
17690 ];
17691 ok_gate(&mut state, Some("0123456789abcdef"));
17692 let body = body_at(&state);
17693 assert!(!body.contains("HTTP tests could not run"), "{body}");
17694 assert!(!body.contains("could not commit"), "{body}");
17696 assert!(
17697 body.contains("passed on an earlier commit (`0123456`)"),
17698 "{body}"
17699 );
17700 assert!(!body.contains("passed on the final commit"), "{body}");
17701
17702 ok_gate(&mut state, None);
17704 let body = body_at(&state);
17705 assert!(
17706 body.contains("passed on a commit that was not recorded"),
17707 "{body}"
17708 );
17709 }
17710
17711 #[test]
17712 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17713 let state = state_with_summary(
17714 "add retries to the client",
17715 "TITLE: feat: retries\n\nDid it.",
17716 );
17717 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17718 assert_eq!(m.title, "feat: retries");
17719 assert!(
17720 m.body.contains("<summary>Original task</summary>"),
17721 "{}",
17722 m.body
17723 );
17724 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17725 assert!(
17726 m.body
17727 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17728 "{}",
17729 m.body
17730 );
17731 }
17732
17733 #[test]
17734 fn pr_message_review_bounds_a_long_english_subject() {
17735 let long = format!("feat: {}", "word ".repeat(100));
17736 let m = pr_message(&review_state(&[&long]), 'A');
17737 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17738 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17739 }
17740
17741 #[test]
17742 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17743 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17744 let m = pr_message(&state, 'A');
17745 assert_eq!(m.title, "add retries");
17746 assert!(m.body.contains("<summary>Original task</summary>"));
17747 assert!(!m.body.contains("Commits under review"));
17748 assert_eq!(landing_subject_source(&state), state.instruction);
17749 }
17750
17751 #[test]
17752 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17753 let state = review_state(&["feat: the change", "fix: typo"]);
17754 let source = landing_subject_source(&state);
17755 assert_eq!(land::merge_subject("", &source), "feat: the change");
17756 assert_eq!(
17757 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17758 "feat: the change"
17759 );
17760 assert_eq!(
17762 land::merge_subject("feat: renamed by hand", &source),
17763 "feat: renamed by hand"
17764 );
17765 let blank = review_state(&["日本語"]);
17766 assert!(
17767 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17768 );
17769 }
17770
17771 #[test]
17772 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17773 let state = review_state(&["feat: the change"]);
17774 let source = landing_subject_source(&state);
17775 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17776 assert_eq!(
17777 land::merge_subject(landing_title(&state, old), &source),
17778 "feat: the change"
17779 );
17780 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17781 let task = state_with_summary("add retries", "");
17782 assert_eq!(landing_title(&task, old), old);
17783 }
17784
17785 #[test]
17786 fn pr_message_describes_the_change_not_the_task() {
17787 let state = state_with_summary(
17788 "今回やってほしいこと: results projector を直す",
17789 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17790 );
17791 let m = pr_message(&state, 'A');
17792 assert_eq!(m.title, "fix(web): batch the runs list reads");
17793 assert!(
17794 m.body.starts_with("## Summary\n\n- reads run.json once"),
17795 "{}",
17796 m.body
17797 );
17798 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17799 let task_at = m.body.find("今回やってほしいこと").unwrap();
17800 let details_at = m.body.find("<details>").unwrap();
17801 assert!(
17802 details_at < task_at,
17803 "the task lives inside <details>: {}",
17804 m.body
17805 );
17806 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17807 assert!(m.body.contains("magi:candidate-a"));
17808 }
17809
17810 #[test]
17811 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17812 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17813 let m = pr_message(&state, 'A');
17814 assert_eq!(m.title, "add retries");
17815 assert!(
17816 m.body.contains("## Summary\n\n- did some things"),
17817 "{}",
17818 m.body
17819 );
17820
17821 let none = RunState::new(
17822 PathBuf::from("/repo"),
17823 "main".to_owned(),
17824 "abc1234".to_owned(),
17825 "add retries".to_owned(),
17826 Config::default(),
17827 );
17828 let m = pr_message(&none, 'A');
17829 assert_eq!(m.title, "add retries");
17830 assert!(!m.body.contains("## Summary"), "{}", m.body);
17831 }
17832
17833 #[test]
17834 fn pr_message_refuses_the_candidate_commit_subject() {
17835 for bad in [
17836 "TITLE: magi: candidate A (uncommitted work)",
17837 "TITLE: chore: stuff (uncommitted work)",
17838 "TITLE: ",
17839 ] {
17840 let state = state_with_summary("add retries", bad);
17841 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17842 }
17843 }
17844
17845 #[test]
17846 fn pr_message_bounds_a_very_long_task_and_title() {
17847 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17848 let state = state_with_summary(&long, "- nothing");
17849 let m = pr_message(&state, 'A');
17850 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17851 assert!(!m.title.contains('\n'));
17852
17853 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17854 let m = pr_message(&state, 'A');
17855 assert!(m.title.starts_with("feat: "));
17856 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17857 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17858 }
17859
17860 fn long_title_of(instruction: &str) -> String {
17861 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17862 }
17863
17864 #[test]
17865 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17866 let first = "Make the landing path keep a readable title for long tasks";
17867 let line = format!(
17868 "{first}. {}",
17869 "And then keep going with more words ".repeat(20)
17870 );
17871 let t = long_title_of(&line);
17872 assert_eq!(t, first);
17873 assert!(!t.starts_with("chore: land"));
17874 }
17875
17876 #[test]
17877 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17878 let line = "word ".repeat(200);
17879 let t = long_title_of(&line);
17880 assert!(t.ends_with("word..."), "{t}");
17881 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17882 }
17883
17884 #[test]
17885 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17886 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17887 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17888 }
17889 }
17890
17891 #[test]
17892 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17893 let mut state = state_with_summary("再試行を追加する", "- nothing");
17894 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17895 state.config.graph.github_language = "ja".into();
17896 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17897 }
17898
17899 #[test]
17900 fn pr_message_title_limit_is_exact() {
17901 let at = "a".repeat(PR_TITLE_MAX);
17902 assert_eq!(long_title_of(&at), at);
17903 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17904 assert!(over.ends_with("..."), "{over}");
17905 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17906 }
17907
17908 #[test]
17909 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17910 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17911 let t = long_title_of(&line);
17912 assert!(t.ends_with("..."), "{t}");
17913 assert!(t.is_ascii(), "{t}");
17914 }
17915
17916 #[test]
17917 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17918 let line = format!(
17919 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17920 "plus more filler words ".repeat(20)
17921 );
17922 let t = long_title_of(&line);
17923 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17924 }
17925
17926 #[test]
17927 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17928 let line = format!("\n\n# {}", "title words ".repeat(40));
17929 let state = state_with_summary(&line, "- nothing");
17930 let m = pr_message(&state, 'A');
17931 assert!(m.title.starts_with("title words"), "{}", m.title);
17932 assert_eq!(
17933 land::merge_subject(&m.title, &landing_subject_source(&state)),
17934 m.title
17935 );
17936 assert_eq!(
17938 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17939 "feat: renamed by hand"
17940 );
17941 }
17942
17943 #[tokio::test]
17944 async fn github_text_rewrite_is_bounded_and_falls_back() {
17945 crate::run::pin_test_home();
17946 for (reply, accepted) in [
17947 (
17948 "TITLE: fix: retries\nAdd retries for failed requests.",
17949 true,
17950 ),
17951 (
17952 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17953 false,
17954 ),
17955 (
17956 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17957 true,
17958 ),
17959 (
17960 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17961 true,
17962 ),
17963 (
17964 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17965 true,
17966 ),
17967 (
17968 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17969 false,
17970 ),
17971 ("TITLE: fix: retries\n", false),
17972 ] {
17973 let dir = tempfile::tempdir().unwrap();
17974 let mut runner = runner_at(RunStatus::Gating);
17975 runner.state = state_with_summary(
17976 "add retries",
17977 "TITLE: fix: retries\n日本語の説明を書きます。",
17978 );
17979 runner.state.repo = dir.path().to_owned();
17980 runner.state.candidates[0].worktree = dir.path().to_owned();
17981 let mut author = spec("alpha");
17982 author.command = vec![
17983 "sh".into(),
17984 "-c".into(),
17985 "printf '%s' \"$REWRITE_REPLY\"".into(),
17986 ];
17987 author.env.insert("REWRITE_REPLY".into(), reply.into());
17988 runner.state.config.agents = vec![author];
17989 seed_public_summary(&mut runner);
17990 let winner = runner.state.candidates[0].clone();
17991 let first = runner
17992 .guarded_pr_message(&winner, None, true)
17993 .await
17994 .unwrap();
17995 let message = if accepted {
17996 first.expect("a passing rewrite never asks")
17997 } else {
17998 assert!(first.is_none());
18000 assert!(runner.state.parked);
18001 let store = ask::Questions::open();
18002 let asked: Vec<_> = store
18003 .list()
18004 .into_iter()
18005 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18006 .collect();
18007 assert_eq!(asked.len(), 1);
18008 assert!(!asked[0].detail.contains("secret"));
18009 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18010 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18011 assert_eq!(
18012 asked[0]
18013 .choices
18014 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18015 rewrite.is_some()
18016 );
18017 store
18020 .update(&asked[0].id, |q| {
18021 q.abandon("test");
18022 Ok(())
18023 })
18024 .unwrap();
18025 let again = runner
18026 .guarded_pr_message(&winner, None, true)
18027 .await
18028 .unwrap()
18029 .expect("an abandoned question falls back");
18030 let n = store
18031 .list()
18032 .into_iter()
18033 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18034 .count();
18035 assert_eq!(n, 1, "asked once per rejected text");
18036 again
18037 };
18038 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18039 assert!(!message.body.contains("secret"));
18040 assert!(!message.body.contains("prefix"));
18041 assert!(!message.title.contains("secret"));
18042 assert_eq!(
18043 message.body.contains("Add retries for failed requests."),
18044 accepted
18045 );
18046 assert_eq!(
18047 runner.state.seats["github-text-rewrite"].turns, 1,
18048 "only one rewrite invocation"
18049 );
18050 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18051 if accepted {
18052 assert!(runner.state.github_text.is_none());
18053 assert!(!runner.state.parked);
18054 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18055 assert!(message.body.contains("## Final verification"));
18056 assert!(message.body.contains("Evidence: final diff"));
18057 assert!(
18058 message
18059 .body
18060 .contains(&format!("magi:run/{}", runner.state.id))
18061 );
18062 }
18063 if !accepted {
18064 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18065 assert!(
18066 message
18067 .body
18068 .contains(&format!("magi:run/{}", runner.state.id))
18069 );
18070 }
18071 }
18072 }
18073
18074 fn scripted(id: &str, reply: &str) -> AgentSpec {
18075 let mut a = spec(id);
18076 a.command = vec![
18077 "sh".into(),
18078 "-c".into(),
18079 "printf '%s' \"$REWRITE_REPLY\"".into(),
18080 ];
18081 a.env.insert("REWRITE_REPLY".into(), reply.into());
18082 a
18083 }
18084
18085 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18086 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18087
18088 fn seed_public_summary(runner: &mut Runner) {
18090 let repo = &runner.state.repo;
18091 let g = |args: &[&str]| {
18092 let out = std::process::Command::new("git")
18093 .args(args)
18094 .current_dir(repo)
18095 .quiet()
18096 .output()
18097 .unwrap();
18098 assert!(out.status.success(), "git {args:?}");
18099 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18100 };
18101 g(&["init", "-q", "-b", "main"]);
18102 g(&[
18103 "-c",
18104 "user.name=Test",
18105 "-c",
18106 "user.email=test@example.invalid",
18107 "commit",
18108 "-q",
18109 "--allow-empty",
18110 "-m",
18111 "feat: fixture",
18112 ]);
18113 let head = g(&["rev-parse", "HEAD"]);
18114 g(&["update-ref", "refs/remotes/origin/main", &head]);
18115 let evidence = crate::pr_outcome::summary_evidence(
18116 &runner.state,
18117 crate::pr_outcome::FinalHead::At(&head),
18118 );
18119 let generated = &runner.state.candidates[0].summary;
18120 runner.state.change_summary = Some(crate::run::ChangeSummary {
18121 base: head.clone(),
18122 head,
18123 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18124 title: summary_title(generated).unwrap(),
18125 body: summary_without_title(generated),
18126 });
18127 }
18128
18129 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18130 let mut runner = runner_at(RunStatus::Gating);
18131 runner.state = state_with_summary("add retries", JA_SUMMARY);
18132 if review_only {
18133 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18134 }
18135 runner.state.repo = dir.to_owned();
18136 runner.state.candidates[0].worktree = dir.to_owned();
18137 runner.state.config.agents = agents;
18138 seed_public_summary(&mut runner);
18139 runner
18140 }
18141
18142 fn gate_questions(runner: &Runner) -> usize {
18143 ask::Questions::open()
18144 .list()
18145 .into_iter()
18146 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18147 .count()
18148 }
18149
18150 #[tokio::test]
18151 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18152 crate::run::pin_test_home();
18153 let dir = tempfile::tempdir().unwrap();
18154 let mut runner = rewrite_runner(
18155 dir.path(),
18156 true,
18157 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18158 );
18159 runner.state.config.roles.language_judge =
18160 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18161 let winner = runner.state.candidates[0].clone();
18162 let msg = runner
18163 .guarded_pr_message(&winner, None, true)
18164 .await
18165 .unwrap()
18166 .expect("an English rewrite posts without asking");
18167 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18168 assert!(msg.body.contains("<details>kept</details>"));
18169 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18170 assert_eq!(gate_questions(&runner), 0);
18171 assert!(runner.state.github_text.is_none());
18172 assert!(!runner.state.seats.contains_key("impl-A"));
18173 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18174 }
18175
18176 #[tokio::test]
18177 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18178 crate::run::pin_test_home();
18179 let dir = tempfile::tempdir().unwrap();
18180 let mut runner = rewrite_runner(
18181 dir.path(),
18182 false,
18183 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18184 );
18185 runner.state.config.roles.language_judge =
18186 Some(crate::config::AgentChoice::One("beta".into()));
18187 runner.state.seats.insert(
18188 "impl-A".into(),
18189 SeatState::new("impl-A", "alpha", runner.state.seed),
18190 );
18191 let winner = runner.state.candidates[0].clone();
18192 let msg = runner
18193 .guarded_pr_message(&winner, None, true)
18194 .await
18195 .unwrap()
18196 .expect("rewritten");
18197 assert!(msg.body.contains("<details>kept</details>"));
18198 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18199 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18200 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18201 }
18202
18203 #[tokio::test]
18204 async fn rewrite_reuses_a_resumable_implementer_session() {
18205 crate::run::pin_test_home();
18206 let dir = tempfile::tempdir().unwrap();
18207 let mut runner = rewrite_runner(
18208 dir.path(),
18209 false,
18210 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18211 );
18212 runner.state.config.roles.language_judge =
18213 Some(crate::config::AgentChoice::One("beta".into()));
18214 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18215 seat.turns = 1;
18216 runner.state.seats.insert("impl-A".into(), seat);
18217 let winner = runner.state.candidates[0].clone();
18218 runner
18219 .guarded_pr_message(&winner, None, true)
18220 .await
18221 .unwrap()
18222 .expect("rewritten");
18223 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18224 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18225 }
18226
18227 #[tokio::test]
18228 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18229 crate::run::pin_test_home();
18230 let dir = tempfile::tempdir().unwrap();
18231 let mut broken = spec("bad");
18232 broken.command = vec!["false".into()];
18233 let mut runner = rewrite_runner(
18234 dir.path(),
18235 true,
18236 vec![
18237 broken,
18238 scripted("noformat", "Add retries."),
18239 scripted("good", EN_REWRITE),
18240 ],
18241 );
18242 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18243 "bad".into(),
18244 "noformat".into(),
18245 "good".into(),
18246 ]));
18247 let winner = runner.state.candidates[0].clone();
18248 let msg = runner
18249 .guarded_pr_message(&winner, None, true)
18250 .await
18251 .unwrap()
18252 .expect("the third agent rewrites");
18253 assert!(msg.body.contains("`src/net.rs`"));
18254 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18255 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18256 }
18257
18258 #[tokio::test]
18259 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18260 crate::run::pin_test_home();
18261 for (agents, role, saved) in [
18263 (
18264 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18265 Some("beta"),
18266 true,
18267 ),
18268 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18269 (Vec::new(), None, false),
18270 ] {
18271 let dir = tempfile::tempdir().unwrap();
18272 let mut runner = rewrite_runner(dir.path(), true, agents);
18273 runner.state.config.roles.language_judge =
18274 role.map(|r| crate::config::AgentChoice::One(r.into()));
18275 let winner = runner.state.candidates[0].clone();
18276 let first = runner
18277 .guarded_pr_message(&winner, None, true)
18278 .await
18279 .unwrap();
18280 assert!(first.is_none(), "withheld text parks the run");
18281 assert!(runner.state.parked);
18282 assert_eq!(gate_questions(&runner), 1);
18283 let g = runner.state.github_text.as_ref().unwrap();
18284 assert_eq!(g.rewrite.is_some(), saved);
18285 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18286 }
18287 }
18288
18289 #[tokio::test]
18290 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18291 crate::run::pin_test_home();
18292 let dir = tempfile::tempdir().unwrap();
18293 let mut runner = runner_at(RunStatus::Gating);
18294 runner.state =
18295 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18296 runner.state.repo = dir.path().to_owned();
18297 runner.state.candidates[0].worktree = dir.path().to_owned();
18298 let mut author = spec("alpha");
18299 author.command = vec![
18300 "sh".into(),
18301 "-c".into(),
18302 "printf '%s' \"$REWRITE_REPLY\"".into(),
18303 ];
18304 author.env.insert(
18305 "REWRITE_REPLY".into(),
18306 format!(
18307 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18308 "English sentence. ".repeat(600)
18309 ),
18310 );
18311 runner.state.config.agents = vec![author];
18312 seed_public_summary(&mut runner);
18313 let winner = runner.state.candidates[0].clone();
18314 assert!(
18315 runner
18316 .guarded_pr_message(&winner, None, true)
18317 .await
18318 .unwrap()
18319 .is_none()
18320 );
18321 let record = runner.state.github_text.clone().unwrap();
18322 let rewrite = record.rewrite.clone().unwrap();
18323 assert_eq!(rewrite.failed, ["body-language"]);
18324 assert!(
18325 rewrite
18326 .body
18327 .ends_with(&format!("magi:run/{}", runner.state.id))
18328 );
18329 let store = ask::Questions::open();
18330 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18331 assert!(
18332 q.choices
18333 .contains(&crate::github_text::USE_REWRITE.to_owned())
18334 );
18335 for label in [
18336 "ORIGINAL",
18337 "FALLBACK",
18338 "REWRITE",
18339 "Rules still failed: body-language",
18340 "truncated",
18341 ] {
18342 assert!(q.detail.contains(label), "{label}");
18343 }
18344 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18345 &runner.state,
18346 &crate::github_text::artifact_name(&record.fingerprint),
18347 ))
18348 .unwrap();
18349 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18350 runner.state =
18352 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18353 store
18354 .update(&q.id, |q| {
18355 q.say("まだ日本語")?;
18356 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18357 })
18358 .unwrap();
18359 assert!(
18360 runner
18361 .guarded_pr_message(&winner, None, true)
18362 .await
18363 .unwrap()
18364 .is_none()
18365 );
18366 let retry = runner.state.github_text.as_ref().unwrap();
18367 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18368 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18369 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18370 assert!(
18371 q.posting_comparison
18372 .as_ref()
18373 .unwrap()
18374 .matches_detail(&q.detail)
18375 );
18376 store
18377 .update(&q.id, |q| {
18378 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18379 })
18380 .unwrap();
18381 let message = runner
18382 .guarded_pr_message(&winner, None, true)
18383 .await
18384 .unwrap()
18385 .unwrap();
18386 assert_eq!(message.title, rewrite.title);
18387 assert_eq!(message.body, rewrite.body);
18388 assert_eq!(
18389 runner.state.github_text.as_ref().unwrap().choice,
18390 Some(GithubTextChoice::Rewrite)
18391 );
18392 let again = runner
18393 .guarded_pr_message(&winner, None, true)
18394 .await
18395 .unwrap()
18396 .unwrap();
18397 assert_eq!(again.title, message.title);
18398 assert_eq!(again.body, message.body);
18399 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18400 assert!(
18401 !runner
18402 .state
18403 .events
18404 .iter()
18405 .any(|e| e.message.contains(&rewrite.title)
18406 || e.message.contains("English sentence."))
18407 );
18408 }
18409
18410 #[tokio::test]
18411 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18412 crate::run::pin_test_home();
18413 let dir = tempfile::tempdir().unwrap();
18414 let mut runner = runner_at(RunStatus::Gating);
18415 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18416 runner.state.repo = dir.path().to_owned();
18417 runner.state.candidates[0].worktree = dir.path().to_owned();
18418 seed_public_summary(&mut runner);
18419 let winner = runner.state.candidates[0].clone();
18420 assert!(
18421 runner
18422 .guarded_pr_message(&winner, None, true)
18423 .await
18424 .unwrap()
18425 .is_none()
18426 );
18427 let record = runner.state.github_text.as_ref().unwrap();
18428 assert!(record.rewrite.is_none());
18429 let mut old = serde_json::to_value(record).unwrap();
18431 old.as_object_mut().unwrap().remove("rewrite");
18432 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18433 assert!(old.rewrite.is_none());
18434 let id = record.question.clone().unwrap();
18435 ask::Questions::open()
18436 .update(&id, |q| {
18437 q.choices.push(crate::github_text::USE_REWRITE.into());
18438 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18439 })
18440 .unwrap();
18441 let message = runner
18442 .guarded_pr_message(&winner, None, true)
18443 .await
18444 .unwrap()
18445 .unwrap();
18446 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18447 assert_eq!(
18448 runner.state.github_text.as_ref().unwrap().choice,
18449 Some(GithubTextChoice::Fallback)
18450 );
18451 }
18452
18453 #[tokio::test]
18454 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18455 crate::run::pin_test_home();
18456 let dir = tempfile::tempdir().unwrap();
18457 let mut runner = runner_at(RunStatus::Gating);
18458 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18459 runner.state.repo = dir.path().to_owned();
18460 runner.state.candidates[0].worktree = dir.path().to_owned();
18461 seed_public_summary(&mut runner);
18462 let winner = runner.state.candidates[0].clone();
18463 assert!(
18464 runner
18465 .guarded_pr_message(&winner, None, true)
18466 .await
18467 .unwrap()
18468 .is_none()
18469 );
18470 let store = ask::Questions::open();
18471 let find = |run: &str| {
18472 store
18473 .list()
18474 .into_iter()
18475 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18476 .unwrap()
18477 };
18478 let run = runner.state.id.clone();
18479 let q = find(&run);
18480 let fp = runner
18481 .state
18482 .github_text
18483 .as_ref()
18484 .unwrap()
18485 .fingerprint
18486 .clone();
18487 let name = crate::github_text::artifact_name(&fp);
18488 let path = crate::run::artifact_path(&runner.state, &name);
18489 assert!(q.detail.contains(&path.display().to_string()));
18490 assert!(q.detail.contains("title-language: title"));
18491 let saved = std::fs::read_to_string(&path).expect("artifact written");
18492 assert!(saved.contains("日本語のタイトル"));
18493 assert!(q.cwd.is_none());
18494 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18495 let say = |id: &str, text: &str, choice: &str| {
18497 store
18498 .update(id, |q| {
18499 q.thread.push(ask::Turn {
18500 who: ask::Who::Operator,
18501 body: text.to_owned(),
18502 at: jiff::Timestamp::now(),
18503 note: None,
18504 });
18505 q.answer(ask::Answer::Choice(choice.to_owned()))
18506 })
18507 .unwrap();
18508 };
18509 say(&q.id, "まだ日本語", "use my text");
18510 assert!(
18511 runner
18512 .guarded_pr_message(&winner, None, true)
18513 .await
18514 .unwrap()
18515 .is_none()
18516 );
18517 let second = find(&run);
18518 assert_ne!(second.id, q.id);
18519 assert!(second.detail.contains("did not pass"));
18520 assert!(second.detail.contains("日本語のタイトル"));
18523 assert!(!second.detail.contains("まだ日本語"));
18524 assert!(second.detail.contains(&path.display().to_string()));
18525 say(&second.id, "fix: retry failed requests", "use my text");
18526 let message = runner
18527 .guarded_pr_message(&winner, None, true)
18528 .await
18529 .unwrap()
18530 .expect("a vetted replacement is posted");
18531 assert_eq!(message.title, "fix: retry failed requests");
18532 assert_eq!(
18533 runner
18534 .state
18535 .github_text
18536 .as_ref()
18537 .unwrap()
18538 .chosen_title
18539 .as_deref(),
18540 Some("fix: retry failed requests")
18541 );
18542 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18543 let again = runner
18545 .guarded_pr_message(&winner, None, true)
18546 .await
18547 .unwrap()
18548 .unwrap();
18549 assert_eq!(again.title, message.title);
18550 assert_eq!(
18551 store
18552 .list()
18553 .into_iter()
18554 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18555 .count(),
18556 2
18557 );
18558 }
18559
18560 #[test]
18561 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18562 let mut state = state_with_summary(
18566 "add retries",
18567 "TITLE: fix(web): batch reads\n- reads run.json once",
18568 );
18569 state.config.graph.language = "ja".to_owned();
18570 let m = pr_message(&state, 'A');
18571 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18572 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18573
18574 let task = "今回やってほしいこと: results projector を直す";
18577 let mut state = state_with_summary(task, "- no title line");
18578 state.config.graph.language = "ja".to_owned();
18579 let m = pr_message(&state, 'A');
18580 assert_eq!(
18581 m.title,
18582 format!("chore: land candidate A of run {}", state.id)
18583 );
18584 assert!(
18585 m.body.contains(&format!(
18586 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18587 )),
18588 "{}",
18589 m.body
18590 );
18591 }
18592
18593 #[test]
18594 fn pr_message_scrubs_home_paths_and_addresses() {
18595 let state = state_with_summary(
18596 "fix it in /Users/someone/src/x",
18597 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18598 );
18599 let m = pr_message(&state, 'A');
18600 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18601 assert!(!m.body.contains(leak), "{}", m.body);
18602 }
18603 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18604 }
18605
18606 #[test]
18607 fn pr_message_survives_a_task_that_closes_details() {
18608 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18609 let m = pr_message(&state, 'A');
18610 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18611 }
18612
18613 #[test]
18614 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18615 let cmd = manual_merge_command(
18616 MergeStyle::Squash,
18617 Path::new("/repo"),
18618 "b",
18619 "fix: \"quoted\" $(x) `y`\n\nbody",
18620 );
18621 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18622 }
18623
18624 #[test]
18625 fn manual_merge_command_matches_the_configured_style() {
18626 let repo = Path::new("/repo");
18627 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18628
18629 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18630 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18631
18632 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18633 assert_eq!(
18634 squash,
18635 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18636 \"Merge magi run 0832 (candidate A)\""
18637 );
18638
18639 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18640 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18641 }
18642
18643 #[test]
18644 fn a_nudge_gets_a_quarter_of_the_budget() {
18645 assert_eq!(retry_budget(secs(1200), true), secs(300));
18647 assert_eq!(retry_budget(secs(3600), true), secs(900));
18648 }
18649
18650 #[test]
18651 fn a_resent_prompt_keeps_the_whole_budget() {
18652 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18655 assert_eq!(retry_budget(secs(60), false), secs(60));
18656 }
18657
18658 #[test]
18659 fn the_floor_never_exceeds_the_original_budget() {
18660 assert_eq!(retry_budget(secs(60), true), secs(60));
18664 assert_eq!(retry_budget(secs(480), true), secs(120));
18665 assert_eq!(retry_budget(secs(0), true), secs(0));
18666 }
18667
18668 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18669 agent::CommandEvidence {
18670 id: "item1".to_owned(),
18671 description: "cargo test".to_owned(),
18672 exit_code,
18673 result_summary: String::new(),
18674 source: "codex".to_owned(),
18675 }
18676 }
18677
18678 #[test]
18679 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18680 assert!(!has_unconfirmed_command(&[]));
18684 }
18685
18686 #[test]
18687 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18688 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18692 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18693 assert!(!has_unconfirmed_command(&[
18694 evidence(Some(0)),
18695 evidence(Some(101))
18696 ]));
18697 }
18698
18699 #[test]
18700 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18701 assert!(has_unconfirmed_command(&[
18702 evidence(Some(0)),
18703 evidence(None)
18704 ]));
18705 }
18706
18707 #[test]
18708 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18709 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18710 assert_eq!(
18711 verified_noop_claim(true, &[], text).as_deref(),
18712 Some("already fixed by b32cfc4, on main.")
18713 );
18714 }
18715
18716 #[test]
18717 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18718 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18721 assert!(verified_noop_claim(false, &[], text).is_none());
18722 }
18723
18724 #[test]
18725 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18726 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18727 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18728 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18730 }
18731
18732 #[test]
18733 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18734 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18735 }
18736
18737 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18740 runner.state.candidates = shape
18741 .iter()
18742 .enumerate()
18743 .map(|(i, &(empty, verified))| Candidate {
18744 index: i,
18745 label: (b'A' + i as u8) as char,
18746 agent: "sonnet".to_owned(),
18747 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18748 worktree: PathBuf::from(format!("/wt/{i}")),
18749 summary: String::new(),
18750 stat: String::new(),
18751 files: 0,
18752 commits: 0,
18753 empty,
18754 failed: None,
18755 verified_noop: verified.map(str::to_owned),
18756 duration_ms: 0,
18757 folded: false,
18758 })
18759 .collect();
18760 }
18761
18762 #[test]
18763 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18764 ask_test_home();
18765 let mut runner = runner_at(RunStatus::Implementing);
18766 set_candidates(
18767 &mut runner,
18768 &[
18769 (true, Some("already on main at b32cfc4")),
18770 (true, Some("same fix, see the existing test")),
18771 ],
18772 );
18773
18774 runner
18775 .after_implement()
18776 .expect("a verified no-op is not an error");
18777
18778 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18779 }
18780
18781 #[test]
18782 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18783 ask_test_home();
18784 let mut runner = runner_at(RunStatus::Implementing);
18785 set_candidates(
18789 &mut runner,
18790 &[(true, Some("already on main at b32cfc4")), (true, None)],
18791 );
18792
18793 let err = runner
18794 .after_implement()
18795 .expect_err("an unverified empty candidate must still fail the run");
18796
18797 assert!(
18798 err.to_string().contains("no candidate produced a change"),
18799 "{err}"
18800 );
18801 assert_eq!(runner.state.status, RunStatus::Failed);
18802 }
18803
18804 #[test]
18805 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18806 ask_test_home();
18807 let mut runner = runner_at(RunStatus::Implementing);
18808 set_candidates(&mut runner, &[(true, None), (true, None)]);
18809
18810 let err = runner
18811 .after_implement()
18812 .expect_err("no candidate declared anything; this is an ordinary failure");
18813
18814 assert!(
18815 err.to_string().contains("no candidate produced a change"),
18816 "{err}"
18817 );
18818 assert_eq!(runner.state.status, RunStatus::Failed);
18819 }
18820}
18821
18822#[derive(Debug)]
18825struct GraftRestoreFailed(String);
18826
18827impl std::fmt::Display for GraftRestoreFailed {
18828 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18829 write!(
18830 f,
18831 "could not restore the winner to its pre-graft commit: {}",
18832 self.0
18833 )
18834 }
18835}
18836
18837impl std::error::Error for GraftRestoreFailed {}
18838
18839#[cfg(test)]
18843mod resource_wait_tests {
18844 use super::*;
18845 use std::collections::BTreeMap;
18846
18847 fn sh(id: &str, script: &str) -> AgentSpec {
18848 AgentSpec {
18849 id: id.to_owned(),
18850 kind: crate::config::AgentKind::Command,
18851 model: None,
18852 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18853 extra_args: Vec::new(),
18854 env: BTreeMap::new(),
18855 prompt_delivery: None,
18856 }
18857 }
18858
18859 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18860 SeatJob {
18861 seat: SeatState::new("impl-A", &spec.id, 1),
18862 spec,
18863 cwd: dir.to_path_buf(),
18864 prompt: "do it".to_owned(),
18865 timeout,
18866 allow_write: true,
18867 sessions: false,
18868 artifacts: dir.join("art"),
18869 stem: "impl-A".to_owned(),
18870 handover: None,
18871 }
18872 }
18873
18874 fn state(resource_wait_secs: u64) -> RunState {
18875 let mut config = Config::default();
18876 config.graph.timeout_resource_wait = resource_wait_secs;
18877 RunState::new(
18878 PathBuf::from("/repo"),
18879 "main".to_owned(),
18880 "abc1234".to_owned(),
18881 "task".to_owned(),
18882 config,
18883 )
18884 }
18885
18886 fn hold(cache: &Path) -> crate::cache::Guard {
18887 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18888 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18889 crate::cache::AcquireOutcome::Acquired(g) => g,
18890 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18891 }
18892 }
18893
18894 async fn run(
18895 jobs: Vec<SeatJob>,
18896 sem: Arc<Semaphore>,
18897 cache: Option<&Path>,
18898 st: &mut RunState,
18899 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18900 let prompts = Prompts::default();
18901 let ctx = WaveCtx {
18902 run: "r",
18903 node: "implement",
18904 prompts: &prompts,
18905 cache,
18906 round: None,
18907 carry_seats: false,
18908 pauses: None,
18909 };
18910 wave(jobs, sem, &ctx, st, 0).await
18911 }
18912
18913 #[tokio::test]
18914 async fn verification_cache_wait_preserves_the_command_budget() {
18915 crate::run::pin_test_home();
18916 let tmp = tempfile::tempdir().unwrap();
18917 let cache = tmp.path().join("target");
18918 let guard = hold(&cache);
18919 let release = tokio::spawn(async move {
18920 tokio::time::sleep(Duration::from_millis(100)).await;
18921 guard.release();
18922 });
18923 let mut st = state(10);
18924 let budget = Duration::from_millis(50);
18925 let wait = repository_wait(&st, None);
18926 let (outcomes, _) = with_cache_lease(
18927 &mut st,
18928 Some(&cache),
18929 "gate",
18930 "gate",
18931 tmp.path(),
18932 "abc1234",
18933 budget,
18934 "test gate",
18935 &wait,
18936 |_, available| async move {
18937 assert_eq!(
18938 available, budget,
18939 "contention must not shorten verification"
18940 );
18941 (Vec::new(), false, Vec::new())
18942 },
18943 )
18944 .await
18945 .unwrap();
18946 release.await.unwrap();
18947 assert!(outcomes.is_empty(), "{outcomes:?}");
18948 assert!(
18949 st.events
18950 .iter()
18951 .any(|event| event.message.contains("waiting for"))
18952 );
18953 assert!(st.resource_wait.is_none());
18954 hold(&cache).release();
18956 }
18957
18958 #[tokio::test]
18959 async fn verification_cache_wait_expires_without_starting_commands() {
18960 crate::run::pin_test_home();
18961 let tmp = tempfile::tempdir().unwrap();
18962 let cache = tmp.path().join("target");
18963 let _held = hold(&cache);
18964 let mut st = state(0);
18965 let wait = repository_wait(&st, None);
18966 let (outcomes, _) = with_cache_lease(
18967 &mut st,
18968 Some(&cache),
18969 "gate",
18970 "gate",
18971 tmp.path(),
18972 "abc1234",
18973 Duration::from_secs(60),
18974 "test gate",
18975 &wait,
18976 |_, _| async { panic!("commands must not start without the cache lease") },
18977 )
18978 .await
18979 .unwrap();
18980 assert_eq!(outcomes.len(), 1);
18981 assert!(outcomes[0].resource_blocked);
18982 assert!(st.resource_wait.is_none());
18983 }
18984
18985 #[tokio::test]
18986 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18987 crate::run::pin_test_home();
18988 let tmp = tempfile::tempdir().unwrap();
18989 let cache = tmp.path().join("target-a");
18990 let guard = hold(&cache);
18991 tokio::spawn(async move {
18993 tokio::time::sleep(Duration::from_millis(1500)).await;
18994 guard.release();
18995 });
18996 let mut st = state(60);
18997 let results = run(
18998 vec![job(
18999 sh("alpha", "echo ok"),
19000 tmp.path(),
19001 Duration::from_secs(1),
19002 )],
19003 Arc::new(Semaphore::new(1)),
19004 Some(&cache),
19005 &mut st,
19006 )
19007 .await;
19008 assert!(
19009 matches!(results[0].2, AgentOutcome::Ok(_)),
19010 "the wait must not be charged to the agent"
19011 );
19012 assert!(st.resource_block.is_none());
19013 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19014 assert!(st.resource_wait.is_none() && st.active.is_empty());
19015 }
19016
19017 #[tokio::test]
19018 #[cfg(unix)]
19019 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19020 use std::os::fd::AsRawFd as _;
19021 crate::run::pin_test_home();
19022 let tmp = tempfile::tempdir().unwrap();
19023 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19024 let common = tmp.path().join(".git");
19025 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19026 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19027 assert_eq!(
19028 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19029 0
19030 );
19031 let mut st = state(1);
19032 st.config.coordination = crate::config::test_coordination();
19033 let mut reader = job(
19034 sh("reader", "echo must-not-run"),
19035 tmp.path(),
19036 Duration::from_secs(30),
19037 );
19038 reader.allow_write = false;
19039 let results = run(
19040 vec![reader.clone()],
19041 Arc::new(Semaphore::new(1)),
19042 None,
19043 &mut st,
19044 )
19045 .await;
19046 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19047 assert_eq!(results[0].1.turns, 0);
19048 assert!(FailClass::of(&results[0].2).is_none());
19049 assert_eq!(
19050 st.resource_block.as_ref().unwrap().kind,
19051 "repository coordination"
19052 );
19053 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19054 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19055 drop(external);
19056 let editor = crate::repo_lock::SeatLease::for_checkout(
19057 tmp.path(),
19058 true,
19059 crate::config::test_coordination().as_ref(),
19060 )
19061 .await
19062 .unwrap();
19063 let pause = Pause::new();
19064 let interrupt = Pause::new();
19065 let prompts = Prompts::default();
19066 let ctx = WaveCtx {
19067 run: "r",
19068 node: "review",
19069 prompts: &prompts,
19070 cache: None,
19071 round: None,
19072 carry_seats: false,
19073 pauses: Some((&pause, &interrupt)),
19074 };
19075 let (results, ()) = tokio::join!(
19076 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19077 async {
19078 tokio::time::sleep(Duration::from_millis(50)).await;
19079 pause.park();
19080 }
19081 );
19082 assert!(
19083 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19084 );
19085 assert_eq!(results[0].1.turns, 0);
19086 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19087 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19088 drop(editor);
19089 }
19090
19091 #[tokio::test]
19092 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19093 crate::run::pin_test_home();
19094 let tmp = tempfile::tempdir().unwrap();
19095 let cache = tmp.path().join("target-b");
19096 let _held = hold(&cache);
19097 let mut st = state(1);
19098 let results = run(
19099 vec![job(
19100 sh("alpha", "echo ok"),
19101 tmp.path(),
19102 Duration::from_secs(1),
19103 )],
19104 Arc::new(Semaphore::new(1)),
19105 Some(&cache),
19106 &mut st,
19107 )
19108 .await;
19109 let (_, seat, out) = &results[0];
19110 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19111 assert!(wave_blocked(&results));
19112 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19113 assert_eq!(seat.turns, 0, "nothing was asked");
19114 assert!(
19115 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19116 "no agent was spawned"
19117 );
19118 assert!(FailClass::of(out).is_none(), "not a failure class");
19119 let block = st.resource_block.as_ref().expect("recorded for the resume");
19120 assert_eq!(block.kind, "shared build cache");
19121 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19122 assert!(st.active.is_empty());
19123 }
19124
19125 #[tokio::test]
19126 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19127 crate::run::pin_test_home();
19128 let tmp = tempfile::tempdir().unwrap();
19129 let mut st = state(1);
19130 let results = run(
19131 vec![job(
19132 sh("alpha", "echo ok"),
19133 tmp.path(),
19134 Duration::from_secs(1),
19135 )],
19136 Arc::new(Semaphore::new(0)),
19137 None,
19138 &mut st,
19139 )
19140 .await;
19141 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19142 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19143 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19144 }
19145
19146 #[tokio::test]
19147 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19148 crate::run::pin_test_home();
19149 let tmp = tempfile::tempdir().unwrap();
19150 let mut st = state(60);
19151 let results = run(
19152 vec![job(
19153 sh("alpha", "sleep 30"),
19154 tmp.path(),
19155 Duration::from_secs(1),
19156 )],
19157 Arc::new(Semaphore::new(1)),
19158 None,
19159 &mut st,
19160 )
19161 .await;
19162 let out = &results[0].2;
19163 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19164 let class = FailClass::of(out).expect("a failure");
19165 assert_eq!(class, FailClass::Timeout);
19166 assert!(should_hand_over(None, &class), "existing handover policy");
19167 assert!(st.resource_block.is_none());
19168 }
19169}