1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's network, Git access and repository coordination before \
526 retrying"
527 )
528}
529
530enum FixStep {
532 Resource,
533 Recorded { report_lost: bool },
534}
535
536struct FixClaim {
553 path: PathBuf,
554}
555
556impl FixClaim {
557 fn acquire(dir: &Path) -> Result<Self> {
558 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559 let path = dir.join("fix.lock");
560 let retry = crate::lock_file::Retry::new(None);
561 match Self::create(&path, retry) {
562 Ok(claim) => Ok(claim),
563 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564 if Self::reclaim_if_dead(&path) {
565 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566 } else {
567 bail!(
568 "another repair or `magi fix` is already running for this run ({} exists)",
569 path.display()
570 )
571 }
572 }
573 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574 }
575 }
576
577 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578 let mut f = retry.create(path)?;
579 use std::io::Write as _;
580 writeln!(f, "{}", std::process::id())?;
582 Ok(Self {
583 path: path.to_owned(),
584 })
585 }
586
587 fn reclaim_if_dead(path: &Path) -> bool {
591 let dead = std::fs::read_to_string(path)
592 .ok()
593 .and_then(|body| body.trim().parse::<u32>().ok())
594 .is_some_and(|pid| !crate::proc::pid_alive(pid));
595 dead && std::fs::remove_file(path).is_ok()
596 }
597}
598
599impl Drop for FixClaim {
600 fn drop(&mut self) {
601 let _ = std::fs::remove_file(&self.path);
602 }
603}
604
605struct DriverExecution<'a> {
608 runner: &'a mut Runner,
609 driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613 fn drop(&mut self) {
614 self.driver.release_local(&mut self.runner.state);
615 }
616}
617
618impl Runner {
619 pub async fn start(
621 repo: &Path,
622 instruction: String,
623 config: Config,
624 origin: Origin,
625 ) -> Result<Self> {
626 Self::start_naming(repo, instruction, "", config, origin).await
627 }
628
629 pub async fn start_naming(
632 repo: &Path,
633 instruction: String,
634 _also_scan: &str,
635 config: Config,
636 origin: Origin,
637 ) -> Result<Self> {
638 let repo = git::toplevel(repo).await?;
639 let missing = agent::missing_programs(&config.agents);
640 if !missing.is_empty() {
641 bail!(
642 "these agent programs are not on PATH: {}. Fix the roster in \
643 magi.toml or install them.",
644 missing.join(", ")
645 );
646 }
647 let base_branch =
648 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649 .await?;
650 let fetched_base = resolve_base(
651 &repo,
652 &base_branch,
653 &config.merge.remote,
654 config.coordination.as_ref(),
655 )
656 .await?;
657 let base_commit = fetched_base.commit.clone();
658 let roles = config.resolve_roles()?;
659 let max_parallel = config.graph.max_parallel.max(1);
660 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661 state.fetched_base = Some(fetched_base);
665 state.origin = Some(origin);
666 state.reserve()?;
667 state.event("start", format!("run {} created", state.id));
668 state.save()?;
669 Ok(Self {
670 state,
671 roles,
672 sem: Arc::new(Semaphore::new(max_parallel)),
673 pause: Pause::new(),
674 interrupt: Pause::new(),
675 })
676 }
677
678 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692 Self::review_taking_over(repo, branch, config, None, origin).await
693 }
694
695 pub async fn review_taking_over(
702 repo: &Path,
703 branch: &str,
704 config: Config,
705 takeover: Option<crate::handover::Takeover>,
706 origin: Origin,
707 ) -> Result<Self> {
708 let repo = git::toplevel(repo).await?;
709 let missing = agent::missing_programs(&config.agents);
710 if !missing.is_empty() {
711 bail!(
712 "these agent programs are not on PATH: {}. Fix the roster in \
713 magi.toml or install them.",
714 missing.join(", ")
715 );
716 }
717 let base_branch =
718 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719 .await?;
720 if base_branch == branch {
721 bail!("`{branch}` is the base branch; there is nothing to review against");
722 }
723 let fetched_base = resolve_base(
724 &repo,
725 &base_branch,
726 &config.merge.remote,
727 config.coordination.as_ref(),
728 )
729 .await?;
730 let base_commit = fetched_base.commit.clone();
731
732 let roles = config.resolve_roles()?;
733 let max_parallel = config.graph.max_parallel.max(1);
734 let mut state = RunState::new(
735 repo.clone(),
736 base_branch,
737 base_commit.clone(),
738 String::new(),
739 config,
740 );
741 state.fetched_base = Some(fetched_base);
742 state.origin = Some(origin);
743 state.reserve()?;
744
745 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752 earlier: Vec::new(),
753 home: crate::run::home(),
754 choice: None,
755 task_instruction: None,
756 });
757 let wait = repository_wait(&state, None);
758 let coordination_config = state.config.coordination.clone();
759 let coordination =
760 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761 .await?;
762 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763 if let Some(released) = &released {
764 state.event(
765 "release",
766 format!(
767 "took `{branch}` over from run {}: its worktree was released: {}",
768 crate::run::short_of(&released.old_id),
769 released.audit
770 ),
771 );
772 }
773 if let Some(choice) = takeover.choice.as_ref()
777 && let Err(e) =
778 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779 .await
780 {
781 if let Some(released) = &released {
782 released.restore(&repo, branch).await;
783 }
784 return Err(e.context("applying the owner's answer about the diverged branch"));
785 }
786 for id in takeover.earlier.iter().rev() {
792 if let Ok(old) = RunState::load_under(id, &takeover.home)
793 && let Some(head) = old.pr_head.clone()
794 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795 {
796 state.pr_head = Some(head);
797 break;
798 }
799 }
800 drop(coordination);
801 let opened = Self::open_review(
802 &repo,
803 branch,
804 state,
805 roles,
806 max_parallel,
807 base_commit,
808 &takeover,
809 )
810 .await;
811 if opened.is_err()
812 && let Some(released) = &released
813 {
814 let _coordination = crate::repo_lock::SeatLease::operation_with(
815 &repo,
816 coordination_config.as_ref(),
817 &wait,
818 )
819 .await?;
820 released.restore(&repo, branch).await;
821 }
822 opened
823 }
824
825 pub async fn recover_pr(
827 repo: &Path,
828 task: &str,
829 selection: crate::pr_recovery::Selection,
830 mut config: Config,
831 ) -> Result<Self> {
832 selection.configure(&mut config)?;
833 selection.check_task(task, &selection.instruction)?;
834 let base_branch =
835 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836 .await?;
837 let fetched_base = resolve_base(
838 repo,
839 &base_branch,
840 &config.merge.remote,
841 config.coordination.as_ref(),
842 )
843 .await?;
844 let base_commit = fetched_base.commit.clone();
845 selection
846 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847 .await?;
848 Box::pin(selection.verify_source(repo)).await?;
849 if let Some(proof) = &selection.source_proof {
850 anyhow::ensure!(
851 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852 "source rebase base is outside the fetched landing base"
853 );
854 }
855 let roles = config.resolve_roles()?;
856 let max_parallel = config.graph.max_parallel.max(1);
857 let mut state = RunState::new(
858 repo.to_path_buf(),
859 base_branch,
860 base_commit.clone(),
861 selection.instruction.clone(),
862 config,
863 );
864 state.fetched_base = Some(fetched_base);
865 state.origin = Some(crate::run::Origin::queue(task));
866 state.pr_head = Some(selection.branch.clone());
867 state.pr_recovery = Some(selection.clone());
868 state.reserve()?;
869 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870 state.discard_reserved();
872 return Err(e);
873 }
874 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875 }
876
877 async fn open_selected(
878 repo: &Path,
879 state: RunState,
880 roles: ResolvedRoles,
881 max_parallel: usize,
882 base_commit: String,
883 selection: crate::pr_recovery::Selection,
884 ) -> Result<Self> {
885 let mut state = state;
886 let branch = state.branch_for('A');
887 let wait = repository_wait(&state, None);
888 let prepared = async {
889 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890 git::git(repo, &["branch", &branch, selection.source()]).await?;
891 drop(coordination);
892 anyhow::Ok(())
893 }
894 .await;
895 if let Err(e) = prepared {
896 state.discard_reserved();
899 return Err(e);
900 }
901 state.event(
902 "recovery",
903 format!(
904 "selected implementation {} for {} (public lease {})",
905 selection.source(),
906 selection.url,
907 selection.head
908 ),
909 );
910 let takeover = crate::handover::Takeover {
911 earlier: Vec::new(),
912 home: crate::run::home(),
913 choice: None,
914 task_instruction: Some(selection.instruction.clone()),
915 };
916 if let Err(e) = state.save() {
917 state.discard_reserved();
918 return Err(e);
919 }
920 let id = state.id.clone();
921 let opened = Self::open_review(
922 repo,
923 &branch,
924 state,
925 roles,
926 max_parallel,
927 base_commit,
928 &takeover,
929 )
930 .await;
931 match opened {
932 Ok(runner) => Ok(runner),
933 Err(e) => {
934 if let Ok(mut partial) = RunState::load(&id) {
935 partial.status = RunStatus::Blocked;
936 partial.driver_exited = true;
937 partial.event(
938 "recovery",
939 format!("startup failed; retained source and candidate {branch}: {e:#}"),
940 );
941 partial.save()?;
942 }
943 Err(crate::pr_recovery::StartupFailed {
944 run: id,
945 cause: format!("{e:#}"),
946 }
947 .into())
948 }
949 }
950 }
951
952 async fn inherit(
957 repo: &Path,
958 branch: &str,
959 base_commit: &str,
960 takeover: &crate::handover::Takeover,
961 ) -> Option<crate::recovery::RecoveryRecord> {
962 use crate::recovery::{Relation, assess};
963 let newest = takeover.earlier.last()?;
964 let mut chosen = None;
965 for id in takeover.earlier.iter().rev() {
966 if let Ok(old) = RunState::load_under(id, &takeover.home)
967 && !old.reviews.is_empty()
968 && (old.winner().is_some_and(|w| w.branch == branch)
969 || old.pr_head.as_deref() == Some(branch))
970 {
971 chosen = Some(old);
972 break;
973 }
974 }
975 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976 .await
977 .unwrap_or_default();
978 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979 Some(last) if last.head == tip => Relation::Same,
980 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981 _ => Relation::Diverged,
982 };
983 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984 Some(assess(from, chosen.as_ref(), base_commit, relation))
985 }
986
987 async fn open_review(
990 repo: &Path,
991 branch: &str,
992 mut state: RunState,
993 roles: ResolvedRoles,
994 max_parallel: usize,
995 base_commit: String,
996 takeover: &crate::handover::Takeover,
997 ) -> Result<Self> {
998 let wait = repository_wait(&state, None);
999 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000 if state.pr_recovery.is_none() {
1001 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002 }
1003 let start = review_base(
1006 repo,
1007 &state.config.merge.remote,
1008 &state.base_branch,
1009 &base_commit,
1010 branch,
1011 )
1012 .await;
1013 let log = git::log_oneline(repo, &start, branch)
1014 .await
1015 .unwrap_or_default();
1016 let log_text = if log.trim().is_empty() {
1017 "(no commit messages)"
1018 } else {
1019 log.trim()
1020 };
1021 let mut instruction = format!(
1022 "Review the work already on branch `{branch}`. There is no task \
1023 statement: what the change claims to do is whatever its commits \
1024 say.\n\n{log_text}"
1025 );
1026 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030 if let Some(task) = takeover
1031 .task_instruction
1032 .as_deref()
1033 .filter(|t| !t.trim().is_empty())
1034 && rec.reviewed_head.is_some()
1035 {
1036 instruction = format!(
1037 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038 task.trim()
1039 );
1040 rec.criteria_inherited = true;
1041 }
1042 state.event("recovery", rec.summary());
1043 state.recovery = Some(rec);
1044 }
1045 if let Some(selected) = &state.pr_recovery {
1046 instruction = selected.instruction.clone();
1047 }
1048 state.task_instruction = takeover.task_instruction.clone();
1049 state.instruction = instruction;
1050 state.reviewed_commits = Some(
1051 git::subjects(repo, &start, branch)
1052 .await
1053 .unwrap_or_default(),
1054 );
1055
1056 for id in &takeover.earlier {
1059 if let Ok(old) = RunState::load_under(id, &takeover.home)
1060 && old.repo == state.repo
1061 && (old.pr_head.as_deref() == Some(branch)
1062 || old.winner().is_some_and(|w| w.branch == branch))
1063 {
1064 for hold in old.branch_owner_holds {
1065 if !state.branch_owner_holds.contains(&hold) {
1066 state.event(
1067 "approval-policy",
1068 format!(
1069 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070 hold.control.key(),
1071 hold.file,
1072 hold.head
1073 ),
1074 );
1075 state.branch_owner_holds.push(hold);
1076 }
1077 }
1078 }
1079 }
1080 let reviewed_head = git::rev_parse(repo, branch).await?;
1081 state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083 let worktree = state.worktree_root().join("under-review");
1086 if let Some(parent) = worktree.parent() {
1087 tokio::fs::create_dir_all(parent).await.ok();
1088 }
1089 let path = worktree.to_string_lossy().to_string();
1090 git::git(repo, &["worktree", "add", &path, branch])
1091 .await
1092 .with_context(|| {
1093 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094 })?;
1095
1096 let setup = match ensure_setup_config(&mut state, repo).await {
1097 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098 Err(e) => Err(e),
1099 };
1100 if let Err(e) = setup {
1101 git::worktree_remove_clean(repo, &worktree).await.ok();
1102 return Err(e);
1103 }
1104 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105 .await
1106 .unwrap_or(0);
1107 if commits == 0 {
1108 git::worktree_remove_clean(repo, &worktree).await.ok();
1109 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110 }
1111 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112 .await
1113 .map(|f| f.len())
1114 .unwrap_or(0);
1115 if files == 0
1116 && let (Ok(head_tree), Ok(base_tree)) = (
1117 git::tree_of(&worktree, "HEAD").await,
1118 git::tree_of(&worktree, &base_commit).await,
1119 )
1120 && head_tree == base_tree
1121 {
1122 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123 git::worktree_remove_clean(repo, &worktree).await.ok();
1124 bail!(
1125 "`{branch}` at {} has a tree identical to base {}; this usually means \
1126 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127 against `{}/{branch}`) rather than an empty change",
1128 short(&head),
1129 short(&base_commit),
1130 state.config.merge.remote
1131 );
1132 }
1133 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134 .await
1135 .unwrap_or_default();
1136
1137 state.candidates.push(Candidate {
1138 index: 0,
1139 label: 'A',
1140 agent: EXISTING_BRANCH.to_owned(),
1143 branch: branch.to_owned(),
1144 worktree,
1145 summary: String::new(),
1146 stat,
1147 files,
1148 commits,
1149 empty: false,
1150 failed: None,
1151 verified_noop: None,
1152 duration_ms: 0,
1153 folded: false,
1154 });
1155 state.tally = Some(Tally {
1156 first_choice: BTreeMap::from([('A', 0)]),
1157 borda: BTreeMap::new(),
1158 winner: 'A',
1159 rankings: 0,
1160 unanimous_initial: false,
1161 deliberated: false,
1162 changed_votes: 0,
1163 unanimous_final: false,
1164 tie_break: None,
1165 judges: 0,
1169 present: 0,
1170 quorum: 0,
1171 met_quorum: true,
1172 uncontested: Some("review-only run: nothing competed".to_owned()),
1173 });
1174 state.status = RunStatus::Reviewing;
1175 state.prep_complete = Some(true);
1176 state.event(
1177 "start",
1178 format!(
1179 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180 state.id
1181 ),
1182 );
1183 state.save()?;
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub fn resume(id: &str) -> Result<Self> {
1195 let state = RunState::load(id)?;
1196 crate::pr_head::ensure_owner(&state)?;
1197 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198 bail!(
1199 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200 );
1201 }
1202 if let Some(to) = &state.released_to {
1203 bail!(
1204 "run {} cannot be resumed: its worktree was released to run {}",
1205 state.short(),
1206 crate::run::short_of(to)
1207 );
1208 }
1209 let roles = state.config.resolve_roles()?;
1210 if let Some(id) = &state.config.graph.selected_implementer
1211 && state.candidates.iter().any(|c| c.agent != *id)
1212 {
1213 bail!(
1214 "replacement implementer pin does not match the saved candidates; task must stay held"
1215 );
1216 }
1217 let max_parallel = state.config.graph.max_parallel.max(1);
1218 Ok(Self {
1219 state,
1220 roles,
1221 sem: Arc::new(Semaphore::new(max_parallel)),
1222 pause: Pause::new(),
1223 interrupt: Pause::new(),
1224 })
1225 }
1226
1227 pub async fn execute(&mut self) -> Result<()> {
1234 if let Some(selection) = &self.state.pr_recovery
1235 && (selection.source_proof.is_some() || selection.source_run.is_some())
1236 {
1237 Box::pin(selection.verify_source(&self.state.repo)).await?;
1238 }
1239 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242 || self.state.land_approval.is_some()
1243 || self.state.merge_decision.is_some()
1244 || !self.state.merge_repairs.is_empty()
1245 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246 {
1247 let claim = FixClaim::acquire(&self.state.dir())?;
1248 let disk = RunState::load(&self.state.id)?;
1249 let checkpoint = |s: &RunState| {
1250 (
1251 s.merge_repairs.len(),
1252 s.merge_repairs
1253 .last()
1254 .map(|r| (r.question.clone(), r.phase)),
1255 s.land_approval
1256 .as_ref()
1257 .map(|a| (a.question.clone(), a.head.clone())),
1258 )
1259 };
1260 if checkpoint(&disk) != checkpoint(&self.state) {
1261 bail!(
1262 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263 );
1264 }
1265 Some(claim)
1266 } else {
1267 None
1268 };
1269 let mut execution = self.claim_driver().await?;
1270 let runner = &mut *execution.runner;
1271 let wait = runner.repository_wait();
1272 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275 let result = match result {
1276 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277 record_repository_block(&mut runner.state, &e);
1278 runner.park_here()?;
1279 Ok(())
1280 }
1281 other => other,
1282 };
1283 execution.driver.finish(&mut runner.state).await?;
1284 let ended = if result.is_err() {
1285 Some({
1286 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1287 if let Err(error) = &result {
1288 notice.message = format!(
1289 "Run {} stopped with an error: {}",
1290 runner.state.id,
1291 error.to_string().lines().next().unwrap_or("unknown error")
1292 );
1293 }
1294 notice
1295 })
1296 } else {
1297 crate::notices::run_ended(&runner.state)
1298 };
1299 if let Some(notice) = ended {
1300 crate::notices::raise(notice);
1301 }
1302 result
1303 }
1304
1305 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1306 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1307 }
1308
1309 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1310 let wait = self.repository_wait();
1311 coordinate_operation(&mut self.state, cwd, &wait).await
1312 }
1313
1314 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1316 let owner = match crate::pr_head::owner_lock_wait(
1317 &crate::run::home(),
1318 &self.state.id,
1319 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1320 )
1321 .await
1322 {
1323 Ok(owner) => owner,
1324 Err(error) => {
1325 let detail = format!("worktree maintenance or ownership update: {error:#}");
1326 self.state.resource_block = Some(crate::run::ResourceBlock {
1330 cause: crate::run::ResourceBlockCause::Other,
1331 node: "start".into(),
1332 kind: "worktree maintenance".into(),
1333 detail: detail.clone(),
1334 at: Timestamp::now(),
1335 });
1336 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1337 }
1338 };
1339 let disk = RunState::load(&self.state.id)?;
1340 crate::pr_head::ensure_owner(&disk)?;
1341 let claimed =
1342 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1343 anyhow::ensure!(
1348 disk.liveness(claimed) != crate::run::Liveness::Live,
1349 "run already has a live driver; wait for its execution to end"
1350 );
1351 if let Some(to) = &disk.released_to {
1352 bail!(
1353 "run {} cannot continue: its worktree was released to run {}",
1354 self.state.short(),
1355 crate::run::short_of(to)
1356 );
1357 }
1358 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1359 let mut execution = DriverExecution {
1360 runner: self,
1361 driver,
1362 };
1363 let _owner = owner;
1366 let state = &mut execution.runner.state;
1367 state.parked = false;
1372 state.resource_wait = None;
1373 state.resource_block = None;
1374 state.clear_active();
1381 let pid = std::process::id();
1396 state.driver_pid = Some(pid);
1397 state.driver_started_at = crate::proc::process_started_at(pid);
1398 state.driver_exited = false;
1399 state.driver_claimed = Some(true);
1400 state.begin_pass();
1401 execution.driver.bind(state);
1402 state.save()?;
1403 execution.driver.record(state)?;
1404 Ok(execution)
1405 }
1406
1407 async fn execute_graph(&mut self) -> Result<()> {
1408 self.state.configure_recovery_resume()?;
1409 self.persist_candidates().await?;
1412 if self
1413 .state
1414 .merge_decision
1415 .as_ref()
1416 .is_some_and(|decision| !decision.closed)
1417 && self.state.status != RunStatus::Landing
1418 {
1419 if crate::merge_decision::resumable_hold(&self.state) {
1420 self.state.merge = Some(MergeOutcome {
1421 mode: MergeMode::Pr,
1422 ok: false,
1423 detail: self
1424 .state
1425 .merge_decision
1426 .as_ref()
1427 .unwrap()
1428 .target
1429 .pr
1430 .clone(),
1431 empty: false,
1432 });
1433 self.run_land().await?;
1434 self.settle_questions();
1435 return Ok(());
1436 }
1437 bail!(
1438 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1439 );
1440 }
1441 if self.state.config.graph.selected_implementer.is_some()
1442 && self.state.status == RunStatus::Blocked
1443 && self.state.tally.is_none()
1444 && self.state.candidate_preservation.is_empty()
1445 && self.state.interrupted_implementations.is_empty()
1446 {
1447 bail!(
1448 "selected implementation is held; explicitly release the owning task to retry it"
1449 );
1450 }
1451
1452 if self.state.merge_repairs.last().is_some_and(|r| {
1453 matches!(
1454 r.phase,
1455 crate::run::MergeRepairPhase::Requested
1456 | crate::run::MergeRepairPhase::Fixing
1457 | crate::run::MergeRepairPhase::Reviewing
1458 )
1459 }) {
1460 if self.repair_merge_review().await? {
1461 self.run_land().await?;
1462 }
1463 self.settle_questions();
1464 return Ok(());
1465 }
1466 if self.state.pr_publication_pending {
1469 self.merge().await?;
1470 if self.resource_halt()? {
1471 return Ok(());
1472 }
1473 self.state.save()?;
1474 self.settle_questions();
1475 return Ok(());
1476 }
1477 if self.state.status == RunStatus::Stalled {
1490 if self.recover_stall().await? {
1491 self.finish_after_tally().await?;
1492 } else {
1493 self.state.save()?;
1495 }
1496 return Ok(());
1497 }
1498 if self.state.status == RunStatus::Landing
1508 || crate::handover::landing_checkpoint(&self.state)
1509 {
1510 self.state.land_sync_pending = true;
1511 self.state.save()?;
1512 self.run_land().await?;
1513 self.settle_questions();
1518 return Ok(());
1519 }
1520 if self.state.github_text.is_some()
1525 && self.state.merge.is_none()
1526 && self.state.status == RunStatus::Gating
1527 {
1528 self.merge().await?;
1529 if self.resource_halt()? {
1530 return Ok(());
1531 }
1532 self.state.save()?;
1533 return Ok(());
1534 }
1535 if let Err(e) = self.prep().await {
1536 if e.is::<crate::repo_lock::ResourceBlocked>() {
1537 return Err(e);
1538 }
1539 self.state.status = RunStatus::Blocked;
1540 self.state
1541 .prep_failure
1542 .get_or_insert_with(|| format!("{e:#}"));
1543 self.state
1544 .event("prep", format!("preparation stopped: {e:#}"));
1545 self.state.save()?;
1546 return Err(e);
1547 }
1548 if self.park_here()? {
1549 return Ok(());
1550 }
1551 self.advise().await?;
1552 if self.park_here()? {
1553 return Ok(());
1554 }
1555 self.implement().await?;
1556 if self.park_here()? {
1557 return Ok(());
1558 }
1559 if self.state.status == RunStatus::VerifiedNoop {
1563 return Ok(());
1564 }
1565 self.judge().await?;
1566 if self.park_here()? {
1567 return Ok(());
1568 }
1569 self.deliberate().await?;
1570 if self.park_here()? {
1571 return Ok(());
1572 }
1573 self.vote().await?;
1574 if self.park_here()? {
1575 return Ok(());
1576 }
1577 self.tally()?;
1578 if self.state.status == RunStatus::Stalled {
1583 self.state.save()?;
1587 return Ok(());
1588 }
1589 self.finish_after_tally().await?;
1590 Ok(())
1591 }
1592
1593 fn stop_for_resource(&mut self) -> Result<()> {
1598 if self.pause.parked() || self.interrupt.parked() {
1601 self.park_here()?;
1602 return Ok(());
1603 }
1604 if self.state.resource_block.is_some() {
1605 self.state.status = RunStatus::Blocked;
1606 }
1607 self.state.save()
1608 }
1609
1610 fn resource_halt(&mut self) -> Result<bool> {
1612 if self.state.resource_block.is_none() {
1613 return Ok(false);
1614 }
1615 self.stop_for_resource()?;
1616 Ok(true)
1617 }
1618
1619 fn park_here(&mut self) -> Result<bool> {
1626 let parking = self.pause.parked() || self.interrupt.parked();
1627 if !parking && self.state.resource_block.is_some() {
1628 self.stop_for_resource()?;
1629 return Ok(true);
1630 }
1631 if !self.pause.parked() && !self.interrupt.parked() {
1636 return Ok(false);
1637 }
1638 if self.state.parked {
1639 return Ok(true);
1640 }
1641 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1642 if let Some(block) = &mut self.state.resource_block
1643 && matches!(
1644 block.cause,
1645 crate::run::ResourceBlockCause::Cancelled { .. }
1646 )
1647 {
1648 block.cause = crate::run::ResourceBlockCause::Cancelled {
1649 park: cause.clone(),
1650 };
1651 }
1652 let why = match cause.as_ref().map(ParkCause::label) {
1653 Some(reason) => format!(
1654 "parked after `{}` ({reason}) — resume to carry on from here",
1655 self.state.status.as_str()
1656 ),
1657 None => format!(
1658 "parked after `{}` — resume to carry on from here",
1659 self.state.status.as_str()
1660 ),
1661 };
1662 self.state.event("park", why);
1663 self.state
1664 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1665 self.state.parked = true;
1666 self.state.save()?;
1667 Ok(true)
1668 }
1669
1670 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1672 if self.state.prep_failure.is_none() {
1673 return Ok(());
1674 }
1675 let failed_at = self
1676 .state
1677 .events
1678 .iter()
1679 .rev()
1680 .find(|event| event.node == "prep")
1681 .map_or(self.state.updated_at, |event| event.at);
1682 anyhow::ensure!(
1683 approval_at > failed_at,
1684 "preparation retry requires a new explicit release after the failure"
1685 );
1686 self.state.prep_failure = None;
1687 self.state.prep_complete = Some(false);
1688 self.state
1689 .event("prep-retry", "operator approved retry of preparation");
1690 Ok(())
1691 }
1692
1693 pub fn on_pause(&mut self, pause: Pause) {
1695 self.pause = pause;
1696 }
1697
1698 pub fn watch_interrupt(&mut self, pause: Pause) {
1704 self.interrupt = pause;
1705 }
1706
1707 fn settle_questions(&mut self) {
1727 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1728 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1729 }
1730 }
1731
1732 async fn finish_after_tally(&mut self) -> Result<()> {
1735 self.graft().await?;
1738 if self.resource_halt()? {
1739 return Ok(());
1740 }
1741 self.fold_losers().await?;
1742 self.sync_to_base().await?;
1747 if self.resource_halt()? {
1748 return Ok(());
1749 }
1750 if self.state.status == RunStatus::AlreadyInBase {
1751 return Ok(());
1752 }
1753 self.review_loop().await?;
1754 if self.resource_halt()? {
1755 return Ok(());
1756 }
1757 self.sync_to_base().await?;
1758 if self.resource_halt()? {
1759 return Ok(());
1760 }
1761 if self.state.status == RunStatus::AlreadyInBase {
1762 return Ok(());
1763 }
1764 self.gate().await?;
1765 self.merge().await?;
1766 if self.resource_halt()? {
1767 return Ok(());
1768 }
1769 self.state.save()?;
1770 Ok(())
1771 }
1772
1773 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1780 let result = match ensure_setup_config(&mut self.state, repo).await {
1781 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1782 Err(e) => Err(e),
1783 };
1784 if let Err(e) = result {
1785 git::worktree_remove_clean(repo, wt).await.ok();
1786 if in_prep {
1787 self.rollback_prep(repo).await;
1788 }
1789 self.state.status = RunStatus::Blocked;
1790 self.state.event(
1791 "setup",
1792 format!("worktree setup failed in {}: {e:#}", wt.display()),
1793 );
1794 self.state.save()?;
1795 return Err(e);
1796 }
1797 Ok(())
1798 }
1799
1800 async fn rollback_prep(&mut self, repo: &Path) {
1802 let root = self.state.worktree_root();
1803 for c in std::mem::take(&mut self.state.candidates) {
1804 if matches!(
1805 git::worktree_remove_clean(repo, &c.worktree).await,
1806 Ok(true)
1807 ) {
1808 git::branch_delete(repo, &c.branch).await.ok();
1809 } else {
1810 self.state.candidates.push(c);
1811 }
1812 }
1813 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1814 for j in 1..=seats {
1815 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1816 .await
1817 .ok();
1818 }
1819 for k in 1..=self.state.config.graph.advisors {
1820 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1821 .await
1822 .ok();
1823 }
1824 }
1825
1826 async fn prep(&mut self) -> Result<()> {
1827 if let Some(reason) = &self.state.prep_failure {
1828 bail!(
1829 "candidate preparation previously failed: {reason}; correct the source before retrying"
1830 );
1831 }
1832 if self
1833 .state
1834 .prep_complete
1835 .unwrap_or(!self.state.candidates.is_empty())
1836 {
1837 return Ok(());
1838 }
1839 let repo = self.state.repo.clone();
1840 let _coordination = self.coordinate(&repo).await?;
1843 let base = self.state.base_commit.clone();
1844 let plan = self.state.sources.plan(&repo, &base).await?;
1845 self.rollback_prep(&repo).await;
1846 if !self.state.candidates.is_empty() {
1847 bail!(
1848 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1849 );
1850 }
1851 self.state.prep_complete = Some(false);
1852 self.state.status = RunStatus::Prep;
1853 self.state.save()?;
1854 let repo = self.state.repo.clone();
1855 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1856 let root = self.state.worktree_root();
1857 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1858
1859 let hooks_dir = self.state.dir().join("hooks");
1862 if self.state.config.blind.commit_msg_hook {
1863 std::fs::create_dir_all(&hooks_dir)
1864 .with_context(|| format!("create {}", hooks_dir.display()))?;
1865 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1866 let path = hooks_dir.join("commit-msg");
1867 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1868 make_executable(&path)?;
1869 git::acquire_worktree_config(&repo).await?;
1877 self.state.enabled_worktree_config = true;
1878 }
1879
1880 ensure_setup_config(&mut self.state, &repo).await?;
1883
1884 for (index, (spec, label)) in self
1885 .roles
1886 .implementers
1887 .clone()
1888 .into_iter()
1889 .zip(labels)
1890 .enumerate()
1891 {
1892 let branch = self.state.branch_for(label);
1893 let worktree = root.join(format!("cand-{label}"));
1894 self.state.event(
1898 "prep-intent",
1899 serde_json::json!({
1900 "branch": branch,
1901 "worktree": worktree,
1902 "start": start,
1903 "picks": plan.picks,
1904 })
1905 .to_string(),
1906 );
1907 self.state.save()?;
1908 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1909 if self.state.config.blind.commit_msg_hook {
1910 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1911 }
1912 git::local_exclude(&worktree, "/.magi/").await?;
1913 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1914 git::worktree_remove_clean(&repo, &worktree).await.ok();
1917 git::branch_delete(&repo, &branch).await.ok();
1918 self.rollback_prep(&repo).await;
1919 self.state.status = RunStatus::Blocked;
1920 self.state
1921 .event("prep", format!("worktree setup failed: {e:#}"));
1922 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1923 self.state.save()?;
1924 return Err(e);
1925 }
1926 self.state.candidates.push(Candidate {
1927 index,
1928 label,
1929 agent: spec.id.clone(),
1930 branch,
1931 worktree,
1932 summary: String::new(),
1933 stat: String::new(),
1934 files: 0,
1935 commits: 0,
1936 empty: false,
1937 failed: None,
1938 verified_noop: None,
1939 duration_ms: 0,
1940 folded: false,
1941 });
1942 self.state.save()?;
1943 for pick in &plan.picks {
1944 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1945 {
1946 self.state.status = RunStatus::Blocked;
1947 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1948 self.state
1949 .event("prep", format!("cannot apply referenced commit: {e}"));
1950 self.state.save()?;
1951 return Err(e);
1952 }
1953 }
1954 }
1955
1956 for j in 1..=self.roles.judges.len() {
1957 let wt = root.join(format!("judge-{j}"));
1958 if !wt.exists() {
1959 git::worktree_add_detached(&repo, &wt, &base).await?;
1960 self.setup_seat_worktree(&repo, &wt, true).await?;
1961 }
1962 }
1963
1964 if self.state.config.graph.advise {
1972 for k in 1..=self.state.config.graph.advisors {
1973 let wt = root.join(format!("advisor-{k}"));
1974 if !wt.exists() {
1975 git::worktree_add_detached(&repo, &wt, &base).await?;
1976 self.setup_seat_worktree(&repo, &wt, true).await?;
1977 }
1978 }
1979 }
1980
1981 let authors: Vec<&str> = self
1986 .roles
1987 .implementers
1988 .iter()
1989 .map(|a| a.id.as_str())
1990 .collect();
1991 let overlap: Vec<String> = self
1992 .roles
1993 .judges
1994 .iter()
1995 .enumerate()
1996 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1997 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1998 .collect();
1999 if !overlap.is_empty() {
2000 let note = format!(
2001 "{} also authored a candidate; blind, but the panel is less \
2002 independent than {} distinct agents would be",
2003 overlap.join(", "),
2004 self.roles.judges.len()
2005 );
2006 self.state.event("prep", note);
2007 }
2008
2009 self.state.event(
2010 "prep",
2011 format!(
2012 "{} candidates, {} judges, base {} ({})",
2013 self.state.candidates.len(),
2014 self.roles.judges.len(),
2015 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2016 self.state.base_branch
2017 ),
2018 );
2019 self.state.prep_complete = Some(true);
2020 self.state.status = RunStatus::Implementing;
2021 self.state.save()?;
2022 Ok(())
2023 }
2024
2025 async fn advise(&mut self) -> Result<()> {
2058 let implement_untouched = self
2059 .state
2060 .candidates
2061 .iter()
2062 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2063 if !self.state.config.graph.advise || self.state.advise_attempted {
2064 return Ok(());
2065 }
2066 if !implement_untouched {
2067 self.state.event(
2068 "advise",
2069 "skipping the design-deliberation stage: at least one \
2070 candidate already shows implementation progress, so this \
2071 run is past the point the stage exists to run before"
2072 .to_owned(),
2073 );
2074 self.state.advise_attempted = true;
2075 self.state.save()?;
2076 return Ok(());
2077 }
2078 let run_id = self.state.id.clone();
2079 let prompts = self.state.config.prompts.clone();
2080 let instruction = self.state.instruction.clone();
2081 let language = self.state.config.graph.language.clone();
2082 let root = self.state.worktree_root();
2083 let n = self.state.config.graph.advisors;
2084 let where_recorded = self.state.dir().join("run.json");
2085
2086 let seats = match self.state.config.advisors() {
2087 Ok(seats) if !seats.is_empty() => seats,
2088 Ok(_) => {
2089 self.state.event(
2090 "advise",
2091 format!(
2092 "[graph] advisors is 0; skipping the design-deliberation \
2093 stage and continuing without a synthesis brief (see {})",
2094 where_recorded.display()
2095 ),
2096 );
2097 self.state.advise_attempted = true;
2098 self.state.save()?;
2099 return Ok(());
2100 }
2101 Err(e) => {
2102 self.state.event(
2103 "advise",
2104 format!(
2105 "could not resolve advisor seats ({e:#}); continuing \
2106 without a design-deliberation brief (see {})",
2107 where_recorded.display()
2108 ),
2109 );
2110 self.state.advise_attempted = true;
2111 self.state.save()?;
2112 return Ok(());
2113 }
2114 };
2115
2116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2117 let artifacts = agent::artifacts_dir(&self.state.dir());
2118 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2119
2120 let mut jobs = Vec::new();
2121 for (i, spec) in seats.iter().cloned().enumerate() {
2122 let seat_key = format!("advisor-{}", i + 1);
2123 let seat = self.seat(&seat_key, &spec.id);
2124 jobs.push(SeatJob {
2125 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2126 spec,
2127 seat,
2128 cwd: worktrees[i % worktrees.len()].clone(),
2129 timeout,
2130 allow_write: false,
2131 sessions: false,
2132 artifacts: artifacts.clone(),
2133 stem: seat_key,
2134 handover: None,
2135 });
2136 }
2137
2138 self.state.event(
2139 "advise",
2140 format!(
2141 "{} advisor seat(s) sketching a design in parallel",
2142 jobs.len()
2143 ),
2144 );
2145 let mut quota_losses = Vec::new();
2146 let cache = self.state.config.cache_dir();
2147 let ctx = WaveCtx {
2148 carry_seats: false,
2149 pauses: Some((&self.pause, &self.interrupt)),
2150 run: &run_id,
2151 node: "advise",
2152 prompts: &prompts,
2153 cache: cache.as_deref(),
2154 round: None,
2155 };
2156 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2157 let results = ask_json_wave::<Proposal>(
2158 jobs,
2159 Arc::clone(&self.sem),
2160 self.state.config.graph.retries,
2161 &advisor_roster,
2162 &ctx,
2163 &mut quota_losses,
2164 &mut self.state,
2165 &|p: &Proposal| p.validate(),
2166 )
2167 .await;
2168 if self.state.resource_block.is_some() {
2169 return self.stop_for_resource();
2170 }
2171 self.state.quota.extend(quota_losses);
2172
2173 let mut records = Vec::with_capacity(results.len());
2174 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2175 let agent_id = seat.agent.clone();
2176 self.state.seats.insert(seat.key.clone(), seat);
2177 match res {
2178 Ok((proposal, out)) => {
2179 self.state
2180 .event("advise", format!("advisor-{} proposed a design", i + 1));
2181 records.push(advise::AdvisorRecord::proposed(
2182 i + 1,
2183 agent_id,
2184 proposal,
2185 out.duration_ms,
2186 ));
2187 }
2188 Err(e) => {
2189 self.state.event(
2190 "advise",
2191 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2192 );
2193 records.push(advise::AdvisorRecord::failed(
2194 i + 1,
2195 agent_id,
2196 e.to_string(),
2197 ));
2198 }
2199 }
2200 }
2201
2202 let mut advice = advise::Advice {
2203 records,
2204 synthesis: None,
2205 };
2206 if advice.proposals().is_empty() {
2207 self.state.event(
2208 "advise",
2209 "no advisor produced a usable proposal; continuing without a \
2210 synthesis brief"
2211 .to_owned(),
2212 );
2213 } else {
2214 match self
2215 .synthesize_brief(
2216 &advice,
2217 &instruction,
2218 &language,
2219 &worktrees[0],
2220 &artifacts,
2221 &run_id,
2222 &prompts,
2223 cache.as_deref(),
2224 )
2225 .await
2226 {
2227 Ok(Some(text)) => {
2228 self.state.event(
2229 "advise",
2230 "synthesized a design brief for the implementer".to_owned(),
2231 );
2232 advice.synthesis = Some(text);
2233 }
2234 Ok(None) => {
2235 self.state.event(
2236 "advise",
2237 "the synthesis seat produced nothing usable; continuing \
2238 without a design brief"
2239 .to_owned(),
2240 );
2241 }
2242 Err(e) => {
2243 if e.is::<crate::repo_lock::ResourceBlocked>() {
2244 return Err(e);
2245 }
2246 self.state.event(
2247 "advise",
2248 format!("could not synthesize a design brief: {e:#}"),
2249 );
2250 }
2251 }
2252 }
2253 advise::apply_reflection(&mut advice);
2254
2255 self.state.advice = Some(advice);
2256 self.state.advise_attempted = true;
2257 self.state.save()?;
2258 Ok(())
2259 }
2260
2261 #[allow(clippy::too_many_arguments)]
2273 async fn synthesize_brief(
2274 &mut self,
2275 advice: &advise::Advice,
2276 instruction: &str,
2277 language: &str,
2278 cwd: &Path,
2279 artifacts: &Path,
2280 run_id: &str,
2281 prompts: &Prompts,
2282 cache: Option<&Path>,
2283 ) -> Result<Option<String>> {
2284 let chain = agent::pick_chain(
2285 &self.state.config.agents,
2286 self.state.config.roles.synthesizer.as_ref(),
2287 &agent::installed,
2288 "synthesizer",
2289 )?;
2290 let proposals = advice.proposals();
2291 let mut prompt = prompt::with_overlay(
2292 prompt::synthesize_brief(instruction, &proposals, language),
2293 prompts.overlay("advise"),
2294 );
2295 if cache.is_some() {
2296 prompt.push('\n');
2301 prompt.push_str(&prompt::build_cache_note("advise", false));
2302 }
2303 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2304 let mut last = None;
2309 for (n, spec) in chain.iter().enumerate() {
2310 if n > 0 {
2311 self.state
2312 .event("advise", format!("synthesis falling back to {}", spec.id));
2313 }
2314 let mut seat = self.seat("advise-synthesis", &spec.id);
2315 let wait = self.repository_wait();
2316 let outcome = wait
2317 .scope(agent::invoke(
2318 spec,
2319 &mut seat,
2320 &Invocation {
2321 coordination: self.state.config.coordination.as_ref(),
2322 fetched_base: None,
2323 cwd,
2324 prompt: &prompt,
2325 timeout,
2326 allow_write: false,
2327 sessions: false,
2328 artifacts,
2329 stem: &if n == 0 {
2330 "advise-synthesis".to_owned()
2331 } else {
2332 format!("advise-synthesis-{}", spec.id)
2333 },
2334 run: run_id,
2335 node: "advise",
2336 cache_dir: None,
2337 attachments: &[],
2338 writable: &[],
2339 },
2340 ))
2341 .await;
2342 if outcome.is_ok() {
2343 self.state.seats.insert(seat.key.clone(), seat);
2344 }
2345 let advance = agent::chain_advances(&outcome);
2346 last = Some(outcome);
2347 if !advance {
2348 break;
2349 }
2350 }
2351 let out = last.expect("a chain holds at least one agent")?;
2354 if !out.usable() {
2355 return Ok(None);
2356 }
2357 let text =
2358 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2359 Ok((!text.trim().is_empty()).then_some(text))
2360 }
2361
2362 async fn implement(&mut self) -> Result<()> {
2365 self.persist_candidates().await?;
2366 let run_id = self.state.id.clone();
2371 let prompts = self.state.config.prompts.clone();
2372 let todo: Vec<usize> = self
2373 .state
2374 .candidates
2375 .iter()
2376 .enumerate()
2377 .filter(|(_, c)| {
2378 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2379 && c.failed.is_none()
2380 && !c.empty
2381 })
2382 .map(|(i, _)| i)
2383 .collect();
2384 if todo.is_empty() {
2385 return self.after_implement();
2386 }
2387 self.state.status = RunStatus::Implementing;
2388
2389 let language = self.state.config.graph.language.clone();
2390 let github_language = self.state.config.graph.github_language.clone();
2391 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2392 let sessions = self.state.config.graph.sessions;
2393 let artifacts = agent::artifacts_dir(&self.state.dir());
2394 let brief = self
2398 .state
2399 .advice
2400 .as_ref()
2401 .and_then(|a| a.synthesis.as_deref())
2402 .map(str::to_owned);
2403 let attachments = self.state.attachments.clone();
2404
2405 let mut jobs = Vec::new();
2406 for &i in &todo {
2407 let (index, label, worktree) = {
2408 let c = &self.state.candidates[i];
2409 (c.index, c.label, c.worktree.clone())
2410 };
2411 let spec = self.roles.implementers[index].clone();
2412 let seat_key = format!("impl-{label}");
2413 let seat = self.seat(&seat_key, &spec.id);
2414 let instruction = seeded_instruction(&self.state);
2415 jobs.push(SeatJob {
2416 spec,
2417 seat,
2418 prompt: prompt::implement_in(
2419 &instruction,
2420 &worktree.to_string_lossy(),
2421 &language,
2422 &github_language,
2423 brief.as_deref(),
2424 &attachments,
2425 ),
2426 cwd: worktree,
2427 timeout,
2428 allow_write: true,
2429 sessions,
2430 artifacts: artifacts.clone(),
2431 stem: format!("impl-{label}"),
2432 handover: None,
2433 });
2434 }
2435
2436 self.state.event(
2437 "implement",
2438 format!("{} candidates in parallel", jobs.len()),
2439 );
2440 let mut sent = jobs.clone();
2446 let cache = self.state.config.cache_dir();
2447 let ctx = WaveCtx {
2448 carry_seats: false,
2449 pauses: Some((&self.pause, &self.interrupt)),
2450 run: &run_id,
2451 node: "implement",
2452 prompts: &prompts,
2453 cache: cache.as_deref(),
2454 round: None,
2455 };
2456 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2457 if !wave_blocked(&results) {
2458 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2459 .await;
2460 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2461 .await?;
2462 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2463 .await;
2464 }
2465 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2466
2467 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2468 && results
2469 .iter()
2470 .any(|(_, _, out)| FailClass::of(out).is_some());
2471 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2472 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2473 if self
2477 .state
2478 .candidate_preservation
2479 .contains_key(&self.state.candidates[i].label)
2480 {
2481 self.state.seats.insert(seat.key.clone(), seat);
2482 }
2483 continue;
2484 }
2485 let seat_key = seat.key.clone();
2486 let agent = seat.agent.clone();
2495 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2496 self.state.seats.insert(seat.key.clone(), seat);
2497 let label = self.state.candidates[i].label;
2498 self.state.interrupted_implementations.remove(&label);
2499
2500 let (summary, duration, failed, verified_claim) = match out {
2501 AgentOutcome::Ok(o) => {
2502 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2503 let failed = (!o.usable()).then(|| {
2504 if o.timed_out {
2505 "agent timed out".to_owned()
2506 } else {
2507 format!(
2508 "agent exited with {}",
2509 crate::notices::exit_status(o.exit_code)
2510 )
2511 }
2512 });
2513 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2514 (text, o.duration_ms, failed, verified_claim)
2515 }
2516 AgentOutcome::Dropped(o) => {
2522 let why = o
2523 .dropped
2524 .as_ref()
2525 .map(|d| d.why.as_str())
2526 .unwrap_or("the CLI ended the stream without delivering its answer");
2527 (
2528 String::new(),
2529 o.duration_ms,
2530 Some(format!("the CLI dropped the stream ({why})")),
2531 None,
2532 )
2533 }
2534 AgentOutcome::Quota(o) => {
2535 self.state.quota.push(QuotaLoss {
2536 seat: seat_key,
2537 node: "implement".to_owned(),
2538 at: Timestamp::now(),
2539 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2540 });
2541 (
2542 String::new(),
2543 o.duration_ms,
2544 Some("rate limited (quota); produced no change".to_owned()),
2545 None,
2546 )
2547 }
2548 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2549 (String::new(), 0, Some(e), None)
2550 }
2551 };
2552
2553 let c = &mut self.state.candidates[i];
2556 if !exhausted_the_fallback_chain {
2557 c.agent = agent;
2558 }
2559 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2560 c.duration_ms = duration;
2561 c.failed = failed;
2562 c.verified_noop = verified_claim;
2563 self.state
2564 .candidate_preservation
2565 .insert(label, "awaiting persistence".to_owned());
2566 }
2567 self.state.save()?;
2568 self.persist_candidates().await?;
2569
2570 if resource_blocked {
2571 return self.stop_for_resource();
2572 }
2573 if selected_failed {
2574 self.state.status = RunStatus::Blocked;
2575 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2576 self.state.save()?;
2577 bail!(
2578 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2579 );
2580 }
2581 self.after_implement()
2582 }
2583
2584 async fn persist_candidates(&mut self) -> Result<()> {
2586 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2587 for label in labels {
2588 let i = self
2589 .state
2590 .candidates
2591 .iter()
2592 .position(|c| c.label == label)
2593 .context("pending candidate is missing")?;
2594 let c = self.state.candidates[i].clone();
2595 let interrupted = self.state.interrupted_implementations.contains(&label);
2596 let result = async {
2597 let wait = crate::repo_lock::WaitPolicy::new(
2598 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2599 || false,
2600 );
2601 let _coordination =
2602 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2603 if let Err(e) = git::snapshot_uncommitted(
2606 &c.worktree,
2607 &self
2608 .state
2609 .dir()
2610 .join("artifacts")
2611 .join(format!("candidate-{label}-recovery")),
2612 )
2613 .await
2614 {
2615 self.state.event(
2616 "preserve",
2617 format!("candidate {label} snapshot incomplete: {e:#}"),
2618 );
2619 }
2620 let rescued = git::rescue_commit(
2621 &c.worktree,
2622 &format!("magi: candidate {label} (uncommitted work)"),
2623 )
2624 .await?;
2625 self.state.note_withheld("implement", &rescued.withheld);
2626 let base = &self.state.base_commit;
2627 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2628 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2629 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2630 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2631 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2632 if c.empty && commits > 0 && !patch.trim().is_empty()
2633 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2634 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2635 || self.state.tally.is_some())
2636 {
2637 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2641 }
2642 let c = &mut self.state.candidates[i];
2643 c.commits = commits;
2644 c.stat = stat;
2645 c.files = files;
2646 let no_changes = commits == 0 || patch.trim().is_empty();
2647 c.empty = no_changes && !interrupted;
2648 if interrupted && no_changes {
2649 c.failed = None;
2650 c.verified_noop = None;
2651 }
2652 if !c.empty {
2653 c.failed = None;
2654 c.verified_noop = None;
2655 }
2656 self.state.event(
2657 "preserve",
2658 format!("candidate {label}: {files} files, {commits} commits"),
2659 );
2660 Ok::<_, anyhow::Error>(())
2661 }
2662 .await;
2663 match result {
2664 Ok(()) => {
2665 self.state.candidate_preservation.remove(&label);
2666 if !interrupted || self.state.candidates[i].files > 0 {
2667 self.state.interrupted_implementations.remove(&label);
2668 }
2669 }
2670 Err(e) => {
2671 let reason = format!("candidate {label} preservation failed: {e:#}");
2672 self.state.event("preserve", &reason);
2673 self.state.candidate_preservation.insert(label, reason);
2674 }
2675 }
2676 if self.state.candidate_preservation.is_empty()
2677 && self.state.status == RunStatus::Blocked
2678 && self.state.candidates.iter().all(|c| c.failed.is_none())
2679 {
2680 self.state.status = RunStatus::Implementing;
2684 self.state.event(
2685 "preserve",
2686 "candidate persistence recovered; implementation can continue",
2687 );
2688 }
2689 self.state.save()?;
2690 }
2691 if !self.state.candidate_preservation.is_empty() {
2692 self.state.status = RunStatus::Blocked;
2693 self.state.save()?;
2694 bail!(
2695 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2696 );
2697 }
2698 Ok(())
2699 }
2700
2701 async fn resume_undelivered(
2729 &mut self,
2730 results: &mut [(usize, SeatState, AgentOutcome)],
2731 sent: &[SeatJob],
2732 prompts: &Prompts,
2733 run_id: &str,
2734 ) {
2735 for (wi, seat, out) in results.iter_mut() {
2736 let Some(dropped) = (match &*out {
2737 AgentOutcome::Dropped(o) => o.dropped.clone(),
2738 _ => None,
2739 }) else {
2740 continue;
2741 };
2742 let Some(job) = sent.get(*wi) else { continue };
2743 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2745 self.state.event(
2746 "implement",
2747 format!(
2748 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2749 work is in the tree",
2750 seat.key, dropped.output_tokens, dropped.why
2751 ),
2752 );
2753 continue;
2754 }
2755 if !has_context(&job.spec, seat, job.sessions) {
2763 self.state.event(
2764 "implement",
2765 format!(
2766 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2767 is no session left to resume",
2768 seat.key, dropped.output_tokens, dropped.why
2769 ),
2770 );
2771 continue;
2772 }
2773 self.state.event(
2774 "implement",
2775 format!(
2776 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2777 conversation",
2778 seat.key, dropped.output_tokens, dropped.why
2779 ),
2780 );
2781 let mut retry = job.clone();
2782 retry.seat = seat.clone();
2783 retry.prompt = prompt::resume_after_drop(&dropped.why);
2784 retry.timeout = retry_budget(job.timeout, true);
2785 retry.stem = format!("{}-resume", job.stem);
2786 let cache = self.state.config.cache_dir();
2787 let ctx = WaveCtx {
2788 carry_seats: false,
2789 pauses: Some((&self.pause, &self.interrupt)),
2790 run: run_id,
2791 node: "implement",
2792 prompts,
2793 cache: cache.as_deref(),
2794 round: None,
2795 };
2796 let (resumed_seat, resumed) =
2797 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2798 *seat = resumed_seat;
2799 *out = resumed;
2800 }
2801 }
2802
2803 async fn resume_seat_handovers(
2868 &mut self,
2869 results: &mut [(usize, SeatState, AgentOutcome)],
2870 sent: &mut [SeatJob],
2871 prompts: &Prompts,
2872 run_id: &str,
2873 ) -> Result<()> {
2874 let instruction = seeded_instruction(&self.state);
2875 let language = self.state.config.graph.language.clone();
2876 let github_language = self.state.config.graph.github_language.clone();
2877 let brief = self
2878 .state
2879 .advice
2880 .as_ref()
2881 .and_then(|a| a.synthesis.as_deref())
2882 .map(str::to_owned);
2883 let attachments = self.state.attachments.clone();
2884 for (wi, seat, out) in results.iter_mut() {
2885 let others: BTreeSet<String> = sent
2888 .iter()
2889 .enumerate()
2890 .filter(|(j, _)| j != wi)
2891 .map(|(_, j)| j.spec.id.clone())
2892 .collect();
2893 let Some(job) = sent.get_mut(*wi) else {
2894 continue;
2895 };
2896 let start = self
2901 .roles
2902 .implementer_roster
2903 .iter()
2904 .position(|s| s.id == job.spec.id)
2905 .unwrap_or(0);
2906 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2907 let mut fallback_attempt = 0usize;
2908 let mut prev: Option<FailClass> = None;
2909 while let Some(cur) = FailClass::of(&*out) {
2910 if !should_hand_over(prev.as_ref(), &cur) {
2911 break;
2912 }
2913 let Some(next) =
2914 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2915 .cloned()
2916 else {
2917 break;
2918 };
2919 tried.insert(next.id.clone());
2920 fallback_attempt += 1;
2921
2922 let coordination = self.coordinate(&job.cwd).await?;
2923 if let Ok(r) = git::rescue_commit(
2924 &job.cwd,
2925 &format!(
2926 "magi: candidate {} (uncommitted work before {} fallback)",
2927 seat.key,
2928 if cur == FailClass::Quota {
2929 "quota"
2930 } else {
2931 "handover"
2932 }
2933 ),
2934 )
2935 .await
2936 {
2937 self.state.note_withheld("implement", &r.withheld);
2938 }
2939 drop(coordination);
2940
2941 record_handover(
2942 &mut self.state,
2943 "implement",
2944 &seat.key,
2945 &seat.agent,
2946 &next.id,
2947 &cur,
2948 &fail_reason(&*out),
2949 );
2950 prev = Some(cur.clone());
2951
2952 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2953 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2954 job.spec = next.clone();
2962 let mut retry = job.clone();
2963 retry.seat = new_seat;
2964 retry.prompt = prompt::implement_in(
2965 &instruction,
2966 &job.cwd.to_string_lossy(),
2967 &language,
2968 &github_language,
2969 brief.as_deref(),
2970 &attachments,
2971 );
2972 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2973 let cache = self.state.config.cache_dir();
2974 let ctx = WaveCtx {
2975 carry_seats: false,
2976 pauses: Some((&self.pause, &self.interrupt)),
2977 run: run_id,
2978 node: "implement",
2979 prompts,
2980 cache: cache.as_deref(),
2981 round: None,
2982 };
2983 let (fallback_seat, fallback_out) = run_one(
2984 retry,
2985 Arc::clone(&self.sem),
2986 &ctx,
2987 &mut self.state,
2988 fallback_attempt,
2989 )
2990 .await;
2991 *seat = fallback_seat;
2992 *out = fallback_out;
2993 }
2994 }
2995 Ok(())
2996 }
2997
2998 async fn resume_unconfirmed_commands(
3022 &mut self,
3023 results: &mut [(usize, SeatState, AgentOutcome)],
3024 sent: &[SeatJob],
3025 prompts: &Prompts,
3026 run_id: &str,
3027 ) {
3028 for (wi, seat, out) in results.iter_mut() {
3029 let AgentOutcome::Ok(o) = &*out else {
3030 continue;
3031 };
3032 if !has_unconfirmed_command(&o.commands) {
3033 continue;
3034 }
3035 let Some(job) = sent.get(*wi) else { continue };
3036 if !has_context(&job.spec, seat, job.sessions) {
3037 self.state.event(
3038 "implement",
3039 format!(
3040 "{}: the reply named a command whose own CLI never confirmed the exit \
3041 status of, but there is no session left to resume",
3042 seat.key
3043 ),
3044 );
3045 continue;
3046 }
3047 self.state.event(
3048 "implement",
3049 format!(
3050 "{}: the reply named a command whose own CLI never confirmed the exit \
3051 status of; resuming the conversation",
3052 seat.key
3053 ),
3054 );
3055 let mut retry = job.clone();
3056 retry.seat = seat.clone();
3057 retry.prompt = prompt::resume_incomplete(
3058 "a command in your last reply had no confirmed exit status",
3059 );
3060 retry.timeout = retry_budget(job.timeout, true);
3061 retry.stem = format!("{}-confirm", job.stem);
3062 let cache = self.state.config.cache_dir();
3063 let ctx = WaveCtx {
3064 carry_seats: false,
3065 pauses: Some((&self.pause, &self.interrupt)),
3066 run: run_id,
3067 node: "implement",
3068 prompts,
3069 cache: cache.as_deref(),
3070 round: None,
3071 };
3072 let (resumed_seat, resumed) =
3073 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3074 *seat = resumed_seat;
3075 *out = resumed;
3076 }
3077 }
3078
3079 async fn continue_fix_report(
3100 &mut self,
3101 mut seat: SeatState,
3102 parse_err: String,
3103 job: &SeatJob,
3104 prompts: &Prompts,
3105 run_id: &str,
3106 round: usize,
3107 ) -> (
3108 SeatState,
3109 Option<FixReport>,
3110 Option<String>,
3111 ContinuationRecord,
3112 ) {
3113 let mut last_err = parse_err;
3114 let mut cumulative_wait_ms = 0u64;
3115 let mut attempts = 0usize;
3116 loop {
3117 if !has_context(&job.spec, &seat, job.sessions) {
3118 self.state.event(
3119 "fix",
3120 format!(
3121 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3122 session left to resume into"
3123 ),
3124 );
3125 let outcome = if attempts == 0 {
3126 ContinuationOutcome::NoSession
3127 } else {
3128 ContinuationOutcome::Exhausted
3129 };
3130 return (
3131 seat,
3132 None,
3133 Some(format!("unparsable fix report: {last_err}")),
3134 ContinuationRecord {
3135 attempts,
3136 cumulative_wait_ms,
3137 outcome,
3138 },
3139 );
3140 }
3141 if attempts >= MAX_FIX_CONTINUATIONS {
3142 self.state.event(
3143 "fix",
3144 format!(
3145 "round {round}: fixer's reply still had no adoption report after \
3146 {attempts} continuation(s) ({last_err}); giving up"
3147 ),
3148 );
3149 return (
3150 seat,
3151 None,
3152 Some(format!(
3153 "unparsable fix report after {attempts} continuation(s): {last_err}"
3154 )),
3155 ContinuationRecord {
3156 attempts,
3157 cumulative_wait_ms,
3158 outcome: ContinuationOutcome::Exhausted,
3159 },
3160 );
3161 }
3162 attempts += 1;
3163 self.state.event(
3164 "fix",
3165 format!(
3166 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3167 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3168 ),
3169 );
3170 let mut retry = job.clone();
3171 retry.seat = seat.clone();
3172 retry.prompt = prompt::resume_incomplete(&last_err);
3173 retry.timeout = retry_budget(job.timeout, true);
3174 retry.stem = format!("{}-continue{attempts}", job.stem);
3175 let cache = self.state.config.cache_dir();
3176 let ctx = WaveCtx {
3177 carry_seats: false,
3178 pauses: Some((&self.pause, &self.interrupt)),
3179 run: run_id,
3180 node: "fix",
3181 prompts,
3182 cache: cache.as_deref(),
3183 round: Some(round),
3184 };
3185 let (resumed_seat, resumed_out) = run_one(
3186 retry,
3187 Arc::clone(&self.sem),
3188 &ctx,
3189 &mut self.state,
3190 attempts,
3191 )
3192 .await;
3193 seat = resumed_seat;
3194 match resumed_out {
3195 AgentOutcome::Ok(o) => {
3196 cumulative_wait_ms += o.duration_ms;
3197 match verdict::extract_json::<FixReport>(&o.text) {
3198 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3199 self.state.event(
3200 "fix",
3201 format!(
3202 "round {round}: fixer's adoption report recovered after \
3203 {attempts} continuation(s)"
3204 ),
3205 );
3206 return (
3207 seat,
3208 Some(report),
3209 None,
3210 ContinuationRecord {
3211 attempts,
3212 cumulative_wait_ms,
3213 outcome: ContinuationOutcome::Resumed,
3214 },
3215 );
3216 }
3217 Ok(_) => {
3225 last_err = "the reply parsed, but it reported a command whose own CLI \
3226 never confirmed an exit status"
3227 .to_owned();
3228 }
3229 Err(e) => last_err = e.to_string(),
3230 }
3231 }
3232 AgentOutcome::Quota(o) => {
3233 cumulative_wait_ms += o.duration_ms;
3234 self.state.quota.push(QuotaLoss {
3235 seat: seat.key.clone(),
3236 node: "fix".to_owned(),
3237 at: Timestamp::now(),
3238 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3239 });
3240 self.state.event(
3241 "fix",
3242 format!(
3243 "round {round}: continuation rate limited (quota); not retrying now"
3244 ),
3245 );
3246 return (
3247 seat,
3248 None,
3249 Some("rate limited (quota) while recovering the fix report".to_owned()),
3250 ContinuationRecord {
3251 attempts,
3252 cumulative_wait_ms,
3253 outcome: ContinuationOutcome::QuotaLost,
3254 },
3255 );
3256 }
3257 AgentOutcome::Dropped(o) => {
3258 cumulative_wait_ms += o.duration_ms;
3259 let why = o
3260 .dropped
3261 .as_ref()
3262 .map(|d| d.why.as_str())
3263 .unwrap_or("the CLI ended the stream without delivering its answer");
3264 last_err = format!("the CLI dropped the stream ({why})");
3265 }
3266 AgentOutcome::Failed(e) => last_err = e,
3267 AgentOutcome::ResourceBlocked(e) => {
3270 return (
3271 seat,
3272 None,
3273 Some(format!("the continuation was not started: {e}")),
3274 ContinuationRecord {
3275 attempts: attempts.saturating_sub(1),
3276 cumulative_wait_ms,
3277 outcome: ContinuationOutcome::NoSession,
3278 },
3279 );
3280 }
3281 }
3282 }
3283 }
3284
3285 fn after_implement(&mut self) -> Result<()> {
3286 if !self.state.candidate_preservation.is_empty() {
3287 bail!("candidate preservation incomplete; judging is unsafe");
3288 }
3289 if self.state.leaks.is_empty() {
3291 let cfg = self.state.config.blind.clone();
3292 let mut leaks = Vec::new();
3293 for c in &self.state.candidates {
3294 let Some(patch) =
3295 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3296 else {
3297 continue;
3298 };
3299 leaks.extend(blind::scan(
3300 &format!("candidate {} patch", c.label),
3301 &patch,
3302 &cfg.vendor_tokens,
3303 ));
3304 }
3305 if !leaks.is_empty() {
3306 let summary = leaks
3307 .iter()
3308 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3309 .collect::<Vec<_>>()
3310 .join(", ");
3311 match cfg.on_leak {
3312 LeakPolicy::Fail => {
3313 self.state.status = RunStatus::Failed;
3314 self.state
3315 .event("blind", format!("vendor text in a patch: {summary}"));
3316 self.state.leaks = leaks;
3317 self.state.save()?;
3318 self.settle_questions();
3319 bail!(
3320 "blind.on_leak = \"fail\" and vendor text reached a \
3321 judged patch: {summary}"
3322 );
3323 }
3324 LeakPolicy::Redact => self.state.event(
3325 "blind",
3326 format!("redacting vendor text for judging: {summary}"),
3327 ),
3328 LeakPolicy::Warn => self.state.event(
3329 "blind",
3330 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3331 ),
3332 }
3333 self.state.leaks = leaks;
3334 }
3335 }
3336
3337 if self.state.viable().is_empty() {
3338 if self.state.all_candidates_verified_noop() {
3339 self.state.status = RunStatus::VerifiedNoop;
3350 self.state.save()?;
3351 self.settle_questions();
3352 return Ok(());
3353 }
3354 self.state.status = RunStatus::Failed;
3355 self.state.save()?;
3356 self.settle_questions();
3357 bail!("no candidate produced a change; nothing to judge");
3358 }
3359 self.state.status = RunStatus::Judging;
3360 self.state.save()?;
3361 Ok(())
3362 }
3363
3364 async fn judge(&mut self) -> Result<()> {
3367 let run_id = self.state.id.clone();
3372 let prompts = self.state.config.prompts.clone();
3373 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3374 return Ok(());
3375 }
3376 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3377 if viable.len() == 1 {
3378 self.state.judge_skipped = true;
3385 self.state.event(
3386 "judge",
3387 format!(
3388 "only candidate {} produced a change; judging skipped",
3389 viable[0].label
3390 ),
3391 );
3392 self.state.save()?;
3393 return Ok(());
3394 }
3395 self.state.status = RunStatus::Judging;
3396
3397 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3398 let language = self.state.config.graph.language.clone();
3399 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3400 let sessions = self.state.config.graph.sessions;
3401 let artifacts = agent::artifacts_dir(&self.state.dir());
3402 let root = self.state.worktree_root();
3403 let base_short = short(&self.state.base_commit);
3404
3405 let mut jobs = Vec::new();
3406 let mut orders = Vec::new();
3407 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3408 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3409 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3410 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3411 let seat_key = format!("judge-{}", j + 1);
3412 let seat = self.seat(&seat_key, &spec.id);
3413 jobs.push(SeatJob {
3414 prompt: prompt::judge(
3415 &self.state.instruction,
3416 &views,
3417 self.roles.judges.len(),
3418 &base_short,
3419 &language,
3420 ),
3421 spec,
3422 seat,
3423 cwd: root.join(format!("judge-{}", j + 1)),
3424 timeout,
3425 allow_write: false,
3426 sessions,
3427 artifacts: artifacts.clone(),
3428 stem: format!("judge-{}", j + 1),
3429 handover: None,
3430 });
3431 }
3432
3433 self.state.event(
3434 "judge",
3435 format!(
3436 "{} judges ranking {} candidates blind",
3437 jobs.len(),
3438 viable.len()
3439 ),
3440 );
3441 let labels_for_check = labels.clone();
3442 let mut quota_losses = Vec::new();
3443 let cache = self.state.config.cache_dir();
3444 let ctx = WaveCtx {
3445 carry_seats: false,
3446 pauses: Some((&self.pause, &self.interrupt)),
3447 run: &run_id,
3448 node: "judge",
3449 prompts: &prompts,
3450 cache: cache.as_deref(),
3451 round: None,
3452 };
3453 let results = ask_json_wave::<Ranking>(
3454 jobs,
3455 Arc::clone(&self.sem),
3456 self.state.config.graph.retries,
3457 &self.roles.judge_roster,
3458 &ctx,
3459 &mut quota_losses,
3460 &mut self.state,
3461 &move |r: &Ranking| r.validate(&labels_for_check),
3462 )
3463 .await;
3464 if self.state.resource_block.is_some() {
3465 return self.stop_for_resource();
3466 }
3467 self.state.quota.extend(quota_losses);
3468
3469 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3470 let agent_id = seat.agent.clone();
3471 self.state.seats.insert(seat.key.clone(), seat);
3472 let mut record = Judgement {
3473 judge: j + 1,
3474 seat: format!("judge-{}", j + 1),
3475 agent: agent_id,
3476 ranking: Vec::new(),
3477 reasons: BTreeMap::new(),
3478 confidence: None,
3479 strengths: BTreeMap::new(),
3480 order: orders[j].clone(),
3481 failed: None,
3482 duration_ms: 0,
3483 };
3484 match res {
3485 Ok((ranking, out)) => {
3486 record.ranking = ranking.normalized();
3487 record.reasons = ranking.reasons;
3488 record.strengths = ranking.strengths;
3489 record.confidence = ranking.confidence;
3490 record.duration_ms = out.duration_ms;
3491 self.state.event(
3492 "judge",
3493 format!(
3494 "judge {} ranked {}",
3495 j + 1,
3496 record.ranking.iter().collect::<String>()
3497 ),
3498 );
3499 }
3500 Err(e) => {
3501 record.failed = Some(e.to_string());
3502 self.state
3503 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3504 }
3505 }
3506 self.state.judgements.push(record);
3507 self.state.save()?;
3508 }
3509 Ok(())
3510 }
3511
3512 async fn deliberate(&mut self) -> Result<()> {
3515 let run_id = self.state.id.clone();
3520 let prompts = self.state.config.prompts.clone();
3521 if !self.state.deliberation.is_empty() {
3522 return Ok(());
3523 }
3524 let tops: Vec<char> = self
3525 .state
3526 .judgements
3527 .iter()
3528 .filter_map(|j| j.ranking.first().copied())
3529 .collect();
3530 let rounds = self.state.config.graph.deliberate_rounds;
3531 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3532 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3533 self.state.event(
3534 "deliberate",
3535 format!("judges agreed on {} outright; no deliberation", tops[0]),
3536 );
3537 }
3538 self.state.status = RunStatus::Voting;
3539 self.state.save()?;
3540 return Ok(());
3541 }
3542
3543 self.state.status = RunStatus::Deliberating;
3544 self.state.event(
3545 "deliberate",
3546 format!(
3547 "split: first choices were {} — opening {rounds} round(s)",
3548 tops.iter().collect::<String>()
3549 ),
3550 );
3551
3552 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3553 let language = self.state.config.graph.language.clone();
3554 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3555 let sessions = self.state.config.graph.sessions;
3556 let artifacts = agent::artifacts_dir(&self.state.dir());
3557 let root = self.state.worktree_root();
3558 let base_short = short(&self.state.base_commit);
3559
3560 for round in 1..=rounds {
3564 let mut turns: Vec<DeliberationTurn> = Vec::new();
3565 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3566 if self.state.judgements[j].failed.is_some() {
3567 continue;
3568 }
3569 let seat_key = format!("judge-{}", j + 1);
3570 let spec = self.occupant(&seat_key, spec);
3571 let mut seat = self.seat(&seat_key, &spec.id);
3572 let transcript = self.transcript(&turns, j);
3573 let build = |context: Option<&str>| {
3574 prompt::deliberate(
3575 &self.state.instruction,
3576 context,
3577 &transcript,
3578 round,
3579 rounds,
3580 &language,
3581 )
3582 };
3583 let block = self.candidate_block(&viable, &base_short);
3584 let full = build(Some(&block));
3585 let text = if has_context(&spec, &seat, sessions) {
3586 build(None)
3587 } else {
3588 full.clone()
3589 };
3590 let job = SeatJob {
3591 spec,
3592 seat: seat.clone(),
3593 prompt: text,
3594 cwd: root.join(format!("judge-{}", j + 1)),
3595 timeout,
3596 allow_write: false,
3597 sessions,
3598 artifacts: artifacts.clone(),
3599 stem: format!("delib-{round}-judge-{}", j + 1),
3600 handover: Some(full),
3601 };
3602 let cache = self.state.config.cache_dir();
3603 let ctx = WaveCtx {
3604 carry_seats: false,
3605 pauses: Some((&self.pause, &self.interrupt)),
3606 run: &run_id,
3607 node: "deliberate",
3608 prompts: &prompts,
3609 cache: cache.as_deref(),
3610 round: None,
3611 };
3612 let mut losses = Vec::new();
3616 let mut results = ask_wave_with::<String>(
3617 vec![job],
3618 Arc::clone(&self.sem),
3619 0,
3620 &self.roles.judge_roster,
3621 &ctx,
3622 &mut losses,
3623 &mut self.state,
3624 &|text: &str| {
3625 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3626 },
3627 )
3628 .await;
3629 self.state.quota.extend(losses);
3630 let (updated, res, _) = results.pop().expect("one job in, one result out");
3631 seat = updated;
3632 let agent_id = seat.agent.clone();
3633 self.state.seats.insert(seat.key.clone(), seat);
3634 let body = match res {
3635 Ok((body, _)) => body,
3636 Err(e) => {
3639 self.state
3640 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3641 continue;
3642 }
3643 };
3644 let tentative = verdict::extract_json::<Position>(&body)
3645 .ok()
3646 .and_then(|p| p.tentative)
3647 .and_then(|s| s.trim().chars().next())
3648 .map(|c| c.to_ascii_uppercase());
3649 self.state.event(
3650 "deliberate",
3651 format!(
3652 "round {round}: judge {} now favours {}",
3653 j + 1,
3654 tentative.map_or("—".to_owned(), |c| c.to_string())
3655 ),
3656 );
3657 turns.push(DeliberationTurn {
3658 judge: j + 1,
3659 agent: agent_id,
3660 body: blind::sanitize_prose(&body, &self.state.config.blind),
3661 tentative,
3662 });
3663 }
3664 self.state
3665 .deliberation
3666 .push(DeliberationRound { round, turns });
3667 self.state.save()?;
3668 }
3669
3670 self.state.status = RunStatus::Voting;
3671 self.state.save()?;
3672 Ok(())
3673 }
3674
3675 async fn vote(&mut self) -> Result<()> {
3678 let run_id = self.state.id.clone();
3683 let prompts = self.state.config.prompts.clone();
3684 if !self.state.votes.is_empty() {
3685 return Ok(());
3686 }
3687 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3688 if viable.len() == 1 {
3689 return Ok(());
3690 }
3691 self.state.status = RunStatus::Voting;
3692
3693 let language = self.state.config.graph.language.clone();
3694 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3695 let sessions = self.state.config.graph.sessions;
3696 let artifacts = agent::artifacts_dir(&self.state.dir());
3697 let root = self.state.worktree_root();
3698 let base_short = short(&self.state.base_commit);
3699 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3700
3701 let mut jobs = Vec::new();
3702 let mut seats_at = Vec::new();
3703 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3704 if self
3705 .state
3706 .judgements
3707 .get(j)
3708 .is_some_and(|r| r.failed.is_some())
3709 {
3710 continue;
3711 }
3712 let seat_key = format!("judge-{}", j + 1);
3713 let spec = self.occupant(&seat_key, spec);
3714 let seat = self.seat(&seat_key, &spec.id);
3715 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3716 let text = if has_context(&spec, &seat, sessions) {
3717 prompt::final_vote(&viable, &language)
3718 } else {
3719 full.clone()
3720 };
3721 jobs.push(SeatJob {
3722 spec,
3723 seat,
3724 prompt: text,
3725 cwd: root.join(format!("judge-{}", j + 1)),
3726 timeout,
3727 allow_write: false,
3728 sessions,
3729 artifacts: artifacts.clone(),
3730 stem: format!("vote-judge-{}", j + 1),
3731 handover: Some(full),
3732 });
3733 seats_at.push(j);
3734 }
3735
3736 self.state.event(
3737 "vote",
3738 format!(
3739 "collecting {} final votes one by one, privately",
3740 jobs.len()
3741 ),
3742 );
3743 let allowed = viable.clone();
3744 let mut quota_losses = Vec::new();
3745 let cache = self.state.config.cache_dir();
3746 let ctx = WaveCtx {
3747 carry_seats: false,
3748 pauses: Some((&self.pause, &self.interrupt)),
3749 run: &run_id,
3750 node: "vote",
3751 prompts: &prompts,
3752 cache: cache.as_deref(),
3753 round: None,
3754 };
3755 let results = ask_json_wave::<FinalVote>(
3756 jobs,
3757 Arc::clone(&self.sem),
3758 self.state.config.graph.retries,
3759 &self.roles.judge_roster,
3760 &ctx,
3761 &mut quota_losses,
3762 &mut self.state,
3763 &move |v: &FinalVote| match v.label() {
3764 Some(c) if allowed.contains(&c) => Ok(()),
3765 other => bail!("vote {other:?} is not one of {allowed:?}"),
3766 },
3767 )
3768 .await;
3769 if self.state.resource_block.is_some() {
3770 return self.stop_for_resource();
3771 }
3772 self.state.quota.extend(quota_losses);
3773
3774 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3775 let agent_id = seat.agent.clone();
3776 self.state.seats.insert(seat.key.clone(), seat);
3777 let initial = self
3778 .state
3779 .judgements
3780 .get(j)
3781 .and_then(|r| r.ranking.first().copied());
3782 let mut record = VoteRecord {
3783 judge: j + 1,
3784 agent: agent_id,
3785 vote: None,
3786 reason: String::new(),
3787 changed: false,
3788 };
3789 match res {
3790 Ok((v, _)) => {
3791 record.vote = v.label();
3792 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3793 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3794 self.state.event(
3795 "vote",
3796 format!(
3797 "judge {} voted {}{}",
3798 j + 1,
3799 record.vote.unwrap_or('?'),
3800 if record.changed { " (changed)" } else { "" }
3801 ),
3802 );
3803 }
3804 Err(e) => {
3805 self.state
3806 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3807 }
3808 }
3809 self.state.votes.push(record);
3810 self.state.save()?;
3811 }
3812 Ok(())
3813 }
3814
3815 fn tally(&mut self) -> Result<()> {
3818 if self.state.tally.is_some() {
3819 return Ok(());
3820 }
3821 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3822 let tops: Vec<char> = self
3823 .state
3824 .judgements
3825 .iter()
3826 .filter_map(|j| j.ranking.first().copied())
3827 .collect();
3828 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3829
3830 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3833 let mut cast: Vec<char> = Vec::new();
3834 for (i, j) in self.state.judgements.iter().enumerate() {
3835 let vote = self
3836 .state
3837 .votes
3838 .iter()
3839 .find(|v| v.judge == i + 1)
3840 .and_then(|v| v.vote)
3841 .or_else(|| j.ranking.first().copied());
3842 if let Some(v) = vote {
3843 *first_choice.entry(v).or_insert(0) += 1;
3844 cast.push(v);
3845 }
3846 }
3847
3848 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3849 for j in &self.state.judgements {
3850 let n = j.ranking.len();
3851 for (pos, label) in j.ranking.iter().enumerate() {
3852 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3853 }
3854 }
3855
3856 let best = first_choice.values().copied().max().unwrap_or(0);
3857 let mut leaders: Vec<char> = first_choice
3858 .iter()
3859 .filter(|(_, v)| **v == best)
3860 .map(|(k, _)| *k)
3861 .collect();
3862 let mut tie_break = None;
3863 if leaders.len() > 1 {
3864 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3865 let borda_leaders: Vec<char> = leaders
3866 .iter()
3867 .copied()
3868 .filter(|l| borda[l] == top_borda)
3869 .collect();
3870 tie_break = Some(if borda_leaders.len() == 1 {
3871 format!(
3872 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3873 leaders.len()
3874 )
3875 } else {
3876 format!(
3877 "{} way tie on both first-choice votes and Borda points, broken by label order",
3878 leaders.len()
3879 )
3880 });
3881 leaders = borda_leaders;
3882 leaders.sort_unstable();
3883 }
3884 let winner = *leaders
3885 .first()
3886 .or(viable.first())
3887 .context("no candidate to declare a winner from")?;
3888
3889 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3890 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3891 let deliberated = !self.state.deliberation.is_empty();
3892
3893 let quota_seats: std::collections::BTreeSet<&str> =
3897 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3898 let mut present = 0usize;
3899 for (i, j) in self.state.judgements.iter().enumerate() {
3900 if quota_seats.contains(j.seat.as_str()) {
3901 continue;
3902 }
3903 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3904 let voted = self
3905 .state
3906 .votes
3907 .iter()
3908 .any(|v| v.judge == i + 1 && v.vote.is_some());
3909 if ranked || voted {
3910 present += 1;
3911 }
3912 }
3913 let needs_quorum = viable.len() > 1;
3919 let judges_total = if needs_quorum {
3920 self.roles.judges.len()
3921 } else {
3922 0
3923 };
3924 let quorum = if needs_quorum {
3925 judges_total / 2 + 1
3926 } else {
3927 0
3928 };
3929 let met_quorum = !needs_quorum || present >= quorum;
3930 let uncontested = (!needs_quorum).then(|| {
3931 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3932 });
3933
3934 self.state.event(
3935 "tally",
3936 match &uncontested {
3937 Some(reason) => format!("winner {winner} — {reason}"),
3938 None => format!(
3939 "winner {winner} — votes {} | initial {} | {} changed | \
3940 {present}/{judges_total} judges{}",
3941 first_choice
3942 .iter()
3943 .map(|(k, v)| format!("{k}:{v}"))
3944 .collect::<Vec<_>>()
3945 .join(" "),
3946 if unanimous_initial {
3947 "unanimous"
3948 } else {
3949 "split"
3950 },
3951 changed_votes,
3952 if met_quorum {
3953 String::new()
3954 } else {
3955 format!(" — below quorum ({quorum} required)")
3956 },
3957 ),
3958 },
3959 );
3960 if !met_quorum {
3961 self.state.event(
3962 "stall",
3963 format!(
3964 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3965 the run stops here, resumable"
3966 ),
3967 );
3968 }
3969 self.state.tally = Some(Tally {
3970 first_choice,
3971 borda,
3972 winner,
3973 rankings: tops.len(),
3974 unanimous_initial,
3975 deliberated,
3976 changed_votes,
3977 unanimous_final,
3978 tie_break,
3979 judges: judges_total,
3980 present,
3981 quorum,
3982 met_quorum,
3983 uncontested,
3984 });
3985 self.state.status = if met_quorum {
3986 RunStatus::Reviewing
3987 } else {
3988 RunStatus::Stalled
3989 };
3990 self.state.save()?;
3991 Ok(())
3992 }
3993
3994 #[allow(clippy::too_many_lines)]
4015 async fn recover_stall(&mut self) -> Result<bool> {
4016 let run_id = self.state.id.clone();
4021 let prompts = self.state.config.prompts.clone();
4022 let quota_seats: BTreeSet<&str> =
4027 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4028 let absent: Vec<String> = self
4029 .state
4030 .judgements
4031 .iter()
4032 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4033 .map(|j| j.seat.clone())
4034 .collect();
4035 if absent.is_empty() {
4036 return Ok(false);
4037 }
4038 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4039 if viable.len() <= 1 {
4040 return Ok(false);
4041 }
4042 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4043 let language = self.state.config.graph.language.clone();
4044 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4045 let sessions = self.state.config.graph.sessions;
4046 let artifacts = agent::artifacts_dir(&self.state.dir());
4047 let root = self.state.worktree_root();
4048 let base_short = short(&self.state.base_commit);
4049 let candidates: Vec<Candidate> = viable.clone();
4050
4051 let mut positions: Vec<usize> = absent
4053 .iter()
4054 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4055 .collect();
4056 if positions.is_empty() {
4057 return Ok(false);
4058 }
4059 positions.sort_unstable();
4060 positions.dedup();
4061
4062 let mut judge_jobs = Vec::new();
4064 for &j in &positions {
4065 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4066 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4067 let seat_key = format!("judge-{}", j + 1);
4068 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4069 let seat = self.seat(&seat_key, &spec.id);
4070 judge_jobs.push(SeatJob {
4071 spec,
4072 seat,
4073 prompt: prompt::judge(
4074 &self.state.instruction,
4075 &views,
4076 self.roles.judges.len(),
4077 &base_short,
4078 &language,
4079 ),
4080 cwd: root.join(seat_key),
4081 timeout,
4082 allow_write: false,
4083 sessions,
4084 artifacts: artifacts.clone(),
4085 stem: format!("judge-{}-recover", j + 1),
4086 handover: None,
4087 });
4088 }
4089
4090 let labels_for_check = labels.clone();
4091 let mut judge_losses = Vec::new();
4092 let retries = self.state.config.graph.retries;
4093 let cache = self.state.config.cache_dir();
4094 let ctx = WaveCtx {
4095 carry_seats: false,
4096 pauses: Some((&self.pause, &self.interrupt)),
4097 run: &run_id,
4098 node: "judge",
4099 prompts: &prompts,
4100 cache: cache.as_deref(),
4101 round: None,
4102 };
4103 let results = ask_json_wave::<Ranking>(
4104 judge_jobs,
4105 Arc::clone(&self.sem),
4106 retries,
4107 &self.roles.judge_roster,
4108 &ctx,
4109 &mut judge_losses,
4110 &mut self.state,
4111 &move |r: &Ranking| r.validate(&labels_for_check),
4112 )
4113 .await;
4114 if self.state.resource_block.is_some() {
4115 self.stop_for_resource()?;
4116 return Ok(false);
4117 }
4118
4119 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4121 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4122 let agent_id = seat.agent.clone();
4123 self.state.seats.insert(seat.key.clone(), seat);
4124 let record = &mut self.state.judgements[j];
4125 match res {
4126 Ok((ranking, out)) => {
4127 record.agent = agent_id;
4128 record.ranking = ranking.normalized();
4129 record.reasons = ranking.reasons;
4130 record.strengths = ranking.strengths;
4131 record.confidence = ranking.confidence;
4132 record.failed = None;
4133 record.duration_ms = out.duration_ms;
4134 recovered.insert(j);
4135 self.state.event(
4136 "recover",
4137 format!("judge {} ranked again after the limit", j + 1),
4138 );
4139 }
4140 Err(e) => {
4141 self.state
4142 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4143 }
4144 }
4145 }
4146
4147 let mut vote_jobs = Vec::new();
4149 let mut vote_pos: Vec<usize> = Vec::new();
4150 for &j in &recovered {
4151 let seat_key = format!("judge-{}", j + 1);
4152 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4153 let seat = self.seat(&seat_key, &spec.id);
4154 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4155 let text = if has_context(&spec, &seat, sessions) {
4156 prompt::final_vote(&labels, &language)
4157 } else {
4158 full.clone()
4159 };
4160 vote_jobs.push(SeatJob {
4161 spec,
4162 seat,
4163 prompt: text,
4164 cwd: root.join(seat_key),
4165 timeout,
4166 allow_write: false,
4167 sessions,
4168 artifacts: artifacts.clone(),
4169 stem: format!("vote-judge-{}-recover", j + 1),
4170 handover: Some(full),
4171 });
4172 vote_pos.push(j);
4173 }
4174 let allowed = labels.clone();
4175 let mut vote_losses = Vec::new();
4176 let vote_retries = self.state.config.graph.retries;
4177 let vote_cache = self.state.config.cache_dir();
4178 let ctx = WaveCtx {
4179 carry_seats: false,
4180 pauses: Some((&self.pause, &self.interrupt)),
4181 run: &run_id,
4182 node: "vote",
4183 prompts: &prompts,
4184 cache: vote_cache.as_deref(),
4185 round: None,
4186 };
4187 let votes = ask_json_wave::<FinalVote>(
4188 vote_jobs,
4189 Arc::clone(&self.sem),
4190 vote_retries,
4191 &self.roles.judge_roster,
4192 &ctx,
4193 &mut vote_losses,
4194 &mut self.state,
4195 &move |v: &FinalVote| match v.label() {
4196 Some(c) if allowed.contains(&c) => Ok(()),
4197 other => bail!("vote {other:?} is not one of {allowed:?}"),
4198 },
4199 )
4200 .await;
4201 if self.state.resource_block.is_some() {
4202 self.stop_for_resource()?;
4203 return Ok(false);
4204 }
4205 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4206 let agent_id = seat.agent.clone();
4207 self.state.seats.insert(seat.key.clone(), seat);
4208 match res {
4209 Ok((v, _)) => {
4210 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4211 rec.vote = v.label();
4212 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4213 } else {
4214 self.state.votes.push(VoteRecord {
4215 judge: j + 1,
4216 agent: agent_id,
4217 vote: v.label(),
4218 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4219 changed: false,
4220 });
4221 }
4222 self.state.event(
4223 "recover",
4224 format!("judge {} voted again after the limit", j + 1),
4225 );
4226 }
4227 Err(e) => {
4228 self.state
4229 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4230 }
4231 }
4232 }
4233
4234 let recovered_keys: BTreeSet<String> = recovered
4238 .iter()
4239 .map(|&j| format!("judge-{}", j + 1))
4240 .collect();
4241 self.state
4242 .quota
4243 .retain(|q| !recovered_keys.contains(&q.seat));
4244 for loss in judge_losses.into_iter().chain(vote_losses) {
4248 if recovered_keys.contains(&loss.seat) {
4249 continue;
4250 }
4251 self.state.quota.retain(|q| q.seat != loss.seat);
4252 self.state.quota.push(loss);
4253 }
4254
4255 self.state.tally = None;
4257 self.tally()?;
4258 Ok(self
4259 .state
4260 .tally
4261 .as_ref()
4262 .map(|t| t.met_quorum)
4263 .unwrap_or(false))
4264 }
4265
4266 async fn graft(&mut self) -> Result<()> {
4282 let Err(e) = self.graft_inner().await else {
4283 return Ok(());
4284 };
4285 if e.is::<crate::repo_lock::ResourceBlocked>() {
4286 return Err(e);
4287 }
4288 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4289 self.state.status = RunStatus::Blocked;
4290 self.state.event("graft", format!("blocked: {e:#}"));
4291 self.state.save()?;
4292 return Err(e);
4293 }
4294 if let Some(rec) = self.state.graft.clone()
4299 && !rec.outcome.settled()
4300 {
4301 if let Some(winner) = self.state.winner().cloned() {
4302 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4303 if re.is::<crate::repo_lock::ResourceBlocked>() {
4304 return Err(re);
4305 }
4306 self.state.status = RunStatus::Blocked;
4307 self.state.event("graft", format!("blocked: {re:#}"));
4308 self.state.save()?;
4309 return Err(re);
4310 }
4311 }
4312 if let Some(g) = self.state.graft.as_mut() {
4313 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4314 }
4315 }
4316 self.state.event("graft", format!("skipped: {e:#}"));
4317 if let Err(e) = self.state.save() {
4318 tracing::warn!("save after graft: {e:#}");
4319 }
4320 Ok(())
4321 }
4322
4323 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4326 let _coordination = self.coordinate(wt).await?;
4327 crate::recovery::restore_repair(wt, pre)
4328 .await
4329 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4330 }
4331
4332 fn graft_skipped(&mut self, why: &str) {
4334 let message = format!("skipped: {why}");
4335 if !self
4336 .state
4337 .events
4338 .iter()
4339 .any(|e| e.node == "graft" && e.message == message)
4340 {
4341 self.state.event("graft", message);
4342 }
4343 }
4344
4345 async fn graft_inner(&mut self) -> Result<()> {
4346 if let Some(rec) = self.state.graft.clone()
4349 && !rec.outcome.settled()
4350 {
4351 if self.state.reviews.is_empty()
4355 && let Some(winner) = self.state.winner().cloned()
4356 {
4357 self.graft_restore(&winner.worktree, &rec.pre).await?;
4358 }
4359 if rec.outcome == GraftOutcome::WaitingForRestore {
4360 self.state.graft = None;
4361 self.state.save()?;
4362 } else {
4363 if let Some(g) = self.state.graft.as_mut() {
4364 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4365 }
4366 self.state.event(
4367 "graft",
4368 "rolled back: the previous round was interrupted; the winner is untouched",
4369 );
4370 self.state.save()?;
4371 return Ok(());
4372 }
4373 }
4374 let plan = match graft::plan(&self.state) {
4375 Ok(plan) => plan,
4376 Err(skip) => {
4377 if skip != graft::Skip::AlreadyDone {
4378 self.graft_skipped(skip.reason());
4379 self.state.save()?;
4380 }
4381 return Ok(());
4382 }
4383 };
4384 let Some(winner) = self.state.winner().cloned() else {
4385 return Ok(());
4386 };
4387 let wt = winner.worktree.clone();
4388 if !git::is_clean(&wt).await? {
4389 self.graft_skipped("the winner's worktree has uncommitted changes");
4390 self.state.save()?;
4391 return Ok(());
4392 }
4393 let pre = git::rev_parse(&wt, "HEAD").await?;
4394 self.state.graft = Some(GraftRecord {
4397 pre: pre.clone(),
4398 post: None,
4399 lines: None,
4400 limit: None,
4401 agent: None,
4402 outcome: GraftOutcome::Running,
4403 requested: graft::requested(&plan),
4404 adoption: None,
4405 adoption_error: None,
4406 });
4407 self.state.event(
4408 "graft",
4409 format!(
4410 "porting strengths of candidate(s) {} onto {}",
4411 plan.strengths
4412 .iter()
4413 .map(|(l, _, _)| l.to_string())
4414 .collect::<Vec<_>>()
4415 .join(", "),
4416 plan.winner
4417 ),
4418 );
4419 self.state.save()?;
4420
4421 let fix_prompt = prompt::graft(
4422 &self.state.instruction,
4423 plan.winner,
4424 &plan.strengths,
4425 &self.state.config.graph.language,
4426 );
4427 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4428 let sessions = self.state.config.graph.sessions;
4429 let artifacts = agent::artifacts_dir(&self.state.dir());
4430 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4431 let (_, seat, out) = self
4432 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4433 SeatJob {
4434 prompt: fix_prompt.clone(),
4435 spec,
4436 seat,
4437 cwd: wt.clone(),
4438 timeout,
4439 allow_write: true,
4440 sessions,
4441 artifacts: artifacts.clone(),
4442 stem: "graft".to_owned(),
4443 handover: None,
4444 }
4445 })
4446 .await;
4447 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4448 self.state.graft.as_mut().expect("graft record").outcome =
4451 GraftOutcome::WaitingForRestore;
4452 return self.stop_for_resource();
4453 }
4454 let mut why: Option<String> = None;
4455 match &out {
4456 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4457 why = Some("the fixer produced nothing usable".to_owned());
4458 }
4459 AgentOutcome::Ok(_) => {}
4460 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4461 AgentOutcome::Quota(o) => {
4462 self.state.quota.push(QuotaLoss {
4463 seat: seat.key.clone(),
4464 node: "graft".to_owned(),
4465 at: Timestamp::now(),
4466 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4467 });
4468 why = Some("rate limited (quota)".to_owned());
4469 }
4470 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4471 why = Some(format!(
4472 "the fixer failed: {}",
4473 e.lines()
4474 .next()
4475 .unwrap_or("")
4476 .chars()
4477 .take(160)
4478 .collect::<String>()
4479 ));
4480 }
4481 }
4482 let agent_id = seat.agent.clone();
4483 self.state.seats.insert(seat.key.clone(), seat);
4484
4485 let mut post = None;
4486 let mut lines = None;
4487 let _coordination = self.coordinate(&wt).await?;
4488 if why.is_none() {
4489 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4490 if let Ok(r) = &rescued {
4491 self.state.note_withheld("graft", &r.withheld);
4492 }
4493 let head = git::rev_parse(&wt, "HEAD").await?;
4494 let dirty = match &rescued {
4498 Ok(r) => git::status_porcelain(&wt)
4499 .await
4500 .map(|st| {
4501 st.lines().any(|l| {
4502 !r.withheld
4503 .iter()
4504 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4505 })
4506 })
4507 .unwrap_or(true),
4508 Err(_) => true,
4509 };
4510 if dirty {
4511 why = Some("the fixer left changes that could not be committed".to_owned());
4512 } else if head == pre {
4513 why = Some("the fixer changed nothing".to_owned());
4514 } else {
4515 lines = git::changed_lines(&wt, &pre, &head).await;
4516 post = Some(head);
4517 }
4518 }
4519 let (adoption, adoption_error) = match &out {
4520 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4521 _ => (None, None),
4522 };
4523 let requested = graft::requested(&plan);
4524 let outcome = match why {
4525 None => {
4526 self.state.event(
4527 "graft",
4528 format!(
4529 "applied: {}",
4530 lines.map_or_else(
4531 || "changed lines unavailable".to_owned(),
4532 |n| format!("{n} changed lines")
4533 )
4534 ),
4535 );
4536 GraftOutcome::Applied
4537 }
4538 Some(why) => {
4539 crate::recovery::restore_repair(&wt, &pre)
4542 .await
4543 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4544 self.state.event(
4545 "graft",
4546 format!("rolled back: {why}; the winner is untouched"),
4547 );
4548 GraftOutcome::RolledBack(why)
4549 }
4550 };
4551 self.state.graft = Some(GraftRecord {
4552 pre,
4553 post,
4554 lines,
4555 limit: None,
4556 agent: Some(agent_id),
4557 outcome,
4558 requested,
4559 adoption,
4560 adoption_error,
4561 });
4562 self.state.save()?;
4563 Ok(())
4564 }
4565
4566 async fn fold_losers(&mut self) -> Result<()> {
4569 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4570 return Ok(());
4571 };
4572 if !self.state.candidate_preservation.is_empty() {
4573 bail!("candidate preservation incomplete; refusing cleanup");
4574 }
4575 let repo = self.state.repo.clone();
4576 let _coordination = self.coordinate(&repo).await?;
4577 let mut folded = Vec::new();
4578 for i in 0..self.state.candidates.len() {
4579 let c = &self.state.candidates[i];
4580 if c.label == winner || c.folded {
4581 continue;
4582 }
4583 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4584 remove_preserved_worktree(&repo, &wt).await?;
4585 if git::branch_exists(&repo, &branch).await?
4586 && !git::branch_delete(&repo, &branch).await?
4587 {
4588 bail!("candidate branch removal refused");
4589 }
4590 self.state.candidates[i].folded = true;
4591 self.state.save()?;
4592 folded.push(label.to_string());
4593 }
4594 let root = self.state.worktree_root();
4596 for j in 1..=self.roles.judges.len() {
4597 let wt = root.join(format!("judge-{j}"));
4598 if wt.exists() {
4599 remove_preserved_worktree(&repo, &wt).await?;
4600 }
4601 }
4602 if self.state.config.graph.advise {
4605 for k in 1..=self.state.config.graph.advisors {
4606 let wt = root.join(format!("advisor-{k}"));
4607 if wt.exists() {
4608 remove_preserved_worktree(&repo, &wt).await?;
4609 }
4610 }
4611 }
4612 if !folded.is_empty() {
4613 self.state
4614 .event("fold", format!("folded candidates {}", folded.join(", ")));
4615 self.state.save()?;
4616 }
4617 Ok(())
4618 }
4619
4620 async fn sync_to_base(&mut self) -> Result<()> {
4660 if self.state.status == RunStatus::AlreadyInBase {
4661 return Ok(());
4662 }
4663 let conflicted = self
4664 .state
4665 .base_sync
4666 .as_ref()
4667 .is_some_and(|s| s.conflict.is_some());
4668 let Some(winner) = self.state.winner().cloned() else {
4669 return Ok(());
4670 };
4671
4672 let repo = self.state.repo.clone();
4673 let remote = self.state.config.merge.remote.clone();
4674 let base_branch = self.state.base_branch.clone();
4675 let tracking = format!("{remote}/{base_branch}");
4676
4677 git::fetch(&repo, &remote, &base_branch).await.ok();
4678 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4683 return Ok(());
4684 };
4685
4686 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4687 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4688 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4689
4690 if (behind > 0 || conflicted || head == tip)
4699 && self
4700 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4701 .await?
4702 {
4703 return Ok(());
4704 }
4705 if conflicted {
4706 return Ok(());
4707 }
4708
4709 if behind == 0 {
4710 if let Some(from) = self
4719 .state
4720 .rebase_fixes
4721 .iter()
4722 .rev()
4723 .find_map(|r| r.from.clone())
4724 && from != head
4725 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4726 .await
4727 .is_ok_and(|o| o.ok())
4728 {
4729 git::sync_to_head(&winner.worktree).await?;
4730 }
4731 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4737 if let Some(why) = &conflict {
4738 self.state.status = RunStatus::Blocked;
4739 self.state.event("land", why.clone());
4740 }
4741 self.state.base_sync = Some(BaseSync {
4742 tip,
4743 behind: 0,
4744 attempts,
4745 conflict,
4746 already_in: None,
4747 });
4748 self.state.save()?;
4749 return Ok(());
4750 }
4751
4752 if attempts >= BASE_SYNC_ROUNDS {
4753 let why = format!(
4754 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4755 rebase(s); rebasing again would only race it",
4756 winner.branch
4757 );
4758 self.state.status = RunStatus::Blocked;
4759 self.state.base_sync = Some(BaseSync {
4760 tip,
4761 behind,
4762 attempts,
4763 conflict: Some(why.clone()),
4764 already_in: None,
4765 });
4766 self.state.event("land", why);
4767 self.state.save()?;
4768 return Ok(());
4769 }
4770
4771 self.state.event(
4772 "land",
4773 format!(
4774 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4775 winner.branch
4776 ),
4777 );
4778 self.state.save()?;
4779
4780 let branch_tracking = format!("{remote}/{}", winner.branch);
4786 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4787 let remote_tip = if self.state.task_pr_branch().is_none()
4788 && matches!(&fetched_branch, Ok(o) if o.ok())
4789 {
4790 git::rev_parse(&repo, &branch_tracking).await.ok()
4791 } else {
4792 None
4793 };
4794 if let Some(theirs) = &remote_tip
4798 && !git::is_ancestor(&repo, theirs, &head).await
4799 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4800 .await
4801 .is_ok_and(|missing| missing.is_empty())
4802 {
4803 let why = format!(
4804 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4805 them",
4806 short(theirs),
4807 winner.branch
4808 );
4809 self.state.status = RunStatus::Blocked;
4810 self.state.base_sync = Some(BaseSync {
4811 tip,
4812 behind,
4813 attempts,
4814 conflict: Some(why.clone()),
4815 already_in: None,
4816 });
4817 self.state.event("land", why);
4818 self.state.save()?;
4819 return Ok(());
4820 }
4821
4822 let scratch = self.state.dir().join("base-sync");
4823 let rebased = match crate::rebase::rebase_with_fixer(
4824 &mut self.state,
4825 &scratch,
4826 &winner.branch,
4827 &tracking,
4828 )
4829 .await
4830 {
4831 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4832 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4833 Err(e) => Err(e),
4834 };
4835 let attempts = attempts + 1;
4836 match rebased {
4837 Ok(None) => {
4838 git::sync_to_head(&winner.worktree).await?;
4842 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4843 let mut conflict = None;
4844 if let Some(pinned) = &remote_tip {
4845 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4846 match pushed {
4847 Ok(o) if o.ok() => self.state.event(
4848 "land",
4849 format!("pushed rebased {} to {remote}", winner.branch),
4850 ),
4851 Ok(o) => {
4852 conflict = Some(format!(
4853 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4854 winner.branch,
4855 short(pinned),
4856 o.stderr.chars().take(600).collect::<String>()
4857 ));
4858 }
4859 Err(e) => {
4860 conflict = Some(format!(
4861 "rebased {} locally but could not push it: {e:#}",
4862 winner.branch
4863 ));
4864 }
4865 }
4866 }
4867 if let Some(why) = &conflict {
4868 self.state.status = RunStatus::Blocked;
4869 self.state.event("land", why.clone());
4870 }
4871 self.state.base_sync = Some(BaseSync {
4872 tip: tip.clone(),
4873 behind: 0,
4874 attempts,
4875 conflict,
4876 already_in: None,
4877 });
4878 self.state
4879 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4880 }
4881 Ok(Some(conflict)) => {
4882 let why = format!(
4883 "{} conflicts with {tracking} and did not rebase: {}",
4884 winner.branch,
4885 conflict.chars().take(600).collect::<String>()
4886 );
4887 self.state.status = RunStatus::Blocked;
4888 self.state.base_sync = Some(BaseSync {
4889 tip,
4890 behind,
4891 attempts,
4892 conflict: Some(why.clone()),
4893 already_in: None,
4894 });
4895 self.state.event("land", why);
4896 }
4897 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4901 self.state.save()?;
4902 return Ok(());
4903 }
4904 Err(e) => {
4905 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4906 self.state.status = RunStatus::Blocked;
4907 self.state.base_sync = Some(BaseSync {
4908 tip,
4909 behind,
4910 attempts,
4911 conflict: Some(why.clone()),
4912 already_in: None,
4913 });
4914 self.state.event("land", why);
4915 }
4916 }
4917 self.state.save()?;
4918 Ok(())
4919 }
4920
4921 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4927 if self.state.task_pr_branch().is_some() {
4930 return None;
4931 }
4932 let repo = self.state.repo.clone();
4933 let remote = self.state.config.merge.remote.clone();
4934 let fetched = git::fetch(&repo, &remote, branch).await;
4935 if !matches!(&fetched, Ok(o) if o.ok()) {
4936 return None;
4937 }
4938 let branch_tracking = format!("{remote}/{branch}");
4939 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4940 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4941 return None;
4942 }
4943 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4944 .await
4945 .is_ok_and(|missing| missing.is_empty())
4946 {
4947 return Some(format!(
4948 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4949 them",
4950 short(&theirs)
4951 ));
4952 }
4953 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4954 Ok(o) if o.ok() => {
4955 self.state
4956 .event("land", format!("pushed rebased {branch} to {remote}"));
4957 None
4958 }
4959 Ok(o) => Some(format!(
4960 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4961 someone may have pushed): {}",
4962 short(&theirs),
4963 o.stderr.chars().take(600).collect::<String>()
4964 )),
4965 Err(e) => Some(format!(
4966 "{branch} is rebased locally but could not be pushed: {e:#}"
4967 )),
4968 }
4969 }
4970
4971 async fn settle_already_in(
4988 &mut self,
4989 branch: &str,
4990 tip: &str,
4991 head: &str,
4992 attempts: usize,
4993 behind: usize,
4994 ) -> Result<bool> {
4995 let repo = self.state.repo.clone();
4996 let remote = self.state.config.merge.remote.clone();
4997 let start = self.state.base_commit.clone();
4998 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4999 Ok(Some(e)) => e,
5000 Ok(None) => return Ok(false),
5001 Err(e) => {
5002 tracing::warn!("already-in-base check for {branch}: {e:#}");
5003 return Ok(false);
5004 }
5005 };
5006 let mut verified = vec![head.to_owned()];
5007 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5008 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5009 if matches!(&fetched, Ok(o) if o.ok())
5010 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5011 && theirs != head
5012 {
5013 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5014 Ok(Some(_)) => verified.push(theirs),
5015 _ => return Ok(false),
5016 }
5017 }
5018 let base_branch = self.state.base_branch.clone();
5019 let message = format!(
5020 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5021 land",
5022 evidence.names(),
5023 evidence.proof.as_str()
5024 );
5025 let closed =
5026 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5027 .await;
5028 match closed {
5029 Ok(Ok(url)) => self
5030 .state
5031 .event("land", format!("closed {url}: superseded on {base_branch}")),
5032 Ok(Err(why)) => self
5033 .state
5034 .event("land", format!("did not close a pull request: {why}")),
5035 Err(e) => {
5036 let why = format!(
5037 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5038 not be closed ({e:#}); resume to retry"
5039 );
5040 self.state.status = RunStatus::Blocked;
5041 self.state.base_sync = Some(BaseSync {
5042 tip: tip.to_owned(),
5043 behind,
5044 attempts,
5045 conflict: Some(why.clone()),
5046 already_in: None,
5047 });
5048 self.state.event("land", why);
5049 self.state.save()?;
5050 return Ok(true);
5051 }
5052 }
5053 self.state.status = RunStatus::AlreadyInBase;
5054 self.state.base_sync = Some(BaseSync {
5055 tip: tip.to_owned(),
5056 behind,
5057 attempts,
5058 conflict: None,
5059 already_in: Some(evidence),
5060 });
5061 self.state.event("land", message);
5062 self.state.save()?;
5063 self.settle_questions();
5064 Ok(true)
5065 }
5066
5067 fn landing_base(&self) -> String {
5077 self.state
5078 .base_sync
5079 .as_ref()
5080 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5081 }
5082
5083 pub async fn fix_selected(
5116 &mut self,
5117 ids: &[String],
5118 reason: &str,
5119 allow_stale: bool,
5120 ) -> Result<()> {
5121 let reason = reason.trim();
5122 if reason.is_empty() {
5123 bail!("a fix request needs a reason — that is the operator's own record of why");
5124 }
5125 if ids.is_empty() {
5126 bail!("no finding id given");
5127 }
5128 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5129 bail!(
5130 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5131 has already concluded — can be given a targeted fix. A run still \
5132 in progress should simply be resumed; a `merged` run's branch has \
5133 already landed, so its answer is a fresh `magi review <branch>`, \
5134 not reopening this run's own record",
5135 self.state.id,
5136 self.state.status.as_str()
5137 );
5138 }
5139 let Some(winner) = self.state.winner().cloned() else {
5140 bail!("run {} has no winning candidate to fix", self.state.id);
5141 };
5142 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5143 bail!(
5144 "branch `{}` no longer exists; this run cannot be extended",
5145 winner.branch
5146 );
5147 }
5148 let home = crate::run::home();
5149 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5150 bail!(
5151 "run {} is currently being worked on by another magi process",
5152 self.state.id
5153 );
5154 }
5155 let _claim = FixClaim::acquire(&self.state.dir())?;
5161
5162 let mut seen = BTreeSet::new();
5166 let mut findings = Vec::new();
5167 let mut missing = Vec::new();
5168 for id in ids {
5169 if !seen.insert(id.clone()) {
5170 continue;
5171 }
5172 match self.state.finding(id) {
5173 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5174 id: f.id.clone(),
5175 severity: f.severity,
5176 reviewer_vote: rec.vote,
5177 round: round.round,
5178 round_head: round.head.clone(),
5179 reviewer: rec.reviewer,
5180 agent: rec.agent.clone(),
5181 file: f.file.clone(),
5182 line: f.line,
5183 title: f.title.clone(),
5184 detail: f.detail.clone(),
5185 outcome: OperatorFixOutcome::Pending,
5186 }),
5187 None => missing.push(id.clone()),
5188 }
5189 }
5190 if !missing.is_empty() {
5191 bail!(
5192 "unknown finding id(s): {}; nothing was changed",
5193 missing.join(", ")
5194 );
5195 }
5196
5197 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5198 let stale_details: Vec<(String, String)> = findings
5199 .iter()
5200 .filter(|f| f.round_head != head_at_request)
5201 .map(|f| (f.id.clone(), f.round_head.clone()))
5202 .collect();
5203 let stale = !stale_details.is_empty();
5204 if stale && !allow_stale {
5205 bail!(
5206 "the branch has moved since some finding(s) were raised — {} — now \
5207 at {}; pass --allow-stale to fix anyway, or re-run review first",
5208 stale_details
5209 .iter()
5210 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5211 .collect::<Vec<_>>()
5212 .join(", "),
5213 short(&head_at_request)
5214 );
5215 }
5216
5217 let request = OperatorFixRequest {
5218 requested_at: Timestamp::now(),
5219 reason: reason.to_owned(),
5220 findings,
5221 head_at_request: head_at_request.clone(),
5222 allow_stale,
5223 stale,
5224 fix: None,
5225 result_head: None,
5226 follow_up_review_run: None,
5227 };
5228 self.state.event(
5229 "fix",
5230 format!(
5231 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5232 request.findings.len(),
5233 request
5234 .findings
5235 .iter()
5236 .map(|f| f.id.as_str())
5237 .collect::<Vec<_>>()
5238 .join(", "),
5239 ),
5240 );
5241 self.state.operator_fixes.push(request);
5248 self.state.save()?;
5249 let request_index = self.state.operator_fixes.len() - 1;
5250
5251 let repo = self.state.repo.clone();
5260 let coordination = self.coordinate(&repo).await?;
5261 if winner.worktree.exists() {
5262 let dirty = git::git(
5265 &winner.worktree,
5266 &["status", "--porcelain", "--untracked-files=all"],
5267 )
5268 .await?;
5269 let only_withheld = dirty.lines().all(|l| {
5270 l.strip_prefix("?? ")
5271 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5272 });
5273 if !only_withheld {
5274 bail!(
5275 "`{}` has uncommitted changes; refusing to touch it — commit or \
5276 discard them first",
5277 winner.worktree.display()
5278 );
5279 }
5280 git::worktree_remove(&self.state.repo, &winner.worktree)
5281 .await
5282 .ok();
5283 }
5284 let fix_worktree = self.state.worktree_root().join("operator-fix");
5285 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5286 git::git(
5287 &self.state.repo,
5288 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5289 )
5290 .await
5291 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5292 if !git::is_clean(&fix_worktree).await? {
5293 git::worktree_remove(&self.state.repo, &fix_worktree)
5294 .await
5295 .ok();
5296 bail!(
5297 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5298 winner.branch
5299 );
5300 }
5301 let repo_for_setup = self.state.repo.clone();
5302 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5303 Ok(()) => {
5304 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5305 }
5306 Err(e) => Err(e),
5307 };
5308 if let Err(e) = setup {
5309 git::worktree_remove(&self.state.repo, &fix_worktree)
5310 .await
5311 .ok();
5312 return Err(e);
5313 }
5314
5315 let run_id = self.state.id.clone();
5316 let prompts = self.state.config.prompts.clone();
5317 let language = self.state.config.graph.language.clone();
5318 let sessions = self.state.config.graph.sessions;
5319 let artifacts = agent::artifacts_dir(&self.state.dir());
5320 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5321 .findings
5322 .iter()
5323 .map(|f| Finding {
5324 id: f.id.clone(),
5325 severity: f.severity,
5326 file: f.file.clone(),
5327 line: f.line,
5328 title: f.title.clone(),
5329 detail: f.detail.clone(),
5330 })
5331 .collect();
5332 let fix_prompt = prompt::operator_fix_in(
5333 &self.state.instruction,
5334 &finding_list,
5335 reason,
5336 &stale_details,
5337 &head_at_request,
5338 &language,
5339 &self.state.config.graph.github_language,
5340 );
5341 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5342 drop(coordination);
5343 let (job, seat, out) = self
5344 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5345 prompt: fix_prompt.clone(),
5346 spec,
5347 seat,
5348 cwd: fix_worktree.clone(),
5349 timeout,
5350 allow_write: true,
5351 sessions,
5352 artifacts: artifacts.clone(),
5353 stem: "operator-fix".to_owned(),
5354 handover: None,
5355 })
5356 .await;
5357 let agent_id = seat.agent.clone();
5358
5359 let mut fix = FixRecord {
5360 agent: agent_id,
5361 addressed: Vec::new(),
5362 rejected: Vec::new(),
5363 notes: String::new(),
5364 committed: false,
5365 failed: None,
5366 duration_ms: 0,
5367 continuation: None,
5368 };
5369 let mut final_seat = seat.clone();
5370 match out {
5371 AgentOutcome::Ok(o) => {
5372 fix.duration_ms = o.duration_ms;
5373 let parsed = verdict::extract_json::<FixReport>(&o.text);
5374 let incomplete_reason = match &parsed {
5375 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5376 "the reply parsed, but it reported a command whose own CLI \
5377 never confirmed an exit status"
5378 .to_owned(),
5379 ),
5380 Ok(_) => None,
5381 Err(e) => Some(e.to_string()),
5382 };
5383 match incomplete_reason {
5384 None => {
5385 let report = parsed.expect("checked Ok above");
5386 fix.addressed = report.addressed;
5387 fix.rejected = report.rejected;
5388 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5389 }
5390 Some(reason) => {
5391 let (resumed_seat, resolved, failure, cont) = self
5392 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5393 .await;
5394 fix.duration_ms += cont.cumulative_wait_ms;
5395 fix.continuation = Some(cont);
5396 final_seat = resumed_seat;
5397 match resolved {
5398 Some(report) => {
5399 fix.addressed = report.addressed;
5400 fix.rejected = report.rejected;
5401 fix.notes =
5402 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5403 }
5404 None => fix.failed = failure,
5405 }
5406 }
5407 }
5408 }
5409 AgentOutcome::Dropped(o) => {
5410 fix.duration_ms = o.duration_ms;
5411 let why = o
5412 .dropped
5413 .as_ref()
5414 .map(|d| d.why.as_str())
5415 .unwrap_or("the CLI ended the stream without delivering its answer");
5416 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5417 }
5418 AgentOutcome::Quota(o) => {
5419 self.state.quota.push(QuotaLoss {
5420 seat: final_seat.key.clone(),
5421 node: "fix".to_owned(),
5422 at: Timestamp::now(),
5423 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5424 });
5425 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5426 }
5427 AgentOutcome::Failed(e) => fix.failed = Some(e),
5428 AgentOutcome::ResourceBlocked(e) => {
5429 fix.failed = Some(format!("the fixer was not started: {e}"));
5430 }
5431 }
5432 if fix.continuation.is_none() {
5433 fix.continuation = Some(ContinuationRecord::not_needed());
5434 }
5435 self.state.seats.insert(final_seat.key.clone(), final_seat);
5436
5437 let rescue_message = format!(
5438 "magi: operator-selected fix ({}) (uncommitted work)",
5439 self.state.operator_fixes[request_index]
5440 .findings
5441 .iter()
5442 .map(|f| f.id.as_str())
5443 .collect::<Vec<_>>()
5444 .join(", ")
5445 );
5446 let repo = self.state.repo.clone();
5447 let coordination = self.coordinate(&repo).await?;
5448 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5449 self.state.note_withheld("fix", &r.withheld);
5450 }
5451 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5452 fix.committed = after != head_at_request;
5453 git::worktree_remove(&self.state.repo, &fix_worktree)
5454 .await
5455 .ok();
5456 drop(coordination);
5457
5458 self.state.event(
5459 "fix",
5460 match &fix.failed {
5461 Some(reason) => format!(
5462 "operator fix: adoption report was lost ({reason}); {}",
5463 if fix.committed {
5464 "committed"
5465 } else {
5466 "NO new commit"
5467 }
5468 ),
5469 None => format!(
5470 "operator fix: {} addressed, {} rejected, {}",
5471 fix.addressed.len(),
5472 fix.rejected.len(),
5473 if fix.committed {
5474 "committed"
5475 } else {
5476 "NO new commit"
5477 }
5478 ),
5479 },
5480 );
5481
5482 for f in &mut self.state.operator_fixes[request_index].findings {
5489 f.outcome = if fix.failed.is_some() {
5490 OperatorFixOutcome::Unreported
5491 } else if fix.addressed.contains(&f.id) {
5492 OperatorFixOutcome::Addressed
5493 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5494 OperatorFixOutcome::Rejected { why: r.why.clone() }
5495 } else {
5496 OperatorFixOutcome::Unreported
5497 };
5498 }
5499
5500 let committed = fix.committed;
5501 if committed {
5502 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5503 }
5504 self.state.operator_fixes[request_index].fix = Some(fix);
5505 self.state.save()?;
5508
5509 if committed {
5510 self.state.event(
5511 "fix",
5512 format!(
5513 "operator fix committed {}; opening a follow-up review-only run",
5514 short(&after)
5515 ),
5516 );
5517 let origin =
5520 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5521 match Self::review(
5522 &self.state.repo,
5523 &winner.branch,
5524 self.state.config.clone(),
5525 origin,
5526 )
5527 .await
5528 {
5529 Ok(mut follow_up) => {
5530 follow_up.state.event(
5531 "start",
5532 format!(
5533 "requested by an operator fix on run {} for finding(s) {}",
5534 self.state.id,
5535 self.state.operator_fixes[request_index]
5536 .findings
5537 .iter()
5538 .map(|f| f.id.as_str())
5539 .collect::<Vec<_>>()
5540 .join(", "),
5541 ),
5542 );
5543 follow_up.state.save()?;
5544 let follow_up_id = follow_up.state.id.clone();
5545 let adopted = match crate::direct::adopt(
5548 &follow_up.state,
5549 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5550 ) {
5551 Ok(a) => a,
5552 Err(e) => {
5553 self.state.event(
5556 "fix",
5557 format!(
5558 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5559 ),
5560 );
5561 self.state.save()?;
5562 return Ok(());
5563 }
5564 };
5565 let executed = follow_up.execute().await;
5566 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5567 if let Some(a) = adopted {
5568 a.finish(&follow_up.state, executed);
5569 }
5570 if let Some(e) = failure {
5571 self.state.event(
5572 "fix",
5573 format!(
5574 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5575 ),
5576 );
5577 }
5578 self.state.operator_fixes[request_index].follow_up_review_run =
5579 Some(follow_up_id);
5580 }
5581 Err(e) => {
5582 self.state.event(
5583 "fix",
5584 format!("committed the fix but could not open a follow-up review: {e:#}"),
5585 );
5586 }
5587 }
5588 self.state.save()?;
5589 }
5590
5591 Ok(())
5592 }
5593
5594 async fn ask_fixer(
5614 &mut self,
5615 winner: &Candidate,
5616 node: &'static str,
5617 round: Option<usize>,
5618 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5619 ) -> (SeatJob, SeatState, AgentOutcome) {
5620 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5621 self.ask_fixer_among(attempts, node, round, None, build)
5622 .await
5623 }
5624
5625 async fn ask_fixer_among(
5630 &mut self,
5631 attempts: Vec<(AgentSpec, String)>,
5632 node: &'static str,
5633 round: Option<usize>,
5634 restore: Option<(&Path, &str)>,
5635 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5636 ) -> (SeatJob, SeatState, AgentOutcome) {
5637 let run_id = self.state.id.clone();
5638 let prompts = self.state.config.prompts.clone();
5639 let cache = self.state.config.cache_dir();
5640 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5641 let mut last = None;
5642 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5643 let seat = self.seat(&key, &spec.id);
5644 let mut job = build(spec.clone(), seat);
5645 if i > 0 {
5646 job.stem = format!("{}-{}", job.stem, spec.id);
5647 }
5648 let ctx = WaveCtx {
5649 carry_seats: false,
5650 pauses: Some((&self.pause, &self.interrupt)),
5651 run: &run_id,
5652 node,
5653 prompts: &prompts,
5654 cache: cache.as_deref(),
5655 round,
5656 };
5657 let (seat, out) =
5658 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5659 let advances = match &out {
5662 AgentOutcome::Ok(o) => agent::output_advances(o),
5663 AgentOutcome::ResourceBlocked(_) => false,
5664 _ => true,
5665 };
5666 if let Some(next) = ids.get(i + 1)
5667 && advances
5668 {
5669 self.state.seats.insert(seat.key.clone(), seat.clone());
5670 let class =
5671 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5672 let reason = match &out {
5673 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5674 other => fail_reason(other),
5675 };
5676 if let Some((wt, rev)) = restore
5680 && let Err(e) = self.graft_restore(wt, rev).await
5681 {
5682 let why =
5683 format!("could not restore the worktree before the next fixer: {e:#}");
5684 tracing::warn!("{why}");
5685 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5686 AgentOutcome::ResourceBlocked(why)
5687 } else {
5688 AgentOutcome::Failed(why)
5689 };
5690 last = Some((job, seat, out));
5691 break;
5692 }
5693 record_handover(
5694 &mut self.state,
5695 node,
5696 &seat.key,
5697 &spec.id,
5698 next,
5699 &class,
5700 &reason,
5701 );
5702 continue;
5703 }
5704 last = Some((job, seat, out));
5705 break;
5706 }
5707 last.expect("the fixer chain always has an entry")
5708 }
5709
5710 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5721 use crate::recovery::{Mode, Repair};
5722 if !self.state.reviews.is_empty() {
5723 return Ok(());
5724 }
5725 let Some(mut rec) = self.state.recovery.clone() else {
5726 return Ok(());
5727 };
5728 if rec.mode != Mode::VerifyRepair {
5729 return Ok(());
5730 }
5731 let wt = winner.worktree.clone();
5732 let mut head = git::rev_parse(&wt, "HEAD").await?;
5733 match rec.repair.clone() {
5734 Repair::Done { .. } => return Ok(()),
5735 Repair::WaitingForRestore { before } => {
5736 self.graft_restore(&wt, &before).await?;
5737 head = before;
5738 rec.repair = Repair::NotStarted;
5739 self.state.recovery = Some(rec.clone());
5740 self.state.save()?;
5741 }
5742 Repair::Started { before } => {
5743 if let Err(e) = self.graft_restore(&wt, &before).await {
5747 if e.is::<crate::repo_lock::ResourceBlocked>() {
5748 return Err(e);
5749 }
5750 return self
5751 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5752 .await;
5753 }
5754 rec.repair = Repair::Done {
5755 before: before.clone(),
5756 after: before,
5757 committed: false,
5758 failed: Some("interrupted; rolled back".to_owned()),
5759 };
5760 self.state
5761 .event("recovery", "the repair was interrupted and rolled back");
5762 self.state.recovery = Some(rec);
5763 self.state.save()?;
5764 return Ok(());
5765 }
5766 Repair::NotStarted => {}
5767 }
5768 if crate::recovery::recheck(&mut rec, &head) {
5769 self.state.event("recovery", rec.summary());
5770 self.state.recovery = Some(rec);
5771 self.state.save()?;
5772 return Ok(());
5773 }
5774 rec.repair = Repair::Started {
5775 before: head.clone(),
5776 };
5777 self.state.recovery = Some(rec.clone());
5778 self.state.save()?;
5779
5780 let language = self.state.config.graph.language.clone();
5781 let sessions = self.state.config.graph.sessions;
5782 let artifacts = agent::artifacts_dir(&self.state.dir());
5783 let fix_prompt = prompt::recovery_fix(
5784 &self.state.instruction,
5785 &rec,
5786 self.state.config.graph.review_rounds,
5787 &language,
5788 );
5789 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5790 self.state.event(
5791 "recovery",
5792 format!(
5793 "repair: the fixer starts from the failed verification on {}",
5794 short(&head)
5795 ),
5796 );
5797 let (_job, seat, out) = self
5798 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5799 prompt: fix_prompt.clone(),
5800 spec,
5801 seat,
5802 cwd: wt.clone(),
5803 timeout,
5804 allow_write: true,
5805 sessions,
5806 artifacts: artifacts.clone(),
5807 stem: "fix-recovery".to_owned(),
5808 handover: None,
5809 })
5810 .await;
5811 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5812 rec.repair = Repair::WaitingForRestore { before: head };
5813 self.state.recovery = Some(rec);
5814 return self.stop_for_resource();
5815 }
5816 let failed = match &out {
5817 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5818 AgentOutcome::Quota(o) => {
5819 self.state.quota.push(QuotaLoss {
5820 seat: seat.key.clone(),
5821 node: "fix".to_owned(),
5822 at: Timestamp::now(),
5823 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5824 });
5825 Some("rate limited (quota); the fixer could not run".to_owned())
5826 }
5827 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5828 AgentOutcome::Failed(e) => Some(e.clone()),
5829 };
5830 self.state.seats.insert(seat.key.clone(), seat);
5831 let coordination = self.coordinate(&wt).await?;
5832 let failed = match failed {
5833 Some(why) => {
5835 if let Err(e) = git::restore_to(&wt, &head).await {
5836 return self
5837 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5838 .await;
5839 }
5840 Some(why)
5841 }
5842 None => {
5843 if let Ok(r) = git::rescue_commit(
5844 &wt,
5845 "magi: recovery repair of the failed verification (uncommitted work)",
5846 )
5847 .await
5848 {
5849 self.state.note_withheld("fix", &r.withheld);
5850 }
5851 None
5852 }
5853 };
5854 let after = git::rev_parse(&wt, "HEAD").await?;
5855 drop(coordination);
5856 self.state.event(
5857 "recovery",
5858 match &failed {
5859 Some(why) => format!("repair: nothing kept ({why})"),
5860 None if after != head => {
5861 format!("repair: committed {}", short(&after))
5862 }
5863 None => "repair: the fixer made no commit".to_owned(),
5864 },
5865 );
5866 rec.repair = Repair::Done {
5867 before: head.clone(),
5868 committed: after != head,
5869 after,
5870 failed,
5871 };
5872 self.state.recovery = Some(rec);
5873 self.state.save()?;
5874 Ok(())
5875 }
5876
5877 async fn stop_recovery(
5880 &mut self,
5881 mut rec: crate::recovery::RecoveryRecord,
5882 head: &str,
5883 why: String,
5884 ) -> Result<()> {
5885 rec.repair = crate::recovery::Repair::Started {
5887 before: head.to_owned(),
5888 };
5889 self.state.recovery = Some(rec);
5890 self.state
5891 .event("recovery", format!("repair could not be undone: {why}"));
5892 self.state.status = RunStatus::Blocked;
5893 self.state.save()?;
5894 bail!("recovery repair could not be rolled back: {why}")
5895 }
5896
5897 async fn review_loop(&mut self) -> Result<()> {
5898 loop {
5899 if self
5900 .state
5901 .base_sync
5902 .as_ref()
5903 .is_some_and(|s| s.conflict.is_some())
5904 {
5905 return Ok(());
5906 }
5907 if self.catchup_repairable() {
5908 let record = self.state.reviews.last().expect("repairable round").clone();
5909 let winner = self
5910 .state
5911 .winner()
5912 .cloned()
5913 .context("catch-up repair has no winner")?;
5914 if !self.fix_review_round(&winner, record).await? {
5915 return Ok(());
5916 }
5917 }
5918 self.review_loop_pass().await?;
5919 if !self.catchup_repairable() {
5920 return Ok(());
5921 }
5922 }
5923 }
5924
5925 fn catchup_repairable(&self) -> bool {
5928 self.state.reviews.len() < self.state.config.graph.review_rounds
5929 && self.state.reviews.last().is_some_and(|r| {
5930 !r.clean
5931 && r.blocking == 0
5932 && r.fix.is_none()
5933 && r.e2e_status() == E2eStatus::Failed
5934 })
5935 }
5936
5937 async fn review_loop_pass(&mut self) -> Result<()> {
5938 if self
5943 .state
5944 .base_sync
5945 .as_ref()
5946 .is_some_and(|s| s.conflict.is_some())
5947 {
5948 return Ok(());
5949 }
5950 let run_id = self.state.id.clone();
5955 let prompts = self.state.config.prompts.clone();
5956 let Some(winner) = self.state.winner().cloned() else {
5957 return Ok(());
5958 };
5959 let max_rounds = self.state.config.graph.review_rounds;
5960 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5970 if status == RunStatus::Gating {
5973 self.record_contested_handoff();
5974 }
5975 self.state.status = status;
5976 self.state.save()?;
5977 return Ok(());
5978 }
5979 self.state.status = RunStatus::Reviewing;
5980 if self.state.reviews.last().is_some_and(|r| {
5990 r.e2e_status() == E2eStatus::ResourceBlocked
5991 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5992 }) {
5993 let shell = self.state.config.shell();
5994 return self
5995 .stop_unresolved_review(
5996 "the last round's own verification never resolved",
5997 &shell,
5998 &winner.worktree,
5999 )
6000 .await;
6001 }
6002
6003 self.recovery_repair(&winner).await?;
6004 if self.resource_halt()? {
6005 return Ok(());
6006 }
6007
6008 let repo = self.state.repo.clone();
6009 let root = self.state.worktree_root();
6010 let language = self.state.config.graph.language.clone();
6011 let sessions = self.state.config.graph.sessions;
6012 let artifacts = self.merge_repair_artifacts();
6013 let base = self.landing_base();
6014 let base_short = short(&base);
6015 let reviewers = self.roles.reviewers.clone();
6016 let shell = self.state.config.shell();
6017
6018 for round in (self.state.reviews.len() + 1)..=max_rounds {
6019 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6020 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6021 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6022 let prev_verification = self
6031 .state
6032 .reviews
6033 .last()
6034 .and_then(|r| r.verification_summary(&head));
6035 let prior_section = match (&self.state.recovery, round) {
6038 (Some(rec), 1) if self.state.reviews.is_empty() => {
6039 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6040 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6041 Some(crate::recovery::head_of(
6042 &git::diff(&winner.worktree, reviewed, "HEAD")
6043 .await
6044 .unwrap_or_default(),
6045 crate::recovery::SINCE_DIFF_MAX,
6046 ))
6047 }
6048 _ => None,
6049 };
6050 Some(prompt::recovery_review(rec, since.as_deref()))
6051 }
6052 _ => None,
6053 };
6054
6055 let mut jobs = Vec::new();
6059 let coordination = self.coordinate(&repo).await?;
6060 for (r, spec) in reviewers.iter().cloned().enumerate() {
6061 let wt = root.join(format!("review-{}", r + 1));
6062 if wt.exists() {
6065 git::reset_detached(&wt, &head).await?;
6066 } else {
6067 git::worktree_add_detached(&repo, &wt, &head).await?;
6068 }
6069 self.setup_seat_worktree(&repo, &wt, false).await?;
6070 let seat_key = format!("review-{}", r + 1);
6071 let spec = pick_start_spec(
6074 &self.roles.reviewer_roster,
6075 spec,
6076 self.state.seat_history.get(&seat_key),
6077 );
6078 let seat = self.seat(&seat_key, &spec.id);
6079 jobs.push(SeatJob {
6080 prompt: prompt::review(&prompt::ReviewCtx {
6081 instruction: &self.state.instruction,
6082 branch: &winner.branch,
6083 base_short: &base_short,
6084 stat: &stat,
6085 patch: &patch,
6086 verification: prev_verification.as_ref(),
6087 prior: prior_section.as_deref(),
6088 reviewers: reviewers.len(),
6089 round,
6090 rounds: max_rounds,
6091 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6094 lens: Lens::for_seat(r),
6095 language: &language,
6096 github_language: &self.state.config.graph.github_language,
6097 }),
6098 spec,
6099 seat,
6100 cwd: wt,
6101 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6102 allow_write: false,
6103 sessions,
6104 artifacts: artifacts.clone(),
6105 stem: format!("review-{round}-{}", r + 1),
6106 handover: None,
6107 });
6108 }
6109
6110 drop(coordination);
6111 self.state.event(
6112 "review",
6113 format!(
6114 "round {round}: {} reviewers on {}",
6115 jobs.len(),
6116 short(&head)
6117 ),
6118 );
6119 let mut quota_losses = Vec::new();
6120 let review_retries = self.state.config.graph.retries;
6121 let review_cache = self.state.config.cache_dir();
6122 let ctx = WaveCtx {
6123 carry_seats: true,
6124 pauses: Some((&self.pause, &self.interrupt)),
6125 run: &run_id,
6126 node: "review",
6127 prompts: &prompts,
6128 cache: review_cache.as_deref(),
6129 round: Some(round),
6130 };
6131 let results = ask_json_wave::<Review>(
6132 jobs,
6133 Arc::clone(&self.sem),
6134 review_retries,
6135 &self.roles.reviewer_roster,
6136 &ctx,
6137 &mut quota_losses,
6138 &mut self.state,
6139 &|_: &Review| Ok(()),
6140 )
6141 .await;
6142 if self.state.resource_block.is_some() {
6143 return self.stop_for_resource();
6144 }
6145 let round_quota_missing = quota_losses.len();
6149 self.state.quota.extend(quota_losses);
6150
6151 let mut records = Vec::new();
6152 let mut all_findings = Vec::new();
6153 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6154 let agent_id = seat.agent.clone();
6155 self.state.seats.insert(seat.key.clone(), seat);
6156 let mut record = ReviewRecord {
6157 reviewer: r + 1,
6158 agent: agent_id,
6159 summary: String::new(),
6160 findings: Vec::new(),
6161 vote: None,
6162 failed: None,
6163 duration_ms: 0,
6164 attempts,
6170 };
6171 match res {
6172 Ok((review, out)) => {
6173 record.summary =
6181 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6182 record.vote = Some(review.vote);
6183 record.duration_ms = out.duration_ms;
6184 for (n, mut f) in review.findings.into_iter().enumerate() {
6185 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6188 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6189 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6190 f.file = f
6196 .file
6197 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6198 all_findings.push(f.clone());
6199 record.findings.push(f);
6200 }
6201 self.state.event(
6202 "review",
6203 format!(
6204 "round {round}: reviewer {} voted {} with {} finding(s)",
6205 r + 1,
6206 review.vote.label(),
6207 record.findings.len()
6208 ),
6209 );
6210 }
6211 Err(e) => {
6212 record.failed = Some(e.to_string());
6213 self.state.event(
6214 "review",
6215 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6216 );
6217 }
6218 }
6219 records.push(record);
6220 }
6221
6222 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6229 let vote_split =
6230 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6231 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6232 if vote_split {
6233 self.state.event(
6234 "review",
6235 format!(
6236 "round {round}: votes split ({}) — one round of reconsideration",
6237 initial_votes
6238 .iter()
6239 .map(|v| v.label())
6240 .collect::<Vec<_>>()
6241 .join(", ")
6242 ),
6243 );
6244 let panel: Vec<ReviewSeatReport<'_>> = records
6247 .iter()
6248 .filter_map(|r| {
6249 r.vote.map(|vote| ReviewSeatReport {
6250 reviewer: r.reviewer,
6251 vote,
6252 summary: &r.summary,
6253 findings: &r.findings,
6254 })
6255 })
6256 .collect();
6257
6258 let mut jobs = Vec::new();
6259 let mut seats_at = Vec::new();
6260 for (r, spec) in reviewers.iter().cloned().enumerate() {
6261 if records[r].vote.is_none() {
6265 continue;
6266 }
6267 let wt = root.join(format!("review-{}", r + 1));
6268 let seat_key = format!("review-{}", r + 1);
6269 let spec = self.occupant(&seat_key, spec);
6270 let seat = self.seat(&seat_key, &spec.id);
6271 let build = |with_patch: bool| {
6279 prompt::review_reconsider(&ReviewReconsiderCtx {
6280 instruction: &self.state.instruction,
6281 reviewer: r + 1,
6282 lens: Lens::for_seat(r),
6283 panel: &panel,
6284 patch: with_patch.then_some(ReviewPatch {
6285 branch: &winner.branch,
6286 base_short: &base_short,
6287 stat: &stat,
6288 patch: &patch,
6289 }),
6290 round,
6291 rounds: max_rounds,
6292 language: &language,
6293 })
6294 };
6295 let full = build(true);
6296 let prompt = if has_context(&spec, &seat, sessions) {
6297 build(false)
6298 } else {
6299 full.clone()
6300 };
6301 jobs.push(SeatJob {
6302 prompt,
6303 spec,
6304 seat,
6305 cwd: wt,
6306 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6307 allow_write: false,
6308 sessions,
6309 artifacts: artifacts.clone(),
6310 stem: format!("review-{round}-reconsider-{}", r + 1),
6311 handover: Some(full),
6312 });
6313 seats_at.push(r);
6314 }
6315
6316 let mut recon_quota_losses = Vec::new();
6317 let recon_cache = self.state.config.cache_dir();
6318 let recon_ctx = WaveCtx {
6319 carry_seats: true,
6320 pauses: Some((&self.pause, &self.interrupt)),
6321 run: &run_id,
6322 node: "review",
6323 prompts: &prompts,
6324 cache: recon_cache.as_deref(),
6325 round: Some(round),
6326 };
6327 let recon_results = ask_json_wave::<ReviewRevote>(
6328 jobs,
6329 Arc::clone(&self.sem),
6330 review_retries,
6331 &self.roles.reviewer_roster,
6332 &recon_ctx,
6333 &mut recon_quota_losses,
6334 &mut self.state,
6335 &|_: &ReviewRevote| Ok(()),
6336 )
6337 .await;
6338 if self.state.resource_block.is_some() {
6339 return self.stop_for_resource();
6340 }
6341 self.state.quota.extend(recon_quota_losses);
6342
6343 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6344 let agent_id = seat.agent.clone();
6345 self.state.seats.insert(seat.key.clone(), seat);
6346 let mut rec = ReviewRevoteRecord {
6347 reviewer: r + 1,
6348 agent: agent_id,
6349 vote: None,
6350 reason: String::new(),
6351 failed: None,
6352 };
6353 match res {
6354 Ok((rv, _)) => {
6355 rec.vote = Some(rv.vote);
6356 rec.reason =
6357 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6358 self.state.event(
6359 "review",
6360 format!(
6361 "round {round}: reviewer {} revoted {}",
6362 r + 1,
6363 rv.vote.label()
6364 ),
6365 );
6366 }
6367 Err(e) => {
6368 rec.failed = Some(e.to_string());
6369 self.state.event(
6370 "review",
6371 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6372 );
6373 }
6374 }
6375 reconsideration.push(rec);
6376 }
6377 } else if initial_votes.len() > 1 {
6378 self.state.event(
6379 "review",
6380 format!(
6381 "round {round}: votes agreed ({}) — no reconsideration",
6382 initial_votes[0].label()
6383 ),
6384 );
6385 }
6386
6387 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6388 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6389 let defer_e2e =
6400 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6401 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6402 let reason =
6403 format!("{blocking} blocking finding(s) already required a fix this round");
6404 self.state.event(
6405 "verify",
6406 format!(
6407 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6408 {}); it will run once a round has none left",
6409 short(&head)
6410 ),
6411 );
6412 (Vec::new(), false, true, Some(reason))
6413 } else {
6414 let e2e_commands = self.state.config.verify.e2e.clone();
6415 let cache_dir = self.state.config.cache_dir();
6416 let context = format!("round {round}");
6417 let wait = self.repository_wait();
6418 let (e2e, verify_retried) = with_cache_lease(
6419 &mut self.state,
6420 cache_dir.as_deref(),
6421 "e2e",
6422 "e2e",
6423 &winner.worktree,
6424 &head,
6425 verify_timeout,
6426 &context,
6427 &wait,
6428 |state, budget| {
6429 let shell = shell.clone();
6430 let e2e_commands = e2e_commands.clone();
6431 let worktree = winner.worktree.clone();
6432 let context = context.clone();
6433 async move {
6434 run_e2e_with_retry(
6435 state,
6436 &shell,
6437 &e2e_commands,
6438 &worktree,
6439 budget,
6440 &context,
6441 )
6442 .await
6443 }
6444 },
6445 )
6446 .await?;
6447 (e2e, verify_retried, false, None)
6448 };
6449
6450 let expected = records.len();
6451 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6452 let incomplete = answered < expected;
6453 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6454 let policy = self.state.config.graph.incomplete_review;
6455 let clean = round_is_clean(
6456 blocking,
6457 e2e_ok,
6458 answered,
6459 expected,
6460 round_quota_missing,
6461 policy,
6462 );
6463
6464 let mut round_record = ReviewRound {
6465 round,
6466 head: head.clone(),
6467 verified_head: None,
6468 verified_at: None,
6469 reviews: records,
6470 e2e,
6471 verify_retried,
6472 e2e_deferred,
6473 e2e_defer_reason,
6474 fix: None,
6475 blocking,
6476 answered,
6477 expected,
6478 clean,
6479 progressed: false,
6480 vote_split,
6481 reconsideration,
6482 verdict: None,
6483 };
6484 round_record.verdict = ReviewVote::worst(
6488 round_record
6489 .final_votes()
6490 .into_iter()
6491 .map(|(_, _, vote)| vote),
6492 );
6493 if !matches!(
6504 round_record.e2e_status(),
6505 E2eStatus::Deferred | E2eStatus::NotConfigured
6506 ) {
6507 round_record.verified_head = Some(head.clone());
6508 round_record.verified_at = Some(Timestamp::now());
6509 }
6510
6511 if incomplete {
6512 let missing: Vec<String> = round_record
6513 .reviews
6514 .iter()
6515 .filter(|r| r.failed.is_some())
6516 .map(|r| format!("review-{}", r.reviewer))
6517 .collect();
6518 self.state.event(
6519 "review",
6520 format!(
6521 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6522 missing.join(", ")
6523 ),
6524 );
6525 }
6526
6527 if clean {
6528 self.state.event(
6529 "review",
6530 if incomplete && policy == IncompleteReviewPolicy::Warn {
6531 format!(
6532 "round {round}: clean (warn policy, incomplete panel) — no \
6533 blocking findings from the seats that answered, verification green"
6534 )
6535 } else if incomplete {
6536 format!(
6537 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6538 quorum) — no blocking findings from the seats that answered, \
6539 verification green",
6540 expected - answered
6541 )
6542 } else {
6543 format!("round {round}: clean — no blocking findings, verification green")
6544 },
6545 );
6546 self.state.reviews.push(round_record);
6547 self.state.status = RunStatus::Gating;
6548 self.state.save()?;
6549 return Ok(());
6550 }
6551
6552 if incomplete && blocking == 0 && e2e_ok {
6560 self.state.reviews.push(round_record);
6561 self.state.save()?;
6562 if round == max_rounds {
6563 self.state.status = RunStatus::Blocked;
6564 self.state.event(
6565 "review",
6566 format!(
6567 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6568 refusing to call it clean",
6569 expected - answered
6570 ),
6571 );
6572 return Ok(());
6573 }
6574 continue;
6575 }
6576
6577 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6589 self.state.reviews.push(round_record);
6590 return self
6591 .stop_unresolved_review(
6592 "the round's own verification could not run",
6593 &shell,
6594 &winner.worktree,
6595 )
6596 .await;
6597 }
6598
6599 if round == max_rounds {
6600 self.state.reviews.push(round_record);
6601 return self
6602 .stop_reviewing(
6603 &format!(
6604 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6605 ),
6606 &shell,
6607 &winner.worktree,
6608 )
6609 .await;
6610 }
6611
6612 if !self.fix_review_round(&winner, round_record).await? {
6613 return Ok(());
6614 }
6615 }
6616 Ok(())
6617 }
6618
6619 async fn fix_review_round(
6622 &mut self,
6623 winner: &Candidate,
6624 mut round_record: ReviewRound,
6625 ) -> Result<bool> {
6626 let round = round_record.round;
6627 let shell = self.state.config.shell();
6628 let step = self.run_fix_round(winner, &mut round_record).await?;
6629 if matches!(step, FixStep::Resource) {
6630 self.stop_for_resource()?;
6631 return Ok(false);
6632 }
6633 if self.state.reviews.len() < round {
6634 self.state.reviews.push(round_record);
6635 } else {
6636 self.state.reviews[round - 1] = round_record;
6637 }
6638 self.state.save()?;
6639
6640 if matches!(step, FixStep::Recorded { report_lost: true }) {
6653 self.stop_reviewing(
6654 "the fixer's adoption report never came back, even after resuming its \
6655 own seat; refusing to start another round against the same worktree \
6656 while that is unresolved",
6657 &shell,
6658 &winner.worktree,
6659 )
6660 .await?;
6661 return Ok(false);
6662 }
6663
6664 let streak = self
6665 .state
6666 .reviews
6667 .iter()
6668 .rev()
6669 .take_while(|r| !r.progressed)
6670 .count();
6671 if streak >= STAGNANT_LIMIT {
6672 self.stop_reviewing(
6673 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6674 &shell,
6675 &winner.worktree,
6676 )
6677 .await?;
6678 return Ok(false);
6679 }
6680 Ok(true)
6681 }
6682
6683 async fn run_fix_round(
6685 &mut self,
6686 winner: &Candidate,
6687 round_record: &mut ReviewRound,
6688 ) -> Result<FixStep> {
6689 let round = round_record.round;
6690 let max_rounds = self.state.config.graph.review_rounds;
6691 let run_id = self.state.id.clone();
6692 let prompts = self.state.config.prompts.clone();
6693 let language = self.state.config.graph.language.clone();
6694 let sessions = self.state.config.graph.sessions;
6695 let artifacts = self.merge_repair_artifacts();
6696 let base = self.landing_base();
6697 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6698 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6699 let this_round_verification = round_record.verification_summary(&head);
6700 let all_findings: Vec<_> = round_record
6701 .reviews
6702 .iter()
6703 .flat_map(|r| r.findings.iter().cloned())
6704 .collect();
6705 let blocking_findings: Vec<_> = all_findings
6708 .iter()
6709 .filter(|f| f.severity.blocks())
6710 .cloned()
6711 .collect();
6712 let fix_prompt = prompt::fix_in(
6713 &self.state.instruction,
6714 &blocking_findings,
6715 this_round_verification.as_ref(),
6716 round,
6717 max_rounds,
6718 &language,
6719 &self.state.config.graph.github_language,
6720 );
6721 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6722 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6723 let (job, seat, out) = self
6724 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6725 prompt: fix_prompt.clone(),
6726 spec,
6727 seat,
6728 cwd: winner.worktree.clone(),
6729 timeout,
6730 allow_write: true,
6731 sessions,
6732 artifacts: artifacts.clone(),
6733 stem: format!("fix-{round}"),
6734 handover: None,
6735 })
6736 .await;
6737 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6740 return Ok(FixStep::Resource);
6741 }
6742 let agent_id = seat.agent.clone();
6743
6744 let mut fix = FixRecord {
6745 agent: agent_id,
6746 addressed: Vec::new(),
6747 rejected: Vec::new(),
6748 notes: String::new(),
6749 committed: false,
6750 failed: None,
6751 duration_ms: 0,
6752 continuation: None,
6753 };
6754 let mut continuation = ContinuationRecord::not_needed();
6755 let mut final_seat = seat.clone();
6756 match out {
6757 AgentOutcome::Ok(o) => {
6758 fix.duration_ms = o.duration_ms;
6759 let parsed = verdict::extract_json::<FixReport>(&o.text);
6760 let incomplete_reason = match &parsed {
6767 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6768 "the reply parsed, but it reported a command whose own CLI never \
6769 confirmed an exit status"
6770 .to_owned(),
6771 ),
6772 Ok(_) => None,
6773 Err(e) => Some(e.to_string()),
6774 };
6775 match incomplete_reason {
6776 None => {
6777 let report = parsed.expect("checked Ok above");
6778 fix.addressed = report.addressed;
6779 fix.rejected = report.rejected;
6780 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6781 }
6782 Some(reason) => {
6783 let (resumed_seat, resolved, failure, cont) = self
6784 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6785 .await;
6786 fix.duration_ms += cont.cumulative_wait_ms;
6787 continuation = cont;
6788 final_seat = resumed_seat;
6789 match resolved {
6790 Some(report) => {
6791 fix.addressed = report.addressed;
6792 fix.rejected = report.rejected;
6793 fix.notes =
6794 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6795 }
6796 None => fix.failed = failure,
6797 }
6798 }
6799 }
6800 }
6801 AgentOutcome::Dropped(o) => {
6803 fix.duration_ms = o.duration_ms;
6804 let why = o
6805 .dropped
6806 .as_ref()
6807 .map(|d| d.why.as_str())
6808 .unwrap_or("the CLI ended the stream without delivering its answer");
6809 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6810 }
6811 AgentOutcome::Quota(o) => {
6812 self.state.quota.push(QuotaLoss {
6813 seat: final_seat.key.clone(),
6814 node: "fix".to_owned(),
6815 at: Timestamp::now(),
6816 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6817 });
6818 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6819 }
6820 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6821 }
6822 fix.continuation = Some(continuation);
6823 self.state.seats.insert(final_seat.key.clone(), final_seat);
6824 let coordination = self.coordinate(&winner.worktree).await?;
6825 if let Ok(r) = git::rescue_commit(
6826 &winner.worktree,
6827 &format!("magi: review round {round} fixes (uncommitted work)"),
6828 )
6829 .await
6830 {
6831 self.state.note_withheld("fix", &r.withheld);
6832 }
6833 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6834 fix.committed = after != before;
6835 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6843 drop(coordination);
6844 let progressed = diff_after != patch;
6845 let commit_note = if fix.committed {
6846 "committed"
6847 } else {
6848 "NO new commit"
6849 };
6850 let tree_note = if progressed {
6851 "changed vs base"
6852 } else {
6853 "unchanged vs base"
6854 };
6855 self.state.event(
6856 "fix",
6857 match &fix.failed {
6858 Some(reason) => {
6864 format!(
6865 "round {round}: fixer's adoption report was lost ({reason}); \
6866 {commit_note}, tree {tree_note}"
6867 )
6868 }
6869 None => format!(
6870 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6871 {tree_note}{}",
6872 fix.addressed.len(),
6873 fix.rejected.len(),
6874 if continuation.outcome == ContinuationOutcome::Resumed {
6875 format!(
6876 " (adoption report recovered after {} continuation(s))",
6877 continuation.attempts
6878 )
6879 } else {
6880 String::new()
6881 },
6882 ),
6883 },
6884 );
6885 round_record.fix = Some(fix);
6886 round_record.progressed = progressed;
6887 Ok(FixStep::Recorded {
6888 report_lost: matches!(
6889 continuation.outcome,
6890 ContinuationOutcome::Exhausted
6891 | ContinuationOutcome::QuotaLost
6892 | ContinuationOutcome::NoSession
6893 ),
6894 })
6895 }
6896
6897 fn record_contested_handoff(&mut self) {
6931 if self.state.contested_handoff.is_some() {
6932 return;
6933 }
6934 let Some(contested) = self
6935 .state
6936 .reviews
6937 .last()
6938 .and_then(ReviewRound::contested_handoff)
6939 else {
6940 return;
6941 };
6942 self.state.event(
6943 "review",
6944 format!(
6945 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6946 wait for the owner's approval",
6947 contested.findings.len(),
6948 contested.rejecters.len()
6949 ),
6950 );
6951 self.state.contested_handoff = Some(contested);
6952 }
6953
6954 async fn stop_unresolved_review(
6955 &mut self,
6956 why: &str,
6957 shell: &[String],
6958 worktree: &Path,
6959 ) -> Result<()> {
6960 self.stop_reviewing_inner(why, shell, worktree, true).await
6961 }
6962
6963 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6964 self.stop_reviewing_inner(why, shell, worktree, false).await
6965 }
6966
6967 async fn stop_reviewing_inner(
6968 &mut self,
6969 why: &str,
6970 shell: &[String],
6971 worktree: &Path,
6972 repairable: bool,
6973 ) -> Result<()> {
6974 let round_idx = self.state.reviews.len() - 1;
6975 let needs_catchup_run = matches!(
6983 self.state.reviews[round_idx].e2e_status(),
6984 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6985 );
6986 if needs_catchup_run {
6987 let round = self.state.reviews[round_idx].round;
6988 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6989 let commands = self.state.config.verify.e2e.clone();
6990 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6991 let cache_dir = self.state.config.cache_dir();
6992 let context = format!(
6993 "round {round}: verification unresolved, catching up before the final decision"
6994 );
6995 let wait = self.repository_wait();
6996 let (outcomes, verify_retried) = with_cache_lease(
6997 &mut self.state,
6998 cache_dir.as_deref(),
6999 "e2e",
7000 "e2e",
7001 worktree,
7002 &attempted_head,
7003 timeout,
7004 &context,
7005 &wait,
7006 |state, budget| {
7007 let shell = shell.to_vec();
7008 let commands = commands.clone();
7009 let context = context.clone();
7010 async move {
7011 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7012 .await
7013 }
7014 },
7015 )
7016 .await?;
7017 let last = &mut self.state.reviews[round_idx];
7018 last.e2e = outcomes;
7019 last.verify_retried = verify_retried;
7020 last.verified_head = Some(attempted_head);
7027 last.verified_at = Some(Timestamp::now());
7028 if verify_inconclusive(&last.e2e) {
7029 self.state.save()?;
7036 return Ok(());
7037 }
7038 last.e2e_deferred = false;
7039 }
7040 let last = &self.state.reviews[round_idx];
7041 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7042
7043 match last.e2e_status() {
7044 E2eStatus::Failed => {
7045 let red: Vec<String> = last
7046 .e2e
7047 .iter()
7048 .filter(|o| !o.ok())
7049 .map(|o| {
7050 format!(
7051 "`{}` -> {:?}\n{}",
7052 o.command,
7053 o.code,
7054 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7055 )
7056 })
7057 .collect();
7058 let can_repair = last.blocking == 0 && last.fix.is_none();
7059 self.state.reviews[round_idx].clean = false;
7060 self.state
7061 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7062 self.state.status = RunStatus::Blocked;
7063 if repairable
7064 && can_repair
7065 && self.state.reviews.len() < self.state.config.graph.review_rounds
7066 {
7067 self.state.status = RunStatus::Reviewing;
7068 }
7069 }
7070 E2eStatus::ResourceBlocked => {
7075 self.state.event(
7076 "review",
7077 format!(
7078 "{why}; e2e could not run (resource unavailable); not \
7079 deciding yet"
7080 ),
7081 );
7082 }
7083 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7084 self.state.event(
7085 "review",
7086 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7087 );
7088 self.record_contested_handoff();
7089 self.state.status = RunStatus::Gating;
7090 }
7091 }
7092 self.state.save()?;
7093 Ok(())
7094 }
7095
7096 async fn gate(&mut self) -> Result<()> {
7099 if self.state.status == RunStatus::Failed
7111 || self
7112 .state
7113 .base_sync
7114 .as_ref()
7115 .is_some_and(|s| s.conflict.is_some())
7116 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7117 != Some(RunStatus::Gating)
7118 {
7119 return Ok(());
7120 }
7121 if self.state.gate_ran {
7122 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7133 self.state.status = RunStatus::Blocked;
7134 self.state.save()?;
7135 }
7136 return Ok(());
7137 }
7138 let Some(winner) = self.state.winner().cloned() else {
7139 return Ok(());
7140 };
7141 self.state.status = RunStatus::Gating;
7142 let mut outcomes = self.run_gate(&winner).await?;
7143 loop {
7144 if verify_inconclusive(&outcomes) {
7155 self.state.save()?;
7156 return Ok(());
7157 }
7158 if outcomes.iter().all(CommandOutcome::ok) {
7159 break;
7160 }
7161 match self.gate_fix_round(&winner, &outcomes).await? {
7162 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7163 GateFix::Stop => break,
7164 GateFix::Defer => {
7165 self.state.save()?;
7166 return Ok(());
7167 }
7168 }
7169 }
7170 let passed = outcomes.iter().all(CommandOutcome::ok);
7171 self.state.gate = outcomes;
7172 self.state.gate_ran = true;
7173 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7174 if !passed {
7175 self.state.status = RunStatus::Blocked;
7176 let spent = self.state.gate_fixes.len();
7177 self.state.event(
7178 "gate",
7179 if spent == 0 {
7180 "gate failed; not merging".to_owned()
7181 } else {
7182 format!("gate failed after {spent} gate-fix round(s); not merging")
7183 },
7184 );
7185 }
7186 self.state.save()?;
7187 Ok(())
7188 }
7189
7190 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7200 let commands = state.config.verify.pre_gate.clone();
7201 if commands.is_empty() {
7202 return;
7203 }
7204 let shell = state.config.shell();
7205 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7206 let (outcomes, _) = run_commands(
7207 state,
7208 "pre_gate",
7209 "pre_gate",
7210 0,
7211 &shell,
7212 &commands,
7213 &winner.worktree,
7214 timeout,
7215 )
7216 .await;
7217 for o in &outcomes {
7218 if !o.ok() {
7219 tracing::warn!(
7220 "pre_gate `{}` failed ({:?}); the gate decides",
7221 o.command,
7222 o.code
7223 );
7224 }
7225 state.event(
7226 "pre_gate",
7227 format!(
7228 "`{}` -> {}",
7229 o.command,
7230 if o.ok() {
7231 "pass".to_owned()
7232 } else {
7233 format!(
7234 "FAIL ({:?})\n{}",
7235 o.code,
7236 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7237 )
7238 }
7239 ),
7240 );
7241 }
7242 state.pre_gate = outcomes;
7243 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7244 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7245 Ok(head) => {
7246 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7247 state.pre_gate_commit = Some(head);
7248 }
7249 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7250 },
7251 Ok(false) => {}
7252 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7253 }
7254 if let Err(e) = state.save() {
7255 tracing::warn!("could not persist the pre_gate record: {e:#}");
7256 }
7257 }
7258
7259 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7262 let wait = self.repository_wait();
7263 run_landing_gate(&mut self.state, winner, &wait).await
7264 }
7265
7266 async fn gate_fix_round(
7279 &mut self,
7280 winner: &Candidate,
7281 outcomes: &[CommandOutcome],
7282 ) -> Result<GateFix> {
7283 let cap = self.state.config.graph.gate_fix_rounds;
7284 let spent = self.state.gate_fixes.len();
7285 if spent >= cap {
7286 if cap > 0 {
7287 self.state.event(
7288 "gate",
7289 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7290 );
7291 }
7292 return Ok(GateFix::Stop);
7293 }
7294 if !gate_fixable(outcomes) {
7295 self.state.event(
7296 "gate",
7297 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7298 command or similar); not spending a fix round on it",
7299 );
7300 return Ok(GateFix::Stop);
7301 }
7302 let min_free = self.state.config.disk.min_free_bytes;
7303 if min_free > 0 {
7304 match crate::disk::free_bytes(&winner.worktree) {
7305 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7306 Ok(free) => {
7307 self.state.event(
7308 "gate",
7309 format!(
7310 "only {free} bytes free ({min_free} required by `[disk] \
7311 min_free_bytes`); not spending a fix round on a failure the disk \
7312 may explain"
7313 ),
7314 );
7315 return Ok(GateFix::Stop);
7316 }
7317 Err(e) => {
7318 self.state.event(
7319 "gate",
7320 format!("free disk space could not be measured ({e:#}); no fix round"),
7321 );
7322 return Ok(GateFix::Stop);
7323 }
7324 }
7325 }
7326
7327 let attempt = spent + 1;
7328 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7329 let base = self.landing_base();
7330 let fix_prompt = prompt::gate_fix_in(
7331 &self.state.instruction,
7332 &failed,
7333 attempt,
7334 cap,
7335 &self.state.config.graph.language,
7336 &self.state.config.graph.github_language,
7337 );
7338 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7339 let sessions = self.state.config.graph.sessions;
7340 let artifacts = self.merge_repair_artifacts();
7341 self.state.event(
7342 "gate",
7343 format!("gate failed; gate-fix round {attempt} of {cap}"),
7344 );
7345 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7346 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7347 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7350 head: before.clone(),
7351 passed: false,
7352 outcomes: Vec::new(),
7353 });
7354 self.state.save()?;
7355 let (_, seat, out) = self
7356 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7357 prompt: fix_prompt.clone(),
7358 spec,
7359 seat,
7360 cwd: winner.worktree.clone(),
7361 timeout,
7362 allow_write: true,
7363 sessions,
7364 artifacts: artifacts.clone(),
7365 stem: format!("gate-fix-{attempt}"),
7366 handover: None,
7367 })
7368 .await;
7369 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7372 return Ok(GateFix::Defer);
7373 }
7374 let mut record = GateFixRecord {
7375 agent: seat.agent.clone(),
7376 failed,
7377 notes: String::new(),
7378 committed: false,
7379 error: None,
7380 };
7381 match out {
7382 AgentOutcome::Ok(o) => {
7383 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7386 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7387 }
7388 }
7389 AgentOutcome::Dropped(_) => {
7390 record.error = Some("the CLI dropped the stream".to_owned());
7391 }
7392 AgentOutcome::Quota(o) => {
7393 self.state.quota.push(QuotaLoss {
7394 seat: seat.key.clone(),
7395 node: "gate-fix".to_owned(),
7396 at: Timestamp::now(),
7397 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7398 });
7399 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7400 }
7401 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7402 }
7403 self.state.seats.insert(seat.key.clone(), seat);
7404 let coordination = self.coordinate(&winner.worktree).await?;
7405 if let Ok(r) = git::rescue_commit(
7406 &winner.worktree,
7407 &format!("magi: gate fix {attempt} (uncommitted work)"),
7408 )
7409 .await
7410 {
7411 self.state.note_withheld("gate-fix", &r.withheld);
7412 }
7413 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7414 record.committed = after != before;
7415 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7416 let note = record.error.clone();
7417 self.state.gate_fixes.push(record);
7418 self.state.save()?;
7419 if !changed {
7420 self.state.event(
7421 "gate",
7422 match note {
7423 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7424 None => format!("gate-fix round {attempt}: the tree did not change"),
7425 },
7426 );
7427 return Ok(GateFix::Stop);
7428 }
7429 self.state.event(
7430 "gate",
7431 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7432 );
7433
7434 drop(coordination);
7435 let wait = self.repository_wait();
7436 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7437 }
7438
7439 async fn guarded_pr_message(
7446 &mut self,
7447 winner: &Candidate,
7448 facts: Option<&BranchFacts>,
7449 posting: bool,
7450 ) -> Result<Option<PrMessage>> {
7451 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7452 let head = match final_head.as_deref() {
7453 Some(h) => crate::pr_outcome::FinalHead::At(h),
7454 None => crate::pr_outcome::FinalHead::Unknown,
7455 };
7456 let summarized = if posting {
7458 Some(self.summarize_change(winner, final_head.as_deref()).await)
7459 } else {
7460 None
7461 };
7462 if self.state.resource_block.is_some() {
7467 self.state.event(
7468 "pr-summary",
7469 "a shared resource was busy; nothing was published, resume to summarize again",
7470 );
7471 return Ok(None);
7472 }
7473 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7474 anyhow::bail!(
7475 "the final commit moved during public description generation; retry before publishing"
7476 );
7477 }
7478 let note = match (&summarized, &self.state.change_summary) {
7479 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7480 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7481 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7482 (None, _) => ChangeNote::NotAsked,
7483 };
7484 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7485 if !posting {
7486 let identity = crate::scrub::Identity::current();
7487 return Ok(Some(PrMessage {
7488 title: crate::scrub::scrub(&pr.title, &identity),
7489 body: crate::scrub::scrub(&pr.body, &identity),
7490 }));
7491 }
7492 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7495 if let Some(known) = self
7496 .state
7497 .github_text
7498 .clone()
7499 .filter(|g| g.fingerprint == raw_fp)
7500 {
7501 return self.settle_withheld(winner, &pr, known).await;
7502 }
7503 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7504 let mut violations =
7505 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7506 let identity = crate::scrub::Identity::current();
7507 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7508 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7509 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7510 {
7511 violations.push(crate::github_text::Violation::SensitiveData);
7512 }
7513 let has_language = violations
7517 .iter()
7518 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7519 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7520 self.state.event(
7521 "github-text",
7522 if has_language {
7523 format!("description rejected: {violations:?}; requesting one rewrite")
7524 } else {
7525 format!("description rejected: {violations:?}; asking the owner")
7526 },
7527 );
7528 let mut rewritten = false;
7529 let mut rejected_rewrite = None;
7530 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7531 {
7532 let output = text;
7533 if let Some(title) = summary_title(&output) {
7534 let raw_body = summary_without_title(&output);
7537 let clean_title = crate::scrub::scrub(&title, &identity);
7538 let mut body = crate::scrub::scrub(&raw_body, &identity);
7539 let redacted = clean_title != title || body != raw_body;
7540 let title = clean_title;
7541 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7542 if !complete {
7543 self.state.event(
7544 "github-text",
7545 "description rewrite omitted required findings",
7546 );
7547 }
7548 if !body.trim().is_empty() {
7549 body.push_str("\n\n");
7550 body.push_str(&public_pr_evidence(&self.state, head));
7551 }
7552 let marker = format!("magi:run/{}", self.state.id);
7553 if !body.trim().is_empty() && !body.contains(&marker) {
7554 body.push_str(&format!("\n\n{marker}"));
7555 }
7556 let rewrite = PrMessage {
7557 title: title.clone(),
7558 body: body.clone(),
7559 };
7560 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7561 let mut failed = crate::github_text::check_for(
7562 &self.state.config.graph,
7563 &title,
7564 &body,
7565 redecision,
7566 );
7567 if (!crate::github_text::shareable(&title)
7568 || !crate::github_text::shareable(&body))
7569 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7570 {
7571 failed.push(crate::github_text::Violation::SensitiveData);
7572 }
7573 if complete
7574 && !title.trim().is_empty()
7575 && !body.trim().is_empty()
7576 && !failed.is_empty()
7577 {
7578 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7579 title: title.clone(),
7580 body: body.clone(),
7581 failed: failed
7582 .iter()
7583 .map(|v| crate::github_text::category(*v).to_owned())
7584 .collect(),
7585 });
7586 }
7587 if complete
7588 && !title.trim().is_empty()
7589 && !raw_body.trim().is_empty()
7590 && failed.is_empty()
7591 {
7592 decision = redecision;
7593 pr = PrMessage { title, body };
7594 rewritten = true;
7595 self.state
7596 .event("github-text", "description rewrite passed");
7597 if redacted {
7598 self.state.event(
7599 "github-text",
7600 "sensitive data removed from description rewrite",
7601 );
7602 }
7603 }
7604 }
7605 }
7606 if !rewritten {
7607 self.state.event(
7608 "github-text",
7609 "description rewrite unavailable or rejected; asking the owner",
7610 );
7611 if let Some(w) =
7612 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7613 {
7614 self.retire_github_text_question("the withheld text changed");
7617 let record = crate::run::GithubTextAsk {
7618 rewrite: rejected_rewrite,
7619 fingerprint: raw_fp,
7620 title: w.title,
7621 body: w.body,
7622 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7623 question: None,
7624 asks: 0,
7625 resolved: false,
7626 chosen_title: None,
7627 choice: None,
7628 };
7629 return self.settle_withheld(winner, &pr, record).await;
7630 }
7631 }
7632 }
7633 let (title, body) =
7634 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7635 Ok(Some(PrMessage { title, body }))
7636 }
7637
7638 async fn summarize_change(
7649 &mut self,
7650 winner: &Candidate,
7651 head: Option<&str>,
7652 ) -> std::result::Result<(), String> {
7653 let Some(head) = head else {
7654 return Err("the final commit could not be read".to_owned());
7655 };
7656 let remote = self.state.config.merge.remote.clone();
7657 let tracking = format!("{remote}/{}", self.state.base_branch);
7658 if !git::rev_exists(&winner.worktree, &tracking).await {
7659 return Err(format!("the remote base `{tracking}` could not be read"));
7660 }
7661 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7662 Ok(b) if !b.is_empty() => b,
7663 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7664 };
7665 let evidence = crate::pr_outcome::summary_evidence(
7666 &self.state,
7667 crate::pr_outcome::FinalHead::At(head),
7668 );
7669 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7670 if self
7671 .state
7672 .change_summary
7673 .as_ref()
7674 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7675 {
7676 return Ok(());
7677 }
7678 if self.state.change_summary.take().is_some() {
7679 self.state.event(
7680 "pr-summary",
7681 "the final diff or evidence changed; the earlier change summary was discarded",
7682 );
7683 }
7684 if let Err(e) = write_artifact(
7686 &self.state,
7687 &format!("public-description-{fingerprint}.md"),
7688 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7689 ) {
7690 tracing::warn!("could not save public description evidence: {e:#}");
7691 }
7692 let chain = match agent::pick_chain(
7693 &self.state.config.agents,
7694 self.state.config.roles.synthesizer.as_ref(),
7695 &agent::installed,
7696 "synthesizer",
7697 ) {
7698 Ok(chain) => chain,
7699 Err(e) => {
7700 tracing::warn!("no agent for the change summary: {e:#}");
7701 self.state.event("pr-summary", "no summary agent available");
7702 return Err("no agent was available to summarize it".to_owned());
7703 }
7704 };
7705 let subjects = git::subjects(&winner.worktree, &base, head)
7706 .await
7707 .unwrap_or_default();
7708 let stat = git::diff_stat(&winner.worktree, &base, head)
7709 .await
7710 .unwrap_or_default();
7711 let stat: String = stat.chars().take(6_000).collect();
7712 let mut prompt = crate::prompt::change_summary_in(
7713 &base,
7714 head,
7715 &subjects,
7716 stat.trim(),
7717 &self.state.config.graph.github_language,
7718 );
7719 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7720 let sessions = self.state.config.graph.sessions;
7721 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7722 let prompts = self.state.config.prompts.clone();
7723 let cache = self.state.config.cache_dir();
7724 let run = self.state.id.clone();
7725 for (i, spec) in chain.into_iter().enumerate() {
7726 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7727 let github_spec = match agent::for_github_text(
7728 &spec,
7729 &self.state.config.graph.github_language,
7730 &winner.worktree,
7731 ) {
7732 Ok(spec) => spec,
7733 Err(e) => {
7734 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7735 continue;
7736 }
7737 };
7738 let job = SeatJob {
7739 spec: github_spec,
7740 seat,
7741 cwd: winner.worktree.clone(),
7742 prompt: prompt.clone(),
7743 timeout,
7744 allow_write: false,
7745 sessions,
7746 artifacts: agent::artifacts_dir(&self.state.dir()),
7747 stem: if i == 0 {
7748 "change-summary".to_owned()
7749 } else {
7750 format!("change-summary-{}", spec.id)
7751 },
7752 handover: None,
7753 };
7754 let ctx = WaveCtx {
7755 run: &run,
7756 node: "pr-summary",
7757 prompts: &prompts,
7758 cache: cache.as_deref(),
7759 round: None,
7760 carry_seats: false,
7761 pauses: Some((&self.pause, &self.interrupt)),
7762 };
7763 let (seat, outcome) =
7764 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7765 self.state.seats.insert(seat.key.clone(), seat);
7766 let text = match outcome {
7767 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7768 AgentOutcome::ResourceBlocked(why) => {
7769 if self.state.resource_block.is_none() {
7772 self.state.resource_block = Some(crate::run::ResourceBlock {
7773 cause: crate::run::ResourceBlockCause::Other,
7774 node: "pr-summary".to_owned(),
7775 kind: "shared resource".to_owned(),
7776 detail: why,
7777 at: Timestamp::now(),
7778 });
7779 }
7780 return Err("a shared resource was busy".to_owned());
7781 }
7782 _ => continue,
7783 };
7784 let blind = &self.state.config.blind;
7785 let Some(title) = summary_title(&text) else {
7786 continue;
7787 };
7788 let body: String = summary_without_title(&text)
7789 .lines()
7790 .filter(|l| !l.contains("magi:run/"))
7791 .collect::<Vec<_>>()
7792 .join("\n");
7793 let body = blind::sanitize_prose(body.trim(), blind);
7794 let title = blind::sanitize_prose(&title, blind);
7795 if body.trim().is_empty() || title.trim().is_empty() {
7796 continue;
7797 }
7798 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7799 continue;
7800 }
7801 if git::rev_parse(&winner.worktree, "HEAD")
7803 .await
7804 .ok()
7805 .as_deref()
7806 != Some(head)
7807 {
7808 return Err("the branch moved while the summary was written".to_owned());
7809 }
7810 if git::merge_base(&winner.worktree, &tracking, head)
7811 .await
7812 .ok()
7813 .as_deref()
7814 != Some(base.as_str())
7815 || crate::pr_outcome::summary_evidence(
7816 &self.state,
7817 crate::pr_outcome::FinalHead::At(head),
7818 ) != evidence
7819 {
7820 return Err("the summary evidence changed while it was written".to_owned());
7821 }
7822 self.state.change_summary = Some(crate::run::ChangeSummary {
7823 head: head.to_owned(),
7824 base,
7825 evidence: fingerprint,
7826 title,
7827 body,
7828 });
7829 self.state.event(
7830 "pr-summary",
7831 format!("change summary written by {} for the final diff", spec.id),
7832 );
7833 if let Err(e) = self.state.save() {
7834 tracing::warn!("could not save the change summary: {e:#}");
7835 }
7836 return Ok(());
7837 }
7838 self.state
7839 .event("pr-summary", "no agent produced a change summary");
7840 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7841 }
7842
7843 async fn rewrite_pr_text(
7853 &mut self,
7854 winner: &Candidate,
7855 pr: &PrMessage,
7856 violations: &[crate::github_text::Violation],
7857 ) -> Option<String> {
7858 let prompt = format!(
7859 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7860 pr.title, pr.body
7861 );
7862 let sessions = self.state.config.graph.sessions;
7863 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7864 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7867 let key = format!("impl-{}", winner.label);
7868 if let Some(spec) = self
7869 .state
7870 .config
7871 .agents
7872 .iter()
7873 .find(|a| a.id == winner.agent)
7874 .cloned()
7875 && let Some(existing) = self.state.seats.get(&key)
7876 && existing.agent == spec.id
7877 && agent::has_session(spec.kind, existing, sessions)
7878 {
7879 attempts.push((spec, Some(existing.clone())));
7880 }
7881 match agent::pick_chain(
7882 &self.state.config.agents,
7883 self.state.config.roles.language_judge.as_ref(),
7884 &agent::installed,
7885 "language_judge",
7886 ) {
7887 Ok(chain) => {
7888 for spec in chain {
7889 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7890 continue;
7891 }
7892 attempts.push((spec, None));
7893 }
7894 }
7895 Err(e) if attempts.is_empty() => {
7896 tracing::warn!("no agent for the description rewrite: {e:#}");
7897 self.state
7898 .event("github-text", "no rewrite agent available");
7899 return None;
7900 }
7901 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7902 }
7903 let prompts = self.state.config.prompts.clone();
7904 let cache = self.state.config.cache_dir();
7905 let run = self.state.id.clone();
7906 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7907 let resumed = resumed_seat.is_some();
7908 let seat = resumed_seat.unwrap_or_else(|| {
7909 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7910 });
7911 let github_spec = match agent::for_github_text(
7912 &spec,
7913 &self.state.config.graph.github_language,
7914 &winner.worktree,
7915 ) {
7916 Ok(spec) => spec,
7917 Err(e) => {
7918 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7919 continue;
7920 }
7921 };
7922 let job = SeatJob {
7923 spec: github_spec,
7924 seat,
7925 cwd: winner.worktree.clone(),
7926 prompt: prompt.clone(),
7927 timeout: retry_budget(timeout_full, resumed),
7928 allow_write: false,
7929 sessions,
7930 artifacts: agent::artifacts_dir(&self.state.dir()),
7931 stem: if i == 0 {
7932 "github-text-rewrite".to_owned()
7933 } else {
7934 format!("github-text-rewrite-{}", spec.id)
7935 },
7936 handover: None,
7937 };
7938 let ctx = WaveCtx {
7939 run: &run,
7940 node: "github-text",
7941 prompts: &prompts,
7942 cache: cache.as_deref(),
7943 round: None,
7944 carry_seats: false,
7945 pauses: Some((&self.pause, &self.interrupt)),
7946 };
7947 let (seat, outcome) =
7948 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7949 self.state.seats.insert(seat.key.clone(), seat);
7950 match outcome {
7951 AgentOutcome::Ok(o)
7952 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7953 {
7954 return Some(o.text);
7955 }
7956 AgentOutcome::ResourceBlocked(_) => return None,
7957 _ => {}
7958 }
7959 }
7960 None
7961 }
7962
7963 fn retire_github_text_question(&mut self, why: &str) {
7965 let Some(id) = self
7966 .state
7967 .github_text
7968 .as_ref()
7969 .and_then(|g| g.question.clone())
7970 else {
7971 return;
7972 };
7973 let _ = ask::Questions::open().update(&id, |q| {
7974 q.abandon(why);
7975 Ok(())
7976 });
7977 }
7978
7979 async fn settle_withheld(
7983 &mut self,
7984 winner: &Candidate,
7985 pr: &PrMessage,
7986 mut g: crate::run::GithubTextAsk,
7987 ) -> Result<Option<PrMessage>> {
7988 use crate::github_text as gt;
7989 let store = ask::Questions::open();
7990 let timeout = self.state.config.graph.answer_timeout;
7991 if !g.resolved {
7992 let mut standing = g
7993 .question
7994 .as_ref()
7995 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7996 if let Some(q) = standing.as_ref()
7997 && q.status.open()
7998 && gt::expired(q, timeout)
7999 {
8000 let why = format!("no answer within {}s of asking", timeout.max(1));
8001 standing = store
8002 .update(&q.id, |q| {
8003 q.abandon(&why);
8004 Ok(())
8005 })
8006 .ok()
8007 .map(|(q, ())| q);
8008 }
8009 let mut ask_again: Option<Option<Vec<String>>> = None;
8010 match standing {
8011 None if g.asks <= 1 => {
8014 g.asks = 0;
8015 ask_again = Some(None);
8016 }
8017 None => {
8018 g.resolved = true;
8019 g.chosen_title = None;
8020 g.choice = Some(GithubTextChoice::Fallback);
8021 }
8022 Some(q) if q.status.open() => {
8023 self.park_on_github_text(&g)?;
8024 return Ok(None);
8025 }
8026 Some(q) => match gt::read_reply(&q) {
8027 gt::Reply::Fallback => {
8028 g.resolved = true;
8029 g.chosen_title = None;
8030 g.choice = Some(GithubTextChoice::Fallback);
8031 }
8032 gt::Reply::Original => {
8033 g.resolved = true;
8034 g.chosen_title = None;
8035 g.choice = Some(GithubTextChoice::Original);
8036 }
8037 gt::Reply::Rewrite => {
8038 g.resolved = true;
8039 g.chosen_title = None;
8040 g.choice = Some(if g.rewrite.is_some() {
8041 GithubTextChoice::Rewrite
8042 } else {
8043 GithubTextChoice::Fallback
8044 });
8045 }
8046 gt::Reply::Title(text) => {
8047 let first = text
8048 .lines()
8049 .map(str::trim)
8050 .find(|l| !l.is_empty())
8051 .unwrap_or("");
8052 let decision = if first.is_empty() {
8053 None
8054 } else {
8055 let candidate = PrMessage {
8056 title: first.to_owned(),
8057 body: String::new(),
8058 };
8059 self.judge_pr_language(&winner.worktree, &candidate).await
8060 };
8061 match gt::vet_title(
8062 &text,
8063 decision,
8064 self.state.config.graph.github_english(),
8065 ) {
8066 Ok(title) => {
8067 g.resolved = true;
8068 g.chosen_title = Some(title);
8069 g.choice = Some(GithubTextChoice::MyTitle);
8070 }
8071 Err(cats) if g.asks < 2 => {
8072 ask_again = Some(Some(
8073 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8074 ))
8075 }
8076 Err(_) => {
8077 g.resolved = true;
8078 g.chosen_title = None;
8079 g.choice = Some(GithubTextChoice::Fallback);
8080 }
8081 }
8082 }
8083 },
8084 }
8085 if let Some(rejected) = ask_again {
8086 let w = gt::Withheld {
8089 title: g.title,
8090 body: g.body,
8091 categories: g.categories.clone(),
8092 };
8093 let artifact = match crate::run::write_artifact(
8096 &self.state,
8097 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8098 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8099 ) {
8100 Ok(p) => Some(p),
8101 Err(e) => {
8102 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8103 None
8104 }
8105 };
8106 let (detail, comparison) = gt::question_content(
8107 &w,
8108 &pr.title,
8109 &pr.body,
8110 &self.state.id,
8111 rejected.as_deref(),
8112 artifact.as_deref(),
8113 g.rewrite.as_ref(),
8114 );
8115 let mut q = ask::Question::new(
8116 self.state.id.clone(),
8117 gt::ASK_NODE.to_owned(),
8118 gt::ASK_SEAT.to_owned(),
8119 gt::question_summary(&self.state.config.graph.language, &w),
8120 detail,
8121 gt::question_choices(&w, g.rewrite.as_ref()),
8122 );
8123 q.posting_comparison = Some(comparison);
8124 q.answer_timeout = timeout;
8125 g.question = Some(q.id.clone());
8128 g.asks += 1;
8129 self.state.github_text = Some(g.clone());
8130 self.state.event(
8131 "github-text",
8132 format!("asking the owner about the withheld text ({})", q.short()),
8133 );
8134 self.state.save()?;
8135 store
8136 .put_executable(&mut q)
8137 .context("file the posting-gate question")?;
8138 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8139 tracing::warn!(
8140 "could not notify about posting-gate question {}: {e:#}",
8141 q.short()
8142 );
8143 }
8144 self.park_on_github_text(&g)?;
8145 return Ok(None);
8146 }
8147 }
8148 let resolved = g.chosen_title.clone();
8149 let original = g.choice == Some(GithubTextChoice::Original);
8150 self.state.event(
8152 "github-text",
8153 if original {
8154 format!(
8155 "posting the original text for: {}",
8156 match (g.title, g.body) {
8157 (true, true) => "title, description",
8158 (true, false) => "title",
8159 _ => "description",
8160 }
8161 )
8162 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8163 "posting the saved rewrite title and description".to_owned()
8164 } else if resolved.is_some() {
8165 "posting the owner's replacement title".to_owned()
8166 } else {
8167 "posting the neutral text for the withheld fields".to_owned()
8168 },
8169 );
8170 self.state.github_text = Some(g.clone());
8171 self.state.save()?;
8172 if g.choice == Some(GithubTextChoice::Rewrite)
8173 && let Some(r) = g.rewrite.as_ref()
8174 {
8175 return Ok(Some(PrMessage {
8176 title: r.title.clone(),
8177 body: r.body.clone(),
8178 }));
8179 }
8180 let w = gt::Withheld {
8181 title: g.title,
8182 body: g.body,
8183 categories: g.categories.clone(),
8184 };
8185 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8186 if let Some(t) = resolved {
8187 title = t;
8188 }
8189 let english = gt::LanguageDecision {
8191 title_english: true,
8192 body_english: true,
8193 body_complete: true,
8194 };
8195 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8196 if original {
8197 if g.title {
8200 title = pr.title.clone();
8201 }
8202 if g.body {
8203 body = pr.body.clone();
8204 }
8205 }
8206 Ok(Some(PrMessage { title, body }))
8207 }
8208
8209 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8210 self.state.github_text = Some(g.clone());
8211 self.state.end_pass_parked(ParkCause::Question);
8212 self.state.parked = true;
8213 self.state.event(
8214 "github-text",
8215 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8216 );
8217 self.state.save()
8218 }
8219
8220 async fn judge_pr_language(
8223 &mut self,
8224 cwd: &Path,
8225 pr: &PrMessage,
8226 ) -> Option<crate::github_text::LanguageDecision> {
8227 self.state.config.roles.language_judge.as_ref()?;
8228 if !self.state.config.graph.language_gate() {
8229 return None;
8230 }
8231 let decision =
8232 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8233 self.state.event(
8234 "github-text",
8235 if decision.is_some() {
8236 "language decided by the language judge"
8237 } else {
8238 "language judge unavailable; using built-in heuristics"
8239 },
8240 );
8241 decision
8242 }
8243
8244 async fn merge(&mut self) -> Result<()> {
8245 if self
8260 .state
8261 .base_sync
8262 .as_ref()
8263 .is_some_and(|s| s.conflict.is_some())
8264 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8265 != Some(RunStatus::Gating)
8266 || !self.state.gate_allows_progress()
8274 {
8275 return Ok(());
8276 }
8277 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8286 return Ok(());
8287 }
8288 let Some(winner) = self.state.winner().cloned() else {
8289 return Ok(());
8290 };
8291 let repo = self.state.repo.clone();
8292 let base = self.state.base_branch.clone();
8293 let mode = self.state.config.merge.mode;
8294 let style = self.state.config.merge.style;
8295 let facts = if is_review_run(&self.state) {
8296 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8297 let start = review_base(
8298 &repo,
8299 &self.state.config.merge.remote,
8300 &base,
8301 &self.state.base_commit,
8302 &winner.branch,
8303 )
8304 .await;
8305 branch_facts(&repo, &start, &winner.branch).await
8306 } else {
8307 None
8308 };
8309 let leaked = if is_review_run(&self.state) {
8312 leaked_subjects(&self.state, &winner.branch).await
8313 } else {
8314 Some(Vec::new())
8315 };
8316 let Some(pr) = self
8320 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8321 .await?
8322 else {
8323 return Ok(());
8324 };
8325 let message = pr.commit_message();
8326
8327 let outcome = match mode {
8328 MergeMode::None => MergeOutcome {
8329 mode,
8330 ok: true,
8331 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8332 empty: false,
8333 },
8334 MergeMode::Pr | MergeMode::Local
8335 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8336 {
8337 MergeOutcome {
8338 mode,
8339 ok: false,
8340 detail: empty_candidate_detail(&self.state, &base),
8341 empty: true,
8342 }
8343 }
8344 MergeMode::Local => {
8345 let on = git::current_branch(&repo).await?;
8346 if on.as_deref() != Some(base.as_str()) {
8347 MergeOutcome {
8348 mode,
8349 ok: false,
8350 detail: format!(
8351 "{} has {} checked out, not the base branch {base}",
8352 repo.display(),
8353 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8354 ),
8355 empty: false,
8356 }
8357 } else if !git::is_clean(&repo).await? {
8358 MergeOutcome {
8359 mode,
8360 ok: false,
8361 detail: format!("{} is dirty; refusing to merge", repo.display()),
8362 empty: false,
8363 }
8364 } else {
8365 let out = match style {
8366 MergeStyle::Merge => {
8367 git::merge_no_ff(&repo, &winner.branch, &message).await?
8368 }
8369 MergeStyle::Squash => {
8370 git::merge_squash(&repo, &winner.branch, &message).await?
8371 }
8372 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8373 };
8374 MergeOutcome {
8375 mode,
8376 ok: out.ok(),
8377 detail: if out.ok() { out.stdout } else { out.stderr },
8378 empty: false,
8379 }
8380 }
8381 }
8382 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8383 self.state.pr_publication_pending = true;
8384 self.state.save()?;
8385 let out =
8386 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8387 self.state.pr_publication_pending = out.is_err();
8388 MergeOutcome {
8389 mode,
8390 ok: out.is_ok(),
8391 detail: match out {
8392 Ok(url) => url,
8393 Err(e) => format!("{e:#}"),
8394 },
8395 empty: false,
8396 }
8397 }
8398 MergeMode::Pr => {
8399 let remote = self.state.config.merge.remote.clone();
8400 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8401 if !pushed.ok() {
8402 MergeOutcome {
8403 mode,
8404 ok: false,
8405 detail: pushed.stderr,
8406 empty: false,
8407 }
8408 } else {
8409 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8415 let out = match pr_merge_plan(found) {
8416 PrPlan::Create => {
8417 gh_pr_create(
8418 &winner.worktree,
8419 &base,
8420 &winner.branch,
8421 &pr.title,
8422 &pr.body,
8423 )
8424 .await
8425 }
8426 PrPlan::Adopt { url, title } => {
8427 self.state
8428 .event("merge", format!("Pr: adopted open pull request {url}"));
8429 if self.state.github_text.as_ref().is_some_and(|g| {
8430 g.resolved
8431 && g.choice == Some(GithubTextChoice::Rewrite)
8432 && g.rewrite.is_some()
8433 }) {
8434 land::set_pr_text_verbatim(
8437 &winner.worktree,
8438 &url,
8439 &pr.title,
8440 &pr.body,
8441 )
8442 .await
8443 .map(|()| url)
8444 } else {
8445 if title != pr.title
8446 && (!is_review_run(&self.state)
8447 || leaked
8448 .as_deref()
8449 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8450 && let Err(e) =
8451 if self.state.github_text.as_ref().is_some_and(|g| {
8452 g.resolved
8453 && g.title
8454 && g.choice == Some(GithubTextChoice::Original)
8455 }) {
8456 land::set_pr_title_verbatim(
8458 &winner.worktree,
8459 &url,
8460 &pr.title,
8461 )
8462 .await
8463 } else {
8464 land::set_pr_title(
8465 &mut self.state,
8466 &winner.worktree,
8467 &url,
8468 &pr.title,
8469 )
8470 .await
8471 }
8472 {
8473 tracing::warn!("could not refresh title of {url}: {e:#}");
8474 self.state
8475 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8476 }
8477 Ok(url)
8478 }
8479 }
8480 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8481 };
8482 match out {
8483 Ok(url) => MergeOutcome {
8484 mode,
8485 ok: true,
8486 detail: url,
8487 empty: false,
8488 },
8489 Err(e) => MergeOutcome {
8490 mode,
8491 ok: false,
8492 detail: e.to_string(),
8493 empty: false,
8494 },
8495 }
8496 }
8497 }
8498 };
8499
8500 self.state.status = match (mode, outcome.ok) {
8501 (MergeMode::None, _) => RunStatus::Ready,
8502 (_, true) if outcome.empty => RunStatus::Ready,
8503 (MergeMode::Pr, true) => RunStatus::Ready,
8504 (_, true) => RunStatus::Merged,
8505 (_, false) => RunStatus::Blocked,
8506 };
8507 self.state.event(
8508 "merge",
8509 format!(
8510 "{:?}: {}",
8511 mode,
8512 outcome.detail.lines().next().unwrap_or("")
8513 ),
8514 );
8515 self.state.merge = Some(outcome);
8516 self.state.save()?;
8517
8518 if self.state.config.graph.land
8524 && mode == MergeMode::Pr
8525 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8526 {
8527 self.run_land().await?;
8528 }
8529 self.settle_questions();
8534 Ok(())
8535 }
8536
8537 async fn run_land(&mut self) -> Result<()> {
8548 crate::pr_head::ensure_owner(&self.state)?;
8549 let url = self
8550 .state
8551 .pr
8552 .as_ref()
8553 .map(|pr| pr.url.clone())
8554 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8555 .unwrap_or_default();
8556 let url = url.lines().next().unwrap_or("").trim().to_owned();
8557 if !url.starts_with("http") {
8558 return Ok(());
8559 }
8560 loop {
8563 match land::land(&mut self.state, &url).await {
8564 Ok(_)
8565 if self
8566 .state
8567 .merge_repairs
8568 .last()
8569 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8570 {
8571 if self.repair_merge_review().await? {
8572 continue;
8573 }
8574 }
8575 Ok(pr) if self.state.parked => {
8576 let _ = pr;
8580 }
8581 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8582 self.state.status = RunStatus::Superseded;
8583 self.state.save()?;
8584 }
8585 Ok(pr) => {
8586 self.state.status = match pr.state {
8587 land::PrLifecycle::Merged => RunStatus::Merged,
8588 _ => RunStatus::Blocked,
8589 };
8590 if bump::should_release_bump(self.state.status)
8597 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8598 {
8599 self.state
8605 .event("bump", format!("release bump skipped: {e:#}"));
8606 }
8607 if self.state.status == RunStatus::Merged {
8610 crate::followup::after_merge(&mut self.state, &pr.url).await;
8611 }
8612 self.state.save()?;
8613 }
8614 Err(e) => {
8615 if e.is::<crate::repo_lock::ResourceBlocked>() {
8616 return Err(e);
8617 }
8618 self.state.status = RunStatus::Blocked;
8619 self.state.event("land", format!("gave up: {e}"));
8620 self.state.save()?;
8621 }
8622 }
8623 break;
8624 }
8625 Ok(())
8626 }
8627
8628 async fn repair_merge_review(&mut self) -> Result<bool> {
8631 use crate::run::MergeRepairPhase as Phase;
8632 crate::pr_head::ensure_owner(&self.state)?;
8633 let i = self.state.merge_repairs.len() - 1;
8634 let request = self.state.merge_repairs[i].clone();
8635 if request.phase == Phase::Fixing {
8636 let restored = match self.state.winner().cloned() {
8637 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8638 None => Err(anyhow::anyhow!("repair has no winning branch")),
8639 };
8640 let why = match restored {
8641 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8642 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8643 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8644 };
8645 self.hold_merge_repair(i, why)?;
8646 return Ok(false);
8647 }
8648 let failure = if i >= self.state.config.graph.land_rounds
8649 || self.state.config.graph.review_rounds == 0
8650 {
8651 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8652 } else {
8653 None
8654 };
8655 if let Some(why) = failure {
8656 self.hold_merge_repair(i, why)?;
8657 return Ok(false);
8658 }
8659 if request.phase == Phase::Requested {
8660 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8661 Ok(pr) => pr,
8662 Err(e) => {
8663 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8664 return Ok(false);
8665 }
8666 };
8667 self.state.merge_repairs[i].phase = Phase::Fixing;
8668 self.state.land_approval = None;
8669 self.state.land_armed_head = None;
8670 self.state.save()?;
8671 let evidence = serde_json::to_string_pretty(&request)?;
8674 let reason = format!(
8675 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8676 );
8677 let budget = self.state.config.graph.land_rounds;
8678 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8679 match result {
8680 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8681 Ok(land::Fixed::Failed(why)) => {
8682 self.hold_merge_repair(
8683 i,
8684 format!(
8685 "repair failed: {why}; inspect the branch and run an explicit review"
8686 ),
8687 )?;
8688 return Ok(false);
8689 }
8690 Err(e) => {
8691 self.hold_merge_repair(
8692 i,
8693 format!(
8694 "repair failed: {e:#}; inspect the branch and run an explicit review"
8695 ),
8696 )?;
8697 return Ok(false);
8698 }
8699 }
8700 self.begin_merge_repair_review(i)?;
8704 }
8705 self.review_loop().await?;
8706 self.gate().await?;
8707 if self.resource_halt()? || self.state.parked {
8708 return Ok(false);
8709 }
8710 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8711 != Some(RunStatus::Gating)
8712 || !self.state.gate_allows_progress()
8713 {
8714 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8715 return Ok(false);
8716 }
8717 let winner = self
8720 .state
8721 .winner()
8722 .cloned()
8723 .context("repair has no winning branch")?;
8724 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8725 if self.state.gate_head.as_deref() != Some(after.as_str())
8726 || !git::is_clean(&winner.worktree).await?
8727 {
8728 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8729 return Ok(false);
8730 }
8731 let expected = self
8732 .state
8733 .pr_pushed_head
8734 .as_deref()
8735 .unwrap_or(&request.head)
8736 .to_owned();
8737 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8738 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8741 let refspec = format!("{after}:refs/heads/{public}");
8742 let push = git::git_raw(
8743 &winner.worktree,
8744 &["push", &lease, &self.state.config.merge.remote, &refspec],
8745 )
8746 .await;
8747 match push {
8748 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8749 Ok(out) => {
8750 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8751 return Ok(false);
8752 }
8753 Err(e) => {
8754 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8755 return Ok(false);
8756 }
8757 }
8758 self.state.merge_repairs[i].phase = Phase::Complete;
8759 self.state.status = RunStatus::Landing;
8760 self.state.event(
8761 "repair",
8762 "new review and gate completed; fresh owner merge decision required",
8763 );
8764 self.state.save()?;
8765 Ok(true)
8766 }
8767
8768 fn merge_repair_artifacts(&self) -> PathBuf {
8769 let artifacts = agent::artifacts_dir(&self.state.dir());
8770 match self
8771 .state
8772 .merge_repairs
8773 .last()
8774 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8775 {
8776 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8777 None => artifacts,
8778 }
8779 }
8780
8781 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8782 self.state.reviews.clear();
8783 self.state.gate.clear();
8784 self.state.gate_ran = false;
8785 self.state.gate_head = None;
8786 self.state.gate_fixes.clear();
8787 self.state.gate_repair_e2e = None;
8788 self.state.pre_gate.clear();
8789 self.state.pre_gate_commit = None;
8790 self.state.contested_handoff = None;
8791 self.state.land_reruns = None;
8792 self.state.land_approval = None;
8793 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8794 self.state.status = RunStatus::Reviewing;
8795 self.state.save()
8796 }
8797
8798 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8799 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8800 self.state.merge_repairs[index].failure = Some(why.clone());
8801 self.state.status = RunStatus::Blocked;
8802 self.state.event("repair", why);
8803 self.state.save()
8804 }
8805
8806 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8810 if let Some(existing) = self.state.seats.get(key)
8811 && existing.agent == agent
8812 {
8813 return existing.clone();
8814 }
8815 let fresh = if self.state.seats.contains_key(key) {
8818 handover_seat(key, agent, self.state.next_seat_seed())
8819 } else {
8820 SeatState::new(key, agent, self.state.seed)
8821 };
8822 self.state.seats.insert(key.to_owned(), fresh.clone());
8823 fresh
8824 }
8825
8826 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8832 match self.state.seats.get(key) {
8833 Some(s) if s.agent != spec.id => {
8834 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8835 }
8836 _ => spec,
8837 }
8838 }
8839
8840 fn view(&self, c: &Candidate) -> CandidateView {
8842 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8843 .unwrap_or_default();
8844 let (patch, _) = blind::sanitize_patch(
8845 &format!("candidate {} patch", c.label),
8846 &raw,
8847 &self.state.config.blind,
8848 );
8849 CandidateView {
8850 label: c.label,
8851 branch: c.branch.clone(),
8852 summary: c.summary.clone(),
8853 stat: c.stat.clone(),
8854 patch,
8855 }
8856 }
8857
8858 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8860 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8861 prompt::judge(
8862 "(see above)",
8863 &views,
8864 self.roles.judges.len(),
8865 base_short,
8866 "en",
8867 )
8868 }
8869
8870 fn vote_prompt_full(
8876 &self,
8877 j: usize,
8878 labels: &[char],
8879 language: &str,
8880 candidates: &[Candidate],
8881 base_short: &str,
8882 ) -> String {
8883 let mut text = format!(
8884 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8885 prompt::final_vote(labels, language),
8886 self.state.instruction,
8887 self.candidate_block(candidates, base_short)
8888 );
8889 if let Some(own) = self
8890 .state
8891 .judgements
8892 .get(j)
8893 .filter(|r| !r.ranking.is_empty())
8894 {
8895 let reasons = own
8896 .reasons
8897 .iter()
8898 .map(|(k, v)| format!("- {k}: {v}"))
8899 .collect::<Vec<_>>()
8900 .join("\n");
8901 text.push_str(&format!(
8902 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8903 own.ranking.iter().collect::<String>(),
8904 if reasons.is_empty() {
8905 ""
8906 } else {
8907 ", because:\n"
8908 }
8909 ));
8910 }
8911 if !self.state.deliberation.is_empty() {
8912 text.push_str("\n# What was argued before this vote\n");
8913 for t in self.transcript(&[], j) {
8914 text.push_str(&format!(
8915 "\n## {}{}\n\n{}\n",
8916 t.who,
8917 if t.is_self { " (you)" } else { "" },
8918 t.body.trim()
8919 ));
8920 }
8921 }
8922 text
8923 }
8924
8925 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8932 let mut turns = Vec::new();
8933 for j in &self.state.judgements {
8934 if j.ranking.is_empty() {
8935 continue;
8936 }
8937 let reasons = j
8938 .reasons
8939 .iter()
8940 .map(|(k, v)| format!("- {k}: {v}"))
8941 .collect::<Vec<_>>()
8942 .join("\n");
8943 turns.push(Turn {
8944 who: format!("Judge {} (opening ranking)", j.judge),
8945 is_self: j.judge == self_idx + 1,
8946 body: format!(
8947 "Ranked {}{}{reasons}",
8948 j.ranking.iter().collect::<String>(),
8949 if reasons.is_empty() {
8950 ""
8951 } else {
8952 ", because:\n"
8953 }
8954 ),
8955 });
8956 }
8957 for t in self
8958 .state
8959 .deliberation
8960 .iter()
8961 .flat_map(|r| r.turns.iter())
8962 .chain(current)
8963 {
8964 turns.push(Turn {
8965 who: format!("Judge {}", t.judge),
8966 is_self: t.judge == self_idx + 1,
8967 body: t.body.clone(),
8968 });
8969 }
8970 turns
8971 }
8972}
8973
8974fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8976 agent::has_session(spec.kind, seat, sessions)
8977}
8978
8979fn next_untried_in_roster<'a>(
9000 roster: &'a [AgentSpec],
9001 start: usize,
9002 tried: &BTreeSet<String>,
9003) -> Option<&'a AgentSpec> {
9004 roster
9005 .get(start + 1..)?
9006 .iter()
9007 .find(|s| !tried.contains(&s.id))
9008}
9009
9010fn pick_successor<'a>(
9031 roster: &'a [AgentSpec],
9032 start: usize,
9033 tried: &BTreeSet<String>,
9034 carried: Option<&BTreeSet<String>>,
9035 others: &BTreeSet<String>,
9036) -> Option<&'a AgentSpec> {
9037 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9038 roster
9039 .get(start + 1..)
9040 .and_then(|tail| {
9041 tail.iter()
9042 .filter(free)
9043 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9044 })
9045 .or_else(|| {
9046 carried
9047 .is_some()
9048 .then(|| roster.iter().find(free))
9049 .flatten()
9050 })
9051 .or_else(|| match carried {
9052 Some(c) => next_for_seat(roster, start, tried, c),
9053 None => next_untried_in_roster(roster, start, tried),
9054 })
9055}
9056
9057fn next_for_seat<'a>(
9067 roster: &'a [AgentSpec],
9068 start: usize,
9069 round_tried: &BTreeSet<String>,
9070 carried_failed: &BTreeSet<String>,
9071) -> Option<&'a AgentSpec> {
9072 roster
9073 .get(start + 1..)?
9074 .iter()
9075 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9076 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9077}
9078
9079fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9085 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9086 return spec;
9087 };
9088 let ok = |id: &str| !h.failed.contains(id);
9089 if let Some(last) = h.last_ok.as_deref()
9090 && ok(last)
9091 && let Some(s) = roster.iter().find(|s| s.id == last)
9092 {
9093 return s.clone();
9094 }
9095 if ok(&spec.id) {
9096 return spec;
9097 }
9098 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9099 roster
9100 .get(from + 1..)
9101 .into_iter()
9102 .flatten()
9103 .chain(roster.iter())
9104 .find(|s| ok(&s.id))
9105 .cloned()
9106 .unwrap_or(spec)
9107}
9108
9109pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9113 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9114}
9115
9116const TIMED_OUT: &str = "timed out";
9119
9120impl FailClass {
9121 fn of(out: &AgentOutcome) -> Option<Self> {
9123 match out {
9124 AgentOutcome::Ok(_) => None,
9125 AgentOutcome::Quota(_) => Some(Self::Quota),
9126 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9127 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9128 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9129 AgentOutcome::ResourceBlocked(_) => None,
9132 }
9133 }
9134
9135 fn stem_word(&self) -> &'static str {
9137 match self {
9138 Self::Quota => "quota",
9139 _ => "handover",
9140 }
9141 }
9142}
9143
9144fn failure_signature(msg: &str) -> String {
9148 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9149 if let Some(code) = line
9152 .strip_prefix("process exited with no usable output (exit code ")
9153 .and_then(|s| s.strip_suffix(')'))
9154 .and_then(|s| s.parse::<i32>().ok())
9155 {
9156 line = format!("exited with some({code}) and no usable output");
9157 } else if matches!(
9158 line.as_str(),
9159 "process exited with no usable output (no exit code reported)"
9160 | "process exited with no usable output (exit code unavailable)"
9161 ) {
9162 line = "exited with none and no usable output".to_owned();
9163 }
9164 let mut out = Vec::new();
9165 for word in line.split_whitespace() {
9166 if word.contains('/') || word.contains('\\') {
9167 out.push("<path>".to_owned());
9168 continue;
9169 }
9170 let mut w = String::new();
9171 let mut in_digits = false;
9172 for c in word.chars() {
9173 if c.is_ascii_digit() {
9174 if !in_digits {
9175 w.push('#');
9176 }
9177 in_digits = true;
9178 } else {
9179 in_digits = false;
9180 w.push(c);
9181 }
9182 }
9183 out.push(w);
9184 }
9185 out.join(" ").chars().take(120).collect()
9186}
9187
9188fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9195 match cur {
9196 FailClass::Quota | FailClass::Timeout => true,
9197 FailClass::Other(_) => prev != Some(cur),
9198 }
9199}
9200
9201fn fail_reason(out: &AgentOutcome) -> String {
9203 match out {
9204 AgentOutcome::Ok(_) => String::new(),
9205 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9206 AgentOutcome::Dropped(o) => format!(
9207 "the CLI dropped the stream ({})",
9208 o.dropped
9209 .as_ref()
9210 .map(|d| d.why.as_str())
9211 .unwrap_or("it ended without delivering its answer")
9212 ),
9213 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9214 e.lines().next().unwrap_or("").chars().take(160).collect()
9215 }
9216 }
9217}
9218
9219pub(crate) fn record_handover(
9222 state: &mut RunState,
9223 node: &str,
9224 seat: &str,
9225 from: &str,
9226 to: &str,
9227 class: &FailClass,
9228 reason: &str,
9229) {
9230 let message = if *class == FailClass::Quota {
9231 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9232 } else {
9233 format!("{seat}: handed over {from} -> {to} ({reason})")
9234 };
9235 state.event(node, message);
9236 state.handovers.push(Handover {
9237 at: Timestamp::now(),
9238 node: node.to_owned(),
9239 seat: seat.to_owned(),
9240 from: from.to_owned(),
9241 to: to.to_owned(),
9242 reason: reason.to_owned(),
9243 });
9244}
9245
9246fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9260 commands.iter().any(|c| c.exit_code.is_none())
9261}
9262
9263fn verified_noop_claim(
9276 usable: bool,
9277 commands: &[agent::CommandEvidence],
9278 text: &str,
9279) -> Option<String> {
9280 (usable && !has_unconfirmed_command(commands))
9281 .then(|| verdict::verified_noop(text))
9282 .flatten()
9283}
9284
9285fn short(commit: &str) -> String {
9286 commit.chars().take(7).collect()
9287}
9288
9289fn make_executable(path: &Path) -> Result<()> {
9290 #[cfg(unix)]
9291 {
9292 use std::os::unix::fs::PermissionsExt as _;
9293 let mut perms = std::fs::metadata(path)?.permissions();
9294 perms.set_mode(0o755);
9295 std::fs::set_permissions(path, perms)?;
9296 }
9297 #[cfg(not(unix))]
9298 {
9299 let _ = path;
9300 }
9301 Ok(())
9302}
9303
9304struct WaveCtx<'a> {
9311 run: &'a str,
9314 node: &'a str,
9316 prompts: &'a Prompts,
9317 cache: Option<&'a Path>,
9319 round: Option<usize>,
9322 carry_seats: bool,
9326 pauses: Option<(&'a Pause, &'a Pause)>,
9329}
9330
9331fn repository_wait(
9333 state: &RunState,
9334 pauses: Option<(&Pause, &Pause)>,
9335) -> crate::repo_lock::WaitPolicy {
9336 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9337 let inherited = crate::repo_lock::WaitPolicy::current();
9338 crate::repo_lock::WaitPolicy::new(
9339 Duration::from_secs(state.config.graph.timeout_resource_wait),
9340 move || {
9341 inherited.cancelled()
9342 || pauses
9343 .as_ref()
9344 .is_some_and(|(a, b)| a.parked() || b.parked())
9345 },
9346 )
9347}
9348
9349fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9350 if error.is::<crate::repo_lock::ResourceBlocked>() {
9351 state.resource_block = Some(crate::run::ResourceBlock {
9352 cause: crate::run::ResourceBlockCause::Other,
9353 node: "repository".into(),
9354 kind: if error.to_string().contains("disk pressure") {
9355 "disk pressure"
9356 } else {
9357 "repository coordination"
9358 }
9359 .into(),
9360 detail: error.to_string(),
9361 at: Timestamp::now(),
9362 });
9363 state.event("resource", error.to_string());
9364 }
9365}
9366
9367async fn coordinate_operation(
9368 state: &mut RunState,
9369 cwd: &Path,
9370 wait: &crate::repo_lock::WaitPolicy,
9371) -> Result<Option<crate::repo_lock::SeatLease>> {
9372 let result =
9373 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9374 .await;
9375 if let Err(e) = &result {
9376 record_repository_block(state, e);
9377 }
9378 result
9379}
9380
9381async fn run_one(
9383 job: SeatJob,
9384 sem: Arc<Semaphore>,
9385 ctx: &WaveCtx<'_>,
9386 state: &mut RunState,
9387 attempt: usize,
9388) -> (SeatState, AgentOutcome) {
9389 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9390 .await
9391 .pop()
9392 .expect("one job in, one result out");
9393 (seat, out)
9394}
9395
9396async fn wave(
9402 jobs: Vec<SeatJob>,
9403 sem: Arc<Semaphore>,
9404 ctx: &WaveCtx<'_>,
9405 state: &mut RunState,
9406 attempt: usize,
9407) -> Vec<(usize, SeatState, AgentOutcome)> {
9408 let WaveCtx {
9409 run,
9410 node,
9411 prompts,
9412 cache,
9413 round,
9414 carry_seats: _,
9415 pauses,
9416 } = *ctx;
9417 for job in &jobs {
9418 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9419 }
9420 if let Err(e) = state.save() {
9421 tracing::warn!("could not persist in-progress seats: {e:#}");
9426 }
9427 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9443 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9447 let cache_guard = if let Some(cache_dir) = cache {
9448 if jobs_had_a_writer {
9449 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9450 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9451 match acquire_cache_lease(
9452 state,
9453 cache_dir,
9454 &owner,
9455 resource_limit,
9456 node,
9457 node,
9458 &cancel,
9459 )
9460 .await
9461 {
9462 Ok(Some(g)) => Some(g),
9463 Ok(None) => {
9466 if let Some(block) = &mut state.resource_block {
9467 block.cause = crate::run::ResourceBlockCause::Cancelled {
9468 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9469 };
9470 }
9471 return block_wave(
9472 jobs,
9473 state,
9474 node,
9475 attempt,
9476 "parked while waiting for the shared build cache",
9477 );
9478 }
9479 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9482 Err(e) => {
9483 let why = format!("{e:#}");
9487 state.event(
9488 "resource",
9489 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9490 );
9491 return block_wave(jobs, state, node, attempt, &why);
9492 }
9493 }
9494 } else {
9495 None
9496 }
9497 } else {
9498 None
9499 };
9500 state.seats_spawning(node, attempt);
9503 if let Err(e) = state.save() {
9504 tracing::warn!("could not persist the start of a wave: {e:#}");
9505 }
9506 let slot_limit = resource_limit
9510 + jobs
9511 .iter()
9512 .map(|j| j.timeout)
9513 .max()
9514 .unwrap_or_default()
9515 .saturating_mul(jobs.len() as u32);
9516 let mut set = tokio::task::JoinSet::new();
9517 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9520 let overlay = prompts.overlay(node);
9521 let repository_wait = repository_wait(state, pauses);
9522 for (i, mut job) in jobs.into_iter().enumerate() {
9523 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9524 if cache.is_some() {
9525 job.prompt.push('\n');
9526 job.prompt
9527 .push_str(&prompt::build_cache_note(node, job.allow_write));
9528 }
9529 let sem = Arc::clone(&sem);
9530 let run = run.to_owned();
9531 let node = node.to_owned();
9532 let attachments = if node == "implement" {
9535 state.attachments.clone()
9536 } else {
9537 Vec::new()
9538 };
9539 let cache = cache
9550 .filter(|_| job.allow_write && cache_guard.is_some())
9551 .map(Path::to_path_buf);
9552 let spawned_tx = spawned_tx.clone();
9553 let repository_wait = repository_wait.clone();
9554 let coordination = state.config.coordination.clone();
9555 let fetched_base = state.fetched_base.clone();
9556 set.spawn(async move {
9557 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9561 Ok(p) => p,
9562 Err(_) => {
9563 return (
9564 i,
9565 job.seat,
9566 AgentOutcome::ResourceBlocked(format!(
9567 "no parallel slot became free within {}s",
9568 slot_limit.as_secs()
9569 )),
9570 false,
9571 );
9572 }
9573 };
9574 spawned_tx.send(job.seat.key.clone()).ok();
9575 let mut seat = job.seat;
9576 let out = repository_wait
9577 .scope(agent::invoke(
9578 &job.spec,
9579 &mut seat,
9580 &Invocation {
9581 coordination: coordination.as_ref(),
9582 fetched_base: fetched_base.as_ref(),
9583 cwd: &job.cwd,
9584 prompt: &job.prompt,
9585 timeout: job.timeout,
9586 allow_write: job.allow_write,
9587 sessions: job.sessions,
9588 artifacts: &job.artifacts,
9589 stem: &job.stem,
9590 run: &run,
9591 node: &node,
9592 cache_dir: cache.as_deref(),
9593 attachments: &attachments,
9594 writable: &[],
9595 },
9596 ))
9597 .await;
9598 let interrupted = out
9599 .as_ref()
9600 .err()
9601 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9602 let out = match out {
9603 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9604 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9605 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9613 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9614 Ok(o) => AgentOutcome::Failed(format!(
9615 "process exited with no usable output ({})",
9616 crate::notices::exit_status(o.exit_code)
9617 )),
9618 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9619 AgentOutcome::ResourceBlocked(e.to_string())
9620 }
9621 Err(e) => AgentOutcome::Failed(e.to_string()),
9622 };
9623 (i, seat, out, interrupted)
9624 });
9625 }
9626 drop(spawned_tx);
9627 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9628 loop {
9629 let joined = tokio::select! {
9630 Some(key) = spawned_rx.recv() => {
9631 state.seat_spawned(&key);
9633 if let Err(e) = state.save() {
9634 tracing::warn!("could not persist a seat's start: {e:#}");
9635 }
9636 continue;
9637 }
9638 joined = set.join_next() => joined,
9639 };
9640 let Some(joined) = joined else {
9641 break;
9642 };
9643 let (i, seat, out, interrupted) = match joined {
9644 Ok(v) => v,
9645 Err(e) => {
9649 tracing::error!("agent task panicked: {e}");
9650 continue;
9651 }
9652 };
9653 state.seat_finished(&seat.key);
9654 if interrupted
9655 && node == "implement"
9656 && let Some(label) = state
9657 .candidates
9658 .iter()
9659 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9660 .map(|candidate| candidate.label)
9661 {
9662 state.candidate_preservation.insert(
9663 label,
9664 "interrupted implementation awaiting persistence".into(),
9665 );
9666 state.interrupted_implementations.insert(label);
9667 }
9668 if let AgentOutcome::ResourceBlocked(why) = &out
9669 && state.resource_block.is_none()
9670 {
9671 state.resource_block = Some(crate::run::ResourceBlock {
9672 cause: crate::run::ResourceBlockCause::Other,
9673 node: node.to_owned(),
9674 kind: if why.contains("disk pressure") {
9675 "disk pressure"
9676 } else if why.contains("repository coordination") {
9677 "repository coordination"
9678 } else {
9679 "parallel slot"
9680 }
9681 .to_owned(),
9682 detail: why.clone(),
9683 at: Timestamp::now(),
9684 });
9685 let activity = if interrupted {
9686 "agent execution interrupted"
9687 } else {
9688 "no agent was started"
9689 };
9690 state.event("resource", format!("{node}: {activity} - {why}"));
9691 }
9692 record_jobs(state, node, round, &seat.key, &out);
9693 if let Err(e) = state.save() {
9694 tracing::warn!("could not persist a seat's completion: {e:#}");
9695 }
9696 if collected.len() <= i {
9697 collected.resize_with(i + 1, || None);
9698 }
9699 collected[i] = Some((i, seat, out));
9700 }
9701 if state
9707 .active
9708 .values()
9709 .any(|a| a.node == node && a.attempt == attempt)
9710 {
9711 state
9712 .active
9713 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9714 if let Err(e) = state.save() {
9715 tracing::warn!("could not persist the end of a wave: {e:#}");
9716 }
9717 }
9718 if let Some(cache_dir) = cache
9725 && jobs_had_a_writer
9726 {
9727 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9728 }
9729 if let Some(guard) = cache_guard {
9730 guard.release();
9731 }
9732 collected.into_iter().flatten().collect()
9733}
9734
9735fn record_jobs(
9746 state: &mut RunState,
9747 node: &str,
9748 round: Option<usize>,
9749 seat: &str,
9750 out: &AgentOutcome,
9751) {
9752 let commands: &[agent::CommandEvidence] = match out {
9753 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9754 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9755 };
9756 let checked_at = Timestamp::now();
9757 for c in commands {
9758 state.jobs.push(JobRecord {
9759 node: node.to_owned(),
9760 round,
9761 seat: seat.to_owned(),
9762 id: c.id.clone(),
9763 description: c.description.clone(),
9764 checked_at,
9765 status: match c.exit_code {
9766 Some(0) => JobStatus::Completed,
9767 Some(_) => JobStatus::Failed,
9768 None => JobStatus::Unknown,
9769 },
9770 exit_code: c.exit_code,
9771 result_summary: c.result_summary.clone(),
9772 source: c.source.clone(),
9773 });
9774 }
9775}
9776
9777fn round_is_clean(
9798 blocking: usize,
9799 e2e_ok: bool,
9800 answered: usize,
9801 expected: usize,
9802 quota_missing: usize,
9803 policy: IncompleteReviewPolicy,
9804) -> bool {
9805 if blocking != 0 || !e2e_ok {
9806 return false;
9807 }
9808 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9809 return true;
9810 }
9811 answered > 0 && expected - answered <= quota_missing
9812}
9813
9814pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9838 if max_rounds == 0
9839 || reviews.last().is_some_and(|r| {
9840 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9841 })
9842 {
9843 return Some(RunStatus::Gating);
9844 }
9845 let last = reviews.last()?;
9846 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9847 if reviews.len() < max_rounds && !stagnant {
9848 return None;
9849 }
9850 if last.incomplete() && last.blocking == 0 {
9851 return Some(RunStatus::Blocked);
9852 }
9853 if last.e2e_status() == E2eStatus::ResourceBlocked
9854 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9855 {
9856 return None;
9857 }
9858 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9859 RunStatus::Gating
9860 } else {
9861 RunStatus::Blocked
9862 })
9863}
9864
9865pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9880 if nudged {
9881 (full / 4).max(Duration::from_secs(120)).min(full)
9882 } else {
9883 full
9884 }
9885}
9886
9887#[allow(clippy::too_many_arguments)]
9916async fn ask_json_wave<T>(
9917 jobs: Vec<SeatJob>,
9918 sem: Arc<Semaphore>,
9919 retries: usize,
9920 roster: &[AgentSpec],
9921 ctx: &WaveCtx<'_>,
9922 losses: &mut Vec<QuotaLoss>,
9923 state: &mut RunState,
9924 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9925) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9926where
9927 T: serde::de::DeserializeOwned + Send + 'static,
9928{
9929 ask_wave_with(
9930 jobs,
9931 sem,
9932 retries,
9933 roster,
9934 ctx,
9935 losses,
9936 state,
9937 &|text: &str| {
9938 let v = verdict::extract_json::<T>(text)?;
9939 validate(&v)?;
9940 Ok(v)
9941 },
9942 )
9943 .await
9944}
9945
9946#[allow(clippy::too_many_arguments)]
9955async fn ask_wave_with<T>(
9956 jobs: Vec<SeatJob>,
9957 sem: Arc<Semaphore>,
9958 retries: usize,
9959 roster: &[AgentSpec],
9960 ctx: &WaveCtx<'_>,
9961 losses: &mut Vec<QuotaLoss>,
9962 state: &mut RunState,
9963 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9964) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9965where
9966 T: Send + 'static,
9967{
9968 let n = jobs.len();
9969 let originals: Vec<SeatJob> = jobs;
9970 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9971 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9972 let mut nudges: Vec<usize> = vec![0; n];
9979 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9983 let mut tried: Vec<BTreeSet<String>> = specs
9984 .iter()
9985 .map(|s| BTreeSet::from([s.id.clone()]))
9986 .collect();
9987 let starts: Vec<usize> = specs
9988 .iter()
9989 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9990 .collect();
9991 let carry = ctx.carry_seats && !roster.is_empty();
9992 let carried: Vec<BTreeSet<String>> = originals
9995 .iter()
9996 .map(|j| {
9997 state
9998 .seat_history
9999 .get(&j.seat.key)
10000 .filter(|_| carry)
10001 .map(|h| h.failed.clone())
10002 .unwrap_or_default()
10003 })
10004 .collect();
10005 let mut prev: Vec<Option<FailClass>> = originals
10006 .iter()
10007 .map(|j| {
10008 state
10009 .seat_history
10010 .get(&j.seat.key)
10011 .filter(|_| carry)
10012 .and_then(|h| h.last_fail.clone())
10013 })
10014 .collect();
10015 let next_agent =
10016 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10017 let others: BTreeSet<String> = specs
10018 .iter()
10019 .enumerate()
10020 .filter(|(j, _)| *j != i)
10021 .map(|(_, s)| s.id.clone())
10022 .collect();
10023 pick_successor(
10024 roster,
10025 starts[i],
10026 tried,
10027 carry.then(|| &carried[i]),
10028 &others,
10029 )
10030 .cloned()
10031 };
10032 let mut fresh: Vec<Option<String>> = vec![None; n];
10033 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10034 let mut pending: Vec<usize> = (0..n).collect();
10035
10036 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10041 for round in 0..max_rounds {
10042 if pending.is_empty() {
10043 break;
10044 }
10045 let mut batch = Vec::with_capacity(pending.len());
10046 let mut renudged: Vec<&str> = Vec::new();
10047 for &i in &pending {
10048 let src = &originals[i];
10049 let full: &str = match &src.handover {
10054 Some(h) if specs[i].id != src.spec.id => h,
10055 _ => &src.prompt,
10056 };
10057 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10060 (
10061 full.to_owned(),
10062 src.timeout,
10063 format!("{}-{word}-{}", src.stem, specs[i].id),
10064 )
10065 } else if nudges[i] == 0 {
10066 (full.to_owned(), src.timeout, src.stem.clone())
10067 } else {
10068 renudged.push(src.seat.key.as_str());
10069 let why = done[i]
10070 .as_ref()
10071 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10072 .unwrap_or_else(|| "no parsable answer".to_owned());
10073 let nudge = prompt::nudge(&why);
10074 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10075 let prompt = if nudged {
10076 nudge
10077 } else {
10078 format!("{full}\n\n---\n\n{nudge}")
10079 };
10080 (
10081 prompt,
10082 retry_budget(src.timeout, nudged),
10083 format!("{}-retry{}", src.stem, nudges[i]),
10084 )
10085 };
10086 batch.push(SeatJob {
10087 spec: specs[i].clone(),
10088 seat: seats[i].clone(),
10089 cwd: src.cwd.clone(),
10090 prompt,
10091 timeout,
10092 allow_write: src.allow_write,
10093 sessions: src.sessions,
10094 artifacts: src.artifacts.clone(),
10095 stem,
10096 handover: None,
10097 });
10098 }
10099
10100 if !renudged.is_empty() {
10101 state.event(
10102 ctx.node,
10103 format!("retry {round}: re-asking {}", renudged.join(", ")),
10104 );
10105 }
10106 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10107 let mut still = Vec::new();
10108 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10109 seats[i] = seat;
10110 if let AgentOutcome::ResourceBlocked(why) = &out {
10113 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10114 continue;
10115 }
10116 let class = FailClass::of(&out);
10117 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10120 && nudges[i] < retries
10121 && !roster.is_empty();
10122 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10123 let next = should_hand_over(prev[i].as_ref(), &cur)
10124 .then(|| next_agent(i, &tried[i], &specs))
10125 .flatten();
10126 if carry {
10127 let h = state
10128 .seat_history
10129 .entry(originals[i].seat.key.clone())
10130 .or_default();
10131 h.failed.insert(specs[i].id.clone());
10132 h.last_fail = Some(cur.clone());
10133 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10134 h.last_ok = None;
10135 }
10136 if let Err(e) = state.save() {
10139 tracing::warn!("could not persist a seat's failure history: {e:#}");
10140 }
10141 }
10142 if let Some(next) = next {
10143 record_handover(
10144 state,
10145 ctx.node,
10146 &originals[i].seat.key,
10147 &specs[i].id,
10148 &next.id,
10149 &cur,
10150 &fail_reason(&out),
10151 );
10152 tried[i].insert(next.id.clone());
10153 prev[i] = Some(cur.clone());
10154 seats[i] =
10155 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10156 specs[i] = next;
10157 fresh[i] = Some(cur.stem_word().to_owned());
10158 nudges[i] = 0;
10159 done[i] = Some(Err(anyhow::anyhow!(
10160 "handed over after: {}",
10161 fail_reason(&out)
10162 )));
10163 still.push(i);
10164 continue;
10165 }
10166 }
10167 if carry
10168 && !nudge_first
10169 && let Some(cur) = class.clone()
10170 {
10171 let h = state
10173 .seat_history
10174 .entry(originals[i].seat.key.clone())
10175 .or_default();
10176 h.failed.insert(specs[i].id.clone());
10177 h.last_fail = Some(cur);
10178 }
10179 let parsed = match out {
10180 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10181 AgentOutcome::Quota(o) => {
10182 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10183 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10184 }
10185 AgentOutcome::Dropped(o) => {
10189 let why = o
10190 .dropped
10191 .as_ref()
10192 .map(|d| d.why.as_str())
10193 .unwrap_or("the CLI ended the stream without delivering its answer");
10194 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10195 }
10196 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10197 Err(anyhow::anyhow!(e))
10198 }
10199 };
10200 let quota = class == Some(FailClass::Quota);
10201 let failed = parsed.is_err();
10202 done[i] = Some(parsed);
10203 if carry && !failed {
10204 let h = state
10205 .seat_history
10206 .entry(originals[i].seat.key.clone())
10207 .or_default();
10208 h.failed.remove(&specs[i].id);
10209 h.last_ok = Some(specs[i].id.clone());
10210 h.last_fail = None;
10211 }
10212 let agent_failure = class.is_some()
10220 && !nudge_first
10221 && !roster.is_empty()
10222 && next_agent(i, &tried[i], &specs).is_some();
10223 if failed && !quota && !agent_failure && nudges[i] < retries {
10224 nudges[i] += 1;
10225 still.push(i);
10226 }
10227 }
10228 pending = still;
10229 }
10230
10231 for (i, q) in last_quota.into_iter().enumerate() {
10234 if let Some(reset) = q {
10235 losses.push(QuotaLoss {
10236 seat: originals[i].seat.key.clone(),
10237 node: ctx.node.to_owned(),
10238 at: Timestamp::now(),
10239 reset,
10240 });
10241 }
10242 }
10243
10244 seats
10245 .into_iter()
10246 .zip(done)
10247 .zip(nudges)
10248 .map(|((seat, res), attempts)| {
10249 (
10250 seat,
10251 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10252 attempts,
10253 )
10254 })
10255 .collect()
10256}
10257
10258fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10260 results
10261 .iter()
10262 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10263}
10264
10265fn block_wave(
10269 jobs: Vec<SeatJob>,
10270 state: &mut RunState,
10271 node: &str,
10272 attempt: usize,
10273 why: &str,
10274) -> Vec<(usize, SeatState, AgentOutcome)> {
10275 state.resource_wait = None;
10276 state
10277 .active
10278 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10279 if let Err(e) = state.save() {
10280 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10281 }
10282 jobs.into_iter()
10283 .enumerate()
10284 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10285 .collect()
10286}
10287
10288async fn acquire_cache_lease(
10301 state: &mut RunState,
10302 cache_dir: &Path,
10303 owner: &crate::cache::Owner,
10304 budget: Duration,
10305 node: &str,
10306 context: &str,
10307 cancel: &(dyn Fn() -> bool + Sync),
10308) -> Result<Option<crate::cache::Guard>> {
10309 let home = crate::run::home();
10310 let started = Instant::now();
10311 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10312 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10313 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10314 Err(e) => {
10315 state.event(
10316 "verify",
10317 format!("{context}: could not check the shared build cache: {e:#}"),
10318 );
10319 if let Err(e2) = state.save() {
10320 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10321 }
10322 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10323 }
10324 };
10325 state.event(
10326 "verify",
10327 format!(
10328 "{context}: waiting for the shared build cache at {} ({})",
10329 cache_dir.display(),
10330 busy.describe()
10331 ),
10332 );
10333 state.resource_wait = Some(crate::run::ResourceWait {
10336 node: context.to_owned(),
10337 kind: "shared build cache".to_owned(),
10338 holder: busy.describe(),
10339 since: Timestamp::now(),
10340 limit_secs: budget.as_secs(),
10341 });
10342 if let Err(e) = state.save() {
10343 tracing::warn!("could not persist a cache wait: {e:#}");
10344 }
10345 let remaining = budget.saturating_sub(started.elapsed());
10346 let waited = crate::cache::wait_for_unless(
10347 &home,
10348 cache_dir,
10349 owner,
10350 remaining,
10351 Duration::from_secs(5),
10352 cancel,
10353 )
10354 .await;
10355 state.resource_wait = None;
10356 let cause = match &waited {
10357 Ok(Some(_)) => None,
10358 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10359 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10360 Some(crate::run::ResourceBlockCause::Timeout {
10361 waited_secs: started.elapsed().as_secs(),
10362 limit_secs: budget.as_secs(),
10363 })
10364 }
10365 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10366 };
10367 if let Some(cause) = cause {
10368 let detail = match &waited {
10369 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10370 Err(error) => format!("{error:#}"),
10371 };
10372 state.resource_block = Some(crate::run::ResourceBlock {
10373 cause,
10374 node: node.to_owned(),
10375 kind: "shared build cache".into(),
10376 detail,
10377 at: Timestamp::now(),
10378 });
10379 state.event(
10380 "resource",
10381 state.resource_block.as_ref().unwrap().description(),
10382 );
10383 }
10384 if let Err(error) = state.save() {
10385 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10386 }
10387 waited
10388}
10389
10390#[derive(Debug)]
10395struct CacheCheckError(String);
10396
10397impl std::fmt::Display for CacheCheckError {
10398 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10399 f.write_str(&self.0)
10400 }
10401}
10402
10403impl std::error::Error for CacheCheckError {}
10404
10405fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10407 state
10408 .resource_block
10409 .get_or_insert_with(|| crate::run::ResourceBlock {
10410 cause: crate::run::ResourceBlockCause::Other,
10411 node: node.to_owned(),
10412 kind: "shared build cache".into(),
10413 detail: format!("{error:#}"),
10414 at: Timestamp::now(),
10415 });
10416 if let Err(error) = state.save() {
10417 tracing::warn!("could not persist cache resource block: {error:#}");
10418 }
10419}
10420
10421#[allow(clippy::too_many_arguments)]
10442async fn with_cache_lease<'s, F, Fut>(
10443 state: &'s mut RunState,
10444 cache_dir: Option<&Path>,
10445 node: &str,
10446 seat: &str,
10447 worktree: &Path,
10448 head: &str,
10449 budget: Duration,
10450 context: &str,
10451 wait: &crate::repo_lock::WaitPolicy,
10452 body: F,
10453) -> Result<(Vec<CommandOutcome>, bool)>
10454where
10455 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10456 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10457{
10458 let started = Instant::now();
10461 let home = crate::run::home();
10462 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10463 let guard = if let Some(cache_dir) = cache_dir {
10464 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10465 match acquire_cache_lease(
10466 state,
10467 cache_dir,
10468 &owner,
10469 resource_limit,
10470 node,
10471 context,
10472 &|| wait.cancelled(),
10473 )
10474 .await
10475 .and_then(|g| {
10476 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10477 }) {
10478 Ok(g) => Some(g),
10479 Err(e) => {
10480 record_cache_error(state, node, &e);
10481 return Ok((
10482 vec![CommandOutcome {
10483 command: "(waiting for the shared build cache)".into(),
10484 code: None,
10485 output_tail: e.to_string(),
10486 duration_ms: started.elapsed().as_millis() as u64,
10487 resource_blocked: true,
10488 }],
10489 false,
10490 ));
10491 }
10492 }
10493 } else {
10494 None
10495 };
10496 let _coordination = coordinate_operation(state, worktree, wait).await?;
10497 if let Some(cache_dir) = cache_dir {
10498 let identity = crate::cache::Identity::new(worktree, head);
10499 if let Err(e) = crate::cache::ensure_fresh(
10500 &home,
10501 cache_dir,
10502 &identity,
10503 &state.config.verify.cargo_manifests(),
10504 ) {
10505 record_cache_error(state, node, &e);
10506 state.event(
10507 "verify",
10508 format!(
10509 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10510 worktree.display(),
10511 short(head)
10512 ),
10513 );
10514 if let Some(guard) = guard {
10515 guard.release();
10516 }
10517 return Ok((
10518 vec![CommandOutcome {
10519 command: "(confirming the shared build cache is fresh)".into(),
10520 code: None,
10521 output_tail: e.to_string(),
10522 duration_ms: started.elapsed().as_millis() as u64,
10523 resource_blocked: true,
10524 }],
10525 false,
10526 ));
10527 }
10528 }
10529 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10530 if !timed_out_pids.is_empty() {
10531 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10532 }
10533 drop(_coordination);
10536 if let Some(guard) = guard {
10537 guard.release();
10538 }
10539 Ok((outcomes, retried))
10540}
10541
10542async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10554 wait_for_pids_with(
10555 pids,
10556 crate::proc::pid_alive,
10557 LEASE_RELEASE_POLL,
10558 LEASE_RELEASE_MAX_WAIT,
10559 )
10560 .await;
10561}
10562
10563async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10569 pids: &[u32],
10570 alive: F,
10571 poll: Duration,
10572 max_wait: Duration,
10573) {
10574 let deadline = Instant::now() + max_wait;
10575 loop {
10576 if pids.iter().all(|&pid| !alive(pid)) {
10577 return;
10578 }
10579 if Instant::now() >= deadline {
10580 return;
10581 }
10582 tokio::time::sleep(poll).await;
10583 }
10584}
10585
10586fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10593 outcomes.iter().any(|o| o.resource_blocked)
10594}
10595
10596pub(crate) enum GateFix {
10598 Retry,
10600 Stop,
10603 Defer,
10606}
10607
10608fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10616 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10617 red.peek().is_some()
10618 && red.all(|o| {
10619 !o.resource_blocked
10620 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10621 && !o.output_tail.trim().is_empty()
10622 })
10623}
10624
10625fn e2e_outcome_label(o: &CommandOutcome) -> String {
10629 if o.ok() {
10630 return "pass".to_owned();
10631 }
10632 let reason = if o.build_failed() {
10633 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10634 } else {
10635 format!("FAIL ({:?})", o.code)
10636 };
10637 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10638}
10639
10640async fn run_e2e_with_retry(
10648 state: &mut RunState,
10649 shell: &[String],
10650 commands: &[String],
10651 worktree: &Path,
10652 timeout: Duration,
10653 context: &str,
10654) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10655 let (mut e2e, mut timed_out_pids) = run_commands(
10656 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10657 )
10658 .await;
10659 for o in &e2e {
10660 state.event(
10661 "verify",
10662 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10663 );
10664 }
10665 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10669 if verify_retried {
10670 state.event(
10671 "verify",
10672 format!(
10673 "{context}: verify could not build/link, not a test result — retrying once \
10674 before concluding"
10675 ),
10676 );
10677 let retried = run_commands(
10678 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10679 )
10680 .await;
10681 e2e = retried.0;
10682 timed_out_pids.extend(retried.1);
10685 for o in &e2e {
10686 state.event(
10687 "verify",
10688 format!(
10689 "{context}: retry `{}` -> {}",
10690 o.command,
10691 e2e_outcome_label(o)
10692 ),
10693 );
10694 }
10695 }
10696 (e2e, verify_retried, timed_out_pids)
10697}
10698
10699#[allow(clippy::too_many_arguments)]
10715async fn run_commands(
10716 state: &mut RunState,
10717 node: &str,
10718 task: &str,
10719 attempt: usize,
10720 shell: &[String],
10721 commands: &[String],
10722 cwd: &Path,
10723 timeout: Duration,
10724) -> (Vec<CommandOutcome>, Vec<u32>) {
10725 if commands.is_empty() {
10726 return (Vec::new(), Vec::new());
10731 }
10732 let mut out = Vec::new();
10733 let mut timed_out_pids = Vec::new();
10734 let total = commands.len();
10735 for (idx, command) in commands.iter().enumerate() {
10736 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10737 if let Err(e) = state.save() {
10738 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10739 }
10740 let started = Instant::now();
10741 let mut cmd = tokio::process::Command::new(&shell[0]);
10742 cmd.quiet();
10743 for key in crate::worker::INVOCATION_ENV {
10747 cmd.env_remove(key);
10748 }
10749 cmd.args(&shell[1..])
10750 .arg(command)
10751 .current_dir(cwd)
10752 .stdin(std::process::Stdio::null())
10753 .stdout(std::process::Stdio::piped())
10754 .stderr(std::process::Stdio::piped())
10755 .kill_on_drop(true);
10756 let mut outputs = Vec::new();
10757 let protected = (|| -> Result<()> {
10758 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10759 outputs.push(guard);
10760 }
10761 if let Some(cache) = state.config.cache_dir() {
10762 outputs.push(crate::retention::OutputLease::cache(
10763 &crate::run::home(),
10764 &cache,
10765 false,
10766 )?);
10767 }
10768 Ok(())
10769 })();
10770 if let Err(e) = protected {
10771 out.push(CommandOutcome {
10772 command: command.clone(),
10773 code: None,
10774 output_tail: format!("output ownership unavailable: {e:#}"),
10775 duration_ms: 0,
10776 resource_blocked: true,
10777 });
10778 continue;
10779 }
10780 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10781 let (code, body) = match spawned {
10782 Ok(mut running) => {
10783 let pid = running.child().id();
10784 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10785 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10786 let mut paths = vec![cwd.to_owned()];
10787 if let Some(cache) = state.config.cache_dir() {
10788 paths.push(cache);
10789 }
10790 let mut pressure = None;
10791 let status = tokio::select! {
10792 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10793 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10794 pressure = Some(reason); None
10795 }
10796 };
10797 let (code, failure) = match status {
10798 Some(Ok(Ok(status))) => (
10799 status.code(),
10800 status
10801 .code()
10802 .is_none()
10803 .then(|| format!("command terminated: {status}")),
10804 ),
10805 Some(Ok(Err(error))) => {
10806 (None, Some(format!("failed waiting for command: {error}")))
10807 }
10808 Some(Err(_)) => {
10809 if !running.coordinated()
10813 && let Some(pid) = pid
10814 {
10815 timed_out_pids.push(pid);
10816 }
10817 (
10818 None,
10819 Some(format!("timed out after {}s", timeout.as_secs())),
10820 )
10821 }
10822 None => (None, None),
10823 };
10824 running.finish().await;
10825 if let Some(reason) = pressure {
10826 state.resource_block = Some(crate::run::ResourceBlock {
10827 cause: crate::run::ResourceBlockCause::Other,
10828 node: node.to_owned(),
10829 kind: "disk pressure".to_owned(),
10830 detail: reason.clone(),
10831 at: Timestamp::now(),
10832 });
10833 out.push(CommandOutcome {
10834 command: command.clone(),
10835 code: None,
10836 output_tail: reason,
10837 duration_ms: started.elapsed().as_millis() as u64,
10838 resource_blocked: true,
10839 });
10840 break;
10841 }
10842 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10843 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10844 if let Some(failure) = failure {
10845 body.push('\n');
10846 body.push_str(&failure);
10847 }
10848 (code, body)
10849 }
10850 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10851 };
10852 out.push(CommandOutcome {
10853 command: command.clone(),
10854 code,
10855 output_tail: tail(&body, OUTPUT_TAIL),
10856 duration_ms: started.elapsed().as_millis() as u64,
10857 resource_blocked: false,
10858 });
10859 }
10860 state.task_finished(task);
10861 if let Err(e) = state.save() {
10862 tracing::warn!("could not persist the end of {task}: {e:#}");
10863 }
10864 (out, timed_out_pids)
10865}
10866
10867fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10879 let repo = repo.display();
10880 match style {
10881 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10882 MergeStyle::Squash => {
10883 let subject = message
10886 .lines()
10887 .next()
10888 .unwrap_or(branch)
10889 .replace(['\\', '"', '$', '`'], "");
10890 format!(
10891 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10892 )
10893 }
10894 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10895 }
10896}
10897
10898const PR_TITLE_MAX: usize = 240;
10909
10910fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10921 let line = queue::first_line(text)?;
10922 let chars: Vec<char> = line.chars().collect();
10923 let (kept, mark) = if chars.len() <= max {
10924 (line.to_owned(), "")
10925 } else if let Some(end) = sentence_end(&chars, max) {
10926 (chars[..end].iter().collect::<String>(), "")
10927 } else {
10928 let room = max.saturating_sub(3);
10929 let cut = if chars[room].is_whitespace() {
10932 room
10933 } else {
10934 chars[..room]
10935 .iter()
10936 .rposition(|c| c.is_whitespace())
10937 .unwrap_or(room)
10938 };
10939 let head: String = chars[..cut].iter().collect();
10940 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10941 (head.to_owned(), "...")
10942 };
10943 if kept.is_empty() {
10944 return None;
10945 }
10946 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10947 return None;
10948 }
10949 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10950 return None;
10951 }
10952 Some(format!("{kept}{mark}"))
10953}
10954
10955fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10959 const MIN: usize = 20;
10960 for i in MIN..max.min(chars.len()) {
10961 if !matches!(chars[i], '.' | '!' | '?') {
10962 continue;
10963 }
10964 let Some(&next) = chars.get(i + 1) else {
10965 continue;
10966 };
10967 if !next.is_whitespace() {
10968 continue;
10969 }
10970 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10971 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10972 continue;
10973 }
10974 let word: String = chars[..i]
10975 .iter()
10976 .rev()
10977 .take_while(|c| !c.is_whitespace())
10978 .collect::<Vec<_>>()
10979 .into_iter()
10980 .rev()
10981 .collect();
10982 let word = word.to_ascii_lowercase();
10983 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10984 continue;
10985 }
10986 let end = chars[..i]
10987 .iter()
10988 .rposition(|c| !c.is_whitespace())
10989 .map_or(i, |p| p + 1);
10990 return Some(end);
10991 }
10992 None
10993}
10994
10995struct PrMessage {
11000 title: String,
11001 body: String,
11002}
11003
11004impl PrMessage {
11005 fn commit_message(&self) -> String {
11009 format!("{}\n\n{}", self.title, self.body)
11010 }
11011}
11012
11013fn title_marker(line: &str) -> Option<&str> {
11015 let line = line.trim();
11016 let head = line.get(..6)?;
11017 head.eq_ignore_ascii_case("title:")
11018 .then(|| line[6..].trim())
11019}
11020
11021fn summary_title(summary: &str) -> Option<String> {
11026 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11027 let raw = title_marker(first)?;
11028 if raw.is_empty() {
11029 return None;
11030 }
11031 let title = queue::title_from(raw, PR_TITLE_MAX);
11032 let lower = title.to_ascii_lowercase();
11033 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11034 return None;
11035 }
11036 Some(title)
11037}
11038
11039const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11041
11042pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11044
11045pub(crate) fn is_review_run(state: &RunState) -> bool {
11049 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11050}
11051
11052fn review_title(state: &RunState) -> Option<String> {
11057 english_subject(
11058 state.reviewed_commits.as_ref()?.first()?,
11059 state.config.graph.github_english(),
11060 )
11061}
11062
11063fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11066 let raw = raw.trim();
11067 if raw.is_empty() {
11068 return None;
11069 }
11070 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11071 return None;
11072 }
11073 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11074 return None;
11075 }
11076 let title = queue::title_from(raw, PR_TITLE_MAX);
11077 let lower = title.to_ascii_lowercase();
11078 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11079 return None;
11080 }
11081 Some(title)
11082}
11083
11084#[derive(Debug, Clone, PartialEq, Eq)]
11087struct BranchFacts {
11088 commits: Vec<(String, String)>,
11090 stat: String,
11092}
11093
11094const STAT_FILE_LINES: usize = 25;
11096const COMMITS_MAX_CHARS: usize = 20_000;
11098
11099async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11102 let commits = git::commit_log(repo, base, branch).await.ok()?;
11103 if commits.is_empty() {
11104 return None;
11105 }
11106 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11107 let lines: Vec<&str> = stat.lines().collect();
11108 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11109 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11110 let more = format!(" ... {omitted} more file(s)");
11111 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11112 kept.push(&more);
11113 kept.push(lines[lines.len() - 1]);
11114 kept.join("\n")
11115 } else {
11116 lines.join("\n")
11117 };
11118 Some(BranchFacts { commits, stat })
11119}
11120
11121fn markdown_safe(text: &str) -> String {
11124 text.replace("</details>", "</details>")
11125 .replace("\x60\x60\x60", "~~~")
11126}
11127
11128fn neutral_title(state: &RunState, winner: char) -> String {
11129 format!(
11130 "chore: land candidate {} of run {}",
11131 winner.to_ascii_uppercase(),
11132 state.id
11133 )
11134}
11135
11136pub(crate) async fn repair_landing_gate(
11139 state: &mut RunState,
11140 winner: &Candidate,
11141 outcomes: &[CommandOutcome],
11142) -> Result<GateFix> {
11143 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11144 return Ok(GateFix::Stop);
11145 }
11146 let mut runner = Runner {
11147 roles: state.config.resolve_roles()?,
11148 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11149 state: state.clone(),
11150 pause: Pause::new(),
11151 interrupt: Pause::new(),
11152 };
11153 let result = runner.gate_fix_round(winner, outcomes).await;
11154 *state = runner.state;
11155 result
11156}
11157
11158pub(crate) async fn check_gate_repair_e2e(
11161 state: &mut RunState,
11162 winner: &Candidate,
11163 wait: &crate::repo_lock::WaitPolicy,
11164) -> Result<GateFix> {
11165 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11166 return Ok(GateFix::Retry);
11167 }
11168 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11169 if state
11170 .gate_repair_e2e
11171 .as_ref()
11172 .is_some_and(|r| r.head == head && r.passed)
11173 {
11174 return Ok(GateFix::Retry);
11175 }
11176 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11177 head: head.clone(),
11178 passed: false,
11179 outcomes: Vec::new(),
11180 });
11181 state.save()?;
11182 let commands = state.config.verify.e2e.clone();
11183 let outcomes = if commands.is_empty() {
11184 Vec::new()
11185 } else {
11186 let shell = state.config.shell();
11187 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11188 let cache_dir = state.config.cache_dir();
11189 let (outcomes, _) = with_cache_lease(
11190 state,
11191 cache_dir.as_deref(),
11192 "e2e",
11193 "e2e",
11194 &winner.worktree,
11195 &head,
11196 timeout,
11197 "gate repair e2e",
11198 wait,
11199 |state, budget| async move {
11200 run_e2e_with_retry(
11201 state,
11202 &shell,
11203 &commands,
11204 &winner.worktree,
11205 budget,
11206 "gate repair e2e",
11207 )
11208 .await
11209 },
11210 )
11211 .await?;
11212 outcomes
11213 };
11214 let deferred = verify_inconclusive(&outcomes);
11215 let passed = outcomes.iter().all(CommandOutcome::ok)
11216 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11217 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11218 head,
11219 passed,
11220 outcomes,
11221 });
11222 state.save()?;
11223 if deferred {
11224 Ok(GateFix::Defer)
11225 } else if !passed {
11226 state.event(
11227 "gate",
11228 "verify.e2e failed after gate repair; nothing published",
11229 );
11230 Ok(GateFix::Stop)
11231 } else {
11232 Ok(GateFix::Retry)
11233 }
11234}
11235
11236pub(crate) async fn run_landing_gate(
11238 state: &mut RunState,
11239 winner: &Candidate,
11240 wait: &crate::repo_lock::WaitPolicy,
11241) -> Result<Vec<CommandOutcome>> {
11242 let shell = state.config.shell();
11243 let gate_commands = state.config.verify.gate.clone();
11244 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11245 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11246 Vec::new()
11247 } else {
11248 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11249 let cache_dir = state.config.cache_dir();
11250 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11251 let (outcomes, _) = with_cache_lease(
11252 state,
11253 cache_dir.as_deref(),
11254 "gate",
11255 "gate",
11256 &winner.worktree,
11257 &head,
11258 timeout,
11259 "final gate",
11260 wait,
11261 |state, budget| {
11262 let shell = shell.clone();
11263 let gate_commands = gate_commands.clone();
11264 let winner = winner.clone();
11265 async move {
11266 Runner::run_pre_gate(state, &winner).await;
11267 let (outcomes, timed_out_pids) = run_commands(
11268 state,
11269 "gate",
11270 "gate",
11271 0,
11272 &shell,
11273 &gate_commands,
11274 &winner.worktree,
11275 budget,
11276 )
11277 .await;
11278 (outcomes, false, timed_out_pids)
11279 }
11280 },
11281 )
11282 .await?;
11283 outcomes
11284 };
11285 if outcomes.is_empty() {
11286 state.event("gate", "no gate commands configured; not tested");
11291 }
11292 for o in &outcomes {
11293 state.event(
11294 "gate",
11295 format!(
11296 "`{}` -> {}",
11297 o.command,
11298 if o.ok() {
11299 "pass".to_owned()
11300 } else {
11301 format!(
11302 "FAIL ({:?})\n{}",
11303 o.code,
11304 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11305 )
11306 }
11307 ),
11308 );
11309 }
11310 Ok(outcomes)
11311}
11312
11313pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11319 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11320 ""
11321 } else {
11322 pr_title
11323 }
11324}
11325
11326pub fn landing_subject_source(state: &RunState) -> String {
11330 if is_review_run(state) {
11331 let winner = state.candidates.first().map_or('A', |c| c.label);
11332 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11333 }
11334 state.instruction.clone()
11335}
11336
11337fn summary_without_title(summary: &str) -> String {
11340 let mut lines = summary.trim().lines().peekable();
11341 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11342 lines.next();
11343 }
11344 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11345}
11346
11347#[cfg(test)]
11359fn pr_message(state: &RunState, winner: char) -> PrMessage {
11360 pr_message_with(state, winner, None)
11361}
11362
11363#[cfg(test)]
11366fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11367 let raw = pr_message_raw(
11368 state,
11369 winner,
11370 facts,
11371 crate::pr_outcome::FinalHead::Unreported,
11372 ChangeNote::NotAsked,
11373 );
11374 let id = crate::scrub::Identity::current();
11375 PrMessage {
11376 title: crate::scrub::scrub(&raw.title, &id),
11377 body: crate::scrub::scrub(&raw.body, &id),
11378 }
11379}
11380
11381#[derive(Debug, Clone, Copy)]
11383enum ChangeNote<'a> {
11384 NotAsked,
11386 Summary(&'a crate::run::ChangeSummary),
11388 Unavailable(&'a str),
11390}
11391
11392fn pr_message_raw(
11393 state: &RunState,
11394 winner: char,
11395 facts: Option<&BranchFacts>,
11396 head: crate::pr_outcome::FinalHead<'_>,
11397 change: ChangeNote<'_>,
11398) -> PrMessage {
11399 if !matches!(change, ChangeNote::NotAsked) {
11400 let (title, mut body) = match change {
11401 ChangeNote::Summary(c) => (
11402 c.title.clone(),
11403 format!("## Summary\n\n{}\n\n", c.body.trim()),
11404 ),
11405 ChangeNote::Unavailable(why) => (
11406 neutral_title(state, winner),
11407 format!(
11408 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11409 ),
11410 ),
11411 ChangeNote::NotAsked => unreachable!(),
11412 };
11413 body.push_str(&public_pr_evidence(state, head));
11414 body.push_str(&format!(
11415 "\n---\nmagi:run/{} magi:candidate-{}\n",
11416 state.id,
11417 winner.to_ascii_lowercase()
11418 ));
11419 return PrMessage { title, body };
11420 }
11421 pr_preview(state, winner, facts, head)
11422}
11423
11424fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11427 let mut body = crate::pr_outcome::review_rounds(state);
11428 body.push_str(&crate::pr_outcome::final_verification(state, head));
11429 let open = state.last_round_findings();
11430 if !open.is_empty() {
11431 body.push_str("## Open review findings\n\n");
11432 for f in open {
11433 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11434 }
11435 body.push('\n');
11436 }
11437 if let Some(c) = state.change_summary.as_ref()
11438 && head == crate::pr_outcome::FinalHead::At(&c.head)
11439 {
11440 body.push_str(&format!(
11441 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11442 c.base, c.head
11443 ));
11444 }
11445 body
11446}
11447
11448fn pr_preview(
11450 state: &RunState,
11451 winner: char,
11452 facts: Option<&BranchFacts>,
11453 head: crate::pr_outcome::FinalHead<'_>,
11454) -> PrMessage {
11455 let summary = state
11456 .candidates
11457 .iter()
11458 .find(|c| c.label == winner)
11459 .map(|c| c.summary.as_str())
11460 .unwrap_or_default();
11461 let review = is_review_run(state);
11464 let title = if review {
11465 facts
11466 .and_then(|f| {
11467 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11468 })
11469 .or_else(|| review_title(state))
11470 .or_else(|| {
11471 state
11472 .candidates
11473 .iter()
11474 .find(|c| c.label == winner)
11475 .filter(|c| !c.branch.starts_with("magi/"))
11476 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11477 })
11478 .unwrap_or_else(|| neutral_title(state, winner))
11479 } else {
11480 summary_title(summary).unwrap_or_else(|| {
11481 english_title(
11482 &state.instruction,
11483 PR_TITLE_MAX,
11484 state.config.graph.github_english(),
11485 )
11486 .unwrap_or_else(|| neutral_title(state, winner))
11487 })
11488 };
11489
11490 let mut body = String::new();
11491 let what = summary_without_title(summary);
11492 if !what.is_empty() {
11493 body.push_str("## Summary\n\n");
11494 body.push_str(&what);
11495 body.push_str("\n\n");
11496 }
11497
11498 body.push_str(&crate::pr_outcome::review_rounds(state));
11500 body.push_str(&crate::pr_outcome::final_verification(state, head));
11501 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11502
11503 let open = state.open_findings();
11504 if !open.is_empty() {
11505 body.push_str("## Open review findings\n\n");
11506 for f in &open {
11507 body.push_str(&format!(
11508 "- `{}` [{:?}] {}\n",
11509 f.id, f.severity, "Context retained in local review records."
11510 ));
11511 }
11512 body.push('\n');
11513 }
11514
11515 if let Some(fix) = fix
11516 && !fix.rejected.is_empty()
11517 {
11518 body.push_str("## Declined by the fixer\n\n");
11519 for r in &fix.rejected {
11520 body.push_str(&format!(
11521 "- `{}`: {}\n",
11522 r.id, "Reason retained in local review records."
11523 ));
11524 }
11525 body.push('\n');
11526 }
11527
11528 if review {
11529 body.push_str("## Commits under review\n\n");
11531 if let Some(facts) = facts {
11532 let mut left = COMMITS_MAX_CHARS;
11533 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11534 let mut entry = format!("- {}\n", markdown_safe(subject));
11535 for l in markdown_safe(text).lines() {
11536 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11537 }
11538 if left == 0 {
11539 body.push_str(&format!(
11540 "- ... {} more commit(s)\n",
11541 facts.commits.len() - i
11542 ));
11543 break;
11544 }
11545 if entry.len() > left {
11546 let mut end = left;
11549 while !entry.is_char_boundary(end) {
11550 end -= 1;
11551 }
11552 entry.truncate(end);
11553 entry.push_str("\n ... (truncated)\n");
11554 left = 0;
11555 } else {
11556 left -= entry.len();
11557 }
11558 body.push_str(&entry);
11559 }
11560 if !facts.stat.trim().is_empty() {
11561 body.push_str(&format!(
11562 "\n## Diff stat\n\n```\n{}\n```\n",
11563 markdown_safe(facts.stat.trim())
11564 ));
11565 }
11566 } else {
11567 match &state.reviewed_commits {
11568 Some(subjects) => {
11569 for s in subjects {
11570 body.push_str(&format!("- {}\n", s.trim()));
11571 }
11572 }
11573 None => {
11574 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11577 body.push_str(rest.trim());
11578 body.push('\n');
11579 }
11580 }
11581 }
11582 } else {
11583 let task = state.instruction.trim();
11584 let task = if task.is_empty() {
11585 "(empty task)"
11586 } else {
11587 task
11588 };
11589 body.push_str(&format!(
11590 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11591 task.replace("</details>", "</details>")
11592 ));
11593 }
11594
11595 body.push_str(&format!(
11596 "\n---\nmagi:run/{} magi:candidate-{}\n",
11597 state.id,
11598 winner.to_ascii_lowercase()
11599 ));
11600
11601 PrMessage { title, body }
11602}
11603
11604fn seeded_instruction(state: &RunState) -> String {
11607 let mut inputs = Vec::new();
11608 if let Some(start) = &state.sources.start_from {
11609 inputs.push(format!(
11610 "Start from `{}` at `{}` (including ancestry).",
11611 start.token, start.sha
11612 ));
11613 }
11614 for pick in &state.sources.pick_commits {
11615 inputs.push(format!(
11616 "Pick `{}` at `{}` (single commit).",
11617 pick.token, pick.sha
11618 ));
11619 }
11620 if inputs.is_empty() {
11621 state.instruction.clone()
11622 } else {
11623 format!(
11624 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11625 state.instruction,
11626 inputs.join("\n")
11627 )
11628 }
11629}
11630
11631async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11635 let base = &state.base_branch;
11636 let mut against = base.clone();
11637 if mode == MergeMode::Pr {
11638 let remote = &state.config.merge.remote;
11642 let tracking = format!("{remote}/{base}");
11643 let fetched = git::fetch(repo, remote, base).await;
11644 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11645 against = tracking;
11646 } else {
11647 return false;
11648 }
11649 }
11650 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11651}
11652
11653fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11656 let mut detail = format!(
11657 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11658 nothing to open a pull request for"
11659 );
11660 detail.push_str(
11661 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11662 );
11663 detail
11664}
11665
11666#[derive(Debug, PartialEq, Eq)]
11669enum PrPlan {
11670 Create,
11671 Adopt { url: String, title: String },
11672 Stop(String),
11673}
11674
11675fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11678 match found {
11679 Ok(land::OpenPr::None) => PrPlan::Create,
11680 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11681 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11682 "several open pull requests exist for this branch, not picking one: {}",
11683 urls.join(" ")
11684 )),
11685 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11686 }
11687}
11688
11689pub(crate) async fn gh_pr_create(
11691 cwd: &Path,
11692 base: &str,
11693 head: &str,
11694 title: &str,
11695 body: &str,
11696) -> Result<String> {
11697 let out = tokio::process::Command::new("gh")
11698 .args([
11699 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11700 ])
11701 .current_dir(cwd)
11702 .quiet()
11703 .stdin(std::process::Stdio::null())
11704 .output()
11705 .await
11706 .context("spawn gh")?;
11707 if out.status.success() {
11708 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11709 } else {
11710 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11711 }
11712}
11713
11714async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11716 match std::fs::symlink_metadata(path) {
11717 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11718 git::worktree_remove_clean(repo, path).await?;
11721 return Ok(());
11722 }
11723 Err(e) => return Err(e.into()),
11724 Ok(_) => {}
11725 }
11726 if !git::is_clean(path).await? {
11727 bail!(
11728 "worktree has uncommitted work; refusing removal: {}",
11729 path.display()
11730 );
11731 }
11732 if !git::worktree_remove_clean(repo, path).await? {
11733 bail!("worktree removal refused: {}", path.display());
11734 }
11735 Ok(())
11736}
11737
11738pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11747 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11748 bail!("refusing fold: {reason}");
11749 }
11750 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11751 bail!("candidate preservation incomplete; refusing cleanup");
11752 }
11753 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11757 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11758 let _driver = crate::driver::maintenance(home, &state.id)?;
11759 let mut output_guards = Vec::new();
11760 for entry in std::fs::read_dir(state.worktree_root())
11761 .into_iter()
11762 .flatten()
11763 .flatten()
11764 {
11765 if entry.path().join(".git").is_file() {
11766 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11767 }
11768 }
11769 let repo = state.repo.clone();
11770 let wait = repository_wait(state, None);
11773 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11774 Ok(lease) => lease,
11775 Err(e) => {
11776 state.save_under(home)?;
11777 return Err(e);
11778 }
11779 };
11780 let _source_outputs = if state.status == RunStatus::Merged
11783 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11784 {
11785 verify_merged_fold_sources(state, drop_winner, home).await?
11786 } else {
11787 Vec::new()
11788 };
11789 let root = state.worktree_root();
11790 let winner = state.tally.as_ref().map(|t| t.winner);
11791 let mut removed = Vec::new();
11792
11793 for i in 0..state.candidates.len() {
11794 let c = state.candidates[i].clone();
11795 let is_winner = Some(c.label) == winner;
11796 if is_winner && !drop_winner {
11797 continue;
11798 }
11799 let existed = c.worktree.exists();
11800 remove_preserved_worktree(&repo, &c.worktree).await?;
11803 if existed {
11804 removed.push(c.worktree.to_string_lossy().into_owned());
11805 }
11806 let handed_over = state.released_branches.contains(&c.branch);
11809 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11810 if !git::branch_delete(&repo, &c.branch).await? {
11811 bail!("candidate branch removal refused");
11812 }
11813 removed.push(c.branch.clone());
11814 }
11815 state.candidates[i].folded = true;
11816 }
11817
11818 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11819 let path = name.path();
11820 let keep = !drop_winner
11821 && winner.is_some_and(|w| {
11822 path.file_name()
11823 .is_some_and(|n| n == format!("cand-{w}").as_str())
11824 });
11825 if keep {
11826 continue;
11827 }
11828 remove_preserved_worktree(&repo, &path).await?;
11829 removed.push(path.to_string_lossy().into_owned());
11830 }
11831
11832 remove_if_empty(&root);
11841
11842 if state.enabled_worktree_config && drop_winner {
11843 git::release_worktree_config(&repo).await.ok();
11847 state.enabled_worktree_config = false;
11848 }
11849 state.save_under(home)?;
11850 Ok(removed)
11851}
11852
11853async fn verify_merged_fold_sources(
11854 state: &RunState,
11855 drop_winner: bool,
11856 home: &Path,
11857) -> Result<Vec<std::fs::File>> {
11858 let winner = state.tally.as_ref().map(|t| t.winner);
11859 let mut heads = BTreeSet::new();
11860 for candidate in &state.candidates {
11861 if Some(candidate.label) == winner && !drop_winner {
11862 continue;
11863 }
11864 anyhow::ensure!(
11865 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11866 "candidate recovery source is outside the inventoried run root: {}",
11867 candidate.worktree.display()
11868 );
11869 match std::fs::symlink_metadata(&candidate.worktree) {
11870 Ok(_) => {}
11871 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11872 Err(error) => return Err(error.into()),
11873 }
11874 if !state.released_branches.contains(&candidate.branch)
11875 && git::branch_exists(&state.repo, &candidate.branch).await?
11876 {
11877 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11878 }
11879 }
11880 let mut cargo_guards = Vec::new();
11881 let entries = match std::fs::read_dir(state.worktree_root()) {
11882 Ok(entries) => Some(entries),
11883 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11884 Err(error) => return Err(error.into()),
11885 };
11886 for entry in entries.into_iter().flatten() {
11887 let path = entry?.path();
11888 let metadata = std::fs::symlink_metadata(&path)?;
11889 anyhow::ensure!(
11890 metadata.is_dir() && !metadata.file_type().is_symlink(),
11891 "non-directory or symlink recovery source retained: {}",
11892 path.display()
11893 );
11894 if !drop_winner
11895 && winner.is_some_and(|w| {
11896 path.file_name()
11897 .is_some_and(|name| name == format!("cand-{w}").as_str())
11898 })
11899 {
11900 continue;
11901 }
11902 anyhow::ensure!(
11903 path.join(".git").is_file(),
11904 "unregistered recovery source retained: {}",
11905 path.display()
11906 );
11907 anyhow::ensure!(
11908 git::git(
11909 &path,
11910 &[
11911 "status",
11912 "--porcelain",
11913 "--untracked-files=all",
11914 "--ignored",
11915 "--",
11916 ".",
11917 ":(exclude)target"
11918 ]
11919 )
11920 .await?
11921 .is_empty(),
11922 "source changes or ignored recovery files retained: {}",
11923 path.display()
11924 );
11925 match std::fs::symlink_metadata(path.join("target")) {
11926 Ok(_) => {
11927 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11928 }
11929 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11930 Err(error) => return Err(error.into()),
11931 }
11932 for checkpoint in [
11933 "rebase-merge",
11934 "rebase-apply",
11935 "sequencer",
11936 "CHERRY_PICK_HEAD",
11937 "MERGE_HEAD",
11938 "REBASE_HEAD",
11939 "REVERT_HEAD",
11940 "BISECT_LOG",
11941 "AUTO_MERGE",
11942 "MERGE_MSG",
11943 ] {
11944 let location = git::git(
11945 &path,
11946 &[
11947 "rev-parse",
11948 "--path-format=absolute",
11949 "--git-path",
11950 checkpoint,
11951 ],
11952 )
11953 .await?;
11954 anyhow::ensure!(
11955 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11956 if e.kind() == std::io::ErrorKind::NotFound),
11957 "Git recovery checkpoint retained: {checkpoint}"
11958 );
11959 }
11960 heads.insert(git::rev_parse(&path, "HEAD").await?);
11961 }
11962 crate::fold_evidence::preserve(state, home, &heads).await?;
11963
11964 Ok(cargo_guards)
11965}
11966
11967fn remove_if_empty(dir: &Path) {
11978 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11979 std::fs::remove_dir(dir).ok();
11980 }
11981}
11982
11983pub fn worst_open(state: &RunState) -> Option<Severity> {
11985 state
11986 .reviews
11987 .last()?
11988 .reviews
11989 .iter()
11990 .flat_map(|r| r.findings.iter())
11991 .map(|f| f.severity)
11992 .max()
11993}
11994
11995async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11999 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12000 git::acquire_worktree_config(repo).await?;
12001 state.enabled_worktree_config = true;
12002 }
12003 Ok(())
12004}
12005
12006#[cfg(test)]
12007mod tests {
12008 #[tokio::test]
12009 async fn launcher_base_fetch_requires_remote_success() {
12010 let tmp = tempfile::tempdir().unwrap();
12011 let repo = tmp.path().join("repo");
12012 let remote = tmp.path().join("remote");
12013 std::fs::create_dir(&repo).unwrap();
12014 std::fs::create_dir(&remote).unwrap();
12015 git::git(&remote, &["init", "--bare", "-b", "main"])
12016 .await
12017 .unwrap();
12018 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12019 git::git(
12020 &repo,
12021 &[
12022 "-c",
12023 "user.name=Fixture",
12024 "-c",
12025 "user.email=fixture@example.invalid",
12026 "commit",
12027 "--allow-empty",
12028 "-m",
12029 "fixture",
12030 ],
12031 )
12032 .await
12033 .unwrap();
12034 git::git(
12035 &repo,
12036 &["remote", "add", "origin", remote.to_str().unwrap()],
12037 )
12038 .await
12039 .unwrap();
12040 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12041 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12042 assert_eq!(
12043 resolve_base(&repo, "main", "origin", None)
12044 .await
12045 .unwrap()
12046 .commit,
12047 head
12048 );
12049 #[cfg(unix)]
12050 {
12051 let policy: crate::config::Coordination = toml::from_str(
12052 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12053 ).unwrap();
12054 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12055 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12056 let held = crate::repo_lock::SeatLease::operation_with(
12057 &repo,
12058 Some(&policy),
12059 &crate::repo_lock::WaitPolicy::default(),
12060 )
12061 .await
12062 .unwrap();
12063 assert!(held.is_some());
12064 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12065 tokio::pin!(fetch);
12066 assert!(
12067 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12068 .await
12069 .is_err()
12070 );
12071 assert!(
12072 !repo.join(".git/FETCH_HEAD").exists(),
12073 "no fetch before admission"
12074 );
12075 drop(held);
12076 let fetched = fetch.await.unwrap();
12077 assert_eq!(fetched.commit, head);
12078 assert_eq!(fetched.remote, "origin");
12079 assert_eq!(fetched.branch, "main");
12080 }
12081 git::git(
12083 &repo,
12084 &[
12085 "remote",
12086 "set-url",
12087 "origin",
12088 tmp.path().join("missing").to_str().unwrap(),
12089 ],
12090 )
12091 .await
12092 .unwrap();
12093 let error = resolve_base(&repo, "main", "origin", None)
12094 .await
12095 .unwrap_err()
12096 .to_string();
12097 assert!(error.contains("cannot read origin/main"));
12098 assert!(error.contains("refusing to branch off"));
12099 }
12100
12101 #[test]
12102 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12103 let leaked = vec!["chore(deps): update a crate".to_owned()];
12104 let own = "fix(daemon): apply a chosen action";
12105 assert!(should_retitle("", own, &leaked));
12106 assert!(should_retitle(
12107 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12108 own,
12109 &leaked
12110 ));
12111 assert!(should_retitle(
12112 "chore: land candidate A of run 1",
12113 own,
12114 &leaked
12115 ));
12116 assert!(should_retitle(
12117 "magi: candidate A (uncommitted work)",
12118 own,
12119 &leaked
12120 ));
12121 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12122 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12123 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12124 }
12125
12126 #[test]
12127 fn pr_merge_plan_creates_adopts_or_stops() {
12128 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12129 assert_eq!(
12130 pr_merge_plan(Ok(land::OpenPr::One {
12131 url: "u".into(),
12132 title: "t".into()
12133 })),
12134 PrPlan::Adopt {
12135 url: "u".into(),
12136 title: "t".into()
12137 }
12138 );
12139 let PrPlan::Stop(many) =
12140 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12141 else {
12142 panic!("many must stop");
12143 };
12144 assert!(many.contains('a') && many.contains('b'));
12145 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12146 panic!("a failed lookup must stop");
12147 };
12148 assert!(err.contains("bad token"));
12149 }
12150
12151 use super::*;
12152 use crate::run::GateStatus;
12153 use std::collections::BTreeMap;
12154 use std::time::Duration;
12155
12156 fn conductor() -> AgentSpec {
12157 AgentSpec {
12158 id: "conductor".to_owned(),
12159 kind: crate::config::AgentKind::Command,
12160 model: None,
12161 command: vec!["true".to_owned()],
12162 extra_args: Vec::new(),
12163 env: BTreeMap::new(),
12164 prompt_delivery: None,
12165 max_turn_secs: None,
12166 }
12167 }
12168
12169 fn spec(id: &str) -> AgentSpec {
12170 AgentSpec {
12171 id: id.to_owned(),
12172 kind: crate::config::AgentKind::Command,
12173 model: None,
12174 command: vec!["true".to_owned()],
12175 extra_args: Vec::new(),
12176 env: BTreeMap::new(),
12177 prompt_delivery: None,
12178 max_turn_secs: None,
12179 }
12180 }
12181
12182 fn ids(xs: &[&str]) -> BTreeSet<String> {
12183 xs.iter().map(|s| (*s).to_owned()).collect()
12184 }
12185
12186 #[test]
12187 fn pick_successor_skips_an_agent_another_seat_holds() {
12188 let roster = [spec("a"), spec("b"), spec("c")];
12190 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12191 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12192 }
12193
12194 #[test]
12195 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12196 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12198 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12199 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12200 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12201 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12202 }
12203
12204 #[test]
12205 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12206 let roster = [spec("a"), spec("b")];
12207 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12208 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12209 let carried = ids(&["b"]);
12210 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12211 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12212 }
12213
12214 #[test]
12215 fn pick_successor_returns_none_without_any_untried_successor() {
12216 let roster = [spec("a"), spec("b")];
12217 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12218 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12219 let carried = ids(&["a"]);
12220 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12221 }
12222
12223 #[test]
12224 fn pick_successor_rescue_avoids_another_seats_occupant() {
12225 let roster = [spec("a"), spec("b"), spec("c")];
12227 let carried = ids(&["b", "c"]);
12228 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12229 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12230 }
12231
12232 #[test]
12233 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12234 let roster = [spec("a"), spec("b"), spec("c")];
12235 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12236 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12237 }
12238
12239 #[test]
12240 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12241 let roster = [spec("a"), spec("b"), spec("c")];
12242 let failed = ids(&["a", "b", "c"]);
12243 let mut tried = ids(&["b"]);
12245 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12246 assert_eq!(first.id, "a");
12247 tried.insert(first.id.clone());
12248 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12249 assert_eq!(second.id, "c");
12250 tried.insert(second.id.clone());
12251 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12252 }
12253
12254 #[test]
12255 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12256 let roster = [spec("a"), spec("b")];
12257 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12258 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12259 }
12260
12261 #[test]
12262 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12263 let roster = [spec("a"), spec("b"), spec("c")];
12264 let h = SeatHistory {
12265 failed: ids(&["a"]),
12266 last_ok: Some("b".to_owned()),
12267 last_fail: None,
12268 };
12269 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12270 let gone = SeatHistory {
12272 last_ok: Some("zzz".to_owned()),
12273 ..h.clone()
12274 };
12275 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12276 let failed = SeatHistory {
12277 failed: ids(&["a", "b"]),
12278 last_ok: Some("b".to_owned()),
12279 last_fail: None,
12280 };
12281 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12282 }
12283
12284 #[test]
12285 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12286 let first = SeatState::new("review-1", "alpha", 7);
12287 let next = handover_seat("review-1", "gamma", 7);
12288 assert_ne!(first.claude_session, next.claude_session);
12289 }
12290
12291 #[test]
12292 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12293 let mut state = state_with_summary("x", "y");
12294 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12295 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12296 assert_ne!(a.claude_session, b.claude_session);
12297 }
12298
12299 #[test]
12300 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12301 let roster = [spec("a"), spec("b")];
12302 let h = SeatHistory {
12303 failed: ids(&["a", "b"]),
12304 ..SeatHistory::default()
12305 };
12306 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12307 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12308 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12309 }
12310
12311 #[test]
12318 fn failure_signature_ignores_numbers_and_paths() {
12319 assert_eq!(
12320 failure_signature("exited with Some(2) and no usable output"),
12321 failure_signature("exited with Some(137) and no usable output")
12322 );
12323 assert_eq!(
12324 failure_signature("process exited with no usable output (exit code 2)"),
12325 failure_signature("process exited with no usable output (exit code 137)")
12326 );
12327 for (old, new) in [
12328 (
12329 "exited with Some(2) and no usable output",
12330 "process exited with no usable output (exit code 2)",
12331 ),
12332 (
12333 "exited with None and no usable output",
12334 "process exited with no usable output (exit code unavailable)",
12335 ),
12336 ] {
12337 let previous = FailClass::Other(failure_signature(old));
12338 let current = FailClass::Other(failure_signature(new));
12339 assert_eq!(previous, current, "resumption retains the failure class");
12340 assert!(!should_hand_over(Some(&previous), ¤t));
12341 }
12342 assert_eq!(
12343 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12344 failure_signature("cannot open /var/x.txt: denied")
12345 );
12346 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12347 }
12348
12349 #[test]
12350 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12351 let other = FailClass::Other("x".into());
12352 assert!(should_hand_over(None, &FailClass::Quota));
12353 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12354 assert!(should_hand_over(
12355 Some(&FailClass::Timeout),
12356 &FailClass::Timeout
12357 ));
12358 assert!(should_hand_over(None, &other));
12359 assert!(!should_hand_over(Some(&other), &other));
12360 assert!(should_hand_over(
12361 Some(&other),
12362 &FailClass::Other("y".into())
12363 ));
12364 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12366 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12367 }
12368
12369 #[test]
12370 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12371 let a = SeatState::new("judge-1", "alpha", 7);
12372 let b = handover_seat("judge-1", "beta", 7);
12373 assert_ne!(a.claude_session, b.claude_session);
12374 assert_eq!(b.turns, 0);
12375 }
12376
12377 #[test]
12378 fn a_timeout_is_classified_apart_from_other_failures() {
12379 assert_eq!(
12380 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12381 Some(FailClass::Timeout)
12382 );
12383 assert!(matches!(
12384 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12385 Some(FailClass::Other(_))
12386 ));
12387 }
12388
12389 #[test]
12390 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12391 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12392 let tried = BTreeSet::from(["beta".to_owned()]);
12393 let next = next_untried_in_roster(&roster, 1, &tried);
12396 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12397 }
12398
12399 #[test]
12400 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12401 let roster = vec![spec("alpha"), spec("beta")];
12402 let tried = BTreeSet::from(["beta".to_owned()]);
12403 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12407 }
12408
12409 #[test]
12410 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12411 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12412 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12413 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12417 }
12418
12419 #[test]
12420 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12421 let roster = vec![spec("a"), spec("a"), spec("b")];
12422 let tried = BTreeSet::from(["a".to_owned()]);
12423 let next = next_untried_in_roster(&roster, 0, &tried);
12424 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12425 }
12426
12427 #[test]
12428 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12429 let roster = vec![spec("a"), spec("b")];
12430 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12431 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12432 }
12433
12434 #[test]
12435 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12436 let dir = tempfile::tempdir().unwrap();
12437 let bay = dir.path().join("ffff");
12438
12439 remove_if_empty(&bay);
12441 assert!(!bay.exists());
12442
12443 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12446 remove_if_empty(&bay);
12447 assert!(bay.exists(), "non-empty directory must survive");
12448
12449 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12451 remove_if_empty(&bay);
12452 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12453 }
12454
12455 #[test]
12464 fn a_full_panel_that_found_nothing_is_clean() {
12465 assert!(round_is_clean(
12466 0,
12467 true,
12468 2,
12469 2,
12470 0,
12471 IncompleteReviewPolicy::Block
12472 ));
12473 }
12474
12475 #[test]
12476 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12477 assert!(!round_is_clean(
12478 0,
12479 true,
12480 1,
12481 2,
12482 0,
12483 IncompleteReviewPolicy::Block
12484 ));
12485 }
12486
12487 #[test]
12488 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12489 assert!(!round_is_clean(
12490 1,
12491 true,
12492 1,
12493 2,
12494 0,
12495 IncompleteReviewPolicy::Warn
12496 ));
12497 }
12498
12499 #[test]
12500 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12501 assert!(round_is_clean(
12502 0,
12503 true,
12504 1,
12505 2,
12506 0,
12507 IncompleteReviewPolicy::Warn
12508 ));
12509 }
12510
12511 #[test]
12512 fn a_full_panel_with_an_open_finding_is_not_clean() {
12513 assert!(!round_is_clean(
12514 1,
12515 true,
12516 2,
12517 2,
12518 0,
12519 IncompleteReviewPolicy::Block
12520 ));
12521 }
12522
12523 #[test]
12524 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12525 assert!(!round_is_clean(
12526 0,
12527 false,
12528 2,
12529 2,
12530 0,
12531 IncompleteReviewPolicy::Block
12532 ));
12533 }
12534
12535 #[test]
12542 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12543 assert!(round_is_clean(
12546 0,
12547 true,
12548 1,
12549 2,
12550 1,
12551 IncompleteReviewPolicy::Block
12552 ));
12553 }
12554
12555 #[test]
12556 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12557 assert!(!round_is_clean(
12560 0,
12561 true,
12562 1,
12563 2,
12564 0,
12565 IncompleteReviewPolicy::Block
12566 ));
12567 }
12568
12569 #[test]
12570 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12571 assert!(!round_is_clean(
12572 1,
12573 true,
12574 1,
12575 2,
12576 1,
12577 IncompleteReviewPolicy::Block
12578 ));
12579 assert!(!round_is_clean(
12580 0,
12581 false,
12582 1,
12583 2,
12584 1,
12585 IncompleteReviewPolicy::Block
12586 ));
12587 }
12588
12589 #[test]
12590 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12591 assert!(!round_is_clean(
12595 0,
12596 true,
12597 0,
12598 2,
12599 2,
12600 IncompleteReviewPolicy::Block
12601 ));
12602 }
12603
12604 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12605 CommandOutcome {
12606 command: "test".to_owned(),
12607 code,
12608 output_tail: String::new(),
12609 duration_ms: 0,
12610 resource_blocked,
12611 }
12612 }
12613
12614 #[test]
12615 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12616 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12617 assert!(
12618 !verify_inconclusive(&[outcome(Some(1), false)]),
12619 "an ordinary failure is still evidence about the patch"
12620 );
12621 assert!(verify_inconclusive(&[outcome(None, true)]));
12622 assert!(
12623 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12624 "one inconclusive outcome taints the whole batch"
12625 );
12626 assert!(!verify_inconclusive(&[]));
12627 }
12628
12629 #[tokio::test]
12630 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12631 let calls = std::sync::atomic::AtomicUsize::new(0);
12635 let started = Instant::now();
12636 wait_for_pids_with(
12637 &[123],
12638 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12639 Duration::from_millis(5),
12640 Duration::from_secs(5),
12641 )
12642 .await;
12643 assert!(
12644 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12645 "must keep checking rather than deciding on the first answer"
12646 );
12647 assert!(
12648 started.elapsed() < Duration::from_secs(1),
12649 "must return the moment it is confirmed dead, not wait out the ceiling"
12650 );
12651 }
12652
12653 #[tokio::test]
12654 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12655 let started = Instant::now();
12656 wait_for_pids_with(
12657 &[123],
12658 |_| true, Duration::from_millis(5),
12660 Duration::from_millis(30),
12661 )
12662 .await;
12663 let elapsed = started.elapsed();
12664 assert!(
12665 elapsed >= Duration::from_millis(30),
12666 "must not give up before its own ceiling: {elapsed:?}"
12667 );
12668 assert!(
12669 elapsed < Duration::from_secs(1),
12670 "must not wait past its own ceiling either: {elapsed:?}"
12671 );
12672 }
12673
12674 #[tokio::test]
12675 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12676 let started = Instant::now();
12677 wait_for_pids_with(
12678 &[],
12679 |_| true,
12680 Duration::from_secs(5),
12681 Duration::from_secs(5),
12682 )
12683 .await;
12684 assert!(
12685 started.elapsed() < Duration::from_millis(200),
12686 "an empty pid list has nothing to confirm"
12687 );
12688 }
12689
12690 fn review_round(
12696 clean: bool,
12697 blocking: usize,
12698 answered: usize,
12699 expected: usize,
12700 progressed: bool,
12701 e2e_ok: bool,
12702 ) -> ReviewRound {
12703 ReviewRound {
12704 round: 1,
12705 head: "h".to_owned(),
12706 verified_head: None,
12707 verified_at: None,
12708 reviews: Vec::new(),
12709 e2e: vec![CommandOutcome {
12710 command: "test".to_owned(),
12711 code: Some(if e2e_ok { 0 } else { 1 }),
12712 output_tail: String::new(),
12713 duration_ms: 0,
12714 resource_blocked: false,
12715 }],
12716 verify_retried: false,
12717 e2e_deferred: false,
12718 e2e_defer_reason: None,
12719 fix: None,
12720 blocking,
12721 answered,
12722 expected,
12723 clean,
12724 progressed,
12725 vote_split: false,
12726 reconsideration: Vec::new(),
12727 verdict: None,
12728 }
12729 }
12730
12731 #[test]
12732 fn review_conclusion_is_none_when_nothing_has_run() {
12733 assert_eq!(review_conclusion(&[], 3), None);
12734 }
12735
12736 #[test]
12737 fn review_conclusion_is_none_while_rounds_remain() {
12738 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12739 assert_eq!(review_conclusion(&rounds, 3), None);
12740 }
12741
12742 #[test]
12743 fn review_conclusion_is_gating_once_a_round_is_clean() {
12744 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12745 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12746 }
12747
12748 #[test]
12749 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12750 let rounds = vec![
12751 review_round(false, 1, 2, 2, true, true),
12752 review_round(false, 1, 2, 2, true, true),
12753 ];
12754 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12755 }
12756
12757 #[test]
12758 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12759 let rounds = vec![
12760 review_round(false, 1, 2, 2, true, true),
12761 review_round(false, 1, 2, 2, true, false),
12762 ];
12763 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12764 }
12765
12766 #[test]
12767 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12768 let mut blocked = review_round(false, 1, 2, 2, true, false);
12775 blocked.e2e[0].resource_blocked = true;
12776 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12777 assert_eq!(review_conclusion(&rounds, 2), None);
12778 }
12779
12780 #[test]
12781 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12782 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12784 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12785 }
12786
12787 #[test]
12788 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12789 let rounds = vec![
12790 review_round(false, 1, 2, 2, false, true),
12791 review_round(false, 1, 2, 2, false, true),
12792 ];
12793 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12794 }
12795
12796 fn secs(n: u64) -> Duration {
12797 Duration::from_secs(n)
12798 }
12799
12800 #[tokio::test]
12803 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12804 ask_test_home();
12805 let tmp = tempfile::tempdir().unwrap();
12806 let repo = tmp.path().join("repo");
12807 std::fs::create_dir_all(&repo).unwrap();
12808 init_repo(&repo);
12809 let g = |args: &[&str]| {
12810 let out = std::process::Command::new("git")
12811 .args(args)
12812 .current_dir(&repo)
12813 .quiet()
12814 .output()
12815 .unwrap();
12816 assert!(
12817 out.status.success(),
12818 "{}",
12819 String::from_utf8_lossy(&out.stderr)
12820 );
12821 };
12822 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12823 g(&["checkout", "-b", "historical"]);
12824 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12825 g(&["add", "."]);
12826 g(&["commit", "-m", "old alternative"]);
12827 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12828 g(&["checkout", "-b", "magi/task-selected", &base]);
12829 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12830 g(&["add", "."]);
12831 g(&["commit", "-m", "measurement infrastructure"]);
12832 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12833 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12834 g(&["add", "."]);
12835 g(&["commit", "-m", "dependent implementation"]);
12836 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12837 g(&["checkout", "main"]);
12838 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12839 g(&["add", "."]);
12840 g(&["commit", "-m", "unrelated base change"]);
12841 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12842 let origin = tmp.path().join("origin.git");
12843 g(&[
12844 "clone",
12845 "--bare",
12846 repo.to_str().unwrap(),
12847 origin.to_str().unwrap(),
12848 ]);
12849 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12850 g(&["fetch", "origin"]);
12851 let selection = crate::pr_recovery::Selection {
12852 url: "https://example.test/project/pull/1".into(),
12853 branch: "magi/task-selected".into(),
12854 head: head.clone(),
12855 source_head: None,
12856 source_run: None,
12857 source_proof: None,
12858 disposition: Some(crate::pr_recovery::Disposition::Review),
12859 instruction: "preserve measured behavior".into(),
12860 approved_at: jiff::Timestamp::now(),
12861 };
12862 let mut template = runner_at(RunStatus::Reviewing);
12863 template.state.repo = repo.clone();
12864 template.state.base_commit = current.clone();
12865 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12866 template.state.origin = Some(crate::run::Origin::queue("selected"));
12867 template.state.pr_head = Some(selection.branch.clone());
12868 template.state.pr_recovery = Some(selection.clone());
12869 let mut runner =
12870 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12871 .await
12872 .unwrap();
12873 let wt = runner.state.winner().unwrap().worktree.clone();
12874 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12875 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12876 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12877 assert!(runner.state.seeds.is_empty());
12878 assert_eq!(runner.state.instruction, "preserve measured behavior");
12879 runner.sync_to_base().await.unwrap();
12880 assert_eq!(
12881 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12882 "prerequisite"
12883 );
12884 assert_eq!(
12885 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12886 "measured implementation"
12887 );
12888 assert_eq!(
12889 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12890 "current main change"
12891 );
12892 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12893 assert_eq!(
12894 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12895 head
12896 );
12897
12898 let mut partial = runner_at(RunStatus::Reviewing);
12900 partial.state.repo = repo.clone();
12901 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12902 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12903 run: Some("exit 1".into()),
12904 ..Default::default()
12905 }];
12906 let selection = runner.state.pr_recovery.clone().unwrap();
12907 partial.state.pr_recovery = Some(selection.clone());
12908 partial.state.pr_head = Some(selection.branch.clone());
12909 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12910 let id = partial.state.id.clone();
12911 let error = match Runner::open_selected(
12912 &repo,
12913 partial.state,
12914 partial.roles,
12915 1,
12916 runner.state.base_commit.clone(),
12917 selection,
12918 )
12919 .await
12920 {
12921 Ok(_) => panic!("setup must fail"),
12922 Err(e) => e,
12923 };
12924 assert_eq!(
12925 error
12926 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12927 .unwrap()
12928 .run,
12929 id
12930 );
12931 let failed = RunState::load(&id).unwrap();
12932 assert_eq!(failed.status, RunStatus::Blocked);
12933 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12934 assert!(Runner::resume(&id).is_err());
12935 runner.state.status = RunStatus::Ready;
12936 runner.state.driver_exited = true;
12937 runner.state.save().unwrap();
12938 let mut retry = runner_at(RunStatus::Reviewing);
12939 retry.state.repo = repo.clone();
12940 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12941 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12942 reselected.approved_at = jiff::Timestamp::now();
12943 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12944 retry.state.pr_head = Some(reselected.branch.clone());
12945 retry.state.pr_recovery = Some(reselected.clone());
12946 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12947 .await
12948 .unwrap();
12949 let restarted = Runner::open_selected(
12950 &repo,
12951 retry.state,
12952 retry.roles,
12953 1,
12954 runner.state.base_commit.clone(),
12955 reselected,
12956 )
12957 .await
12958 .unwrap();
12959 assert_eq!(
12960 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12961 .await
12962 .unwrap(),
12963 head
12964 );
12965 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12966 assert_eq!(
12967 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12968 head
12969 );
12970 }
12971
12972 #[tokio::test]
12973 #[cfg(unix)]
12974 async fn fallback_and_interrupted_restoration_require_coordination() {
12975 use std::os::fd::AsRawFd as _;
12976 ask_test_home();
12977 let tmp = tempfile::tempdir().unwrap();
12978 let mut runner = registered_runner(tmp.path()).await;
12979 runner.state.config.graph.timeout_resource_wait = 1;
12980 let winner = runner.state.candidates[0].clone();
12981 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12982 let leftover = winner.worktree.join("leftover");
12983 std::fs::write(&leftover, "untrusted edits").unwrap();
12984 let next = tmp.path().join("next-fixer");
12985 let mut failed = spec("first");
12986 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12987 let mut successor = spec("second");
12988 successor.command = vec![
12989 "sh".into(),
12990 "-c".into(),
12991 format!("touch '{}'; echo done", next.display()),
12992 ];
12993 let attempts = vec![
12994 (failed.clone(), "first".into()),
12995 (successor.clone(), "second".into()),
12996 ];
12997 let lock_path = runner.state.repo.join(".git/updater.lock");
12998 let external = std::fs::File::create(&lock_path).unwrap();
12999 assert_eq!(
13000 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13001 0
13002 );
13003 let artifacts = tmp.path().join("artifacts");
13004 let build = |spec, seat| SeatJob {
13005 spec,
13006 seat,
13007 cwd: winner.worktree.clone(),
13008 prompt: "fixture".into(),
13009 timeout: Duration::from_secs(10),
13010 allow_write: true,
13011 sessions: false,
13012 artifacts: artifacts.clone(),
13013 stem: "fix".into(),
13014 handover: None,
13015 };
13016 let (_, _, out) = runner
13017 .ask_fixer_among(
13018 attempts.clone(),
13019 "graft",
13020 None,
13021 Some((&winner.worktree, &head)),
13022 build,
13023 )
13024 .await;
13025 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13026 assert!(leftover.exists(), "contention must prevent clean");
13027 assert!(!next.exists());
13028 assert!(runner.state.handovers.is_empty());
13029 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13031 from_run: "earlier".into(),
13032 generation: 1,
13033 mode: crate::recovery::Mode::VerifyRepair,
13034 reasons: Vec::new(),
13035 reviewed_head: None,
13036 rounds: Vec::new(),
13037 open_findings: Vec::new(),
13038 evidence: None,
13039 criteria_inherited: true,
13040 repair: crate::recovery::Repair::Started {
13041 before: head.clone(),
13042 },
13043 });
13044 assert!(
13045 runner
13046 .recovery_repair(&winner)
13047 .await
13048 .unwrap_err()
13049 .is::<crate::repo_lock::ResourceBlocked>()
13050 );
13051 assert!(matches!(
13052 runner.state.recovery.as_ref().unwrap().repair,
13053 crate::recovery::Repair::Started { .. }
13054 ));
13055 assert!(leftover.exists());
13056 drop(external);
13057 runner.recovery_repair(&winner).await.unwrap();
13058 assert!(!leftover.exists());
13059 assert!(matches!(
13060 runner.state.recovery.as_ref().unwrap().repair,
13061 crate::recovery::Repair::Done { .. }
13062 ));
13063 std::fs::remove_file(&lock_path).unwrap();
13065 std::fs::create_dir(&lock_path).unwrap();
13066 std::fs::write(&leftover, "untrusted edits").unwrap();
13067 let (_, _, out) = runner
13068 .ask_fixer_among(
13069 attempts,
13070 "graft",
13071 None,
13072 Some((&winner.worktree, &head)),
13073 build,
13074 )
13075 .await;
13076 assert!(matches!(out, AgentOutcome::Failed(_)));
13077 assert!(leftover.exists());
13078 assert!(!next.exists());
13079 assert_eq!(
13080 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13081 head
13082 );
13083 }
13084
13085 #[cfg(unix)]
13086 async fn registered_runner(root: &Path) -> Runner {
13087 ask_test_home();
13088 let repo = root.join("repo");
13089 std::fs::create_dir_all(&repo).unwrap();
13090 init_repo(&repo);
13091 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13092 let mut runner = runner_at(RunStatus::Prep);
13093 runner.state.repo = repo.clone();
13094 runner.state.config.coordination = crate::config::test_coordination();
13095 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13096 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13097 runner.state.config.graph.advise = false;
13098 runner.roles.implementers = vec![spec("implement")];
13099 runner.roles.judges = vec![spec("judge")];
13100 runner.prep().await.unwrap();
13101 runner
13102 }
13103
13104 #[tokio::test]
13105 #[cfg(unix)]
13106 async fn parked_implementer_persists_completed_work() {
13107 for registered in [false, true] {
13108 let tmp = tempfile::tempdir().unwrap();
13109 let mut runner = registered_runner(tmp.path()).await;
13110 if !registered {
13111 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13112 }
13113 let marker = tmp.path().join("started");
13114 let release = tmp.path().join("release");
13115 let mut implementer = spec("implement");
13116 implementer.command = vec![
13117 "sh".into(),
13118 "-c".into(),
13119 format!(
13120 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13121 marker.display(),
13122 release.display()
13123 ),
13124 ];
13125 runner.roles.implementers = vec![implementer.clone()];
13126 runner.state.config.agents = vec![implementer];
13127 let pause = runner.pause.clone();
13128 let (result, ()) = tokio::join!(runner.implement(), async {
13129 tokio::time::timeout(Duration::from_secs(10), async {
13130 while !marker.exists() {
13131 tokio::time::sleep(Duration::from_millis(10)).await;
13132 }
13133 })
13134 .await
13135 .unwrap();
13136 pause.park();
13137 std::fs::write(&release, "go").unwrap();
13138 });
13139 result.unwrap();
13140 let candidate = &runner.state.candidates[0];
13141 assert!(candidate.commits > 0, "{candidate:?}");
13142 assert!(candidate.summary.contains("saved-result"));
13143 assert_eq!(candidate.files, 1);
13144 let before = candidate.commits;
13145 runner.implement().await.unwrap();
13146 assert_eq!(runner.state.candidates[0].commits, before);
13147 }
13148 }
13149
13150 #[tokio::test]
13151 #[cfg(unix)]
13152 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13153 let tmp = tempfile::tempdir().unwrap();
13154 let mut runner = registered_runner(tmp.path()).await;
13155 let winner = runner.state.candidates[0].clone();
13156 let cache = tmp.path().join("cache");
13157 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13158 runner.state.config.graph.timeout_resource_wait = 0;
13159 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13160 let crate::cache::AcquireOutcome::Acquired(held) =
13161 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13162 else {
13163 panic!("cache must be free")
13164 };
13165 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13166 from_run: "earlier".into(),
13167 generation: 1,
13168 mode: crate::recovery::Mode::VerifyRepair,
13169 reasons: Vec::new(),
13170 reviewed_head: None,
13171 rounds: Vec::new(),
13172 open_findings: Vec::new(),
13173 evidence: None,
13174 criteria_inherited: true,
13175 repair: crate::recovery::Repair::NotStarted,
13176 });
13177 runner.recovery_repair(&winner).await.unwrap();
13178 assert!(matches!(
13179 runner.state.recovery.as_ref().unwrap().repair,
13180 crate::recovery::Repair::WaitingForRestore { .. }
13181 ));
13182 drop(held);
13183 runner.state.resource_block = None;
13184 runner.state.config.graph.timeout_resource_wait = 1;
13185 runner.recovery_repair(&winner).await.unwrap();
13186 assert!(matches!(
13187 runner.state.recovery.as_ref().unwrap().repair,
13188 crate::recovery::Repair::Done { .. }
13189 ));
13190 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13191 }
13192
13193 #[tokio::test]
13194 #[cfg(unix)]
13195 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13196 use std::os::fd::AsRawFd as _;
13197 for graft in [false, true] {
13198 let tmp = tempfile::tempdir().unwrap();
13199 let mut runner = registered_runner(tmp.path()).await;
13200 runner.state.config.graph.timeout_resource_wait = 1;
13201 let winner = runner.state.candidates[0].clone();
13202 let marker = tmp.path().join("fixer-started");
13203 let mut fixer = spec("fixer");
13204 fixer.command = vec![
13205 "sh".into(),
13206 "-c".into(),
13207 format!("echo started >> '{}'; echo done", marker.display()),
13208 ];
13209 runner.roles.fixer = Some(vec![fixer]);
13210 if graft {
13211 let mut loser = winner.clone();
13212 loser.label = 'B';
13213 loser.index = 1;
13214 runner.state.candidates.push(loser);
13215 runner.state.config.graph.graft_runner_up = true;
13216 runner.state.tally = Some(Tally {
13217 first_choice: BTreeMap::new(),
13218 borda: BTreeMap::new(),
13219 winner: winner.label,
13220 rankings: 1,
13221 unanimous_initial: true,
13222 deliberated: false,
13223 changed_votes: 0,
13224 unanimous_final: true,
13225 tie_break: None,
13226 judges: 1,
13227 present: 1,
13228 quorum: 1,
13229 met_quorum: true,
13230 uncontested: None,
13231 });
13232 runner.state.judgements.push(crate::run::Judgement {
13233 judge: 1,
13234 seat: "judge-1".into(),
13235 agent: "judge".into(),
13236 ranking: vec![winner.label, 'B'],
13237 reasons: BTreeMap::new(),
13238 confidence: None,
13239 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13240 order: vec![0, 1],
13241 failed: None,
13242 duration_ms: 0,
13243 });
13244 } else {
13245 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13246 from_run: "earlier".into(),
13247 generation: 1,
13248 mode: crate::recovery::Mode::VerifyRepair,
13249 reasons: Vec::new(),
13250 reviewed_head: None,
13251 rounds: Vec::new(),
13252 open_findings: Vec::new(),
13253 evidence: None,
13254 criteria_inherited: true,
13255 repair: crate::recovery::Repair::NotStarted,
13256 });
13257 }
13258 let reader = crate::repo_lock::SeatLease::for_checkout(
13259 &winner.worktree,
13260 false,
13261 crate::config::test_coordination().as_ref(),
13262 )
13263 .await
13264 .unwrap();
13265 let pause = runner.pause.clone();
13266 let (result, ()) = tokio::join!(
13267 async {
13268 if graft {
13269 runner.graft().await
13270 } else {
13271 runner.recovery_repair(&winner).await
13272 }
13273 },
13274 async {
13275 tokio::time::sleep(Duration::from_millis(100)).await;
13276 pause.park();
13277 }
13278 );
13279 result.unwrap();
13280 assert!(!marker.exists(), "a blocked seat must not spawn");
13281 runner.state = RunState::load(&runner.state.id).unwrap();
13283 if graft {
13284 assert_eq!(
13285 runner.state.graft.as_ref().unwrap().outcome,
13286 GraftOutcome::WaitingForRestore
13287 );
13288 } else {
13289 assert!(matches!(
13290 runner.state.recovery.as_ref().unwrap().repair,
13291 crate::recovery::Repair::WaitingForRestore { .. }
13292 ));
13293 }
13294 drop(reader);
13295 runner.pause = Pause::new();
13296 runner.state.resource_block = None;
13297 let external =
13299 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13300 assert_eq!(
13301 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13302 0
13303 );
13304 let leftover = winner.worktree.join("leftover");
13305 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13306 let result = if graft {
13307 runner.graft().await
13308 } else {
13309 runner.recovery_repair(&winner).await
13310 };
13311 assert!(
13312 result
13313 .unwrap_err()
13314 .is::<crate::repo_lock::ResourceBlocked>()
13315 );
13316 assert!(leftover.exists());
13317 assert!(!marker.exists());
13318 drop(external);
13319 runner.state.resource_block = None;
13320 let result = if graft {
13321 runner.graft().await
13322 } else {
13323 runner.recovery_repair(&winner).await
13324 };
13325 result.unwrap();
13326 assert!(!leftover.exists(), "restore must precede retry");
13327 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13328 if graft {
13329 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13330 } else {
13331 assert!(matches!(
13332 runner.state.recovery.as_ref().unwrap().repair,
13333 crate::recovery::Repair::Done { .. }
13334 ));
13335 }
13336 }
13337 }
13338
13339 #[tokio::test]
13340 #[cfg(unix)]
13341 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13342 use std::os::fd::AsRawFd as _;
13343 let tmp = tempfile::tempdir().unwrap();
13344 let mut runner = registered_runner(tmp.path()).await;
13345 runner.state.config.graph.timeout_resource_wait = 1;
13346 let winner = runner.state.candidates[0].clone();
13347 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13348 runner.state.tally = Some(Tally {
13349 first_choice: BTreeMap::new(),
13350 borda: BTreeMap::new(),
13351 winner: winner.label,
13352 rankings: 0,
13353 unanimous_initial: false,
13354 deliberated: false,
13355 changed_votes: 0,
13356 unanimous_final: false,
13357 tie_break: None,
13358 judges: 0,
13359 present: 0,
13360 quorum: 0,
13361 met_quorum: true,
13362 uncontested: Some("fixture".into()),
13363 });
13364 runner.state.graft = Some(GraftRecord {
13365 pre: pre.clone(),
13366 post: None,
13367 lines: None,
13368 limit: None,
13369 agent: None,
13370 outcome: GraftOutcome::Running,
13371 requested: Vec::new(),
13372 adoption: None,
13373 adoption_error: None,
13374 });
13375 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13376 git::commit_all(&winner.worktree, "unfinished graft")
13377 .await
13378 .unwrap();
13379 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13380 let leftover = winner.worktree.join("leftover");
13381 std::fs::write(&leftover, "untrusted edits").unwrap();
13382 let path = runner.state.repo.join(".git/updater.lock");
13383 let external = std::fs::File::create(&path).unwrap();
13384 assert_eq!(
13385 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13386 0
13387 );
13388 assert!(
13389 runner
13390 .graft()
13391 .await
13392 .unwrap_err()
13393 .is::<crate::repo_lock::ResourceBlocked>()
13394 );
13395 assert!(matches!(
13396 runner.state.graft.as_ref().unwrap().outcome,
13397 GraftOutcome::Running
13398 ));
13399 assert_eq!(
13400 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13401 unfinished
13402 );
13403 assert!(leftover.exists());
13404 drop(external);
13405 std::fs::remove_file(&path).unwrap();
13406 std::fs::create_dir(&path).unwrap();
13407 assert!(runner.graft().await.is_err());
13408 assert!(matches!(
13409 runner.state.graft.as_ref().unwrap().outcome,
13410 GraftOutcome::Running
13411 ));
13412 assert_eq!(
13413 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13414 unfinished
13415 );
13416 assert!(leftover.exists());
13417 std::fs::remove_dir(&path).unwrap();
13418 runner.graft().await.unwrap();
13419 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13420 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13421 assert!(!leftover.exists());
13422 }
13423
13424 #[tokio::test]
13425 #[cfg(unix)]
13426 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13427 use std::os::fd::AsRawFd as _;
13428 let tmp = tempfile::tempdir().unwrap();
13429 let mut runner = registered_runner(tmp.path()).await;
13430 runner.state.prep_complete = Some(false);
13431 runner.state.config.graph.timeout_resource_wait = 1;
13432 runner.state.save().unwrap();
13433 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13434 assert_eq!(
13435 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13436 0
13437 );
13438 runner.execute().await.unwrap();
13439 let disk = RunState::load(&runner.state.id).unwrap();
13440 assert!(disk.driver_exited);
13441 assert_eq!(disk.driver_pid, None);
13442 assert_eq!(runner.state.status, RunStatus::Blocked);
13443 assert_eq!(
13444 runner.state.resource_block.as_ref().unwrap().kind,
13445 "repository coordination"
13446 );
13447 assert!(runner.state.prep_failure.is_none());
13448 assert!(runner.state.seats.is_empty());
13449 runner.pause.park();
13450 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13453 .await
13454 .unwrap()
13455 .unwrap();
13456 assert!(runner.state.parked);
13457 assert!(runner.state.prep_failure.is_none());
13458 let disk = RunState::load(&runner.state.id).unwrap();
13459 assert!(disk.driver_exited);
13460 assert_eq!(disk.driver_pid, None);
13461 }
13462
13463 #[tokio::test]
13464 #[cfg(unix)]
13465 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13466 use std::os::fd::AsRawFd as _;
13467 let tmp = tempfile::tempdir().unwrap();
13468 let mut runner = registered_runner(tmp.path()).await;
13469 runner.state.prep_complete = Some(false);
13470 runner.state.config.graph.timeout_resource_wait = 30;
13471 runner.state.save().unwrap();
13472 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13473 assert_eq!(
13474 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13475 0
13476 );
13477 assert!(
13478 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13479 .await
13480 .is_err()
13481 );
13482 let disk = RunState::load(&runner.state.id).unwrap();
13483 assert!(disk.driver_exited);
13484 assert_eq!(disk.driver_pid, None);
13485 assert!(disk.status.resumable());
13486 assert!(runner.state.driver_exited);
13487 assert_eq!(runner.state.driver_pid, None);
13488 runner.state.save().unwrap();
13489 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13490 }
13491
13492 #[tokio::test]
13493 #[cfg(unix)]
13494 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13495 let tmp = tempfile::tempdir().unwrap();
13496 let mut runner = registered_runner(tmp.path()).await;
13497 let winner = runner.state.candidates[0].clone();
13498 let cache = tmp.path().join("cache");
13499 let started = tmp.path().join("pre-gate-started");
13500 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13501 runner.state.config.verify.gate =
13502 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13503 let home = crate::run::home();
13504 let owner =
13505 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13506 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13507 crate::cache::AcquireOutcome::Acquired(g) => g,
13508 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13509 };
13510 let mut gate = Box::pin(runner.run_gate(&winner));
13511 assert!(
13512 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13513 .await
13514 .is_err()
13515 );
13516 assert!(!started.exists(), "pre-gate must wait for the cache too");
13517 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13521 let editor = crate::repo_lock::SeatLease::checkout_with(
13522 &winner.worktree,
13523 true,
13524 crate::config::test_coordination().as_ref(),
13525 &wait,
13526 )
13527 .await
13528 .unwrap();
13529 drop(editor);
13530 guard.release();
13531 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13532 .await
13533 .unwrap()
13534 .unwrap();
13535 drop(gate);
13536 assert!(outcomes.iter().all(CommandOutcome::ok));
13537 assert!(started.exists());
13538 }
13539
13540 #[tokio::test]
13541 #[cfg(unix)]
13542 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13543 use std::os::fd::AsRawFd as _;
13544
13545 ask_test_home();
13546 let tmp = tempfile::tempdir().unwrap();
13547 let repo = tmp.path().join("repo");
13548 std::fs::create_dir_all(&repo).unwrap();
13549 init_repo(&repo);
13550 let common = repo.join(".git");
13551 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13552 let mut runner = runner_at(RunStatus::Prep);
13553 runner.state.repo = repo.clone();
13554 runner.state.config.coordination = crate::config::test_coordination();
13555 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13556 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13557 runner.state.config.graph.advise = false;
13558 runner.roles.implementers = vec![spec("implement")];
13559 runner.roles.judges = vec![spec("judge")];
13560 runner.prep().await.unwrap();
13561 let winner = runner.state.candidates[0].clone();
13562 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13565 let completed = tmp.path().join("seat-completed");
13566 runner.roles.implementers[0].command = vec![
13567 "sh".into(),
13568 "-c".into(),
13569 format!("touch '{}'; echo completed", completed.display()),
13570 ];
13571 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13572 assert_eq!(
13573 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13574 0
13575 );
13576 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13577 let mut implement = Box::pin(runner.implement());
13578 tokio::select! {
13579 result = &mut implement => panic!("rescue must wait: {result:?}"),
13580 result = tokio::time::timeout(Duration::from_secs(10), async {
13581 while !completed.exists() {
13582 tokio::time::sleep(Duration::from_millis(20)).await;
13583 }
13584 }) => result.unwrap(),
13585 }
13586 assert!(
13587 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13588 .await
13589 .is_err()
13590 );
13591 assert_eq!(
13592 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13593 before
13594 );
13595 assert_eq!(
13596 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13597 0
13598 );
13599 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13600 .await
13601 .unwrap()
13602 .unwrap();
13603 drop(implement);
13604 assert_ne!(
13605 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13606 before
13607 );
13608 assert!(git::is_clean(&winner.worktree).await.unwrap());
13609
13610 let started = tmp.path().join("pre-gate-started");
13611 let proceed = tmp.path().join("pre-gate-proceed");
13612 let checked = tmp.path().join("gate-ran");
13613 runner.state.config.verify.pre_gate = vec![format!(
13614 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13615 started.display(),
13616 proceed.display()
13617 )];
13618 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13619 assert_eq!(
13620 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13621 0
13622 );
13623 let mut gate = Box::pin(runner.run_gate(&winner));
13624 assert!(
13625 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13626 .await
13627 .is_err()
13628 );
13629 assert!(!started.exists());
13630 assert!(!checked.exists());
13631 assert!(git::is_clean(&winner.worktree).await.unwrap());
13632 assert_eq!(
13633 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13634 0
13635 );
13636 tokio::select! {
13637 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13638 result = tokio::time::timeout(Duration::from_secs(10), async {
13639 while !started.exists() {
13640 tokio::time::sleep(Duration::from_millis(20)).await;
13641 }
13642 }) => result.unwrap(),
13643 }
13644 assert_eq!(
13645 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13646 -1,
13647 "mechanical edits must hold the real lock"
13648 );
13649 std::fs::write(proceed, "ready").unwrap();
13650 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13651 .await
13652 .unwrap()
13653 .unwrap();
13654 drop(gate);
13655 assert!(outcomes.iter().all(CommandOutcome::ok));
13656 assert!(checked.exists());
13657 assert!(runner.state.pre_gate_commit.is_some());
13658 assert!(git::is_clean(&winner.worktree).await.unwrap());
13659 assert_eq!(
13660 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13661 0,
13662 "gate completion must release coordination"
13663 );
13664 drop(outsider);
13665 std::fs::remove_file(common.join("updater.lock")).unwrap();
13666 std::fs::create_dir(common.join("updater.lock")).unwrap();
13667 std::fs::remove_file(&started).unwrap();
13668 std::fs::remove_file(&checked).unwrap();
13669 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13670 let error = runner.run_gate(&winner).await.unwrap_err();
13671 assert!(
13672 error
13673 .to_string()
13674 .contains("acquire mandatory repository lock")
13675 );
13676 assert!(
13677 !started.exists(),
13678 "failed acquisition must prevent pre-gate"
13679 );
13680 assert!(!checked.exists(), "failed acquisition must prevent gate");
13681 assert!(git::is_clean(&winner.worktree).await.unwrap());
13682 assert_eq!(
13683 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13684 head
13685 );
13686 }
13687
13688 #[tokio::test]
13689 #[cfg(unix)]
13690 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13691 use std::os::fd::AsRawFd as _;
13692 use std::time::Duration;
13693
13694 ask_test_home();
13695 let tmp = tempfile::tempdir().unwrap();
13696 let repo = tmp.path().join("repo");
13697 std::fs::create_dir_all(&repo).unwrap();
13698 init_repo(&repo);
13699 let common = repo.join(".git");
13700 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13701 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13702 assert_eq!(
13703 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13704 0
13705 );
13706 let mut runner = runner_at(RunStatus::Prep);
13707 runner.state.repo = repo.clone();
13708 runner.state.config.coordination = crate::config::test_coordination();
13709 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13710 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13711 runner.state.config.graph.advise = true;
13712 runner.state.config.graph.advisors = 1;
13713 runner.roles.implementers = vec![spec("implement")];
13714 runner.roles.judges = vec![spec("judge")];
13715 let root = runner.state.worktree_root();
13716 let started = tmp.path().join("setup-started");
13717 let proceed = tmp.path().join("setup-proceed");
13718 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13719 run: Some(format!(
13720 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13721 started.display(),
13722 proceed.display()
13723 )),
13724 ..Default::default()
13725 }];
13726 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13727 let config = git::git(&repo, &["config", "--local", "--list"])
13728 .await
13729 .unwrap();
13730 let mut prep = Box::pin(runner.prep());
13731 assert!(
13732 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13733 .await
13734 .is_err()
13735 );
13736 assert!(
13737 !root.exists(),
13738 "contention must prevent even checkout creation"
13739 );
13740 assert!(!started.exists(), "contention must prevent setup");
13741 assert_eq!(
13742 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13743 branches
13744 );
13745 assert_eq!(
13746 git::git(&repo, &["config", "--local", "--list"])
13747 .await
13748 .unwrap(),
13749 config
13750 );
13751 drop(lock);
13752 tokio::select! {
13753 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13754 result = tokio::time::timeout(Duration::from_secs(10), async {
13755 while !started.exists() {
13756 tokio::time::sleep(Duration::from_millis(20)).await;
13757 }
13758 }) => result.unwrap(),
13759 }
13760 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13761 assert_eq!(
13762 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13763 -1
13764 );
13765 assert!(
13766 tokio::time::timeout(
13767 Duration::from_millis(100),
13768 crate::repo_lock::SeatLease::for_checkout(
13769 &repo,
13770 true,
13771 crate::config::test_coordination().as_ref()
13772 )
13773 )
13774 .await
13775 .is_err(),
13776 "preparation must also exclude local editors"
13777 );
13778 std::fs::write(proceed, "ready").unwrap();
13779 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13780 .await
13781 .unwrap()
13782 .unwrap();
13783 drop(prep);
13784 assert_eq!(runner.state.prep_complete, Some(true));
13785 assert!(root.join("judge-1").exists());
13786 assert!(root.join("advisor-1").exists());
13787 assert_eq!(
13788 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13789 0
13790 );
13791 }
13792
13793 #[tokio::test]
13794 #[cfg(unix)]
13795 async fn preparation_lock_failure_prevents_repository_changes() {
13796 ask_test_home();
13797 let tmp = tempfile::tempdir().unwrap();
13798 let repo = tmp.path().join("repo");
13799 std::fs::create_dir_all(&repo).unwrap();
13800 init_repo(&repo);
13801 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13802 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13805 let mut runner = runner_at(RunStatus::Prep);
13806 runner.state.repo = repo.clone();
13807 runner.state.config.coordination = crate::config::test_coordination();
13808 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13809 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13810 runner.roles.implementers = vec![spec("implement")];
13811 let error = runner.prep().await.unwrap_err();
13812 assert!(
13813 error
13814 .to_string()
13815 .contains("acquire mandatory repository lock")
13816 );
13817 assert!(!runner.state.worktree_root().exists());
13818 assert!(runner.state.candidates.is_empty());
13819 assert!(!repo.join(".git/config.worktree").exists());
13820 assert_eq!(
13821 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13822 "* main"
13823 );
13824 }
13825
13826 #[tokio::test]
13827 async fn preparation_records_creation_intent_before_git_can_fail() {
13828 ask_test_home();
13829 let tmp = tempfile::tempdir().unwrap();
13830 let repo = tmp.path().join("repo");
13831 std::fs::create_dir_all(&repo).unwrap();
13832 init_repo(&repo);
13833 let mut runner = runner_at(RunStatus::Prep);
13834 runner.state.repo = repo.clone();
13835 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13836 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13837 runner.state.config.blind.commit_msg_hook = false;
13838 runner.roles.implementers = vec![spec("one")];
13839 let label = blind::assign_labels(1, runner.state.seed)[0];
13840 let branch = runner.state.branch_for(label);
13841 git::git(&repo, &["branch", &branch]).await.unwrap();
13844 assert!(runner.prep().await.is_err());
13845 let saved = RunState::load(&runner.state.id).unwrap();
13846 assert!(saved.candidates.is_empty());
13847 let event = saved
13848 .events
13849 .iter()
13850 .find(|e| e.node == "prep-intent")
13851 .unwrap();
13852 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13853 assert_eq!(intent["branch"], branch);
13854 assert_eq!(intent["start"], runner.state.base_commit);
13855 assert_eq!(
13856 intent["worktree"],
13857 runner
13858 .state
13859 .worktree_root()
13860 .join(format!("cand-{label}"))
13861 .to_string_lossy()
13862 .as_ref()
13863 );
13864 assert_eq!(intent["picks"], serde_json::json!([]));
13865 }
13866
13867 #[tokio::test]
13868 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13869 ask_test_home();
13870 let tmp = tempfile::tempdir().unwrap();
13871 let repo = tmp.path().join("repo");
13872 std::fs::create_dir_all(&repo).unwrap();
13873 init_repo(&repo);
13874 let mut runner = runner_at(RunStatus::Prep);
13875 runner.state.repo = repo.clone();
13876 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13877 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13878 runner.state.sources.pick_commits.push(refs::Input {
13879 token: "missing".into(),
13880 sha: "0".repeat(40),
13881 provenance: refs::Provenance::ExplicitCli,
13882 });
13883 assert!(runner.prep().await.is_err());
13884 assert!(runner.state.candidates.is_empty());
13885 assert!(!runner.state.worktree_root().exists());
13886 }
13887
13888 #[tokio::test]
13889 async fn prose_and_legacy_inference_never_import_into_candidates() {
13890 ask_test_home();
13891 let tmp = tempfile::tempdir().unwrap();
13892 let repo = tmp.path().join("repo");
13893 std::fs::create_dir_all(&repo).unwrap();
13894 init_repo(&repo);
13895 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13896 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13897 .await
13898 .unwrap();
13899 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13900 git::git(&repo, &["add", "."]).await.unwrap();
13901 git::git(&repo, &["commit", "-m", "historical implementation"])
13902 .await
13903 .unwrap();
13904 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13905 git::git(&repo, &["checkout", "main"]).await.unwrap();
13906 for instruction in [
13907 format!("Example: magi/history/A at {source}"),
13908 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13909 format!("Never import magi/history/A or {source}"),
13910 format!("Conversation answer: I mentioned {source} only as background"),
13911 ] {
13912 let mut runner = runner_at(RunStatus::Prep);
13913 runner.state.repo = repo.clone();
13914 runner.state.base_commit = base.clone();
13915 runner.state.instruction = instruction.clone();
13916 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13917 runner.state.config.blind.commit_msg_hook = false;
13918 runner.state.seeds = vec![refs::Seed {
13919 token: "magi/history/A".into(),
13920 sha: source.clone(),
13921 kind: refs::SeedKind::Unmerged,
13922 branch: true,
13923 detail: String::new(),
13924 }];
13925 runner.roles.implementers = vec![spec("one")];
13926 assert_eq!(seeded_instruction(&runner.state), instruction);
13927 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13929 legacy.as_object_mut().unwrap().remove("sources");
13930 runner.state = serde_json::from_value(legacy).unwrap();
13931 assert_eq!(runner.state.sources, refs::Sources::default());
13932 runner.state.save().unwrap();
13933 runner.state = RunState::load(&runner.state.id).unwrap();
13934 runner.prep().await.unwrap();
13935 for candidate in &runner.state.candidates {
13936 assert_eq!(
13937 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13938 base
13939 );
13940 assert!(!candidate.worktree.join("unwanted").exists());
13941 }
13942 }
13943 }
13944
13945 #[tokio::test]
13946 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13947 ask_test_home();
13948 let tmp = tempfile::tempdir().unwrap();
13949 let repo = tmp.path().join("repo");
13950 std::fs::create_dir_all(&repo).unwrap();
13951 init_repo(&repo);
13952 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13953 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13954 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13955 git::git(&repo, &["add", "."]).await.unwrap();
13956 git::git(&repo, &["commit", "-m", "required seed"])
13957 .await
13958 .unwrap();
13959 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13960 git::git(&repo, &["checkout", "main"]).await.unwrap();
13961 let mut runner = runner_at(RunStatus::Prep);
13962 runner.state.repo = repo.clone();
13963 runner.state.base_commit = base.clone();
13964 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13965 runner.state.config.graph.advise = false;
13966 runner.state.config.blind.commit_msg_hook = false;
13967 runner.state.sources =
13968 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13969 .await
13970 .unwrap();
13971 runner.roles.implementers = vec![spec("one"), spec("two")];
13972 runner.roles.judges = vec![spec("judge")];
13973 let label = blind::assign_labels(2, runner.state.seed)[0];
13974 let branch = runner.state.branch_for(label);
13975 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13976 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13977 .await
13978 .unwrap();
13979 runner.state.candidates.push(Candidate {
13980 index: 0,
13981 label,
13982 agent: "one".into(),
13983 branch,
13984 worktree,
13985 summary: String::new(),
13986 stat: String::new(),
13987 files: 0,
13988 commits: 0,
13989 empty: false,
13990 failed: None,
13991 verified_noop: None,
13992 duration_ms: 0,
13993 folded: false,
13994 });
13995 runner.state.save().unwrap();
13996 runner.state = RunState::load(&runner.state.id).unwrap();
13998 assert_eq!(runner.state.prep_complete, Some(false));
13999 runner.prep().await.unwrap();
14000 assert_eq!(runner.state.prep_complete, Some(true));
14001 assert_eq!(runner.state.candidates.len(), 2);
14002 for candidate in &runner.state.candidates {
14003 assert_eq!(
14004 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14005 git::tree_of(&repo, &seed).await.unwrap()
14006 );
14007 assert_eq!(
14008 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14009 "required commit"
14010 );
14011 }
14012 assert!(runner.state.worktree_root().join("judge-1").exists());
14013 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14014 runner.prep().await.unwrap();
14015 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14016
14017 runner.state.prep_complete = Some(false);
14018 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14019 run: Some(format!(
14020 "test -f '{}'",
14021 tmp.path().join("dependency-ready").display()
14022 )),
14023 ..Default::default()
14024 }];
14025 assert!(runner.prep().await.is_err());
14026 assert!(runner.state.prep_failure.is_some());
14027 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14028 runner.state = RunState::load(&runner.state.id).unwrap();
14029 assert!(
14030 runner.prep().await.is_err(),
14031 "correcting the cause alone must not auto-retry"
14032 );
14033 let failed_at = runner.state.events.last().unwrap().at;
14034 assert!(
14035 runner.retry_preparation(failed_at).is_err(),
14036 "old approvals cannot unlock the hold"
14037 );
14038 let mut task = crate::queue::Task::new(
14039 "retry".into(),
14040 "retry setup".into(),
14041 repo.clone(),
14042 crate::queue::Source::Human,
14043 );
14044 task.start(runner.state.id.clone());
14045 task.hold_machine(Some("setup failed".into()));
14046 task.release_operator("cli", None);
14047 runner
14048 .retry_preparation(task.resumption.as_ref().unwrap().at)
14049 .unwrap();
14050 runner.prep().await.unwrap();
14051 assert!(runner.state.prep_failure.is_none());
14052 assert_eq!(runner.state.prep_complete, Some(true));
14053 assert_eq!(runner.state.candidates.len(), 2);
14054 assert!(runner.state.worktree_root().join("judge-1").exists());
14055 for candidate in &runner.state.candidates {
14056 assert_eq!(
14057 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14058 git::tree_of(&repo, &seed).await.unwrap()
14059 );
14060 }
14061 }
14062
14063 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14064 let mut heads = Vec::new();
14065 for candidate in candidates {
14066 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14067 }
14068 heads
14069 }
14070
14071 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14072 ask_test_home();
14073 let tmp = tempfile::tempdir().unwrap();
14074 let repo = tmp.path().join("repo");
14075 std::fs::create_dir(&repo).unwrap();
14076 init_repo(&repo);
14077 let mut runner = runner_at(RunStatus::Reviewing);
14078 runner.state.repo = repo.clone();
14079 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14080 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14081 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14082 let branch = format!("candidate-{label}");
14083 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14084 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14085 .await
14086 .unwrap();
14087 if label == 'B' {
14088 std::fs::write(worktree.join("change"), "winner").unwrap();
14089 git::rescue_commit(&worktree, "candidate change")
14090 .await
14091 .unwrap();
14092 }
14093 runner.state.candidates.push(Candidate {
14094 index,
14095 label,
14096 agent: "alpha".into(),
14097 branch,
14098 worktree,
14099 summary: "completed reply".into(),
14100 stat: String::new(),
14101 files: usize::from(label == 'B'),
14102 commits: usize::from(label == 'B'),
14103 empty: label == 'A',
14104 failed: None,
14105 verified_noop: None,
14106 duration_ms: 1,
14107 folded: false,
14108 });
14109 }
14110 runner.state.judge_skipped = true;
14111 runner.state.tally = Some(Tally {
14112 first_choice: BTreeMap::new(),
14113 borda: BTreeMap::new(),
14114 winner: 'B',
14115 rankings: 0,
14116 unanimous_initial: false,
14117 deliberated: false,
14118 changed_votes: 0,
14119 unanimous_final: false,
14120 tie_break: None,
14121 judges: 0,
14122 present: 0,
14123 quorum: 0,
14124 met_quorum: true,
14125 uncontested: Some("only B produced a change".into()),
14126 });
14127 runner.state.save().unwrap();
14128 (tmp, runner)
14129 }
14130
14131 #[tokio::test]
14132 #[cfg(unix)]
14133 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14134 for (solo, failed_checkpoint) in
14135 [(true, false), (false, false), (true, true), (false, true)]
14136 {
14137 let (_tmp, mut runner) = preservation_resume_fixture().await;
14138 if solo {
14139 runner.state.candidates.retain(|c| c.label == 'A');
14140 }
14141 runner.state.candidates[0].empty = false;
14142 runner.state.candidates[0].summary.clear();
14143 runner.state.tally = None;
14144 runner.state.judge_skipped = false;
14145 runner.state.status = RunStatus::Implementing;
14146 runner.state.config.disk.min_free_bytes = u64::MAX;
14147 let mut implementer = spec("alpha");
14148 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14149 runner.roles.implementers = vec![implementer];
14150 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14151 if failed_checkpoint {
14152 std::fs::create_dir_all(&patch).unwrap();
14153 }
14154 runner.state.save().unwrap();
14155 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14156 runner.state = RunState::load(&runner.state.id).unwrap();
14157 assert_eq!(runner.state.status, RunStatus::Blocked);
14158 assert!(runner.state.interrupted_implementations.contains(&'A'));
14159 assert_eq!(runner.state.candidates[0].commits, 0);
14160 assert!(!runner.state.candidates[0].empty);
14161 assert!(runner.state.candidates[0].failed.is_none());
14162 if failed_checkpoint {
14163 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14164 std::fs::remove_dir(patch).unwrap();
14165 }
14166 runner.state.resource_block = None;
14167 runner.state.config.disk.min_free_bytes = 0;
14168 runner.roles.implementers[0].command = vec![
14169 "sh".into(),
14170 "-c".into(),
14171 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14172 ];
14173 runner.implement().await.unwrap();
14174 assert!(runner.state.interrupted_implementations.is_empty());
14175 assert!(runner.state.candidate_preservation.is_empty());
14176 assert_eq!(runner.state.candidates[0].commits, 1);
14177 assert!(!runner.state.candidates[0].empty);
14178 assert!(
14179 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14180 .unwrap()
14181 .contains("retried empty interrupted seat")
14182 );
14183 if !solo {
14184 assert_eq!(runner.state.candidates[1].commits, 1);
14185 }
14186 }
14187 }
14188
14189 #[tokio::test]
14190 #[cfg(unix)]
14191 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14192 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14193 let (_tmp, mut runner) = preservation_resume_fixture().await;
14194 runner.state.candidates.retain(|c| c.label == 'A');
14195 runner.state.candidates[0].empty = false;
14196 runner.state.candidates[0].summary.clear();
14197 runner.state.tally = None;
14198 runner.state.judge_skipped = false;
14199 runner.state.status = RunStatus::Implementing;
14200 runner.state.config.disk.min_free_bytes = u64::MAX;
14203 let wt = runner.state.candidates[0].worktree.clone();
14204 std::fs::write(
14205 wt.join("interrupted-source"),
14206 "retain implementation work\n",
14207 )
14208 .unwrap();
14209 if committed {
14210 git::rescue_commit(&wt, "implementation checkpoint")
14211 .await
14212 .unwrap();
14213 }
14214 let lock = git::git(
14215 &wt,
14216 &[
14217 "rev-parse",
14218 "--path-format=absolute",
14219 "--git-path",
14220 "index.lock",
14221 ],
14222 )
14223 .await
14224 .unwrap();
14225 if block_rescue {
14226 std::fs::write(&lock, "fixture-owned lock").unwrap();
14227 }
14228 let mut implementer = spec("alpha");
14229 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14230 runner.roles.implementers = vec![implementer];
14231 runner.state.save().unwrap();
14232 let result = runner.implement().await;
14233 assert_eq!(result.is_err(), block_rescue);
14234 assert_eq!(
14235 runner.state.resource_block.as_ref().unwrap().kind,
14236 "disk pressure"
14237 );
14238 assert!(
14239 runner
14240 .state
14241 .events
14242 .iter()
14243 .any(|e| e.message.contains("agent execution interrupted"))
14244 );
14245 assert!(
14246 runner
14247 .state
14248 .dir()
14249 .join("artifacts/candidate-A-recovery/staged.patch")
14250 .exists()
14251 );
14252 if block_rescue {
14253 let saved = RunState::load(&runner.state.id).unwrap();
14254 assert!(saved.candidate_preservation.contains_key(&'A'));
14255 assert!(!saved.candidates[0].empty);
14256 assert_eq!(
14257 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14258 "retain implementation work\n"
14259 );
14260 std::fs::remove_file(lock).unwrap();
14261 runner.state = saved;
14262 } else {
14263 assert_eq!(runner.state.status, RunStatus::Blocked);
14264 assert_eq!(runner.state.candidates[0].commits, 1);
14265 }
14266 runner.roles.implementers.clear();
14269 runner.state.config.disk.min_free_bytes = 0;
14270 runner.state.resource_block = None;
14271 runner.implement().await.unwrap();
14272 assert!(runner.state.candidate_preservation.is_empty());
14273 assert_eq!(runner.state.candidates[0].commits, 1);
14274 assert!(!runner.state.candidates[0].empty);
14275 assert!(git::is_clean(&wt).await.unwrap());
14276 assert!(
14277 crate::run::read_artifact(&runner.state, "cand-A.patch")
14278 .unwrap()
14279 .contains("retain implementation work")
14280 );
14281 }
14282 }
14283
14284 #[tokio::test]
14285 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14286 let (_tmp, mut runner) = preservation_resume_fixture().await;
14287 runner.implement().await.unwrap();
14289 assert!(runner.state.candidate_preservation.is_empty());
14290 assert!(
14291 !runner
14292 .state
14293 .dir()
14294 .join("artifacts/candidate-A-recovery")
14295 .exists()
14296 );
14297 runner.fold_losers().await.unwrap();
14298 assert!(!runner.state.candidates[0].worktree.exists());
14299 for schema in [14, crate::run::SCHEMA] {
14300 runner.state.schema = schema;
14301 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14302 runner.state.save().unwrap();
14303 runner.state = RunState::load(&runner.state.id).unwrap();
14304 runner.implement().await.unwrap();
14305 assert!(runner.state.candidate_preservation.is_empty());
14306 assert!(runner.state.candidates[0].folded);
14307 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14308 }
14309 }
14310
14311 #[tokio::test]
14312 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14313 let (_tmp, mut runner) = preservation_resume_fixture().await;
14314 runner.state.candidates.retain(|c| c.label == 'A');
14315 runner.state.tally = None;
14316 runner.state.judge_skipped = false;
14317 let candidate = &mut runner.state.candidates[0];
14318 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14319 candidate.failed = Some("deployment validation unavailable".into());
14320 std::fs::write(
14321 candidate.worktree.join("repair"),
14322 "future launcher policy\n",
14323 )
14324 .unwrap();
14325 runner
14326 .state
14327 .candidate_preservation
14328 .insert('A', "awaiting persistence".into());
14329 runner.persist_candidates().await.unwrap();
14330 let candidate = &runner.state.candidates[0];
14331 assert!(!candidate.empty);
14332 assert_eq!(candidate.commits, 1);
14333 assert_eq!(candidate.files, 1);
14334 assert!(candidate.failed.is_none());
14335 assert!(candidate.summary.contains("EPERM"));
14336 assert!(
14337 crate::run::read_artifact(&runner.state, "cand-A.patch")
14338 .unwrap()
14339 .contains("future launcher policy")
14340 );
14341 assert!(runner.state.candidate_preservation.is_empty());
14342 }
14343
14344 #[tokio::test]
14345 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14346 let (_tmp, mut runner) = preservation_resume_fixture().await;
14347 let wt = runner.state.candidates[0].worktree.clone();
14348 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14349 runner.state.schema = 14;
14350 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14351 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14352 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14353 for _ in 0..2 {
14354 assert!(runner.implement().await.is_err());
14355 assert_eq!(runner.state.status, RunStatus::Blocked);
14356 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14357 assert!(
14358 runner.state.candidates[0].empty,
14359 "old verdict must remain unusable"
14360 );
14361 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14362 assert!(runner.state.judge_skipped);
14363 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14364 assert!(
14365 crate::run::read_artifact(&runner.state, "cand-A.patch")
14366 .unwrap()
14367 .contains("owned edits")
14368 );
14369 assert!(wt.exists());
14370 assert_eq!(
14371 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14372 .await
14373 .unwrap(),
14374 1
14375 );
14376 runner.state = RunState::load(&runner.state.id).unwrap();
14377 }
14378 }
14379
14380 #[tokio::test]
14381 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14382 let (_tmp, mut runner) = preservation_resume_fixture().await;
14383 let wt = runner.state.candidates[0].worktree.clone();
14384 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14385 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14386 let index_lock = PathBuf::from(
14387 git::git(
14388 &wt,
14389 &[
14390 "rev-parse",
14391 "--path-format=absolute",
14392 "--git-path",
14393 "index.lock",
14394 ],
14395 )
14396 .await
14397 .unwrap(),
14398 );
14399 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14400 runner.state.schema = 14;
14401 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14402 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14403 assert!(runner.implement().await.is_err());
14404 assert!(runner.after_implement().is_err());
14405 assert!(runner.fold_losers().await.is_err());
14406 assert_eq!(
14407 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14408 "failed seat edits"
14409 );
14410 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14411 runner.state = RunState::load(&runner.state.id).unwrap();
14412 std::fs::remove_file(index_lock).unwrap();
14413 assert!(runner.implement().await.is_err());
14415 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14416 assert!(runner.state.candidates[0].empty);
14417 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14418 assert!(
14419 crate::run::read_artifact(&runner.state, "cand-A.patch")
14420 .unwrap()
14421 .contains("failed seat edits")
14422 );
14423 assert!(wt.exists());
14424 }
14425
14426 #[tokio::test]
14427 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14428 let (_tmp, mut runner) = preservation_resume_fixture().await;
14429 let repo = runner.state.repo.clone();
14430 let common = git::git(
14431 &repo,
14432 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14433 )
14434 .await
14435 .unwrap();
14436 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14437 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14438 assert!(runner.fold_losers().await.is_err());
14439 assert!(!runner.state.candidates[0].worktree.exists());
14440 assert!(!runner.state.candidates[0].folded);
14441 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14442 runner.state = RunState::load(&runner.state.id).unwrap();
14443 std::fs::remove_file(branch_lock).unwrap();
14444 let judge = runner.state.worktree_root().join("judge-1");
14445 git::worktree_add_detached(&repo, &judge, "main")
14446 .await
14447 .unwrap();
14448 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14449 runner.roles.judges = vec![spec("judge")];
14450 assert!(runner.fold_losers().await.is_err());
14451 runner.state = RunState::load(&runner.state.id).unwrap();
14452 assert!(
14453 runner.state.candidates[0].folded,
14454 "candidate progress saved before judge failure"
14455 );
14456 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14457 assert!(judge.join("uncommitted").exists());
14458 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14459 runner.fold_losers().await.unwrap();
14460 assert!(!judge.exists());
14461 runner.fold_losers().await.unwrap();
14462 }
14463
14464 #[tokio::test]
14465 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14466 let (_tmp, mut runner) = preservation_resume_fixture().await;
14467 runner.state.candidates.retain(|c| c.label == 'B');
14468 runner.state.tally = None;
14469 runner.state.judge_skipped = false;
14470 runner.state.prep_complete = Some(true);
14471 runner.state.config.graph.advise = false;
14472 runner.state.config.graph.selected_implementer = Some("alpha".into());
14473 let wt = runner.state.candidates[0].worktree.clone();
14474 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14475 runner
14476 .state
14477 .candidate_preservation
14478 .insert('B', "awaiting persistence".into());
14479 let index_lock = git::git(
14480 &wt,
14481 &[
14482 "rev-parse",
14483 "--path-format=absolute",
14484 "--git-path",
14485 "index.lock",
14486 ],
14487 )
14488 .await
14489 .unwrap();
14490 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14491 runner.state.save().unwrap();
14492 assert!(runner.execute_graph().await.is_err());
14493 assert_eq!(runner.state.status, RunStatus::Blocked);
14494 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14495 std::fs::remove_file(index_lock).unwrap();
14496 for _ in 0..2 {
14499 runner.state = RunState::load(&runner.state.id).unwrap();
14500 runner.pause.park();
14501 runner.execute_graph().await.unwrap();
14502 assert_eq!(runner.state.status, RunStatus::Implementing);
14503 assert!(runner.state.candidate_preservation.is_empty());
14504 assert_eq!(runner.state.candidates[0].commits, 2);
14505 assert!(runner.state.tally.is_none());
14506 }
14507 runner.implement().await.unwrap();
14508 assert_eq!(runner.state.status, RunStatus::Judging);
14509 runner.state.status = RunStatus::Blocked;
14511 runner.state.candidates[0].failed = Some("implementation failed".into());
14512 runner.state.save().unwrap();
14513 let error = runner.execute_graph().await.unwrap_err();
14514 assert!(
14515 error
14516 .to_string()
14517 .contains("selected implementation is held")
14518 );
14519 }
14520
14521 #[tokio::test]
14522 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14523 let (_tmp, mut runner) = preservation_resume_fixture().await;
14524 let repo = runner.state.repo.clone();
14525 let candidate = runner.state.candidates[0].clone();
14526 let registered = git::worktree_holding(&repo, &candidate.branch)
14529 .await
14530 .unwrap()
14531 .unwrap();
14532 assert_eq!(
14533 registered.canonicalize().unwrap(),
14534 candidate.worktree.canonicalize().unwrap()
14535 );
14536 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14537 assert_eq!(
14538 git::worktree_holding(&repo, &candidate.branch)
14539 .await
14540 .unwrap(),
14541 Some(registered)
14542 );
14543 assert!(
14544 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14545 "the stale registration initially keeps the branch checked out"
14546 );
14547 runner.fold_losers().await.unwrap();
14548 assert!(runner.state.candidates[0].folded);
14549 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14550 assert_eq!(
14551 git::worktree_holding(&repo, &candidate.branch)
14552 .await
14553 .unwrap(),
14554 None
14555 );
14556 runner.state = RunState::load(&runner.state.id).unwrap();
14557 runner.fold_losers().await.unwrap();
14558 }
14559
14560 #[tokio::test]
14561 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14562 let (_tmp, mut runner) = preservation_resume_fixture().await;
14563 let repo = runner.state.repo.clone();
14564 let candidate = runner.state.candidates[0].clone();
14565 let winner = runner.state.candidates[1].clone();
14566 let registered = git::worktree_holding(&repo, &candidate.branch)
14567 .await
14568 .unwrap()
14569 .unwrap();
14570 assert_eq!(
14571 registered.canonicalize().unwrap(),
14572 candidate.worktree.canonicalize().unwrap()
14573 );
14574 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14575 assert_eq!(
14576 git::worktree_holding(&repo, &candidate.branch)
14577 .await
14578 .unwrap(),
14579 Some(registered)
14580 );
14581 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14582
14583 for _ in 0..2 {
14584 fold_run(&mut runner.state, false, &crate::run::home())
14585 .await
14586 .unwrap();
14587 runner.state = RunState::load(&runner.state.id).unwrap();
14588 assert!(runner.state.candidates[0].folded);
14589 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14590 assert_eq!(
14591 git::worktree_holding(&repo, &candidate.branch)
14592 .await
14593 .unwrap(),
14594 None
14595 );
14596 assert!(!runner.state.candidates[1].folded);
14597 assert!(winner.worktree.exists());
14598 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14599 }
14600 }
14601
14602 #[tokio::test]
14603 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14604 ask_test_home();
14605 let tmp = tempfile::tempdir().unwrap();
14606 let repo = tmp.path().join("repo");
14607 std::fs::create_dir(&repo).unwrap();
14608 init_repo(&repo);
14609 let mut runner = runner_at(RunStatus::Implementing);
14610 runner.state.repo = repo.clone();
14611 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14612 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14613 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14614 let branch = format!("candidate-{label}");
14615 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14616 git::worktree_add_branch(&repo, &wt, &branch, "main")
14617 .await
14618 .unwrap();
14619 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14620 git::git(&wt, &["add", "README.md"]).await.unwrap();
14621 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14622 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14623 runner.state.candidates.push(Candidate {
14624 index,
14625 label,
14626 agent: "alpha".to_owned(),
14627 branch,
14628 worktree: wt,
14629 summary: "completed implementation".to_owned(),
14630 stat: String::new(),
14631 files: 0,
14632 commits: 0,
14633 empty: false,
14634 failed: None,
14635 verified_noop: None,
14636 duration_ms: 42,
14637 folded: false,
14638 });
14639 runner
14640 .state
14641 .candidate_preservation
14642 .insert(label, "awaiting persistence".to_owned());
14643 }
14644 let wt = runner.state.candidates[0].worktree.clone();
14645 let index_lock = git::git(
14646 &wt,
14647 &[
14648 "rev-parse",
14649 "--path-format=absolute",
14650 "--git-path",
14651 "index.lock",
14652 ],
14653 )
14654 .await
14655 .unwrap();
14656 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14657 runner.state.save().unwrap();
14658 assert!(runner.persist_candidates().await.is_err());
14659 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14660 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14661 assert_eq!(
14662 runner.state.candidates[1].commits, 1,
14663 "other results are still collected"
14664 );
14665 assert!(
14666 !runner.state.candidates[0].empty,
14667 "failed rescue is not no change"
14668 );
14669 assert_eq!(
14670 runner.state.candidates[0].summary,
14671 "completed implementation"
14672 );
14673 assert!(runner.after_implement().is_err());
14674 let home = tmp.path().join("fold-home");
14675 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14676 assert!(wt.join("new-file").exists());
14677 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14678 assert!(recovery.join("staged.patch").exists());
14679 assert!(recovery.join("unstaged.patch").exists());
14680 assert_eq!(
14681 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14682 "untracked edits\n"
14683 );
14684 runner.state.begin_pass();
14685 runner.state.end_pass_parked(ParkCause::Upgrade {
14686 from: "1.0".into(),
14687 to: Some("1.1".into()),
14688 });
14689 runner.state.save().unwrap();
14690 runner.state = RunState::load(&runner.state.id).unwrap();
14691 assert!(matches!(
14692 runner.state.passes.last().unwrap().end,
14693 Some(crate::run::PassEnd::Parked {
14694 cause: ParkCause::Upgrade { .. }
14695 })
14696 ));
14697 std::fs::remove_file(index_lock).unwrap();
14698 runner.implement().await.unwrap();
14700 assert!(runner.state.candidate_preservation.is_empty());
14701 assert!(
14702 runner
14703 .state
14704 .candidates
14705 .iter()
14706 .all(|c| c.commits == 1 && !c.empty)
14707 );
14708 assert!(git::is_clean(&wt).await.unwrap());
14709 assert!(
14710 crate::run::read_artifact(&runner.state, "cand-A.patch")
14711 .unwrap()
14712 .contains("later tracked edits")
14713 );
14714 }
14715
14716 #[tokio::test]
14717 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14718 ask_test_home();
14719 let tmp = tempfile::tempdir().unwrap();
14720 let repo = tmp.path().join("repo");
14721 std::fs::create_dir(&repo).unwrap();
14722 init_repo(&repo);
14723 let wt = tmp.path().join("candidate");
14724 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14725 .await
14726 .unwrap();
14727 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14728 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14729 assert!(wt.join("uncommitted").exists());
14730 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14731 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14732 let mut runner = runner_at(RunStatus::Implementing);
14733 runner.state.repo = repo.clone();
14734 runner.state.base_commit = "missing-base".to_owned();
14735 runner.state.candidates.push(Candidate {
14736 index: 0,
14737 label: 'A',
14738 agent: "alpha".into(),
14739 branch: "candidate".into(),
14740 worktree: wt.clone(),
14741 summary: "completed".into(),
14742 stat: String::new(),
14743 files: 0,
14744 commits: 0,
14745 empty: false,
14746 failed: None,
14747 verified_noop: None,
14748 duration_ms: 1,
14749 folded: false,
14750 });
14751 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14752 assert!(
14753 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14754 .await
14755 .is_err()
14756 );
14757 assert!(!runner.state.candidates[0].folded);
14758 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14759 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14760 runner
14761 .state
14762 .candidate_preservation
14763 .insert('A', "pending".into());
14764 assert!(runner.persist_candidates().await.is_err());
14765 assert!(!runner.state.candidates[0].empty);
14766 assert!(!runner.state.candidates[0].folded);
14767 assert!(wt.exists());
14768 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14769 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14770 std::fs::create_dir_all(&patch_path).unwrap();
14771 assert!(
14772 runner.persist_candidates().await.is_err(),
14773 "artifact failure stays pending"
14774 );
14775 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14776 assert!(!runner.state.candidates[0].empty);
14777 std::fs::remove_dir(patch_path).unwrap();
14778 runner.persist_candidates().await.unwrap();
14779 assert!(
14780 runner.state.candidates[0].empty,
14781 "only successful inspection confirms no change"
14782 );
14783 }
14784
14785 fn init_repo(dir: &Path) {
14786 let run = |args: &[&str]| {
14787 let out = std::process::Command::new("git")
14788 .args(args)
14789 .current_dir(dir)
14790 .quiet()
14791 .output()
14792 .expect("spawn git");
14793 assert!(
14794 out.status.success(),
14795 "git {args:?} failed: {}",
14796 String::from_utf8_lossy(&out.stderr)
14797 );
14798 };
14799 run(&["init", "-b", "main"]);
14800 run(&["config", "user.name", "magi test"]);
14801 run(&["config", "user.email", "magi@example.com"]);
14802 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14803 run(&["add", "-A"]);
14804 run(&["commit", "-m", "init"]);
14805 }
14806
14807 fn ask_test_home() {
14815 crate::run::pin_test_home();
14816 }
14817
14818 fn runner_at(status: RunStatus) -> Runner {
14821 let mut state = RunState::new(
14822 PathBuf::from("/nonexistent/repo"),
14823 "main".to_owned(),
14824 "deadbeef".to_owned(),
14825 "task".to_owned(),
14826 Config::default(),
14827 );
14828 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14831 state.id = format!(
14832 "20000101-000000-{:04x}",
14833 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14834 );
14835 state.status = status;
14836 Runner {
14837 state,
14838 roles: ResolvedRoles {
14839 implementers: Vec::new(),
14840 judges: Vec::new(),
14841 reviewers: Vec::new(),
14842 fixer: None,
14843 conductor: conductor(),
14844 implementer_roster: Vec::new(),
14845 judge_roster: Vec::new(),
14846 reviewer_roster: Vec::new(),
14847 },
14848 sem: Arc::new(Semaphore::new(1)),
14849 pause: Pause::new(),
14850 interrupt: Pause::new(),
14851 }
14852 }
14853
14854 #[tokio::test]
14855 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14856 crate::run::pin_test_home();
14857 let mut runner = runner_at(RunStatus::Gating);
14858 runner.state.save().unwrap();
14859 let id = runner.state.id.clone();
14860 let passes = runner.state.passes.len();
14861 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14862 let mut claiming = Box::pin(runner.claim_driver());
14863 std::future::poll_fn(|cx| {
14864 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14865 std::task::Poll::Ready(())
14866 })
14867 .await;
14868 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14869 drop(owner);
14870 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14871 .await
14872 .unwrap()
14873 .unwrap();
14874 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14875 assert!(execution.runner.state.resource_block.is_none());
14876 assert_eq!(
14877 execution.runner.state.liveness(false),
14878 crate::run::Liveness::Live
14879 );
14880 drop(execution);
14881 assert!(RunState::load(&id).unwrap().driver_exited);
14882 }
14883
14884 #[tokio::test]
14885 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14886 crate::run::pin_test_home();
14887 let mut runner = runner_at(RunStatus::Gating);
14888 runner.state.save().unwrap();
14889 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14892 assert!(runner.claim_driver().await.is_err());
14893 assert!(runner.state.driver_exited);
14894 assert_eq!(runner.state.driver_pid, None);
14895 let disk = RunState::load(&runner.state.id).unwrap();
14896 assert!(disk.driver_exited);
14897 assert_eq!(disk.driver_pid, None);
14898 assert_eq!(disk.status, RunStatus::Gating);
14899 }
14900
14901 #[tokio::test]
14902 async fn an_early_graph_error_releases_its_driver() {
14903 crate::run::pin_test_home();
14904 let mut runner = runner_at(RunStatus::Blocked);
14905 runner.state.config.graph.selected_implementer = Some("selected".into());
14906 runner.state.save().unwrap();
14907 assert!(runner.execute().await.is_err());
14908 let disk = RunState::load(&runner.state.id).unwrap();
14909 assert!(disk.driver_exited);
14910 assert_eq!(disk.driver_pid, None);
14911 assert_eq!(disk.status, RunStatus::Blocked);
14912 }
14913
14914 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14915 crate::run::MergeRepair {
14916 question: "repair-question".into(),
14917 head: "old-head".into(),
14918 pr: "https://github.com/o/r/pull/42".into(),
14919 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14920 gate: vec![outcome(Some(0), false)],
14921 gate_head: Some("old-head".into()),
14922 gate_fixes: Vec::new(),
14923 pre_gate: Vec::new(),
14924 phase,
14925 failure: None,
14926 }
14927 }
14928
14929 #[tokio::test]
14930 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14931 crate::run::pin_test_home();
14932 let mut runner = runner_at(RunStatus::Landing);
14933 runner.state.merge_repairs.push(merge_repair_request(
14934 crate::run::MergeRepairPhase::Requested,
14935 ));
14936 runner.state.save().unwrap();
14937 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14938 assert!(
14939 runner
14940 .execute()
14941 .await
14942 .unwrap_err()
14943 .to_string()
14944 .contains("already running")
14945 );
14946 assert_eq!(
14947 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14948 crate::run::MergeRepairPhase::Requested
14949 );
14950 drop(claim);
14951 let mut disk = runner.state.clone();
14952 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14953 disk.save().unwrap();
14954 assert!(
14955 runner
14956 .execute()
14957 .await
14958 .unwrap_err()
14959 .to_string()
14960 .contains("decision changed")
14961 );
14962 assert_eq!(
14963 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14964 crate::run::MergeRepairPhase::Complete
14965 );
14966 }
14967
14968 #[test]
14969 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14970 crate::run::pin_test_home();
14971 let mut runner = runner_at(RunStatus::Landing);
14972 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14973 runner.state.reviews = request.reviews.clone();
14974 runner.state.gate = request.gate.clone();
14975 runner.state.gate_ran = true;
14976 runner.state.land_approval = Some(crate::run::LandApproval {
14977 question: "old".into(),
14978 head: "old-head".into(),
14979 });
14980 runner.state.merge_repairs.push(request);
14981 runner.begin_merge_repair_review(0).unwrap();
14982 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14983 assert!(!runner.state.gate_status().ok());
14984 assert!(runner.state.land_approval.is_none());
14985 let disk = RunState::load(&runner.state.id).unwrap();
14986 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14987 assert!(disk.merge_repairs[0].gate[0].ok());
14988 assert_eq!(
14989 disk.merge_repairs[0].phase,
14990 crate::run::MergeRepairPhase::Reviewing
14991 );
14992 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14993 assert!(disk.reviews.is_empty());
14994 }
14995
14996 #[tokio::test]
14997 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14998 crate::run::pin_test_home();
14999 for phase in [
15000 crate::run::MergeRepairPhase::Fixing,
15001 crate::run::MergeRepairPhase::Requested,
15002 ] {
15003 let mut runner = runner_at(RunStatus::Landing);
15004 runner.state.config.graph.land_rounds = 0;
15005 runner.state.merge_repairs.push(merge_repair_request(phase));
15006 runner.state.save().unwrap();
15007 assert!(!runner.repair_merge_review().await.unwrap());
15008 assert_eq!(runner.state.status, RunStatus::Blocked);
15009 assert_eq!(
15010 runner.state.merge_repairs[0].phase,
15011 crate::run::MergeRepairPhase::Failed
15012 );
15013 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
15014 assert!(
15015 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
15016 "interrupted"
15017 } else {
15018 "budget exhausted"
15019 })
15020 );
15021 assert!(runner.state.merge.is_none());
15022 }
15023 }
15024
15025 #[test]
15029 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15030 crate::run::pin_test_home();
15031 let mut runner = runner_at(RunStatus::Implementing);
15032 let interrupt = Pause::new();
15033 runner.watch_interrupt(interrupt.clone());
15034
15035 interrupt.park_because("task a1b2 asked to run first");
15036
15037 assert!(runner.park_here().expect("park_here"));
15038 assert!(runner.state.parked);
15039 let last = runner.state.events.last().expect("a park event");
15040 assert_eq!(last.node, "park");
15041 assert!(
15042 last.message.contains("task a1b2 asked to run first"),
15043 "expected the interrupt reason in {:?}",
15044 last.message
15045 );
15046 }
15047
15048 #[test]
15051 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15052 crate::run::pin_test_home();
15053 let mut runner = runner_at(RunStatus::Implementing);
15054 runner.state.begin_pass();
15055 let shutdown = Pause::new();
15056 runner.on_pause(shutdown.clone());
15057 let upgrade = ParkCause::Upgrade {
15058 from: "0.121.0".to_owned(),
15059 to: Some("0.121.1".to_owned()),
15060 };
15061 shutdown.park_for(upgrade.clone());
15062 shutdown.park_for(ParkCause::Operator);
15063
15064 assert!(runner.park_here().expect("park_here"));
15065 assert_eq!(
15066 runner.state.passes[0].end,
15067 Some(crate::run::PassEnd::Parked { cause: upgrade })
15068 );
15069 let last = runner.state.events.last().expect("a park event");
15070 assert!(last.message.contains("0.121.0"), "{}", last.message);
15071 }
15072
15073 #[tokio::test]
15074 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15075 crate::run::pin_test_home();
15076 let mut runner = runner_at(RunStatus::Reviewing);
15077 runner.state.save().unwrap();
15078 let mut execution = runner.claim_driver().await.expect("first execution");
15079 assert_eq!(execution.runner.state.passes.len(), 1);
15080 execution
15081 .driver
15082 .finish(&mut execution.runner.state)
15083 .await
15084 .unwrap();
15085 }
15086
15087 #[tokio::test]
15088 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15089 crate::run::pin_test_home();
15090 let mut runner = runner_at(RunStatus::Reviewing);
15091 runner.state.config.agents = vec![conductor()];
15092 runner.state.config.graph.timeout_resource_wait = 0;
15093 runner.state.save().unwrap();
15094 let id = runner.state.id.clone();
15095 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15096 assert!(runner.claim_driver().await.is_err());
15097 drop(owner);
15098 drop(runner);
15099 let mut resumed = Runner::resume(&id).unwrap();
15100 assert_eq!(resumed.state.driver_claimed, Some(false));
15101 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15102 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15103 let mut execution = resumed
15104 .claim_driver()
15105 .await
15106 .expect("retry initial claim after reload");
15107 assert_eq!(execution.runner.state.driver_claimed, Some(true));
15108 assert_eq!(execution.runner.state.passes.len(), 1);
15109 execution
15110 .driver
15111 .finish(&mut execution.runner.state)
15112 .await
15113 .unwrap();
15114 }
15115
15116 #[tokio::test]
15117 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15118 crate::run::pin_test_home();
15119 let mut runner = runner_at(RunStatus::Reviewing);
15120 runner.state.config.agents = vec![conductor()];
15121 runner.state.save().unwrap();
15122 let id = runner.state.id.clone();
15123 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15124 let mut resumed = Runner::resume(&id).unwrap();
15125 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15126 assert!(resumed.claim_driver().await.is_err());
15127 assert!(RunState::load(&id).unwrap().passes.is_empty());
15128 drop(lease);
15129 let mut execution = resumed.claim_driver().await.unwrap();
15130 execution
15131 .driver
15132 .finish(&mut execution.runner.state)
15133 .await
15134 .unwrap();
15135 }
15136
15137 #[tokio::test]
15138 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15139 crate::run::pin_test_home();
15140 let mut runner = runner_at(RunStatus::Reviewing);
15141 let state = &mut runner.state;
15142 state.config.agents = vec![conductor()];
15143 state.driver_claimed = None;
15144 state.driver_pid = Some(std::process::id());
15145 state.driver_started_at = None;
15146 state.begin_pass();
15147 state.save().unwrap();
15148 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15149 std::fs::write(
15150 state.dir().join("driver.json"),
15151 serde_json::to_vec(&serde_json::json!({
15152 "pid": state.driver_pid, "started_at": null,
15153 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15154 }))
15155 .unwrap(),
15156 )
15157 .unwrap();
15158 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15159 drop(lease);
15160 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15161 let mut resumed = Runner::resume(&state.id).unwrap();
15162 assert!(!resumed.state.driver_exited);
15163 let mut execution = resumed
15164 .claim_driver()
15165 .await
15166 .expect("lease proves previous execution ended");
15167 assert_eq!(execution.runner.state.passes.len(), 2);
15168 execution
15169 .driver
15170 .finish(&mut execution.runner.state)
15171 .await
15172 .unwrap();
15173 }
15174
15175 #[tokio::test]
15176 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15177 crate::run::pin_test_home();
15178 let mut runner = runner_at(RunStatus::Reviewing);
15179 let mut json = serde_json::to_value(&runner.state).unwrap();
15180 json.as_object_mut().unwrap().remove("driver_claimed");
15181 runner.state = serde_json::from_value(json).unwrap();
15182 assert_eq!(runner.state.driver_claimed, None);
15183 runner.state.save().unwrap();
15184 let mut execution = runner
15185 .claim_driver()
15186 .await
15187 .expect("the execution lease, not a missing marker, excludes a second driver");
15188 assert_eq!(execution.runner.state.passes.len(), 1);
15189 execution
15190 .driver
15191 .finish(&mut execution.runner.state)
15192 .await
15193 .unwrap();
15194 }
15195
15196 #[tokio::test]
15197 async fn driver_admission_refuses_a_proven_live_driver() {
15198 crate::run::pin_test_home();
15199 let mut runner = runner_at(RunStatus::Reviewing);
15200 let pid = std::process::id();
15201 runner.state.driver_pid = Some(pid);
15202 runner.state.driver_started_at = crate::proc::process_started_at(pid);
15203 assert!(runner.state.driver_started_at.is_some());
15204 runner.state.save().unwrap();
15205 let error = runner
15206 .claim_driver()
15207 .await
15208 .err()
15209 .expect("a live driver must refuse");
15210 assert!(error.to_string().contains("live driver"), "{error:#}");
15211 let saved = RunState::load(&runner.state.id).unwrap();
15212 assert_eq!(saved.driver_pid, Some(pid));
15213 assert_eq!(saved.status, RunStatus::Reviewing);
15214 assert!(saved.passes.is_empty());
15215 }
15216
15217 #[test]
15218 fn parking_resource_halt_persists_the_pass_once() {
15219 crate::run::pin_test_home();
15220 let mut runner = runner_at(RunStatus::Reviewing);
15221 runner.state.begin_pass();
15222 runner.state.resource_block = Some(crate::run::ResourceBlock {
15223 node: "fix".into(),
15224 kind: "shared build cache".into(),
15225 detail: "cancelled".into(),
15226 at: Timestamp::now(),
15227 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15228 });
15229 let cause = ParkCause::Upgrade {
15230 from: "old".into(),
15231 to: Some("new".into()),
15232 };
15233 runner.pause.park_for(cause.clone());
15234 runner.stop_for_resource().unwrap();
15237 assert!(runner.resource_halt().unwrap());
15238 assert!(runner.park_here().unwrap());
15239 let saved = RunState::load(&runner.state.id).unwrap();
15240 assert!(saved.parked);
15241 assert_eq!(saved.status, RunStatus::Reviewing);
15242 assert!(saved.passes[0].ended_at.is_some());
15243 assert_eq!(
15244 saved.passes[0].end,
15245 Some(crate::run::PassEnd::Parked {
15246 cause: cause.clone()
15247 })
15248 );
15249 assert_eq!(
15250 saved.resource_block.unwrap().cause,
15251 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15252 );
15253 assert_eq!(
15254 saved
15255 .events
15256 .iter()
15257 .filter(|event| event.node == "park")
15258 .count(),
15259 1
15260 );
15261 }
15262
15263 #[test]
15271 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15272 crate::run::pin_test_home();
15273 let mut runner = runner_at(RunStatus::Implementing);
15274 let shutdown = Pause::new();
15275 runner.on_pause(shutdown.clone());
15276 let interrupt = Pause::new();
15277 runner.watch_interrupt(interrupt.clone());
15278
15279 assert!(!runner.park_here().expect("park_here"));
15281 assert!(!runner.state.parked);
15282
15283 interrupt.park_because("test");
15285 assert!(!shutdown.parked());
15286 assert!(runner.park_here().expect("park_here"));
15287 }
15288
15289 #[tokio::test]
15303 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15304 crate::run::pin_test_home();
15305 let mut runner = runner_at(RunStatus::Implementing);
15306 let interrupt = Pause::new();
15307 runner.watch_interrupt(interrupt.clone());
15308
15309 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15310 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15311
15312 let node = async move {
15316 started_tx.send(()).expect("send started");
15317 finish_rx.await.expect("recv finish");
15318 "node finished"
15319 };
15320
15321 let interrupter = async move {
15322 started_rx.await.expect("recv started");
15323 interrupt.park_because("higher-priority task waiting");
15325 tokio::task::yield_now().await;
15329 finish_tx.send(()).expect("send finish");
15330 };
15331
15332 let (node_result, ()) = tokio::join!(node, interrupter);
15333 assert_eq!(
15334 node_result, "node finished",
15335 "the in-flight call ran to completion"
15336 );
15337
15338 assert!(runner.park_here().expect("park_here"));
15341 assert!(runner.state.parked);
15342 }
15343
15344 #[test]
15350 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15351 crate::run::pin_test_home();
15352 let mut runner = runner_at(RunStatus::Judging);
15353 runner.state.config.agents = vec![conductor()];
15357 runner.state.candidates = vec![Candidate {
15358 index: 0,
15359 label: 'A',
15360 agent: "alpha".to_owned(),
15361 branch: "magi/x/A".to_owned(),
15362 worktree: PathBuf::from("/nonexistent/worktree"),
15363 summary: "did the thing".to_owned(),
15364 stat: "1 file changed".to_owned(),
15365 files: 1,
15366 commits: 1,
15367 empty: false,
15368 failed: None,
15369 verified_noop: None,
15370 duration_ms: 1234,
15371 folded: false,
15372 }];
15373 let run_id = runner.state.id.clone();
15374
15375 let interrupt = Pause::new();
15376 runner.watch_interrupt(interrupt.clone());
15377 interrupt.park_because("task c3d4 asked to run first");
15378 assert!(runner.park_here().expect("park_here"));
15379
15380 let resumed = Runner::resume(&run_id).expect("resume");
15381 assert_eq!(resumed.state.candidates.len(), 1);
15382 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15383 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15384 assert_eq!(resumed.state.status, runner.state.status);
15385 assert!(
15386 resumed.state.parked,
15387 "still parked until `execute` actually walks the graph again"
15388 );
15389 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15390 }
15391
15392 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15394 let mut q = ask::Question::new(
15395 run.to_owned(),
15396 "implement".to_owned(),
15397 "impl-A".to_owned(),
15398 "Which storage backend should the cache use?".to_owned(),
15399 String::new(),
15400 vec!["SQLite".to_owned(), "Redis".to_owned()],
15401 );
15402 store.put(&mut q).unwrap();
15403 q
15404 }
15405
15406 #[test]
15407 fn a_failed_runs_open_question_is_abandoned() {
15408 ask_test_home();
15409 let store = ask::Questions::open();
15410 let mut runner = runner_at(RunStatus::Failed);
15411 let run = runner.state.id.clone();
15412 let q = ask_open_question(&store, &run);
15413
15414 runner.settle_questions();
15415
15416 let back = store.get(&q.id).unwrap();
15417 assert!(
15418 !back.status.open(),
15419 "the seat that asked died with the run; nobody is left to read an answer"
15420 );
15421 assert!(
15422 back.detail.contains(&run) && back.detail.contains("failed"),
15423 "the reason names what the run became, not just that it is gone: {}",
15424 back.detail
15425 );
15426 }
15427
15428 #[test]
15429 fn a_merged_runs_open_question_is_abandoned_too() {
15430 ask_test_home();
15431 let store = ask::Questions::open();
15432 for status in [RunStatus::Merged, RunStatus::Ready] {
15435 let mut runner = runner_at(status);
15436 let run = runner.state.id.clone();
15437 let q = ask_open_question(&store, &run);
15438
15439 runner.settle_questions();
15440
15441 let back = store.get(&q.id).unwrap();
15442 assert!(
15443 !back.status.open(),
15444 "{status:?} run's question must not outlive the run"
15445 );
15446 }
15447 }
15448
15449 #[test]
15450 fn a_still_resumable_runs_open_question_is_left_alone() {
15451 ask_test_home();
15452 let store = ask::Questions::open();
15453 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15459 let mut runner = runner_at(status);
15460 let run = runner.state.id.clone();
15461 let q = ask_open_question(&store, &run);
15462
15463 runner.settle_questions();
15464
15465 let back = store.get(&q.id).unwrap();
15466 assert!(
15467 back.status.open(),
15468 "{status:?} is still alive; the question must still be waiting"
15469 );
15470 }
15471 }
15472
15473 #[test]
15474 fn settle_questions_never_touches_an_already_answered_question() {
15475 ask_test_home();
15476 let store = ask::Questions::open();
15477 let mut runner = runner_at(RunStatus::Failed);
15478 let run = runner.state.id.clone();
15479 let mut q = ask_open_question(&store, &run);
15480 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15481 .unwrap();
15482 store.put(&mut q).unwrap();
15483
15484 runner.settle_questions();
15489 runner.settle_questions();
15490
15491 let back = store.get(&q.id).unwrap();
15492 assert_eq!(
15493 back.status,
15494 ask::QuestionStatus::Answered,
15495 "a real answer is a decision on record, never overwritten by a sweep"
15496 );
15497 }
15498
15499 #[tokio::test]
15510 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15511 crate::run::pin_test_home();
15512 let tmp = tempfile::tempdir().expect("tempdir");
15513 let repo = tmp.path().join("repo");
15514 std::fs::create_dir_all(&repo).unwrap();
15515 init_repo(&repo);
15516
15517 let mut config = Config::default();
15518 config.graph.worktree_root = Some(tmp.path().join("wt"));
15519
15520 let mut state = RunState::new(
15521 repo.clone(),
15522 "main".to_owned(),
15523 "deadbeef".to_owned(),
15524 "task".to_owned(),
15525 config,
15526 );
15527 let root = state.worktree_root();
15528 let wt_a = root.join("cand-A");
15529 let wt_b = root.join("cand-B");
15530 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15531 .await
15532 .expect("worktree A");
15533 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15534 .await
15535 .expect("worktree B");
15536
15537 state.candidates = vec![
15538 Candidate {
15539 index: 0,
15540 label: 'A',
15541 agent: "alpha".to_owned(),
15542 branch: "magi/x/A".to_owned(),
15543 worktree: wt_a.clone(),
15544 summary: String::new(),
15545 stat: String::new(),
15546 files: 0,
15547 commits: 0,
15548 empty: false,
15549 failed: None,
15550 verified_noop: None,
15551 duration_ms: 0,
15552 folded: false,
15553 },
15554 Candidate {
15555 index: 1,
15556 label: 'B',
15557 agent: "beta".to_owned(),
15558 branch: "magi/x/B".to_owned(),
15559 worktree: wt_b.clone(),
15560 summary: String::new(),
15561 stat: String::new(),
15562 files: 0,
15563 commits: 0,
15564 empty: false,
15565 failed: None,
15566 verified_noop: None,
15567 duration_ms: 0,
15568 folded: false,
15569 },
15570 ];
15571 state.tally = Some(Tally {
15572 first_choice: BTreeMap::from([('A', 1)]),
15573 borda: BTreeMap::new(),
15574 winner: 'A',
15575 rankings: 1,
15576 unanimous_initial: true,
15577 deliberated: false,
15578 changed_votes: 0,
15579 unanimous_final: true,
15580 tie_break: None,
15581 judges: 1,
15582 present: 1,
15583 quorum: 1,
15584 met_quorum: true,
15585 uncontested: None,
15586 });
15587 state.status = RunStatus::Ready;
15588
15589 fold_run(&mut state, false, &crate::run::home())
15590 .await
15591 .expect("fold_run");
15592
15593 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15594 assert!(
15595 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15596 "the unmerged winner's branch survives"
15597 );
15598 assert!(
15599 !state.candidates[0].folded,
15600 "the winner is not marked folded"
15601 );
15602
15603 assert!(!wt_b.exists(), "the loser's worktree is removed");
15604 assert!(
15605 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15606 "the loser's branch is removed"
15607 );
15608 assert!(state.candidates[1].folded, "the loser is marked folded");
15609 }
15610
15611 #[tokio::test]
15612 #[cfg(unix)]
15613 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15614 use std::os::fd::AsRawFd as _;
15615 use std::time::Duration;
15616
15617 let tmp = tempfile::tempdir().unwrap();
15618 let repo = tmp.path().join("repo");
15619 std::fs::create_dir_all(&repo).unwrap();
15620 init_repo(&repo);
15621 let common = repo.join(".git");
15622 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15623 let home = tmp.path().join("home");
15624 let mut config = Config::default();
15625 config.coordination = crate::config::test_coordination();
15626 config.graph.worktree_root = Some(tmp.path().join("wt"));
15627 config.graph.timeout_resource_wait = 1;
15628 let mut state = RunState::new(
15629 repo.clone(),
15630 "main".into(),
15631 "deadbeef".into(),
15632 "task".into(),
15633 config,
15634 );
15635 let root = state.worktree_root();
15636 let candidate = root.join("cand-A");
15637 let reviewer = root.join("review-1");
15638 let branch = state.branch_for('A');
15639 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15640 .await
15641 .unwrap();
15642 git::worktree_add_detached(&repo, &reviewer, "main")
15643 .await
15644 .unwrap();
15645 git::acquire_worktree_config(&repo).await.unwrap();
15646 state.enabled_worktree_config = true;
15647 state.status = RunStatus::Ready;
15648 state.candidates.push(Candidate {
15649 index: 0,
15650 label: 'A',
15651 agent: "implement".into(),
15652 branch: branch.clone(),
15653 worktree: candidate.clone(),
15654 summary: String::new(),
15655 stat: String::new(),
15656 files: 0,
15657 commits: 0,
15658 empty: false,
15659 failed: None,
15660 verified_noop: None,
15661 duration_ms: 0,
15662 folded: false,
15663 });
15664 state.save_under(&home).unwrap();
15665 let config_before = std::fs::read(common.join("config")).unwrap();
15666 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15667 .await
15668 .unwrap();
15669 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15672 assert_eq!(
15673 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15674 0
15675 );
15676 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15677 assert!(
15678 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15679 .await
15680 .is_err()
15681 );
15682 assert!(candidate.exists());
15683 assert!(reviewer.exists());
15684 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15685 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15686 assert_eq!(
15687 git::git(&repo, &["worktree", "list", "--porcelain"])
15688 .await
15689 .unwrap(),
15690 worktrees_before
15691 );
15692 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15693 .await
15694 .unwrap()
15695 .unwrap_err();
15696 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15697 drop(fold);
15698 assert!(candidate.exists() && reviewer.exists());
15699 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15700 assert_eq!(
15701 state.resource_block.as_ref().unwrap().kind,
15702 "repository coordination"
15703 );
15704 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15705 drop(external);
15706 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15707 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15708 .await
15709 .unwrap()
15710 .unwrap();
15711 drop(fold);
15712 assert!(removed.contains(&branch));
15713 assert!(!root.exists());
15714 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15715 assert!(state.candidates[0].folded);
15716 assert!(!state.enabled_worktree_config);
15717 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15718 let enabled = git::git_raw(
15719 &repo,
15720 &["config", "--local", "--get", "extensions.worktreeConfig"],
15721 )
15722 .await
15723 .unwrap();
15724 assert!(!enabled.ok(), "the last config reference was released");
15725 }
15726
15727 #[tokio::test]
15730 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15731 let tmp = tempfile::tempdir().expect("tempdir");
15732 let repo = tmp.path().join("repo");
15733 std::fs::create_dir_all(&repo).unwrap();
15734 init_repo(&repo);
15735 let home = tmp.path().join("home");
15736
15737 let mut config = Config::default();
15738 config.graph.worktree_root = Some(tmp.path().join("wt"));
15739 let mut state = RunState::new(
15740 repo.clone(),
15741 "main".to_owned(),
15742 "deadbeef".to_owned(),
15743 "task".to_owned(),
15744 config,
15745 );
15746 git::git(&repo, &["branch", "magi/x/A", "main"])
15748 .await
15749 .expect("branch");
15750 state.candidates = vec![Candidate {
15751 index: 0,
15752 label: 'A',
15753 agent: "alpha".to_owned(),
15754 branch: "magi/x/A".to_owned(),
15755 worktree: state.worktree_root().join("cand-A"),
15756 summary: String::new(),
15757 stat: String::new(),
15758 files: 0,
15759 commits: 0,
15760 empty: false,
15761 failed: None,
15762 verified_noop: None,
15763 duration_ms: 0,
15764 folded: true,
15765 }];
15766 state.released_to = Some("20260901-000000-new1".to_owned());
15767 state.released_branches = vec!["magi/x/A".to_owned()];
15768
15769 fold_run(&mut state, true, &home).await.expect("fold_run");
15770 state.status = RunStatus::Merged;
15771 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15772 state.merge = Some(crate::run::MergeOutcome {
15773 mode: MergeMode::Pr,
15774 ok: true,
15775 detail: "confirmed merged".into(),
15776 empty: false,
15777 });
15778 fold_run(&mut state, true, &home)
15779 .await
15780 .expect("merged fold");
15781
15782 assert!(
15783 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15784 "the handed-over branch survives a fold"
15785 );
15786 }
15787
15788 #[tokio::test]
15792 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15793 let tmp = tempfile::tempdir().expect("tempdir");
15794 let repo = tmp.path().join("repo");
15795 std::fs::create_dir_all(&repo).unwrap();
15796 init_repo(&repo);
15797 let run = |args: &[&str]| {
15798 let out = std::process::Command::new("git")
15799 .quiet()
15800 .args(args)
15801 .current_dir(&repo)
15802 .output()
15803 .expect("spawn git");
15804 assert!(out.status.success(), "git {args:?}");
15805 };
15806 run(&["branch", "magi/x/A"]);
15807 run(&["checkout", "-q", "-b", "magi/x/B"]);
15808 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15809 run(&["add", "-A"]);
15810 run(&["commit", "-q", "-m", "work"]);
15811 run(&["checkout", "-q", "main"]);
15812
15813 let probe = RunState::new(
15817 repo.clone(),
15818 "main".to_owned(),
15819 "deadbeef".to_owned(),
15820 "task".to_owned(),
15821 Config::default(),
15822 );
15823 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15824 let bare = tmp.path().join("origin.git");
15825 let out = std::process::Command::new("git")
15826 .quiet()
15827 .args(["init", "-q", "--bare"])
15828 .arg(&bare)
15829 .output()
15830 .expect("spawn git");
15831 assert!(out.status.success(), "git init --bare");
15832 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15833 run(&["push", "-q", "origin", "main"]);
15834
15835 let mut state = RunState::new(
15836 repo.clone(),
15837 "main".to_owned(),
15838 "deadbeef".to_owned(),
15839 "task".to_owned(),
15840 Config::default(),
15841 );
15842 state.seeds = vec![refs::Seed {
15843 token: "magi/27b2/A".to_owned(),
15844 kind: refs::SeedKind::Unresolved,
15845 sha: String::new(),
15846 branch: true,
15847 detail: "no branch or commit named magi/27b2/A".to_owned(),
15848 }];
15849
15850 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15851 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15852 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15853 let detail = empty_candidate_detail(&state, "main");
15854 assert!(detail.starts_with("empty candidate"), "{detail}");
15855 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15856 assert!(detail.contains("--start-from"), "{detail}");
15857 }
15858
15859 #[tokio::test]
15868 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15869 let tmp = tempfile::tempdir().expect("tempdir");
15870 let repo = tmp.path().join("repo");
15871 std::fs::create_dir_all(&repo).unwrap();
15872 init_repo(&repo);
15873
15874 let mut config = Config::default();
15875 config.merge.mode = MergeMode::Local;
15876
15877 let mut state = RunState::new(
15878 repo.clone(),
15879 "main".to_owned(),
15880 "deadbeef".to_owned(),
15881 "task".to_owned(),
15882 config,
15883 );
15884 state.candidates = vec![Candidate {
15885 index: 0,
15886 label: 'A',
15887 agent: "alpha".to_owned(),
15888 branch: "does-not-exist".to_owned(),
15889 worktree: repo.clone(),
15890 summary: String::new(),
15891 stat: String::new(),
15892 files: 0,
15893 commits: 0,
15894 empty: false,
15895 failed: None,
15896 verified_noop: None,
15897 duration_ms: 0,
15898 folded: false,
15899 }];
15900 state.tally = Some(Tally {
15901 first_choice: BTreeMap::from([('A', 1)]),
15902 borda: BTreeMap::new(),
15903 winner: 'A',
15904 rankings: 1,
15905 unanimous_initial: true,
15906 deliberated: false,
15907 changed_votes: 0,
15908 unanimous_final: true,
15909 tie_break: None,
15910 judges: 0,
15911 present: 0,
15912 quorum: 0,
15913 met_quorum: true,
15914 uncontested: Some("only candidate A produced a change".to_owned()),
15915 });
15916 state.reviews = vec![ReviewRound {
15917 round: 1,
15918 head: "deadbeef".to_owned(),
15919 verified_head: None,
15920 verified_at: None,
15921 reviews: Vec::new(),
15922 e2e: Vec::new(),
15923 fix: None,
15924 blocking: 0,
15925 answered: 0,
15926 expected: 0,
15927 clean: true,
15928 verify_retried: false,
15929 e2e_deferred: false,
15930 e2e_defer_reason: None,
15931 progressed: false,
15932 vote_split: false,
15933 reconsideration: Vec::new(),
15934 verdict: None,
15935 }];
15936 state.gate = vec![CommandOutcome {
15937 command: "test".to_owned(),
15938 code: Some(0),
15939 output_tail: String::new(),
15940 duration_ms: 0,
15941 resource_blocked: false,
15942 }];
15943 state.gate_ran = true;
15944 state.status = RunStatus::Ready;
15949 state.merge = Some(MergeOutcome {
15950 mode: MergeMode::Local,
15951 ok: false,
15952 detail: "already concluded".to_owned(),
15953 empty: false,
15954 });
15955
15956 let mut runner = Runner {
15957 state,
15958 roles: ResolvedRoles {
15959 implementers: Vec::new(),
15960 judges: Vec::new(),
15961 reviewers: Vec::new(),
15962 fixer: None,
15963 conductor: conductor(),
15964 implementer_roster: Vec::new(),
15965 judge_roster: Vec::new(),
15966 reviewer_roster: Vec::new(),
15967 },
15968 sem: Arc::new(Semaphore::new(1)),
15969 pause: Pause::new(),
15970 interrupt: Pause::new(),
15971 };
15972
15973 runner.merge().await.expect("merge");
15974
15975 assert_eq!(
15976 runner.state.status,
15977 RunStatus::Ready,
15978 "a concluded run's status must not change on reentry"
15979 );
15980 assert_eq!(
15981 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15982 Some("already concluded"),
15983 "merge must not run again once the node already recorded an outcome"
15984 );
15985 }
15986
15987 #[tokio::test]
15996 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15997 let tmp = tempfile::tempdir().expect("tempdir");
15998 let repo = tmp.path().join("repo");
15999 std::fs::create_dir_all(&repo).unwrap();
16000 init_repo(&repo);
16001
16002 let mut config = Config::default();
16003 config.merge.mode = MergeMode::Local;
16004
16005 let mut state = RunState::new(
16006 repo.clone(),
16007 "main".to_owned(),
16008 "deadbeef".to_owned(),
16009 "task".to_owned(),
16010 config,
16011 );
16012 state.candidates = vec![Candidate {
16013 index: 0,
16014 label: 'A',
16015 agent: "alpha".to_owned(),
16016 branch: "does-not-exist".to_owned(),
16017 worktree: repo.clone(),
16018 summary: String::new(),
16019 stat: String::new(),
16020 files: 0,
16021 commits: 0,
16022 empty: false,
16023 failed: None,
16024 verified_noop: None,
16025 duration_ms: 0,
16026 folded: false,
16027 }];
16028 state.tally = Some(Tally {
16029 first_choice: BTreeMap::from([('A', 1)]),
16030 borda: BTreeMap::new(),
16031 winner: 'A',
16032 rankings: 1,
16033 unanimous_initial: true,
16034 deliberated: false,
16035 changed_votes: 0,
16036 unanimous_final: true,
16037 tie_break: None,
16038 judges: 0,
16039 present: 0,
16040 quorum: 0,
16041 met_quorum: true,
16042 uncontested: Some("only candidate A produced a change".to_owned()),
16043 });
16044 state.reviews = vec![ReviewRound {
16045 round: 1,
16046 head: "deadbeef".to_owned(),
16047 verified_head: None,
16048 verified_at: None,
16049 reviews: Vec::new(),
16050 e2e: Vec::new(),
16051 fix: None,
16052 blocking: 0,
16053 answered: 0,
16054 expected: 0,
16055 clean: true,
16056 verify_retried: false,
16057 e2e_deferred: false,
16058 e2e_defer_reason: None,
16059 progressed: false,
16060 vote_split: false,
16061 reconsideration: Vec::new(),
16062 verdict: None,
16063 }];
16064 state.gate = Vec::new();
16066 state.gate_ran = false;
16067 state.status = RunStatus::Gating;
16068
16069 let mut runner = Runner {
16070 state,
16071 roles: ResolvedRoles {
16072 implementers: Vec::new(),
16073 judges: Vec::new(),
16074 reviewers: Vec::new(),
16075 fixer: None,
16076 conductor: conductor(),
16077 implementer_roster: Vec::new(),
16078 judge_roster: Vec::new(),
16079 reviewer_roster: Vec::new(),
16080 },
16081 sem: Arc::new(Semaphore::new(1)),
16082 pause: Pause::new(),
16083 interrupt: Pause::new(),
16084 };
16085
16086 runner.merge().await.expect("merge");
16087
16088 assert!(
16089 runner.state.merge.is_none(),
16090 "an empty gate must never be read as a passing one: {:?}",
16091 runner.state.merge
16092 );
16093 }
16094
16095 #[tokio::test]
16102 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16103 let tmp = tempfile::tempdir().expect("tempdir");
16104 let repo = tmp.path().join("repo");
16105 std::fs::create_dir_all(&repo).unwrap();
16106 init_repo(&repo);
16107
16108 let config = Config::default();
16110
16111 let mut state = RunState::new(
16112 repo.clone(),
16113 "main".to_owned(),
16114 "deadbeef".to_owned(),
16115 "task".to_owned(),
16116 config,
16117 );
16118 state.candidates = vec![Candidate {
16119 index: 0,
16120 label: 'A',
16121 agent: "alpha".to_owned(),
16122 branch: "does-not-exist".to_owned(),
16123 worktree: repo.clone(),
16124 summary: String::new(),
16125 stat: String::new(),
16126 files: 0,
16127 commits: 0,
16128 empty: false,
16129 failed: None,
16130 verified_noop: None,
16131 duration_ms: 0,
16132 folded: false,
16133 }];
16134 state.tally = Some(Tally {
16135 first_choice: BTreeMap::from([('A', 1)]),
16136 borda: BTreeMap::new(),
16137 winner: 'A',
16138 rankings: 1,
16139 unanimous_initial: true,
16140 deliberated: false,
16141 changed_votes: 0,
16142 unanimous_final: true,
16143 tie_break: None,
16144 judges: 0,
16145 present: 0,
16146 quorum: 0,
16147 met_quorum: true,
16148 uncontested: Some("only candidate A produced a change".to_owned()),
16149 });
16150 state.reviews = vec![ReviewRound {
16151 round: 1,
16152 head: "deadbeef".to_owned(),
16153 verified_head: None,
16154 verified_at: None,
16155 reviews: Vec::new(),
16156 e2e: Vec::new(),
16157 fix: None,
16158 blocking: 0,
16159 answered: 0,
16160 expected: 0,
16161 clean: true,
16162 verify_retried: false,
16163 e2e_deferred: false,
16164 e2e_defer_reason: None,
16165 progressed: false,
16166 vote_split: false,
16167 reconsideration: Vec::new(),
16168 verdict: None,
16169 }];
16170
16171 let mut runner = Runner {
16172 state,
16173 roles: ResolvedRoles {
16174 implementers: Vec::new(),
16175 judges: Vec::new(),
16176 reviewers: Vec::new(),
16177 fixer: None,
16178 conductor: conductor(),
16179 implementer_roster: Vec::new(),
16180 judge_roster: Vec::new(),
16181 reviewer_roster: Vec::new(),
16182 },
16183 sem: Arc::new(Semaphore::new(1)),
16184 pause: Pause::new(),
16185 interrupt: Pause::new(),
16186 };
16187
16188 runner.gate().await.expect("gate");
16189 assert!(
16190 runner.state.gate_ran,
16191 "zero configured commands is still a real attempt, not an unrun gate"
16192 );
16193 assert!(runner.state.gate.is_empty());
16194 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16195 assert_ne!(
16196 runner.state.status,
16197 RunStatus::Blocked,
16198 "a gate with nothing to check must not read as failed"
16199 );
16200
16201 runner.merge().await.expect("merge");
16202 assert_eq!(
16203 runner.state.status,
16204 RunStatus::Ready,
16205 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16206 );
16207 }
16208
16209 #[tokio::test]
16220 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16221 crate::run::pin_test_home();
16222 let home = crate::run::home();
16223
16224 let tmp = tempfile::tempdir().expect("tempdir");
16225 let repo = tmp.path().join("repo");
16226 std::fs::create_dir_all(&repo).unwrap();
16227 init_repo(&repo);
16228 let cache_dir = tmp.path().join("target");
16231
16232 let mut config = Config::default();
16233 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16234 config.graph.timeout_verify = Some(2);
16237 config.graph.timeout_resource_wait = 2;
16238
16239 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16240 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16241 .expect("no io error acquiring directly")
16242 {
16243 crate::cache::AcquireOutcome::Acquired(g) => g,
16244 crate::cache::AcquireOutcome::Busy(b) => {
16245 panic!("expected the direct acquire to win the lease first: {b:?}")
16246 }
16247 };
16248
16249 let mut state = RunState::new(
16250 repo.clone(),
16251 "main".to_owned(),
16252 "deadbeef".to_owned(),
16253 "task".to_owned(),
16254 config,
16255 );
16256 state.candidates = vec![Candidate {
16257 index: 0,
16258 label: 'A',
16259 agent: "alpha".to_owned(),
16260 branch: "does-not-exist".to_owned(),
16261 worktree: repo.clone(),
16262 summary: String::new(),
16263 stat: String::new(),
16264 files: 0,
16265 commits: 0,
16266 empty: false,
16267 failed: None,
16268 verified_noop: None,
16269 duration_ms: 0,
16270 folded: false,
16271 }];
16272 state.tally = Some(Tally {
16273 first_choice: BTreeMap::from([('A', 1)]),
16274 borda: BTreeMap::new(),
16275 winner: 'A',
16276 rankings: 1,
16277 unanimous_initial: true,
16278 deliberated: false,
16279 changed_votes: 0,
16280 unanimous_final: true,
16281 tie_break: None,
16282 judges: 0,
16283 present: 0,
16284 quorum: 0,
16285 met_quorum: true,
16286 uncontested: Some("only candidate A produced a change".to_owned()),
16287 });
16288 state.reviews = vec![ReviewRound {
16289 round: 1,
16290 head: "deadbeef".to_owned(),
16291 verified_head: None,
16292 verified_at: None,
16293 reviews: Vec::new(),
16294 e2e: Vec::new(),
16295 fix: None,
16296 blocking: 0,
16297 answered: 0,
16298 expected: 0,
16299 clean: true,
16300 verify_retried: false,
16301 e2e_deferred: false,
16302 e2e_defer_reason: None,
16303 progressed: false,
16304 vote_split: false,
16305 reconsideration: Vec::new(),
16306 verdict: None,
16307 }];
16308
16309 let mut runner = Runner {
16310 state,
16311 roles: ResolvedRoles {
16312 implementers: Vec::new(),
16313 judges: Vec::new(),
16314 reviewers: Vec::new(),
16315 fixer: None,
16316 conductor: conductor(),
16317 implementer_roster: Vec::new(),
16318 judge_roster: Vec::new(),
16319 reviewer_roster: Vec::new(),
16320 },
16321 sem: Arc::new(Semaphore::new(1)),
16322 pause: Pause::new(),
16323 interrupt: Pause::new(),
16324 };
16325
16326 let started = std::time::Instant::now();
16327 runner.gate().await.expect("gate");
16328 assert!(
16329 started.elapsed() < Duration::from_secs(1),
16330 "a gate with nothing to run must never wait on a lease it never needed"
16331 );
16332 assert!(
16333 runner.state.gate_ran,
16334 "zero commands is still a real, immediate attempt"
16335 );
16336 assert!(runner.state.gate.is_empty());
16337 assert_ne!(
16338 runner.state.status,
16339 RunStatus::Blocked,
16340 "must not read as resource-blocked on a lease it never asked for"
16341 );
16342 }
16343
16344 #[cfg(unix)]
16347 #[tokio::test]
16348 async fn verification_is_standalone_inside_a_live_worker() {
16349 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
16350 if std::env::var_os(CHILD).is_none() {
16351 let home = tempfile::tempdir().unwrap();
16352 let generation = crate::worker::pin_current(home.path()).unwrap();
16353 let output = std::process::Command::new(std::env::current_exe().unwrap())
16354 .args([
16355 "--exact",
16356 "graph::tests::verification_is_standalone_inside_a_live_worker",
16357 "--nocapture",
16358 ])
16359 .env(CHILD, "1")
16360 .env("MAGI_HOME", home.path())
16361 .env("MAGI_WORKER_HOME", home.path())
16362 .env("MAGI_WORKER_ID", "abcdef")
16363 .env("MAGI_WORKER_GENERATION", &generation.id)
16364 .env("MAGI_WORKER_EXE", &generation.executable)
16365 .env("MAGI_RUN", "verification-parent")
16366 .env("MAGI_NODE", "gate")
16367 .env("MAGI_SEAT", "worker-fixture")
16368 .env("MAGI_PROCESS_OWNER", "fixture-owner")
16369 .env("MAGI_NO_AUTOUPDATE", "0")
16370 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
16371 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
16372 .env(
16373 "VERIFY_EXPECT_TOOLCHAIN",
16374 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
16375 )
16376 .env(
16377 "VERIFY_EXPECT_CACHE",
16378 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
16379 )
16380 .output()
16381 .unwrap();
16382 assert!(
16383 output.status.success()
16384 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16385 "{}\n{}",
16386 String::from_utf8_lossy(&output.stdout),
16387 String::from_utf8_lossy(&output.stderr)
16388 );
16389 return;
16390 }
16391 let home = crate::run::home();
16392 let directory = home.join("workers/jobs/abcdef");
16393 std::fs::create_dir_all(&directory).unwrap();
16394 let job = crate::worker::Job {
16395 id: "abcdef".into(),
16396 sequence: 1,
16397 kind: "task".into(),
16398 key: "task:verification".into(),
16399 payload: serde_json::json!({}),
16400 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
16401 version: Some(env!("CARGO_PKG_VERSION").into()),
16402 status: crate::worker::Status::Running,
16403 result: None,
16404 pid: Some(std::process::id()),
16405 created_at: 0,
16406 finished_at: None,
16407 };
16408 std::fs::write(
16409 directory.join("job.json"),
16410 serde_json::to_vec(&job).unwrap(),
16411 )
16412 .unwrap();
16413 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
16414 let assert_worker = || {
16415 assert_eq!(
16416 crate::worker::current_job(&home).unwrap().unwrap().id,
16417 job.id
16418 );
16419 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
16420 assert_eq!(
16421 crate::worker::helper_executable(&home).unwrap(),
16422 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
16423 );
16424 assert!(
16425 crate::updater::Checker::new(&crate::config::Update {
16426 mode: crate::config::UpdateMode::Notify,
16427 interval: None,
16428 auto_restart: false,
16429 })
16430 .is_none(),
16431 "the driver remains pinned, even while tests are standalone"
16432 );
16433 };
16434 assert_worker();
16435 let repo = home.join("repo");
16436 std::fs::create_dir_all(&repo).unwrap();
16437 init_repo(&repo);
16438 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
16441 test -z "${MAGI_WORKER_GENERATION+x}" &&
16442 test -z "${MAGI_WORKER_HOME+x}" &&
16443 test -z "${MAGI_WORKER_EXE+x}" &&
16444 test "$MAGI_HOME/repo" -ef . &&
16445 test "$MAGI_RUN" = verification-parent &&
16446 test "$MAGI_NODE" = gate &&
16447 test "$MAGI_SEAT" = worker-fixture &&
16448 test -n "$MAGI_PROCESS_OWNER" &&
16449 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
16450 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
16451 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
16452 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
16453 assert!(
16455 !std::process::Command::new("sh")
16456 .args(["-c", probe])
16457 .current_dir(&repo)
16458 .status()
16459 .unwrap()
16460 .success()
16461 );
16462 let unrelated_home = tempfile::tempdir().unwrap();
16463 assert!(
16464 crate::worker::current_job(unrelated_home.path()).is_err(),
16465 "an inherited worker id cannot be resolved in a unit fixture's home"
16466 );
16467 let mut runner = runner_at(RunStatus::Gating);
16468 runner.state.repo = repo.clone();
16469 runner.state.config.disk.min_free_bytes = 0;
16470 runner.state.config.coordination = crate::config::test_coordination();
16471 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
16472 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
16473 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
16474 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
16475 let winner = Candidate {
16476 index: 0,
16477 label: 'A',
16478 agent: "fixture".into(),
16479 branch: "main".into(),
16480 worktree: repo.clone(),
16481 summary: String::new(),
16482 stat: String::new(),
16483 files: 0,
16484 commits: 0,
16485 empty: false,
16486 failed: None,
16487 verified_noop: None,
16488 duration_ms: 0,
16489 folded: false,
16490 };
16491 let (e2e, retried, pids) = run_e2e_with_retry(
16492 &mut runner.state,
16493 &["sh".into(), "-c".into()],
16494 &[format!("{probe} && printf initial")],
16495 &repo,
16496 Duration::from_secs(10),
16497 "initial",
16498 )
16499 .await;
16500 assert!(!retried && pids.is_empty());
16501 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
16502 assert_eq!(e2e[0].output_tail, "initial");
16503 let gate = runner.run_gate(&winner).await.unwrap();
16504 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
16505 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
16508 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
16509 passed: false,
16510 outcomes: Vec::new(),
16511 });
16512 runner.state.save().unwrap();
16513 let mut resumed = RunState::load(&runner.state.id).unwrap();
16514 let wait = crate::repo_lock::WaitPolicy::default();
16515 assert!(matches!(
16516 check_gate_repair_e2e(&mut resumed, &winner, &wait)
16517 .await
16518 .unwrap(),
16519 GateFix::Retry
16520 ));
16521 let saved = RunState::load(&resumed.id).unwrap();
16522 let repair = saved.gate_repair_e2e.unwrap();
16523 assert!(repair.passed);
16524 assert_eq!(repair.outcomes.len(), 1);
16525 assert!(repair.outcomes[0].ok());
16526 let landing = run_landing_gate(&mut resumed, &winner, &wait)
16527 .await
16528 .unwrap();
16529 assert_eq!(landing.len(), 1);
16530 assert!(landing[0].ok(), "{landing:?}");
16531 assert_eq!(
16532 std::fs::read_to_string(repo.join("phases")).unwrap(),
16533 "pre\ngate\ne2e\npre\ngate\n"
16534 );
16535 assert_worker();
16536 }
16537
16538 #[cfg(unix)]
16539 #[tokio::test]
16540 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16541 crate::run::pin_test_home();
16542 let tmp = tempfile::tempdir().unwrap();
16543 let mut config = Config::default();
16544 config.disk.min_free_bytes = 0;
16545 let mut state = RunState::new(
16546 tmp.path().to_owned(),
16547 "main".into(),
16548 "base".into(),
16549 "verification evidence".into(),
16550 config,
16551 );
16552 let (outcomes, pids) = run_commands(
16553 &mut state,
16554 "gate",
16555 "gate",
16556 1,
16557 &["sh".into(), "-c".into()],
16558 &[
16559 "printf before-timeout; exec sleep 30".into(),
16560 "kill -TERM $$".into(),
16561 "exit 7".into(),
16562 "printf success".into(),
16563 ],
16564 tmp.path(),
16565 Duration::from_secs(1),
16566 )
16567 .await;
16568 wait_for_timed_out_children_to_die(&pids).await;
16569 assert_eq!(
16570 pids.len(),
16571 1,
16572 "unleased timeout retains its death-check PID"
16573 );
16574 assert_eq!(outcomes.len(), 4);
16575 assert_eq!(outcomes[0].code, None);
16576 assert!(outcomes[0].output_tail.contains("before-timeout"));
16577 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16578 assert_eq!(outcomes[1].code, None);
16579 assert!(outcomes[1].output_tail.contains("command terminated:"));
16580 assert!(!outcomes[1].output_tail.contains("timed out"));
16581 assert_eq!(outcomes[2].code, Some(7));
16582 assert!(!outcomes[2].output_tail.contains("timed out"));
16583 assert_eq!(outcomes[3].code, Some(0));
16584 assert_eq!(outcomes[3].output_tail, "success");
16585 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16586 }
16587
16588 #[tokio::test]
16598 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16599 crate::run::pin_test_home();
16600
16601 let tmp = tempfile::tempdir().expect("tempdir");
16602 let repo = tmp.path().join("repo");
16603 std::fs::create_dir_all(&repo).unwrap();
16604 init_repo(&repo);
16605
16606 let mut config = Config::default();
16607 config.verify.gate = vec![
16608 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16609 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16610 .to_owned(),
16611 ];
16612
16613 let mut state = RunState::new(
16614 repo.clone(),
16615 "main".to_owned(),
16616 "deadbeef".to_owned(),
16617 "task".to_owned(),
16618 config,
16619 );
16620 let run_id = state.id.clone();
16621 state.candidates = vec![Candidate {
16622 index: 0,
16623 label: 'A',
16624 agent: "alpha".to_owned(),
16625 branch: "does-not-exist".to_owned(),
16626 worktree: repo.clone(),
16627 summary: String::new(),
16628 stat: String::new(),
16629 files: 0,
16630 commits: 0,
16631 empty: false,
16632 failed: None,
16633 verified_noop: None,
16634 duration_ms: 0,
16635 folded: false,
16636 }];
16637 state.tally = Some(Tally {
16638 first_choice: BTreeMap::from([('A', 1)]),
16639 borda: BTreeMap::new(),
16640 winner: 'A',
16641 rankings: 1,
16642 unanimous_initial: true,
16643 deliberated: false,
16644 changed_votes: 0,
16645 unanimous_final: true,
16646 tie_break: None,
16647 judges: 0,
16648 present: 0,
16649 quorum: 0,
16650 met_quorum: true,
16651 uncontested: Some("only candidate A produced a change".to_owned()),
16652 });
16653 state.reviews = vec![ReviewRound {
16654 round: 1,
16655 head: "deadbeef".to_owned(),
16656 verified_head: None,
16657 verified_at: None,
16658 reviews: Vec::new(),
16659 e2e: Vec::new(),
16660 fix: None,
16661 blocking: 0,
16662 answered: 0,
16663 expected: 0,
16664 clean: true,
16665 verify_retried: false,
16666 e2e_deferred: false,
16667 e2e_defer_reason: None,
16668 progressed: false,
16669 vote_split: false,
16670 reconsideration: Vec::new(),
16671 verdict: None,
16672 }];
16673
16674 let mut runner = Runner {
16675 state,
16676 roles: ResolvedRoles {
16677 implementers: Vec::new(),
16678 judges: Vec::new(),
16679 reviewers: Vec::new(),
16680 fixer: None,
16681 conductor: conductor(),
16682 implementer_roster: Vec::new(),
16683 judge_roster: Vec::new(),
16684 reviewer_roster: Vec::new(),
16685 },
16686 sem: Arc::new(Semaphore::new(1)),
16687 pause: Pause::new(),
16688 interrupt: Pause::new(),
16689 };
16690
16691 let started_marker = repo.join("started.marker");
16692 let release_marker = repo.join("release.marker");
16693 let poller = tokio::spawn(async move {
16694 for _ in 0..100 {
16699 if started_marker.exists()
16700 && let Ok(s) = crate::run::RunState::load(&run_id)
16701 && let Some(a) = s.active.get("gate")
16702 {
16703 std::fs::write(&release_marker, b"go").expect("release marker");
16704 return Some(a.clone());
16705 }
16706 tokio::time::sleep(Duration::from_millis(50)).await;
16707 }
16708 None
16709 });
16710
16711 runner.gate().await.expect("gate");
16712 let captured = poller.await.expect("poller task");
16713 let captured = captured.expect(
16714 "the poller never saw a `gate` task entry in run.json while the command was \
16715 still blocked on its own release marker",
16716 );
16717
16718 assert_eq!(captured.task.as_deref(), Some("gate"));
16719 assert_eq!(captured.node, "gate");
16720 assert_eq!(captured.index, Some(1));
16721 assert_eq!(captured.total, Some(1));
16722 assert!(
16723 captured
16724 .command
16725 .as_deref()
16726 .is_some_and(|c| c.contains("started.marker")),
16727 "{captured:?}"
16728 );
16729
16730 assert!(
16731 runner.state.active.is_empty(),
16732 "the entry must be cleared once the command actually finished: {:?}",
16733 runner.state.active
16734 );
16735 assert!(runner.state.gate_ran);
16736 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16737 }
16738
16739 #[tokio::test]
16742 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16743 use crate::verdict::{Finding, ReviewVote, Severity};
16744 crate::run::pin_test_home();
16745 let tmp = tempfile::tempdir().expect("tempdir");
16746 let repo = tmp.path().join("repo");
16747 std::fs::create_dir_all(&repo).unwrap();
16748 init_repo(&repo);
16749
16750 for (severity, vote, expect) in [
16751 (Severity::Major, ReviewVote::Reject, true),
16752 (Severity::Minor, ReviewVote::Reject, false),
16753 (Severity::Major, ReviewVote::Approve, false),
16754 ] {
16755 let mut round = review_round(false, 1, 1, 1, false, true);
16756 round.reviews = vec![ReviewRecord {
16757 reviewer: 1,
16758 agent: "alpha".to_owned(),
16759 summary: String::new(),
16760 findings: vec![Finding {
16761 id: "R1-1-1".to_owned(),
16762 severity,
16763 file: None,
16764 line: None,
16765 title: "t".to_owned(),
16766 detail: String::new(),
16767 }],
16768 vote: Some(vote),
16769 failed: None,
16770 duration_ms: 0,
16771 attempts: 0,
16772 }];
16773 let mut state = RunState::new(
16774 repo.clone(),
16775 "main".to_owned(),
16776 "deadbeef".to_owned(),
16777 "task".to_owned(),
16778 Config::default(),
16779 );
16780 state.reviews = vec![round];
16781 let mut runner = Runner {
16782 state,
16783 roles: ResolvedRoles {
16784 implementers: Vec::new(),
16785 judges: Vec::new(),
16786 reviewers: Vec::new(),
16787 fixer: None,
16788 conductor: conductor(),
16789 implementer_roster: Vec::new(),
16790 judge_roster: Vec::new(),
16791 reviewer_roster: Vec::new(),
16792 },
16793 sem: Arc::new(Semaphore::new(1)),
16794 pause: Pause::new(),
16795 interrupt: Pause::new(),
16796 };
16797 let shell = runner.state.config.shell();
16798 runner
16799 .stop_reviewing("round budget spent", &shell, &repo)
16800 .await
16801 .expect("stop_reviewing");
16802 assert_eq!(runner.state.status, RunStatus::Gating);
16803 assert_eq!(
16804 runner.state.contested_handoff.is_some(),
16805 expect,
16806 "{severity:?} + {vote:?}"
16807 );
16808 }
16809 }
16810
16811 #[tokio::test]
16824 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16825 crate::run::pin_test_home();
16826 let home = crate::run::home();
16827
16828 let tmp = tempfile::tempdir().expect("tempdir");
16829 let repo = tmp.path().join("repo");
16830 std::fs::create_dir_all(&repo).unwrap();
16831 init_repo(&repo);
16832 let head = crate::git::rev_parse(&repo, "HEAD")
16833 .await
16834 .expect("rev-parse");
16835 let cache_dir = tmp.path().join("target");
16838
16839 let mut config = Config::default();
16840 config.verify.e2e = vec![format!(
16841 "CARGO_TARGET_DIR='{}' test -f README.md",
16842 cache_dir.display()
16843 )];
16844 config.graph.review_rounds = 1;
16845 config.graph.timeout_verify = Some(2);
16848 config.graph.timeout_resource_wait = 2;
16849
16850 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16851 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16852 .expect("no io error acquiring directly")
16853 {
16854 crate::cache::AcquireOutcome::Acquired(g) => g,
16855 crate::cache::AcquireOutcome::Busy(b) => {
16856 panic!("expected the direct acquire to win the lease first: {b:?}")
16857 }
16858 };
16859
16860 let mut state = RunState::new(
16861 repo.clone(),
16862 "main".to_owned(),
16863 head.clone(),
16864 "task".to_owned(),
16865 config,
16866 );
16867 state.candidates = vec![Candidate {
16868 index: 0,
16869 label: 'A',
16870 agent: "alpha".to_owned(),
16871 branch: "does-not-exist".to_owned(),
16872 worktree: repo.clone(),
16873 summary: String::new(),
16874 stat: String::new(),
16875 files: 0,
16876 commits: 0,
16877 empty: false,
16878 failed: None,
16879 verified_noop: None,
16880 duration_ms: 0,
16881 folded: false,
16882 }];
16883 state.tally = Some(Tally {
16884 first_choice: BTreeMap::from([('A', 1)]),
16885 borda: BTreeMap::new(),
16886 winner: 'A',
16887 rankings: 1,
16888 unanimous_initial: true,
16889 deliberated: false,
16890 changed_votes: 0,
16891 unanimous_final: true,
16892 tie_break: None,
16893 judges: 0,
16894 present: 0,
16895 quorum: 0,
16896 met_quorum: true,
16897 uncontested: Some("only candidate A produced a change".to_owned()),
16898 });
16899 state.reviews = vec![ReviewRound {
16903 round: 1,
16904 head: head.clone(),
16905 verified_head: None,
16906 verified_at: None,
16907 reviews: Vec::new(),
16908 e2e: Vec::new(),
16909 fix: None,
16910 blocking: 1,
16911 answered: 1,
16912 expected: 1,
16913 clean: false,
16914 verify_retried: false,
16915 e2e_deferred: true,
16916 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16917 progressed: false,
16918 vote_split: false,
16919 reconsideration: Vec::new(),
16920 verdict: None,
16921 }];
16922
16923 let mut runner = Runner {
16924 state,
16925 roles: ResolvedRoles {
16926 implementers: Vec::new(),
16927 judges: Vec::new(),
16928 reviewers: Vec::new(),
16929 fixer: None,
16930 conductor: conductor(),
16931 implementer_roster: Vec::new(),
16932 judge_roster: Vec::new(),
16933 reviewer_roster: Vec::new(),
16934 },
16935 sem: Arc::new(Semaphore::new(1)),
16936 pause: Pause::new(),
16937 interrupt: Pause::new(),
16938 };
16939
16940 let shell = runner.state.config.shell();
16941 runner
16942 .stop_reviewing("round budget spent", &shell, &repo)
16943 .await
16944 .expect("stop_reviewing");
16945
16946 let last = runner.state.reviews.last().expect("round record");
16947 assert_eq!(
16948 last.e2e_status(),
16949 E2eStatus::ResourceBlocked,
16950 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16951 failed: {last:?}"
16952 );
16953 assert_eq!(
16954 last.verified_head.as_deref(),
16955 Some(head.as_str()),
16956 "which commit this attempt targeted is known even though nothing finished checking \
16957 it"
16958 );
16959 let first_attempt_at = last
16960 .verified_at
16961 .expect("when this attempt ran is known too");
16962 assert_ne!(
16963 runner.state.status,
16964 RunStatus::Blocked,
16965 "contention is evidence about the machine, not the patch — it must not settle the \
16966 run as blocked: {:?}",
16967 runner.state.status
16968 );
16969 assert!(
16970 !runner
16971 .state
16972 .events
16973 .iter()
16974 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16975 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16976 runner.state.events
16977 );
16978
16979 runner
16984 .stop_reviewing("round budget spent", &shell, &repo)
16985 .await
16986 .expect("stop_reviewing retry");
16987 assert_eq!(
16988 runner.state.reviews.len(),
16989 1,
16990 "no new round was started: {:?}",
16991 runner.state.reviews
16992 );
16993 let last = runner.state.reviews.last().expect("round record");
16994 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16995 assert!(
16996 last.verified_at.expect("still known") > first_attempt_at,
16997 "a second reentry must be a fresh attempt, not a stale copy of the first"
16998 );
16999 assert_ne!(runner.state.status, RunStatus::Blocked);
17000
17001 held.release();
17002 }
17003
17004 #[cfg(unix)]
17005 #[tokio::test]
17006 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
17007 ask_test_home();
17008 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
17009 let tmp = tempfile::tempdir().unwrap();
17010 let repo = tmp.path().join("repo");
17011 std::fs::create_dir(&repo).unwrap();
17012 init_repo(&repo);
17013 let run_git = |args: &[&str]| {
17014 let out = std::process::Command::new("git")
17015 .args(args)
17016 .current_dir(&repo)
17017 .output()
17018 .unwrap();
17019 assert!(
17020 out.status.success(),
17021 "{args:?}: {}",
17022 String::from_utf8_lossy(&out.stderr)
17023 );
17024 };
17025 let origin = tmp.path().join("origin.git");
17026 run_git(&["init", "--bare", origin.to_str().unwrap()]);
17027 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17028 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17029 run_git(&["add", "spec.rs"]);
17030 run_git(&["commit", "-m", "initial spec"]);
17031 run_git(&["push", "origin", "main"]);
17032 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17033 run_git(&["checkout", "-b", "fix/review"]);
17034 std::fs::write(
17035 repo.join("check.rs"),
17036 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17037 )
17038 .unwrap();
17039 run_git(&["add", "check.rs"]);
17040 run_git(&["commit", "-m", "use spec"]);
17041 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17042 run_git(&["checkout", "main"]);
17043 std::fs::write(
17044 repo.join("spec.rs"),
17045 "struct AgentSpec { max_turn_secs: u64 }\n",
17046 )
17047 .unwrap();
17048 run_git(&["add", "spec.rs"]);
17049 run_git(&["commit", "-m", "extend spec"]);
17050 run_git(&["push", "origin", "main"]);
17051 run_git(&["checkout", "fix/review"]);
17052
17053 let mut runner = runner_at(RunStatus::Reviewing);
17054 runner.state.repo = repo.clone();
17055 runner.state.base_commit = base;
17056 runner.state.pr_head = Some("fix/review".into());
17057 runner.state.config.graph.review_rounds = max_rounds;
17058 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17059 runner.state.config.graph.sessions = false;
17060 runner.state.config.disk.min_free_bytes = 0;
17061 runner.state.config.verify.e2e =
17062 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17063 runner.state.candidates = vec![Candidate {
17064 index: 0,
17065 label: 'A',
17066 agent: "fixture".into(),
17067 branch: "fix/review".into(),
17068 worktree: repo.clone(),
17069 summary: String::new(),
17070 stat: String::new(),
17071 files: 1,
17072 commits: 1,
17073 empty: false,
17074 failed: None,
17075 verified_noop: None,
17076 duration_ms: 0,
17077 folded: false,
17078 }];
17079 runner.state.tally = Some(Tally {
17080 first_choice: BTreeMap::new(),
17081 borda: BTreeMap::new(),
17082 winner: 'A',
17083 rankings: 0,
17084 unanimous_initial: true,
17085 deliberated: false,
17086 changed_votes: 0,
17087 unanimous_final: true,
17088 tie_break: None,
17089 judges: 0,
17090 present: 0,
17091 quorum: 0,
17092 met_quorum: true,
17093 uncontested: Some("review-only".into()),
17094 });
17095 let mut last = review_round(false, 0, 1, 1, false, false);
17096 last.head = reviewed.clone();
17097 last.e2e_deferred = true;
17098 last.e2e[0].resource_blocked = true;
17099 last.e2e[0].output_tail = "waiting for the shared build cache".into();
17100 if max_rounds == 1 {
17101 last.e2e.clear();
17102 }
17103 runner.state.reviews = vec![last];
17104 let calls = tmp.path().join("fix-calls");
17105 let mut fixer = spec("repair");
17106 fixer
17107 .env
17108 .insert("FIX_CALLS".into(), calls.display().to_string());
17109 fixer.command = vec!["sh".into(), "-c".into(), r#"
17110 echo fix >> "$FIX_CALLS"
17111 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
17112 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
17113 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
17114 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
17115 "#.into()];
17116 let mut reviewer = spec("reviewer");
17117 reviewer.command = vec![
17118 "sh".into(),
17119 "-c".into(),
17120 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
17121 .into(),
17122 ];
17123 runner.roles.fixer = Some(vec![fixer]);
17124 runner.roles.reviewers = vec![reviewer];
17125
17126 runner.sync_to_base().await.unwrap();
17127 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
17128 assert_ne!(rebased, reviewed);
17129 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
17130 if reentry {
17131 let shell = runner.state.config.shell();
17134 runner
17135 .stop_unresolved_review("deferred verification", &shell, &repo)
17136 .await
17137 .unwrap();
17138 assert!(runner.catchup_repairable());
17139 assert!(!calls.exists());
17140 runner.state = RunState::load(&runner.state.id).unwrap();
17141 assert!(runner.catchup_repairable());
17142 }
17143 runner.review_loop().await.unwrap();
17144 let first = &runner.state.reviews[0];
17145 assert_eq!(first.head, reviewed);
17146 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
17147 assert!(
17148 first.verify_retried,
17149 "a reproduced build failure gets exactly one retry: {:?}",
17150 first.e2e
17151 );
17152 assert!(first.e2e[0].output_tail.contains("E0063"));
17153 assert!(!runner.catchup_repairable());
17154 if max_rounds == 2 {
17155 assert_eq!(
17156 runner.state.status,
17157 RunStatus::Gating,
17158 "{}",
17159 crate::report::run(&runner.state)
17160 );
17161 assert_eq!(runner.state.reviews.len(), 2);
17162 assert!(first.fix.as_ref().unwrap().committed);
17163 assert!(runner.state.reviews[1].clean);
17164 assert_eq!(
17165 runner
17166 .state
17167 .reviews
17168 .iter()
17169 .filter(|r| r.fix.is_some())
17170 .count(),
17171 1
17172 );
17173 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
17174 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
17175 } else {
17176 assert_eq!(runner.state.status, RunStatus::Blocked);
17177 assert!(first.fix.is_none());
17178 assert!(!calls.exists());
17179 let recovery = crate::recovery::assess(
17180 "prior",
17181 Some(&runner.state),
17182 &runner.landing_base(),
17183 crate::recovery::Relation::Diverged,
17184 );
17185 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
17186 let reasons = recovery.reasons.join("\n");
17187 assert!(
17188 reasons.contains(&reviewed) && reasons.contains(&rebased),
17189 "{reasons}"
17190 );
17191 assert!(
17192 reasons.contains("base sync to")
17193 && reasons.contains("additional review is required"),
17194 "{reasons}"
17195 );
17196 }
17197 }
17198 }
17199
17200 #[tokio::test]
17212 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
17213 crate::run::pin_test_home();
17214 let home = crate::run::home();
17215
17216 let tmp = tempfile::tempdir().expect("tempdir");
17217 let repo = tmp.path().join("repo");
17218 std::fs::create_dir_all(&repo).unwrap();
17219 init_repo(&repo);
17220 let head = crate::git::rev_parse(&repo, "HEAD")
17221 .await
17222 .expect("rev-parse");
17223 let cache_dir = tmp.path().join("target");
17224
17225 let mut config = Config::default();
17226 config.verify.e2e = vec![format!(
17227 "CARGO_TARGET_DIR='{}' test -f README.md",
17228 cache_dir.display()
17229 )];
17230 config.graph.review_rounds = 1;
17231 config.graph.timeout_verify = Some(2);
17232 config.graph.timeout_resource_wait = 2;
17233
17234 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17235 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17236 .expect("no io error acquiring directly")
17237 {
17238 crate::cache::AcquireOutcome::Acquired(g) => g,
17239 crate::cache::AcquireOutcome::Busy(b) => {
17240 panic!("expected the direct acquire to win the lease first: {b:?}")
17241 }
17242 };
17243
17244 let mut state = RunState::new(
17245 repo.clone(),
17246 "main".to_owned(),
17247 head.clone(),
17248 "task".to_owned(),
17249 config,
17250 );
17251 state.candidates = vec![Candidate {
17252 index: 0,
17253 label: 'A',
17254 agent: "alpha".to_owned(),
17255 branch: "does-not-exist".to_owned(),
17256 worktree: repo.clone(),
17257 summary: String::new(),
17258 stat: String::new(),
17259 files: 0,
17260 commits: 0,
17261 empty: false,
17262 failed: None,
17263 verified_noop: None,
17264 duration_ms: 0,
17265 folded: false,
17266 }];
17267 state.tally = Some(Tally {
17268 first_choice: BTreeMap::from([('A', 1)]),
17269 borda: BTreeMap::new(),
17270 winner: 'A',
17271 rankings: 1,
17272 unanimous_initial: true,
17273 deliberated: false,
17274 changed_votes: 0,
17275 unanimous_final: true,
17276 tie_break: None,
17277 judges: 0,
17278 present: 0,
17279 quorum: 0,
17280 met_quorum: true,
17281 uncontested: Some("only candidate A produced a change".to_owned()),
17282 });
17283 state.reviews = vec![ReviewRound {
17287 round: 1,
17288 head: head.clone(),
17289 verified_head: Some(head.clone()),
17290 verified_at: Some(jiff::Timestamp::now()),
17291 reviews: Vec::new(),
17292 e2e: vec![CommandOutcome {
17293 command: format!(
17294 "CARGO_TARGET_DIR='{}' test -f README.md",
17295 cache_dir.display()
17296 ),
17297 code: None,
17298 output_tail: "waiting for the shared build cache".to_owned(),
17299 duration_ms: 0,
17300 resource_blocked: true,
17301 }],
17302 fix: None,
17303 blocking: 1,
17304 answered: 1,
17305 expected: 1,
17306 clean: false,
17307 verify_retried: false,
17308 e2e_deferred: false,
17309 e2e_defer_reason: None,
17310 progressed: false,
17311 vote_split: false,
17312 reconsideration: Vec::new(),
17313 verdict: None,
17314 }];
17315
17316 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17317 let mut runner = Runner {
17318 state,
17319 roles: ResolvedRoles {
17320 implementers: Vec::new(),
17321 judges: Vec::new(),
17322 reviewers: Vec::new(),
17323 fixer: None,
17324 conductor: conductor(),
17325 implementer_roster: Vec::new(),
17326 judge_roster: Vec::new(),
17327 reviewer_roster: Vec::new(),
17328 },
17329 sem: Arc::new(Semaphore::new(1)),
17330 pause: Pause::new(),
17331 interrupt: Pause::new(),
17332 };
17333
17334 runner.review_loop().await.expect("review_loop");
17339
17340 assert_eq!(
17341 runner.state.reviews.len(),
17342 1,
17343 "no new round was started on top of the unresolved one: {:?}",
17344 runner.state.reviews
17345 );
17346 let last = &runner.state.reviews[0];
17347 assert_eq!(
17348 last.e2e_status(),
17349 E2eStatus::ResourceBlocked,
17350 "still contended: {last:?}"
17351 );
17352 assert!(
17353 last.verified_at.expect("still known") > first_attempt_at,
17354 "review_loop must have actually retried the check, not left it exactly as found"
17355 );
17356 assert_ne!(
17357 runner.state.status,
17358 RunStatus::Blocked,
17359 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17360 runner.state.status
17361 );
17362
17363 held.release();
17364 }
17365
17366 #[tokio::test]
17367 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17368 crate::run::pin_test_home();
17369 for status in [RunStatus::Blocked, RunStatus::Gating] {
17370 let mut runner = runner_at(status);
17371 runner.state.config.graph.land = true;
17372 runner.state.gate_ran = true;
17373 runner.state.merge = Some(MergeOutcome {
17374 mode: MergeMode::Pr,
17375 ok: false,
17376 detail: "the fixer produced no commit".into(),
17377 empty: false,
17378 });
17379 let url = "https://example.invalid/o/r/pull/1";
17380 runner.state.pr = Some(crate::run::PrRecord {
17381 url: url.into(),
17382 number: 1,
17383 state: "open".into(),
17384 checks: "red".into(),
17385 round: 1,
17386 rounds: 1,
17387 red_at_merge: Vec::new(),
17388 });
17389 runner.state.save().unwrap();
17390 runner.execute_graph().await.unwrap();
17394 assert_eq!(runner.state.status, RunStatus::Blocked);
17395 assert!(runner.state.land_sync_pending);
17396 assert!(runner.state.events.iter().any(|event|
17397 event.node == "land" && event.message == format!("watching {url}")));
17398 assert_eq!(
17399 runner.state.merge.as_ref().unwrap().detail,
17400 "the fixer produced no commit"
17401 );
17402 }
17403 }
17404
17405 #[tokio::test]
17406 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17407 crate::run::pin_test_home();
17408 let tmp = tempfile::tempdir().expect("tempdir");
17409 let repo = tmp.path().join("repo");
17410 std::fs::create_dir_all(&repo).unwrap();
17411 init_repo(&repo);
17412
17413 let mut config = Config::default();
17414 config.merge.mode = MergeMode::Pr;
17415 config.graph.land = true;
17416 config.graph.land_approval = false;
17417
17418 let mut state = RunState::new(
17419 repo.clone(),
17420 "main".to_owned(),
17421 "deadbeef".to_owned(),
17422 "task".to_owned(),
17423 config,
17424 );
17425 state.candidates = vec![Candidate {
17426 index: 0,
17427 label: 'A',
17428 agent: "alpha".to_owned(),
17429 branch: "does-not-exist".to_owned(),
17430 worktree: repo.clone(),
17431 summary: String::new(),
17432 stat: String::new(),
17433 files: 0,
17434 commits: 0,
17435 empty: false,
17436 failed: None,
17437 verified_noop: None,
17438 duration_ms: 0,
17439 folded: false,
17440 }];
17441 state.tally = Some(Tally {
17442 first_choice: BTreeMap::from([('A', 1)]),
17443 borda: BTreeMap::new(),
17444 winner: 'A',
17445 rankings: 1,
17446 unanimous_initial: true,
17447 deliberated: false,
17448 changed_votes: 0,
17449 unanimous_final: true,
17450 tie_break: None,
17451 judges: 0,
17452 present: 0,
17453 quorum: 0,
17454 met_quorum: true,
17455 uncontested: Some("only candidate A produced a change".to_owned()),
17456 });
17457 state.reviews = vec![ReviewRound {
17458 round: 1,
17459 head: "deadbeef".to_owned(),
17460 verified_head: None,
17461 verified_at: None,
17462 reviews: Vec::new(),
17463 e2e: Vec::new(),
17464 fix: None,
17465 blocking: 0,
17466 answered: 0,
17467 expected: 0,
17468 clean: true,
17469 verify_retried: false,
17470 e2e_deferred: false,
17471 e2e_defer_reason: None,
17472 progressed: false,
17473 vote_split: false,
17474 reconsideration: Vec::new(),
17475 verdict: None,
17476 }];
17477 state.gate = vec![CommandOutcome {
17478 command: "test".to_owned(),
17479 code: Some(0),
17480 output_tail: String::new(),
17481 duration_ms: 0,
17482 resource_blocked: false,
17483 }];
17484 state.gate_ran = true;
17485 state.status = RunStatus::Landing;
17489 state.merge = Some(MergeOutcome {
17490 mode: MergeMode::Pr,
17491 ok: true,
17492 detail: "https://example.invalid/x/y/pull/1".to_owned(),
17493 empty: false,
17494 });
17495
17496 ask_test_home();
17500 state.save().expect("save landing run");
17503 let store = ask::Questions::open();
17504 let q = ask_open_question(&store, &state.id);
17505
17506 let mut runner = Runner {
17507 state,
17508 roles: ResolvedRoles {
17509 implementers: Vec::new(),
17510 judges: Vec::new(),
17511 reviewers: Vec::new(),
17512 fixer: None,
17513 conductor: conductor(),
17514 implementer_roster: Vec::new(),
17515 judge_roster: Vec::new(),
17516 reviewer_roster: Vec::new(),
17517 },
17518 sem: Arc::new(Semaphore::new(1)),
17519 pause: Pause::new(),
17520 interrupt: Pause::new(),
17521 };
17522
17523 runner.execute().await.expect("execute");
17528
17529 assert_eq!(
17530 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17531 Some("https://example.invalid/x/y/pull/1"),
17532 "reentry must not push again or open a second pull request over the \
17533 one `land` is already watching"
17534 );
17535 assert_ne!(
17536 runner.state.status,
17537 RunStatus::Landing,
17538 "land could not actually reach the fake pull request, so it must \
17539 have given up rather than left the run silently parked forever"
17540 );
17541 assert_eq!(runner.state.status, RunStatus::Blocked);
17545 assert!(
17546 store.get(&q.id).unwrap().status.open(),
17547 "Blocked is still alive; settle_questions must have been a no-op here"
17548 );
17549 }
17550
17551 fn state_with_round(round: ReviewRound) -> RunState {
17552 let mut s = RunState::new(
17553 PathBuf::from("/repo"),
17554 "main".to_owned(),
17555 "abc1234".to_owned(),
17556 "add retries".to_owned(),
17557 Config::default(),
17558 );
17559 s.reviews = vec![round];
17560 s
17561 }
17562
17563 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17564 crate::verdict::Finding {
17565 id: id.to_owned(),
17566 severity,
17567 file: None,
17568 line: None,
17569 title: title.to_owned(),
17570 detail: String::new(),
17571 }
17572 }
17573
17574 #[test]
17575 fn pr_body_names_open_findings_and_declined_ones() {
17576 let round = ReviewRound {
17577 round: 2,
17578 head: "deadbee".to_owned(),
17579 verified_head: None,
17580 verified_at: None,
17581 reviews: vec![ReviewRecord {
17582 attempts: 0,
17583 reviewer: 1,
17584 agent: "alpha".to_owned(),
17585 summary: String::new(),
17586 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17587 vote: None,
17588 failed: None,
17589 duration_ms: 0,
17590 }],
17591 e2e: vec![CommandOutcome {
17592 command: "cargo test".to_owned(),
17593 code: Some(0),
17594 output_tail: String::new(),
17595 duration_ms: 0,
17596 resource_blocked: false,
17597 }],
17598 verify_retried: false,
17599 e2e_deferred: false,
17600 e2e_defer_reason: None,
17601 fix: Some(FixRecord {
17602 agent: "alpha".to_owned(),
17603 addressed: Vec::new(),
17604 rejected: vec![crate::verdict::Rejection {
17605 id: "R1-1-1".to_owned(),
17606 why: "not reachable from any caller".to_owned(),
17607 }],
17608 notes: String::new(),
17609 committed: true,
17610 failed: None,
17611 duration_ms: 0,
17612 continuation: None,
17613 }),
17614 blocking: 0,
17615 answered: 1,
17616 expected: 1,
17617 clean: false,
17618 progressed: true,
17619 vote_split: false,
17620 reconsideration: Vec::new(),
17621 verdict: None,
17622 };
17623 let state = state_with_round(round);
17624 let body = pr_message(&state, 'A').body;
17625
17626 assert!(body.contains("add retries"), "the task must still be there");
17627 assert!(body.contains("R2-1-1"), "{body}");
17628 assert!(!body.contains("unused import"), "{body}");
17629 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17630 assert!(
17631 !body.contains("not reachable from any caller"),
17632 "the reason it was declined: {body}"
17633 );
17634 }
17635
17636 #[test]
17637 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17638 let round = ReviewRound {
17639 round: 1,
17640 head: "deadbee".to_owned(),
17641 verified_head: None,
17642 verified_at: None,
17643 reviews: vec![ReviewRecord {
17644 attempts: 0,
17645 reviewer: 1,
17646 agent: "alpha".to_owned(),
17647 summary: String::new(),
17648 findings: Vec::new(),
17649 vote: None,
17650 failed: None,
17651 duration_ms: 0,
17652 }],
17653 e2e: Vec::new(),
17654 verify_retried: false,
17655 e2e_deferred: false,
17656 e2e_defer_reason: None,
17657 fix: None,
17658 blocking: 0,
17659 answered: 1,
17660 expected: 1,
17661 clean: true,
17662 progressed: false,
17663 vote_split: false,
17664 reconsideration: Vec::new(),
17665 verdict: None,
17666 };
17667 let state = state_with_round(round);
17668 let body = pr_message(&state, 'A').body;
17669 assert!(!body.contains("Open review findings"), "{body}");
17670 assert!(!body.contains("Declined"), "{body}");
17671 }
17672
17673 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17674 let mut state = RunState::new(
17675 PathBuf::from("/repo"),
17676 "main".to_owned(),
17677 "abc1234".to_owned(),
17678 instruction.to_owned(),
17679 Config::default(),
17680 );
17681 state.candidates.push(Candidate {
17682 index: 0,
17683 label: 'A',
17684 agent: "alpha".to_owned(),
17685 branch: "magi/x/A".to_owned(),
17686 worktree: PathBuf::from("/wt"),
17687 summary: summary.to_owned(),
17688 stat: String::new(),
17689 files: 1,
17690 commits: 1,
17691 empty: false,
17692 failed: None,
17693 verified_noop: None,
17694 folded: false,
17695 duration_ms: 0,
17696 });
17697 state
17698 }
17699
17700 fn review_state(subjects: &[&str]) -> RunState {
17701 let mut state = state_with_summary(
17702 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17703 "",
17704 );
17705 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17706 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17707 state
17708 }
17709
17710 #[tokio::test]
17713 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17714 ask_test_home();
17715 let tmp = tempfile::tempdir().unwrap();
17716 let repo = tmp.path().join("repo");
17717 std::fs::create_dir_all(&repo).unwrap();
17718 init_repo(&repo);
17719 let origin = tmp.path().join("origin.git");
17720 let g = |dir: &Path, args: &[&str]| {
17721 let out = std::process::Command::new("git")
17722 .args(args)
17723 .current_dir(dir)
17724 .quiet()
17725 .output()
17726 .expect("spawn git");
17727 assert!(
17728 out.status.success(),
17729 "git {args:?}: {}",
17730 String::from_utf8_lossy(&out.stderr)
17731 );
17732 };
17733 g(
17734 tmp.path(),
17735 &[
17736 "clone",
17737 "--bare",
17738 "-q",
17739 repo.to_str().unwrap(),
17740 origin.to_str().unwrap(),
17741 ],
17742 );
17743 g(
17744 &repo,
17745 &["remote", "add", "origin", origin.to_str().unwrap()],
17746 );
17747 let c1 = git::rev_parse(&repo, "main").await.unwrap();
17748
17749 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17751 g(&repo, &["add", "-A"]);
17752 g(
17753 &repo,
17754 &["commit", "-q", "-m", "chore(deps): update a crate"],
17755 );
17756 g(&repo, &["push", "-q", "origin", "main"]);
17757 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17758 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17759 g(&repo, &["add", "-A"]);
17760 g(
17761 &repo,
17762 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17763 );
17764 g(&repo, &["checkout", "-q", "main"]);
17765
17766 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17767 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17768 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17770 assert_eq!(fallback, c1);
17771
17772 let mut state = review_state(&[
17773 "chore(deps): update a crate",
17774 "fix(daemon): apply a chosen action",
17775 ]);
17776 state.repo = repo.clone();
17777 state.base_branch = "main".to_owned();
17778 state.base_commit = c1;
17779 refresh_reviewed_commits(&mut state, "feat/own").await;
17780 assert_eq!(
17781 state.reviewed_commits,
17782 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17783 );
17784 assert_eq!(
17785 review_title(&state).as_deref(),
17786 Some("fix(daemon): apply a chosen action")
17787 );
17788 assert_eq!(
17789 leaked_subjects(&state, "feat/own").await,
17790 Some(vec!["chore(deps): update a crate".to_owned()])
17791 );
17792 state.config.merge.remote = "nowhere".to_owned();
17795 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17796 }
17797
17798 struct SummaryFixture {
17802 _tmp: tempfile::TempDir,
17803 repo: PathBuf,
17804 reply: PathBuf,
17805 runner: Runner,
17806 }
17807
17808 fn summary_fixture() -> SummaryFixture {
17809 ask_test_home();
17810 let tmp = tempfile::tempdir().unwrap();
17811 let repo = tmp.path().join("repo");
17812 std::fs::create_dir_all(&repo).unwrap();
17813 init_repo(&repo);
17814 let origin = tmp.path().join("origin.git");
17815 let g = |args: &[&str]| {
17816 let out = std::process::Command::new("git")
17817 .args(args)
17818 .current_dir(&repo)
17819 .quiet()
17820 .output()
17821 .expect("spawn git");
17822 assert!(
17823 out.status.success(),
17824 "git {args:?}: {}",
17825 String::from_utf8_lossy(&out.stderr)
17826 );
17827 };
17828 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17829 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17830 g(&["push", "-q", "origin", "main"]);
17831 g(&["checkout", "-q", "-b", "feat/own"]);
17832 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17833 g(&["add", "-A"]);
17834 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17835 let reply = tmp.path().join("reply.txt");
17836 std::fs::write(
17837 &reply,
17838 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17839 )
17840 .unwrap();
17841 let mut runner = runner_at(RunStatus::Gating);
17842 let mut state = review_state(&["feat(own): add the own file"]);
17843 state.repo = repo.clone();
17844 state.base_branch = "main".to_owned();
17845 state.candidates[0].branch = "feat/own".to_owned();
17846 state.candidates[0].worktree = repo.clone();
17847 let mut author = spec("alpha");
17848 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17849 author
17850 .env
17851 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17852 state.config.agents = vec![author];
17853 runner.state = state;
17854 SummaryFixture {
17855 _tmp: tmp,
17856 repo,
17857 reply,
17858 runner,
17859 }
17860 }
17861
17862 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17863 let winner = f.runner.state.candidates[0].clone();
17864 f.runner
17865 .guarded_pr_message(&winner, None, true)
17866 .await
17867 .unwrap()
17868 .expect("an English description never parks")
17869 }
17870
17871 fn clean_round_with_minor_findings() -> ReviewRound {
17872 let mut round = passing_round(2, None);
17873 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17874 round.reviews = vec![ReviewRecord {
17875 attempts: 0,
17876 reviewer: 1,
17877 agent: "alpha".into(),
17878 summary: "内部レビュー".into(),
17879 findings: vec![
17880 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17881 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17882 ],
17883 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17884 failed: None,
17885 duration_ms: 0,
17886 }];
17887 round
17888 }
17889
17890 #[tokio::test]
17891 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17892 for review_only in [false, true] {
17893 let mut f = summary_fixture();
17894 if !review_only {
17895 f.runner.state.candidates[0].agent = "alpha".into();
17896 }
17897 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17898 assert!(f.runner.state.reviews[0].clean);
17899 assert!(f.runner.state.reviews[0].fix.is_none());
17900 assert!(f.runner.state.open_findings().is_empty());
17901 assert_eq!(
17902 crate::pr_outcome::required_finding_ids(&f.runner.state),
17903 ["R2-1-1", "R2-1-2"]
17904 );
17905 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17906 let evidence = crate::pr_outcome::summary_evidence(
17907 &f.runner.state,
17908 crate::pr_outcome::FinalHead::At(&head),
17909 );
17910 assert!(
17911 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17912 );
17913 let omitted = posted(&mut f).await;
17915 assert!(omitted.body.contains("No description"));
17916 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17917 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17918 assert!(f.runner.state.change_summary.is_none());
17919 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17920 let described = posted(&mut f).await;
17921 assert!(described.body.contains("improve documentation for callers"));
17922 assert!(described.body.contains("remaining edge case"));
17923 assert!(described.body.contains("## Open review findings"));
17924 assert!(!described.body.contains("未修正"));
17925 }
17926 }
17927
17928 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17929 let mut f = summary_fixture();
17930 if !review_only {
17931 f.runner.state.candidates[0].agent = "alpha".into();
17932 }
17933 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17934 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17935 fixed
17936 .fix
17937 .as_mut()
17938 .unwrap()
17939 .rejected
17940 .push(crate::verdict::Rejection {
17941 id: "R1-1-1".into(),
17942 why: "呼び出されないコードのため辞退".into(),
17943 });
17944 let mut clean = clean_round_with_minor_findings();
17945 clean.verified_head = Some(head.clone());
17946 f.runner.state.reviews = vec![fixed, clean];
17947 ok_gate(&mut f.runner.state, Some(&head));
17948 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17949 f.runner.state.config.agents.push(scripted("beta", reply));
17950 f.runner.state.config.roles.language_judge =
17951 Some(crate::config::AgentChoice::One("beta".into()));
17952 f
17953 }
17954
17955 #[tokio::test]
17956 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17957 for review_only in [false, true] {
17958 let mut f = rewrite_fixture_with_findings(review_only,
17959 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17960 let m = posted(&mut f).await;
17961 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17962 let expected =
17963 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17964 assert!(m.body.contains(&expected));
17965 assert!(m.body.contains("## Review rounds"));
17966 assert!(
17967 m.body
17968 .contains("Gate (`verify.gate`): passed on the final commit")
17969 );
17970 assert!(
17971 m.body
17972 .contains("Tests (`verify.e2e`): passed on the final commit")
17973 );
17974 let c = f.runner.state.change_summary.as_ref().unwrap();
17975 assert!(
17976 m.body
17977 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17978 );
17979 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17980 assert!(crate::pr_outcome::covers_required_findings(
17981 &f.runner.state,
17982 &m.body
17983 ));
17984 assert!(m.body.contains("documentation remains incomplete"));
17985 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17986 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17987 assert!(f.runner.state.github_text.is_none());
17988 }
17989 }
17990
17991 #[tokio::test]
17992 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17993 for review_only in [false, true] {
17994 for reply in [
17995 "TITLE: fix: public description\nA short English description.",
17996 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17997 ] {
17998 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17999 let winner = f.runner.state.candidates[0].clone();
18000 let posted = f
18001 .runner
18002 .guarded_pr_message(&winner, None, true)
18003 .await
18004 .unwrap();
18005 assert!(posted.is_none());
18006 assert!(f.runner.state.parked);
18007 let withheld = f.runner.state.github_text.as_ref().unwrap();
18008 assert!(
18009 withheld.rewrite.is_none(),
18010 "an incomplete rewrite must not be selectable"
18011 );
18012 let q = ask::Questions::open()
18013 .get(withheld.question.as_ref().unwrap())
18014 .unwrap();
18015 assert!(
18016 !q.choices
18017 .contains(&crate::github_text::USE_REWRITE.to_owned())
18018 );
18019 assert!(
18020 q.detail.contains("R1-1-1")
18021 && q.detail.contains("R2-1-1")
18022 && q.detail.contains("R2-1-2")
18023 );
18024 }
18025 }
18026 }
18027
18028 #[tokio::test]
18029 async fn japanese_reports_stay_local_in_both_public_run_modes() {
18030 for review_only in [false, true] {
18031 let mut f = summary_fixture();
18032 if !review_only {
18033 f.runner.state.candidates[0].agent = "alpha".into();
18034 }
18035 f.runner.state.instruction = "内部タスクの説明".into();
18036 f.runner.state.candidates[0].summary = "内部の候補報告".into();
18037 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18038 f.runner.state.reviews = vec![
18039 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18040 round_with_notes(2, Some("二回目の内部修正報告です。")),
18041 passing_round(3, Some(&head)),
18042 ];
18043 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18044 ok_gate(&mut f.runner.state, Some(&head));
18045 let captured = f.repo.join("public-prompt.txt");
18046 let author = &mut f.runner.state.config.agents[0];
18047 author.command = vec![
18048 "sh".into(),
18049 "-c".into(),
18050 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18051 ];
18052 author.env.insert(
18053 "PROMPT_FILE".into(),
18054 captured.to_string_lossy().into_owned(),
18055 );
18056 let m = posted(&mut f).await;
18057 assert!(m.body.contains("Adds `own.txt`"));
18058 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18059 assert!(m.body.contains("R1-1-1"));
18060 assert!(
18061 m.body
18062 .contains("Gate (`verify.gate`): passed on the final commit")
18063 );
18064 assert!(
18065 m.body
18066 .contains("Tests (`verify.e2e`): passed on the final commit")
18067 );
18068 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18069 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18070 let prompt = std::fs::read_to_string(captured).unwrap();
18071 assert!(prompt.contains(&format!(
18072 "git diff {} {head}",
18073 f.runner.state.change_summary.as_ref().unwrap().base
18074 )));
18075 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18076 assert_eq!(
18077 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18078 "二回目の内部修正報告です。"
18079 );
18080 }
18081 }
18082
18083 #[tokio::test]
18084 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18085 for review_only in [false, true] {
18086 let mut f = summary_fixture();
18087 if !review_only {
18088 f.runner.state.candidates[0].agent = "alpha".into();
18089 }
18090 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18091 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18092 std::fs::write(&f.reply, "no usable description").unwrap();
18093 let failed = posted(&mut f).await;
18094 assert!(
18095 failed
18096 .body
18097 .contains("No description of what this change does could be generated")
18098 );
18099 assert!(failed.body.is_ascii());
18100 assert!(f.runner.state.github_text.is_none());
18101 assert!(f.runner.state.change_summary.is_none());
18102 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18103 let evidence = crate::pr_outcome::summary_evidence(
18104 &f.runner.state,
18105 crate::pr_outcome::FinalHead::At(&head),
18106 );
18107 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18108 assert!(
18109 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
18110 .exists()
18111 );
18112 f.runner.state =
18113 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
18114 std::fs::write(
18115 &f.reply,
18116 "TITLE: feat(own): add the own file\nAdds the committed own file.",
18117 )
18118 .unwrap();
18119 let retried = posted(&mut f).await;
18120 assert!(retried.body.contains("Adds the committed own file."));
18121 assert_eq!(
18122 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
18123 "日本語の修正報告"
18124 );
18125 }
18126 }
18127
18128 #[tokio::test]
18129 async fn same_head_verification_and_findings_invalidate_public_prose() {
18130 let mut f = summary_fixture();
18131 posted(&mut f).await;
18132 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18133 let original = f
18134 .runner
18135 .state
18136 .change_summary
18137 .as_ref()
18138 .unwrap()
18139 .evidence
18140 .clone();
18141 ok_gate(&mut f.runner.state, Some(&head));
18142 std::fs::write(
18143 &f.reply,
18144 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
18145 )
18146 .unwrap();
18147 let verified = posted(&mut f).await;
18148 assert!(verified.body.contains("now verified"));
18149 assert_ne!(
18150 f.runner.state.change_summary.as_ref().unwrap().evidence,
18151 original
18152 );
18153 let mut round = round_with_notes(1, Some("内部報告"));
18154 round.reviews.push(ReviewRecord {
18155 attempts: 0,
18156 reviewer: 1,
18157 agent: "alpha".into(),
18158 summary: "内部レビュー".into(),
18159 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
18160 vote: None,
18161 failed: None,
18162 duration_ms: 0,
18163 });
18164 round
18165 .fix
18166 .as_mut()
18167 .unwrap()
18168 .rejected
18169 .push(crate::verdict::Rejection {
18170 id: "R1-1-1".into(),
18171 why: "呼び出し元から到達できない".into(),
18172 });
18173 f.runner.state.reviews = vec![round];
18174 let omitted = posted(&mut f).await;
18176 assert!(omitted.body.contains("No description"));
18177 assert!(omitted.body.contains("R1-1-1"));
18178 assert!(!omitted.body.contains("到達"));
18179 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
18180 let described = posted(&mut f).await;
18181 assert!(described.body.contains("unreachable from callers"));
18182 assert!(described.body.contains("R1-1-1"));
18183 assert!(!described.body.contains("到達"));
18184 let evidence = f
18185 .runner
18186 .state
18187 .change_summary
18188 .as_ref()
18189 .unwrap()
18190 .evidence
18191 .clone();
18192 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
18193 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
18194 assert!(posted(&mut f).await.body.contains("rationale updated"));
18195 assert_ne!(
18196 f.runner.state.change_summary.as_ref().unwrap().evidence,
18197 evidence
18198 );
18199 f.runner
18201 .state
18202 .change_summary
18203 .as_mut()
18204 .unwrap()
18205 .evidence
18206 .clear();
18207 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
18208 assert!(posted(&mut f).await.body.contains("current specification"));
18209 }
18210
18211 #[tokio::test]
18212 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
18213 let mut f = summary_fixture();
18214 let m = posted(&mut f).await;
18215 assert_eq!(m.title, "feat(own): add the own file");
18216 assert!(
18217 m.body
18218 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
18219 "{}",
18220 m.body
18221 );
18222 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
18223 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
18224 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18225 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18226 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
18227 let kept = f.runner.state.change_summary.as_ref().unwrap();
18228 assert_eq!(
18229 (kept.head.as_str(), kept.base.as_str()),
18230 (head.as_str(), base.as_str())
18231 );
18232 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
18233 }
18234
18235 #[tokio::test]
18236 async fn review_only_summary_receives_the_configured_github_language() {
18237 let mut f = summary_fixture();
18238 f.runner.state.config.graph.github_language = "ja".into();
18239 let captured = f.repo.join("summary-prompt.txt");
18240 let author = &mut f.runner.state.config.agents[0];
18241 author.command = vec![
18242 "sh".into(),
18243 "-c".into(),
18244 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18245 ];
18246 author.env.insert(
18247 "PROMPT_FILE".into(),
18248 captured.to_string_lossy().into_owned(),
18249 );
18250 std::fs::write(
18251 &f.reply,
18252 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18253 )
18254 .unwrap();
18255 let m = posted(&mut f).await;
18256 let prompt = std::fs::read_to_string(captured).unwrap();
18257 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18258 assert!(!prompt.contains("in English"));
18259 assert_eq!(m.title, "feat(own): ファイルを追加");
18260 assert!(m.body.contains("own.txt を追加します。"));
18261 }
18262
18263 #[tokio::test]
18264 async fn review_fixes_come_after_the_whole_change_summary() {
18265 let mut f = summary_fixture();
18266 f.runner.state.reviews = vec![
18267 round_with_notes(1, Some("handled the PATH case")),
18268 round_with_notes(2, None),
18269 ];
18270 let m = posted(&mut f).await;
18271 let summary = m.body.find("## Summary").expect("summary");
18272 let fixes = m.body.find("## Review rounds").expect("review rounds");
18273 assert!(summary < fixes, "{}", m.body);
18274 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18275 assert!(m.body.contains("## Final verification"), "{}", m.body);
18276 }
18277
18278 #[tokio::test]
18279 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18280 let mut f = summary_fixture();
18281 let first = posted(&mut f).await;
18282 let first_head = f.runner.state.change_summary.clone().unwrap().head;
18283 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18285 let again = posted(&mut f).await;
18286 assert_eq!((again.title, again.body), (first.title, first.body));
18287
18288 let g = |args: &[&str]| {
18289 let out = std::process::Command::new("git")
18290 .args(args)
18291 .current_dir(&f.repo)
18292 .quiet()
18293 .output()
18294 .expect("spawn git");
18295 assert!(out.status.success(), "git {args:?}");
18296 };
18297 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18298 g(&["add", "-A"]);
18299 g(&["commit", "-q", "-m", "fix(own): handle more"]);
18300 let second = posted(&mut f).await;
18301 let kept = f.runner.state.change_summary.clone().unwrap();
18302 assert_ne!(kept.head, first_head);
18303 assert!(second.body.contains("Not read yet."), "{}", second.body);
18304 assert!(!second.body.contains("so the first summary is read"));
18305 assert_eq!(second.title, "feat(own): other");
18306 }
18307
18308 #[tokio::test]
18309 async fn an_unavailable_summary_is_stated_and_never_invented() {
18310 let mut f = summary_fixture();
18311 std::fs::write(&f.reply, "no title line here").unwrap();
18312 let m = posted(&mut f).await;
18313 assert!(
18314 m.body
18315 .contains("No description of what this change does could be generated"),
18316 "{}",
18317 m.body
18318 );
18319 assert!(m.body.contains("internal reports remain in local records"));
18320 assert!(f.runner.state.change_summary.is_none());
18321 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18322 let winner = f.runner.state.candidates[0].clone();
18324 let preview = f
18325 .runner
18326 .guarded_pr_message(&winner, None, false)
18327 .await
18328 .unwrap()
18329 .unwrap();
18330 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18331 }
18332
18333 #[test]
18334 fn pr_message_review_single_commit_uses_its_subject() {
18335 let state = review_state(&["feat(nats): per-role user"]);
18336 let m = pr_message(&state, 'A');
18337 assert_eq!(m.title, "feat(nats): per-role user");
18338 assert!(!m.body.contains("Review the work already"), "{}", m.body);
18339 assert!(m.body.contains("## Commits under review"), "{}", m.body);
18340 }
18341
18342 #[test]
18343 fn pr_message_review_multi_commit_takes_the_oldest() {
18344 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18345 let m = pr_message(&state, 'A');
18346 assert_eq!(m.title, "feat: the change");
18347 for s in ["feat: the change", "fix: typo", "fix: again"] {
18348 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18349 }
18350 }
18351
18352 #[test]
18353 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18354 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18355 let mut state = review_state(&[first, "fix: later fixup"]);
18356 state.candidates[0].branch = "機能/ログイン".to_owned();
18357 let m = pr_message(&state, 'A');
18358 assert!(
18359 m.title.starts_with("chore: land candidate A of run"),
18360 "{}",
18361 m.title
18362 );
18363 }
18364 }
18365
18366 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18367 BranchFacts {
18368 commits: commits
18369 .iter()
18370 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18371 .collect(),
18372 stat: stat.to_owned(),
18373 }
18374 }
18375
18376 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18377 let mut r = review_round(true, 0, 1, 1, true, true);
18378 r.round = round;
18379 r.fix = notes.map(|n| FixRecord {
18380 agent: "fixer".to_owned(),
18381 addressed: Vec::new(),
18382 rejected: Vec::new(),
18383 notes: n.to_owned(),
18384 committed: true,
18385 failed: None,
18386 duration_ms: 0,
18387 continuation: None,
18388 });
18389 r
18390 }
18391
18392 #[test]
18393 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18394 let state = review_state(&["ignored"]);
18395 let f = facts(
18396 &[
18397 (
18398 "fix(login): resolve PATH on macOS",
18399 "Login shells skip rc files.",
18400 ),
18401 ("fix: address review", ""),
18402 ],
18403 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18404 );
18405 let m = pr_message_with(&state, 'A', Some(&f));
18406 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18407 assert!(
18408 m.body
18409 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
18410 "{}",
18411 m.body
18412 );
18413 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18414 assert!(m.body.contains("## Diff stat"), "{}", m.body);
18415 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18416 for banned in [
18417 "Review the work already",
18418 "no task statement",
18419 "Original task",
18420 ] {
18421 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18422 }
18423 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18424 }
18425
18426 #[test]
18427 fn pr_message_review_truncates_a_huge_first_commit_body() {
18428 let state = review_state(&["ignored"]);
18429 let f = facts(
18430 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18431 "s",
18432 );
18433 let m = pr_message_with(&state, 'A', Some(&f));
18434 assert!(m.body.len() < 30_000, "{}", m.body.len());
18435 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18436 assert!(
18437 m.body.contains("- ... 1 more commit(s)"),
18438 "{}",
18439 m.body.len()
18440 );
18441 assert!(m.body.ends_with("magi:candidate-a\n"));
18442 }
18443
18444 #[test]
18445 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18446 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18447 assert_eq!(m.title, "feat: x");
18448 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18449 assert!(!m.body.contains("Diff stat"), "{}", m.body);
18450 assert!(!m.body.contains("no task statement"), "{}", m.body);
18451 }
18452
18453 #[test]
18454 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18455 let mut state = review_state(&["日本語の件名"]);
18456 state.candidates[0].branch = "fix/macos-login-path".to_owned();
18457 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18458 state.candidates[0].branch = "機能/ログイン".to_owned();
18459 assert!(
18460 pr_message(&state, 'A')
18461 .title
18462 .starts_with("chore: land candidate A")
18463 );
18464 }
18465
18466 #[test]
18467 fn pr_message_review_fixes_survive_a_clean_final_round() {
18468 let mut state = review_state(&["feat: x"]);
18469 state.reviews = vec![
18470 round_with_notes(1, Some("handled the PATH case")),
18471 round_with_notes(2, None),
18472 ];
18473 let body = pr_message(&state, 'A').body;
18474 assert!(
18475 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18476 "{body}"
18477 );
18478 assert!(body.contains("### Round 1"), "{body}");
18479
18480 state.reviews = vec![
18481 round_with_notes(1, Some("first fix")),
18482 round_with_notes(2, Some("")),
18483 round_with_notes(3, Some("second fix")),
18484 round_with_notes(4, None),
18485 ];
18486 let body = pr_message(&state, 'A').body;
18487 assert!(
18488 body.contains("### Round 1") && !body.contains("first fix"),
18489 "{body}"
18490 );
18491 assert!(
18492 body.contains("### Round 3") && !body.contains("second fix"),
18493 "{body}"
18494 );
18495 assert!(body.contains("### Round 2"), "{body}");
18496 }
18497
18498 const FINAL: &str = "abcdef1234567890";
18499
18500 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18501 let mut r = round_with_notes(round, None);
18502 r.verified_head = verified.map(str::to_owned);
18503 r
18504 }
18505
18506 fn ok_gate(state: &mut RunState, head: Option<&str>) {
18507 state.gate = vec![CommandOutcome {
18508 command: "gate".to_owned(),
18509 code: Some(0),
18510 output_tail: String::new(),
18511 duration_ms: 0,
18512 resource_blocked: false,
18513 }];
18514 state.gate_ran = true;
18515 state.gate_head = head.map(str::to_owned);
18516 }
18517
18518 fn body_at(state: &RunState) -> String {
18519 pr_message_raw(
18520 state,
18521 'A',
18522 None,
18523 crate::pr_outcome::FinalHead::At(FINAL),
18524 ChangeNote::NotAsked,
18525 )
18526 .body
18527 }
18528
18529 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18530
18531 #[test]
18532 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18533 let mut state = review_state(&["feat: x"]);
18534 state.reviews = vec![
18535 round_with_notes(1, Some(SANDBOXED)),
18536 passing_round(2, Some(FINAL)),
18537 ];
18538 ok_gate(&mut state, Some(FINAL));
18539 let body = body_at(&state);
18540 assert!(!body.contains("Handled the PATH case."), "{body}");
18541 assert!(!body.contains("could not commit"), "{body}");
18542 assert!(!body.contains("could not run"), "{body}");
18543 assert!(
18544 body.contains("The fix is committed on the branch"),
18545 "{body}"
18546 );
18547 assert!(!body.contains("sandbox"), "{body}");
18548 assert!(
18549 body.contains("Tests (`verify.e2e`): passed on the final commit"),
18550 "{body}"
18551 );
18552 assert!(
18553 body.contains("Gate (`verify.gate`): passed on the final commit"),
18554 "{body}"
18555 );
18556 assert!(
18558 state.reviews[0]
18559 .fix
18560 .as_ref()
18561 .unwrap()
18562 .notes
18563 .contains("could not commit")
18564 );
18565 }
18566
18567 #[test]
18568 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18569 let mut state = review_state(&["feat: x"]);
18570 let mut r1 = round_with_notes(1, Some(SANDBOXED));
18571 r1.fix.as_mut().unwrap().committed = false;
18572 let mut r2 = passing_round(2, Some(FINAL));
18573 r2.e2e[0].code = Some(1);
18574 state.reviews = vec![r1, r2];
18575 let body = body_at(&state);
18576 assert!(!body.contains("could not commit"), "{body}");
18577 assert!(!body.contains("could not run"), "{body}");
18578 assert!(body.contains("This round produced no commit"), "{body}");
18579 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18580 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18581 assert!(!body.contains("passed"), "{body}");
18582 }
18583
18584 #[test]
18585 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18586 let mut state = review_state(&["feat: x"]);
18587 state.reviews = vec![
18588 round_with_notes(1, Some(SANDBOXED)),
18589 passing_round(2, Some("0123456789abcdef")),
18590 ];
18591 ok_gate(&mut state, Some("0123456789abcdef"));
18592 let body = body_at(&state);
18593 assert!(!body.contains("HTTP tests could not run"), "{body}");
18594 assert!(!body.contains("could not commit"), "{body}");
18596 assert!(
18597 body.contains("passed on an earlier commit (`0123456`)"),
18598 "{body}"
18599 );
18600 assert!(!body.contains("passed on the final commit"), "{body}");
18601
18602 ok_gate(&mut state, None);
18604 let body = body_at(&state);
18605 assert!(
18606 body.contains("passed on a commit that was not recorded"),
18607 "{body}"
18608 );
18609 }
18610
18611 #[test]
18612 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18613 let state = state_with_summary(
18614 "add retries to the client",
18615 "TITLE: feat: retries\n\nDid it.",
18616 );
18617 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18618 assert_eq!(m.title, "feat: retries");
18619 assert!(
18620 m.body.contains("<summary>Original task</summary>"),
18621 "{}",
18622 m.body
18623 );
18624 assert!(!m.body.contains("Commits under review"), "{}", m.body);
18625 assert!(
18626 m.body
18627 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18628 "{}",
18629 m.body
18630 );
18631 }
18632
18633 #[test]
18634 fn pr_message_review_bounds_a_long_english_subject() {
18635 let long = format!("feat: {}", "word ".repeat(100));
18636 let m = pr_message(&review_state(&[&long]), 'A');
18637 assert!(m.title.starts_with("feat: word"), "{}", m.title);
18638 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18639 }
18640
18641 #[test]
18642 fn pr_message_implementation_run_is_unchanged_by_review_support() {
18643 let state = state_with_summary("add retries\n\ndetails", "- did some things");
18644 let m = pr_message(&state, 'A');
18645 assert_eq!(m.title, "add retries");
18646 assert!(m.body.contains("<summary>Original task</summary>"));
18647 assert!(!m.body.contains("Commits under review"));
18648 assert_eq!(landing_subject_source(&state), state.instruction);
18649 }
18650
18651 #[test]
18652 fn review_run_squash_subject_is_the_change_not_the_prompt() {
18653 let state = review_state(&["feat: the change", "fix: typo"]);
18654 let source = landing_subject_source(&state);
18655 assert_eq!(land::merge_subject("", &source), "feat: the change");
18656 assert_eq!(
18657 land::merge_subject("magi: candidate A (uncommitted work)", &source),
18658 "feat: the change"
18659 );
18660 assert_eq!(
18662 land::merge_subject("feat: renamed by hand", &source),
18663 "feat: renamed by hand"
18664 );
18665 let blank = review_state(&["日本語"]);
18666 assert!(
18667 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18668 );
18669 }
18670
18671 #[test]
18672 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18673 let state = review_state(&["feat: the change"]);
18674 let source = landing_subject_source(&state);
18675 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18676 assert_eq!(
18677 land::merge_subject(landing_title(&state, old), &source),
18678 "feat: the change"
18679 );
18680 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18681 let task = state_with_summary("add retries", "");
18682 assert_eq!(landing_title(&task, old), old);
18683 }
18684
18685 #[test]
18686 fn pr_message_describes_the_change_not_the_task() {
18687 let state = state_with_summary(
18688 "今回やってほしいこと: results projector を直す",
18689 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18690 );
18691 let m = pr_message(&state, 'A');
18692 assert_eq!(m.title, "fix(web): batch the runs list reads");
18693 assert!(
18694 m.body.starts_with("## Summary\n\n- reads run.json once"),
18695 "{}",
18696 m.body
18697 );
18698 assert!(!m.body.contains("TITLE:"), "{}", m.body);
18699 let task_at = m.body.find("今回やってほしいこと").unwrap();
18700 let details_at = m.body.find("<details>").unwrap();
18701 assert!(
18702 details_at < task_at,
18703 "the task lives inside <details>: {}",
18704 m.body
18705 );
18706 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18707 assert!(m.body.contains("magi:candidate-a"));
18708 }
18709
18710 #[test]
18711 fn pr_message_falls_back_to_the_task_without_a_title_line() {
18712 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18713 let m = pr_message(&state, 'A');
18714 assert_eq!(m.title, "add retries");
18715 assert!(
18716 m.body.contains("## Summary\n\n- did some things"),
18717 "{}",
18718 m.body
18719 );
18720
18721 let none = RunState::new(
18722 PathBuf::from("/repo"),
18723 "main".to_owned(),
18724 "abc1234".to_owned(),
18725 "add retries".to_owned(),
18726 Config::default(),
18727 );
18728 let m = pr_message(&none, 'A');
18729 assert_eq!(m.title, "add retries");
18730 assert!(!m.body.contains("## Summary"), "{}", m.body);
18731 }
18732
18733 #[test]
18734 fn pr_message_refuses_the_candidate_commit_subject() {
18735 for bad in [
18736 "TITLE: magi: candidate A (uncommitted work)",
18737 "TITLE: chore: stuff (uncommitted work)",
18738 "TITLE: ",
18739 ] {
18740 let state = state_with_summary("add retries", bad);
18741 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18742 }
18743 }
18744
18745 #[test]
18746 fn pr_message_bounds_a_very_long_task_and_title() {
18747 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18748 let state = state_with_summary(&long, "- nothing");
18749 let m = pr_message(&state, 'A');
18750 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18751 assert!(!m.title.contains('\n'));
18752
18753 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18754 let m = pr_message(&state, 'A');
18755 assert!(m.title.starts_with("feat: "));
18756 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18757 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18758 }
18759
18760 fn long_title_of(instruction: &str) -> String {
18761 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18762 }
18763
18764 #[test]
18765 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18766 let first = "Make the landing path keep a readable title for long tasks";
18767 let line = format!(
18768 "{first}. {}",
18769 "And then keep going with more words ".repeat(20)
18770 );
18771 let t = long_title_of(&line);
18772 assert_eq!(t, first);
18773 assert!(!t.starts_with("chore: land"));
18774 }
18775
18776 #[test]
18777 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18778 let line = "word ".repeat(200);
18779 let t = long_title_of(&line);
18780 assert!(t.ends_with("word..."), "{t}");
18781 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18782 }
18783
18784 #[test]
18785 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18786 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18787 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18788 }
18789 }
18790
18791 #[test]
18792 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18793 let mut state = state_with_summary("再試行を追加する", "- nothing");
18794 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18795 state.config.graph.github_language = "ja".into();
18796 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18797 }
18798
18799 #[test]
18800 fn pr_message_title_limit_is_exact() {
18801 let at = "a".repeat(PR_TITLE_MAX);
18802 assert_eq!(long_title_of(&at), at);
18803 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18804 assert!(over.ends_with("..."), "{over}");
18805 assert_eq!(over.chars().count(), PR_TITLE_MAX);
18806 }
18807
18808 #[test]
18809 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18810 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18811 let t = long_title_of(&line);
18812 assert!(t.ends_with("..."), "{t}");
18813 assert!(t.is_ascii(), "{t}");
18814 }
18815
18816 #[test]
18817 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18818 let line = format!(
18819 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18820 "plus more filler words ".repeat(20)
18821 );
18822 let t = long_title_of(&line);
18823 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18824 }
18825
18826 #[test]
18827 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18828 let line = format!("\n\n# {}", "title words ".repeat(40));
18829 let state = state_with_summary(&line, "- nothing");
18830 let m = pr_message(&state, 'A');
18831 assert!(m.title.starts_with("title words"), "{}", m.title);
18832 assert_eq!(
18833 land::merge_subject(&m.title, &landing_subject_source(&state)),
18834 m.title
18835 );
18836 assert_eq!(
18838 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18839 "feat: renamed by hand"
18840 );
18841 }
18842
18843 #[tokio::test]
18844 async fn github_text_rewrite_is_bounded_and_falls_back() {
18845 crate::run::pin_test_home();
18846 for (reply, accepted) in [
18847 (
18848 "TITLE: fix: retries\nAdd retries for failed requests.",
18849 true,
18850 ),
18851 (
18852 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18853 false,
18854 ),
18855 (
18856 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18857 true,
18858 ),
18859 (
18860 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18861 true,
18862 ),
18863 (
18864 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18865 true,
18866 ),
18867 (
18868 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18869 false,
18870 ),
18871 ("TITLE: fix: retries\n", false),
18872 ] {
18873 let dir = tempfile::tempdir().unwrap();
18874 let mut runner = runner_at(RunStatus::Gating);
18875 runner.state = state_with_summary(
18876 "add retries",
18877 "TITLE: fix: retries\n日本語の説明を書きます。",
18878 );
18879 runner.state.repo = dir.path().to_owned();
18880 runner.state.candidates[0].worktree = dir.path().to_owned();
18881 let mut author = spec("alpha");
18882 author.command = vec![
18883 "sh".into(),
18884 "-c".into(),
18885 "printf '%s' \"$REWRITE_REPLY\"".into(),
18886 ];
18887 author.env.insert("REWRITE_REPLY".into(), reply.into());
18888 runner.state.config.agents = vec![author];
18889 seed_public_summary(&mut runner);
18890 let winner = runner.state.candidates[0].clone();
18891 let first = runner
18892 .guarded_pr_message(&winner, None, true)
18893 .await
18894 .unwrap();
18895 let message = if accepted {
18896 first.expect("a passing rewrite never asks")
18897 } else {
18898 assert!(first.is_none());
18900 assert!(runner.state.parked);
18901 let store = ask::Questions::open();
18902 let asked: Vec<_> = store
18903 .list()
18904 .into_iter()
18905 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18906 .collect();
18907 assert_eq!(asked.len(), 1);
18908 assert!(!asked[0].detail.contains("secret"));
18909 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18910 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18911 assert_eq!(
18912 asked[0]
18913 .choices
18914 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18915 rewrite.is_some()
18916 );
18917 store
18920 .update(&asked[0].id, |q| {
18921 q.abandon("test");
18922 Ok(())
18923 })
18924 .unwrap();
18925 let again = runner
18926 .guarded_pr_message(&winner, None, true)
18927 .await
18928 .unwrap()
18929 .expect("an abandoned question falls back");
18930 let n = store
18931 .list()
18932 .into_iter()
18933 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18934 .count();
18935 assert_eq!(n, 1, "asked once per rejected text");
18936 again
18937 };
18938 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18939 assert!(!message.body.contains("secret"));
18940 assert!(!message.body.contains("prefix"));
18941 assert!(!message.title.contains("secret"));
18942 assert_eq!(
18943 message.body.contains("Add retries for failed requests."),
18944 accepted
18945 );
18946 assert_eq!(
18947 runner.state.seats["github-text-rewrite"].turns, 1,
18948 "only one rewrite invocation"
18949 );
18950 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18951 if accepted {
18952 assert!(runner.state.github_text.is_none());
18953 assert!(!runner.state.parked);
18954 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18955 assert!(message.body.contains("## Final verification"));
18956 assert!(message.body.contains("Evidence: final diff"));
18957 assert!(
18958 message
18959 .body
18960 .contains(&format!("magi:run/{}", runner.state.id))
18961 );
18962 }
18963 if !accepted {
18964 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18965 assert!(
18966 message
18967 .body
18968 .contains(&format!("magi:run/{}", runner.state.id))
18969 );
18970 }
18971 }
18972 }
18973
18974 fn scripted(id: &str, reply: &str) -> AgentSpec {
18975 let mut a = spec(id);
18976 a.command = vec![
18977 "sh".into(),
18978 "-c".into(),
18979 "printf '%s' \"$REWRITE_REPLY\"".into(),
18980 ];
18981 a.env.insert("REWRITE_REPLY".into(), reply.into());
18982 a
18983 }
18984
18985 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18986 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18987
18988 fn seed_public_summary(runner: &mut Runner) {
18990 let repo = &runner.state.repo;
18991 let g = |args: &[&str]| {
18992 let out = std::process::Command::new("git")
18993 .args(args)
18994 .current_dir(repo)
18995 .quiet()
18996 .output()
18997 .unwrap();
18998 assert!(out.status.success(), "git {args:?}");
18999 String::from_utf8_lossy(&out.stdout).trim().to_owned()
19000 };
19001 g(&["init", "-q", "-b", "main"]);
19002 g(&[
19003 "-c",
19004 "user.name=Test",
19005 "-c",
19006 "user.email=test@example.invalid",
19007 "commit",
19008 "-q",
19009 "--allow-empty",
19010 "-m",
19011 "feat: fixture",
19012 ]);
19013 let head = g(&["rev-parse", "HEAD"]);
19014 g(&["update-ref", "refs/remotes/origin/main", &head]);
19015 let evidence = crate::pr_outcome::summary_evidence(
19016 &runner.state,
19017 crate::pr_outcome::FinalHead::At(&head),
19018 );
19019 let generated = &runner.state.candidates[0].summary;
19020 runner.state.change_summary = Some(crate::run::ChangeSummary {
19021 base: head.clone(),
19022 head,
19023 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
19024 title: summary_title(generated).unwrap(),
19025 body: summary_without_title(generated),
19026 });
19027 }
19028
19029 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19030 let mut runner = runner_at(RunStatus::Gating);
19031 runner.state = state_with_summary("add retries", JA_SUMMARY);
19032 if review_only {
19033 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19034 }
19035 runner.state.repo = dir.to_owned();
19036 runner.state.candidates[0].worktree = dir.to_owned();
19037 runner.state.config.agents = agents;
19038 seed_public_summary(&mut runner);
19039 runner
19040 }
19041
19042 fn gate_questions(runner: &Runner) -> usize {
19043 ask::Questions::open()
19044 .list()
19045 .into_iter()
19046 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19047 .count()
19048 }
19049
19050 #[tokio::test]
19051 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19052 crate::run::pin_test_home();
19053 let dir = tempfile::tempdir().unwrap();
19054 let mut runner = rewrite_runner(
19055 dir.path(),
19056 true,
19057 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19058 );
19059 runner.state.config.roles.language_judge =
19060 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19061 let winner = runner.state.candidates[0].clone();
19062 let msg = runner
19063 .guarded_pr_message(&winner, None, true)
19064 .await
19065 .unwrap()
19066 .expect("an English rewrite posts without asking");
19067 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19068 assert!(msg.body.contains("<details>kept</details>"));
19069 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19070 assert_eq!(gate_questions(&runner), 0);
19071 assert!(runner.state.github_text.is_none());
19072 assert!(!runner.state.seats.contains_key("impl-A"));
19073 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19074 }
19075
19076 #[tokio::test]
19077 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19078 crate::run::pin_test_home();
19079 let dir = tempfile::tempdir().unwrap();
19080 let mut runner = rewrite_runner(
19081 dir.path(),
19082 false,
19083 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19084 );
19085 runner.state.config.roles.language_judge =
19086 Some(crate::config::AgentChoice::One("beta".into()));
19087 runner.state.seats.insert(
19088 "impl-A".into(),
19089 SeatState::new("impl-A", "alpha", runner.state.seed),
19090 );
19091 let winner = runner.state.candidates[0].clone();
19092 let msg = runner
19093 .guarded_pr_message(&winner, None, true)
19094 .await
19095 .unwrap()
19096 .expect("rewritten");
19097 assert!(msg.body.contains("<details>kept</details>"));
19098 assert_eq!(runner.state.seats["impl-A"].turns, 0);
19099 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19100 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19101 }
19102
19103 #[tokio::test]
19104 async fn rewrite_reuses_a_resumable_implementer_session() {
19105 crate::run::pin_test_home();
19106 let dir = tempfile::tempdir().unwrap();
19107 let mut runner = rewrite_runner(
19108 dir.path(),
19109 false,
19110 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
19111 );
19112 runner.state.config.roles.language_judge =
19113 Some(crate::config::AgentChoice::One("beta".into()));
19114 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19115 seat.turns = 1;
19116 runner.state.seats.insert("impl-A".into(), seat);
19117 let winner = runner.state.candidates[0].clone();
19118 runner
19119 .guarded_pr_message(&winner, None, true)
19120 .await
19121 .unwrap()
19122 .expect("rewritten");
19123 assert_eq!(runner.state.seats["impl-A"].turns, 2);
19124 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
19125 }
19126
19127 #[tokio::test]
19128 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
19129 crate::run::pin_test_home();
19132 let dir = tempfile::tempdir().unwrap();
19133 let mut exhausted = spec("alpha");
19134 exhausted.command = vec!["false".into()];
19135 let mut runner = rewrite_runner(
19136 dir.path(),
19137 false,
19138 vec![exhausted, scripted("beta", EN_REWRITE)],
19139 );
19140 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19142 "alpha".into(),
19143 "beta".into(),
19144 ]));
19145 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19146 seat.turns = 1;
19147 runner.state.seats.insert("impl-A".into(), seat);
19148 let winner = runner.state.candidates[0].clone();
19149 let msg = runner
19150 .guarded_pr_message(&winner, None, true)
19151 .await
19152 .unwrap()
19153 .expect("the role rewrites after the implementer fails");
19154 assert!(msg.body.contains("`src/net.rs`"));
19155 assert_eq!(
19156 runner.state.seats["impl-A"].turns, 2,
19157 "the implementer was tried once"
19158 );
19159 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19160 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19161 }
19162
19163 #[tokio::test]
19164 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
19165 crate::run::pin_test_home();
19166 let dir = tempfile::tempdir().unwrap();
19167 let mut broken = spec("bad");
19168 broken.command = vec!["false".into()];
19169 let mut runner = rewrite_runner(
19170 dir.path(),
19171 true,
19172 vec![
19173 broken,
19174 scripted("noformat", "Add retries."),
19175 scripted("good", EN_REWRITE),
19176 ],
19177 );
19178 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19179 "bad".into(),
19180 "noformat".into(),
19181 "good".into(),
19182 ]));
19183 let winner = runner.state.candidates[0].clone();
19184 let msg = runner
19185 .guarded_pr_message(&winner, None, true)
19186 .await
19187 .unwrap()
19188 .expect("the third agent rewrites");
19189 assert!(msg.body.contains("`src/net.rs`"));
19190 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
19191 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
19192 }
19193
19194 #[tokio::test]
19195 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
19196 crate::run::pin_test_home();
19197 for (agents, role, saved) in [
19199 (
19200 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
19201 Some("beta"),
19202 true,
19203 ),
19204 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
19205 (Vec::new(), None, false),
19206 ] {
19207 let dir = tempfile::tempdir().unwrap();
19208 let mut runner = rewrite_runner(dir.path(), true, agents);
19209 runner.state.config.roles.language_judge =
19210 role.map(|r| crate::config::AgentChoice::One(r.into()));
19211 let winner = runner.state.candidates[0].clone();
19212 let first = runner
19213 .guarded_pr_message(&winner, None, true)
19214 .await
19215 .unwrap();
19216 assert!(first.is_none(), "withheld text parks the run");
19217 assert!(runner.state.parked);
19218 assert_eq!(gate_questions(&runner), 1);
19219 let g = runner.state.github_text.as_ref().unwrap();
19220 assert_eq!(g.rewrite.is_some(), saved);
19221 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
19222 }
19223 }
19224
19225 #[tokio::test]
19226 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
19227 crate::run::pin_test_home();
19228 let dir = tempfile::tempdir().unwrap();
19229 let mut runner = runner_at(RunStatus::Gating);
19230 runner.state =
19231 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
19232 runner.state.repo = dir.path().to_owned();
19233 runner.state.candidates[0].worktree = dir.path().to_owned();
19234 let mut author = spec("alpha");
19235 author.command = vec![
19236 "sh".into(),
19237 "-c".into(),
19238 "printf '%s' \"$REWRITE_REPLY\"".into(),
19239 ];
19240 author.env.insert(
19241 "REWRITE_REPLY".into(),
19242 format!(
19243 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19244 "English sentence. ".repeat(600)
19245 ),
19246 );
19247 runner.state.config.agents = vec![author];
19248 seed_public_summary(&mut runner);
19249 let winner = runner.state.candidates[0].clone();
19250 assert!(
19251 runner
19252 .guarded_pr_message(&winner, None, true)
19253 .await
19254 .unwrap()
19255 .is_none()
19256 );
19257 let record = runner.state.github_text.clone().unwrap();
19258 let rewrite = record.rewrite.clone().unwrap();
19259 assert_eq!(rewrite.failed, ["body-language"]);
19260 assert!(
19261 rewrite
19262 .body
19263 .ends_with(&format!("magi:run/{}", runner.state.id))
19264 );
19265 let store = ask::Questions::open();
19266 let q = store.get(record.question.as_ref().unwrap()).unwrap();
19267 assert!(
19268 q.choices
19269 .contains(&crate::github_text::USE_REWRITE.to_owned())
19270 );
19271 for label in [
19272 "ORIGINAL",
19273 "FALLBACK",
19274 "REWRITE",
19275 "Rules still failed: body-language",
19276 "truncated",
19277 ] {
19278 assert!(q.detail.contains(label), "{label}");
19279 }
19280 let artifact = std::fs::read_to_string(crate::run::artifact_path(
19281 &runner.state,
19282 &crate::github_text::artifact_name(&record.fingerprint),
19283 ))
19284 .unwrap();
19285 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19286 runner.state =
19288 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19289 store
19290 .update(&q.id, |q| {
19291 q.say("まだ日本語")?;
19292 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19293 })
19294 .unwrap();
19295 assert!(
19296 runner
19297 .guarded_pr_message(&winner, None, true)
19298 .await
19299 .unwrap()
19300 .is_none()
19301 );
19302 let retry = runner.state.github_text.as_ref().unwrap();
19303 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19304 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19305 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19306 assert!(
19307 q.posting_comparison
19308 .as_ref()
19309 .unwrap()
19310 .matches_detail(&q.detail)
19311 );
19312 store
19313 .update(&q.id, |q| {
19314 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19315 })
19316 .unwrap();
19317 let message = runner
19318 .guarded_pr_message(&winner, None, true)
19319 .await
19320 .unwrap()
19321 .unwrap();
19322 assert_eq!(message.title, rewrite.title);
19323 assert_eq!(message.body, rewrite.body);
19324 assert_eq!(
19325 runner.state.github_text.as_ref().unwrap().choice,
19326 Some(GithubTextChoice::Rewrite)
19327 );
19328 let again = runner
19329 .guarded_pr_message(&winner, None, true)
19330 .await
19331 .unwrap()
19332 .unwrap();
19333 assert_eq!(again.title, message.title);
19334 assert_eq!(again.body, message.body);
19335 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19336 assert!(
19337 !runner
19338 .state
19339 .events
19340 .iter()
19341 .any(|e| e.message.contains(&rewrite.title)
19342 || e.message.contains("English sentence."))
19343 );
19344 }
19345
19346 #[tokio::test]
19347 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19348 crate::run::pin_test_home();
19349 let dir = tempfile::tempdir().unwrap();
19350 let mut runner = runner_at(RunStatus::Gating);
19351 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19352 runner.state.repo = dir.path().to_owned();
19353 runner.state.candidates[0].worktree = dir.path().to_owned();
19354 seed_public_summary(&mut runner);
19355 let winner = runner.state.candidates[0].clone();
19356 assert!(
19357 runner
19358 .guarded_pr_message(&winner, None, true)
19359 .await
19360 .unwrap()
19361 .is_none()
19362 );
19363 let record = runner.state.github_text.as_ref().unwrap();
19364 assert!(record.rewrite.is_none());
19365 let mut old = serde_json::to_value(record).unwrap();
19367 old.as_object_mut().unwrap().remove("rewrite");
19368 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19369 assert!(old.rewrite.is_none());
19370 let id = record.question.clone().unwrap();
19371 ask::Questions::open()
19372 .update(&id, |q| {
19373 q.choices.push(crate::github_text::USE_REWRITE.into());
19374 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19375 })
19376 .unwrap();
19377 let message = runner
19378 .guarded_pr_message(&winner, None, true)
19379 .await
19380 .unwrap()
19381 .unwrap();
19382 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19383 assert_eq!(
19384 runner.state.github_text.as_ref().unwrap().choice,
19385 Some(GithubTextChoice::Fallback)
19386 );
19387 }
19388
19389 #[tokio::test]
19390 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19391 crate::run::pin_test_home();
19392 let dir = tempfile::tempdir().unwrap();
19393 let mut runner = runner_at(RunStatus::Gating);
19394 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19395 runner.state.repo = dir.path().to_owned();
19396 runner.state.candidates[0].worktree = dir.path().to_owned();
19397 seed_public_summary(&mut runner);
19398 let winner = runner.state.candidates[0].clone();
19399 assert!(
19400 runner
19401 .guarded_pr_message(&winner, None, true)
19402 .await
19403 .unwrap()
19404 .is_none()
19405 );
19406 let store = ask::Questions::open();
19407 let find = |run: &str| {
19408 store
19409 .list()
19410 .into_iter()
19411 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19412 .unwrap()
19413 };
19414 let run = runner.state.id.clone();
19415 let q = find(&run);
19416 let fp = runner
19417 .state
19418 .github_text
19419 .as_ref()
19420 .unwrap()
19421 .fingerprint
19422 .clone();
19423 let name = crate::github_text::artifact_name(&fp);
19424 let path = crate::run::artifact_path(&runner.state, &name);
19425 assert!(q.detail.contains(&path.display().to_string()));
19426 assert!(q.detail.contains("title-language: title"));
19427 let saved = std::fs::read_to_string(&path).expect("artifact written");
19428 assert!(saved.contains("日本語のタイトル"));
19429 assert!(q.cwd.is_none());
19430 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19431 let say = |id: &str, text: &str, choice: &str| {
19433 store
19434 .update(id, |q| {
19435 q.thread.push(ask::Turn {
19436 who: ask::Who::Operator,
19437 body: text.to_owned(),
19438 at: jiff::Timestamp::now(),
19439 note: None,
19440 });
19441 q.answer(ask::Answer::Choice(choice.to_owned()))
19442 })
19443 .unwrap();
19444 };
19445 say(&q.id, "まだ日本語", "use my text");
19446 assert!(
19447 runner
19448 .guarded_pr_message(&winner, None, true)
19449 .await
19450 .unwrap()
19451 .is_none()
19452 );
19453 let second = find(&run);
19454 assert_ne!(second.id, q.id);
19455 assert!(second.detail.contains("did not pass"));
19456 assert!(second.detail.contains("日本語のタイトル"));
19459 assert!(!second.detail.contains("まだ日本語"));
19460 assert!(second.detail.contains(&path.display().to_string()));
19461 say(&second.id, "fix: retry failed requests", "use my text");
19462 let message = runner
19463 .guarded_pr_message(&winner, None, true)
19464 .await
19465 .unwrap()
19466 .expect("a vetted replacement is posted");
19467 assert_eq!(message.title, "fix: retry failed requests");
19468 assert_eq!(
19469 runner
19470 .state
19471 .github_text
19472 .as_ref()
19473 .unwrap()
19474 .chosen_title
19475 .as_deref(),
19476 Some("fix: retry failed requests")
19477 );
19478 assert!(runner.state.github_text.as_ref().unwrap().resolved);
19479 let again = runner
19481 .guarded_pr_message(&winner, None, true)
19482 .await
19483 .unwrap()
19484 .unwrap();
19485 assert_eq!(again.title, message.title);
19486 assert_eq!(
19487 store
19488 .list()
19489 .into_iter()
19490 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19491 .count(),
19492 2
19493 );
19494 }
19495
19496 #[test]
19497 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19498 let mut state = state_with_summary(
19502 "add retries",
19503 "TITLE: fix(web): batch reads\n- reads run.json once",
19504 );
19505 state.config.graph.language = "ja".to_owned();
19506 let m = pr_message(&state, 'A');
19507 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19508 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19509
19510 let task = "今回やってほしいこと: results projector を直す";
19513 let mut state = state_with_summary(task, "- no title line");
19514 state.config.graph.language = "ja".to_owned();
19515 let m = pr_message(&state, 'A');
19516 assert_eq!(
19517 m.title,
19518 format!("chore: land candidate A of run {}", state.id)
19519 );
19520 assert!(
19521 m.body.contains(&format!(
19522 "<summary>Original task</summary>\n\n{task}\n\n</details>"
19523 )),
19524 "{}",
19525 m.body
19526 );
19527 }
19528
19529 #[test]
19530 fn pr_message_scrubs_home_paths_and_addresses() {
19531 let state = state_with_summary(
19532 "fix it in /Users/someone/src/x",
19533 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19534 );
19535 let m = pr_message(&state, 'A');
19536 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19537 assert!(!m.body.contains(leak), "{}", m.body);
19538 }
19539 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19540 }
19541
19542 #[test]
19543 fn pr_message_survives_a_task_that_closes_details() {
19544 let state = state_with_summary("a </details> b", "TITLE: fix: x");
19545 let m = pr_message(&state, 'A');
19546 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19547 }
19548
19549 #[test]
19550 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19551 let cmd = manual_merge_command(
19552 MergeStyle::Squash,
19553 Path::new("/repo"),
19554 "b",
19555 "fix: \"quoted\" $(x) `y`\n\nbody",
19556 );
19557 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19558 }
19559
19560 #[test]
19561 fn manual_merge_command_matches_the_configured_style() {
19562 let repo = Path::new("/repo");
19563 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19564
19565 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19566 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19567
19568 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19569 assert_eq!(
19570 squash,
19571 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19572 \"Merge magi run 0832 (candidate A)\""
19573 );
19574
19575 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19576 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19577 }
19578
19579 #[test]
19580 fn a_nudge_gets_a_quarter_of_the_budget() {
19581 assert_eq!(retry_budget(secs(1200), true), secs(300));
19583 assert_eq!(retry_budget(secs(3600), true), secs(900));
19584 }
19585
19586 #[test]
19587 fn a_resent_prompt_keeps_the_whole_budget() {
19588 assert_eq!(retry_budget(secs(1200), false), secs(1200));
19591 assert_eq!(retry_budget(secs(60), false), secs(60));
19592 }
19593
19594 #[test]
19595 fn the_floor_never_exceeds_the_original_budget() {
19596 assert_eq!(retry_budget(secs(60), true), secs(60));
19600 assert_eq!(retry_budget(secs(480), true), secs(120));
19601 assert_eq!(retry_budget(secs(0), true), secs(0));
19602 }
19603
19604 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19605 agent::CommandEvidence {
19606 id: "item1".to_owned(),
19607 description: "cargo test".to_owned(),
19608 exit_code,
19609 result_summary: String::new(),
19610 source: "codex".to_owned(),
19611 }
19612 }
19613
19614 #[test]
19615 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19616 assert!(!has_unconfirmed_command(&[]));
19620 }
19621
19622 #[test]
19623 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19624 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19628 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19629 assert!(!has_unconfirmed_command(&[
19630 evidence(Some(0)),
19631 evidence(Some(101))
19632 ]));
19633 }
19634
19635 #[test]
19636 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19637 assert!(has_unconfirmed_command(&[
19638 evidence(Some(0)),
19639 evidence(None)
19640 ]));
19641 }
19642
19643 #[test]
19644 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19645 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19646 assert_eq!(
19647 verified_noop_claim(true, &[], text).as_deref(),
19648 Some("already fixed by b32cfc4, on main.")
19649 );
19650 }
19651
19652 #[test]
19653 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19654 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19657 assert!(verified_noop_claim(false, &[], text).is_none());
19658 }
19659
19660 #[test]
19661 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19662 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19663 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19664 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19666 }
19667
19668 #[test]
19669 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19670 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19671 }
19672
19673 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19676 runner.state.candidates = shape
19677 .iter()
19678 .enumerate()
19679 .map(|(i, &(empty, verified))| Candidate {
19680 index: i,
19681 label: (b'A' + i as u8) as char,
19682 agent: "sonnet".to_owned(),
19683 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19684 worktree: PathBuf::from(format!("/wt/{i}")),
19685 summary: String::new(),
19686 stat: String::new(),
19687 files: 0,
19688 commits: 0,
19689 empty,
19690 failed: None,
19691 verified_noop: verified.map(str::to_owned),
19692 duration_ms: 0,
19693 folded: false,
19694 })
19695 .collect();
19696 }
19697
19698 #[test]
19699 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19700 ask_test_home();
19701 let mut runner = runner_at(RunStatus::Implementing);
19702 set_candidates(
19703 &mut runner,
19704 &[
19705 (true, Some("already on main at b32cfc4")),
19706 (true, Some("same fix, see the existing test")),
19707 ],
19708 );
19709
19710 runner
19711 .after_implement()
19712 .expect("a verified no-op is not an error");
19713
19714 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19715 }
19716
19717 #[test]
19718 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19719 ask_test_home();
19720 let mut runner = runner_at(RunStatus::Implementing);
19721 set_candidates(
19725 &mut runner,
19726 &[(true, Some("already on main at b32cfc4")), (true, None)],
19727 );
19728
19729 let err = runner
19730 .after_implement()
19731 .expect_err("an unverified empty candidate must still fail the run");
19732
19733 assert!(
19734 err.to_string().contains("no candidate produced a change"),
19735 "{err}"
19736 );
19737 assert_eq!(runner.state.status, RunStatus::Failed);
19738 }
19739
19740 #[test]
19741 fn after_implement_still_fails_an_ordinary_all_empty_run() {
19742 ask_test_home();
19743 let mut runner = runner_at(RunStatus::Implementing);
19744 set_candidates(&mut runner, &[(true, None), (true, None)]);
19745
19746 let err = runner
19747 .after_implement()
19748 .expect_err("no candidate declared anything; this is an ordinary failure");
19749
19750 assert!(
19751 err.to_string().contains("no candidate produced a change"),
19752 "{err}"
19753 );
19754 assert_eq!(runner.state.status, RunStatus::Failed);
19755 }
19756}
19757
19758#[derive(Debug)]
19761struct GraftRestoreFailed(String);
19762
19763impl std::fmt::Display for GraftRestoreFailed {
19764 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19765 write!(
19766 f,
19767 "could not restore the winner to its pre-graft commit: {}",
19768 self.0
19769 )
19770 }
19771}
19772
19773impl std::error::Error for GraftRestoreFailed {}
19774
19775#[cfg(test)]
19779mod resource_wait_tests {
19780 use super::*;
19781 use std::collections::BTreeMap;
19782
19783 fn sh(id: &str, script: &str) -> AgentSpec {
19784 AgentSpec {
19785 id: id.to_owned(),
19786 kind: crate::config::AgentKind::Command,
19787 model: None,
19788 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19789 extra_args: Vec::new(),
19790 env: BTreeMap::new(),
19791 prompt_delivery: None,
19792 max_turn_secs: None,
19793 }
19794 }
19795
19796 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19797 SeatJob {
19798 seat: SeatState::new("impl-A", &spec.id, 1),
19799 spec,
19800 cwd: dir.to_path_buf(),
19801 prompt: "do it".to_owned(),
19802 timeout,
19803 allow_write: true,
19804 sessions: false,
19805 artifacts: dir.join("art"),
19806 stem: "impl-A".to_owned(),
19807 handover: None,
19808 }
19809 }
19810
19811 fn state(resource_wait_secs: u64) -> RunState {
19812 let mut config = Config::default();
19813 config.graph.timeout_resource_wait = resource_wait_secs;
19814 RunState::new(
19815 PathBuf::from("/repo"),
19816 "main".to_owned(),
19817 "abc1234".to_owned(),
19818 "task".to_owned(),
19819 config,
19820 )
19821 }
19822
19823 fn hold(cache: &Path) -> crate::cache::Guard {
19824 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19825 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19826 crate::cache::AcquireOutcome::Acquired(g) => g,
19827 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19828 }
19829 }
19830
19831 async fn run(
19832 jobs: Vec<SeatJob>,
19833 sem: Arc<Semaphore>,
19834 cache: Option<&Path>,
19835 st: &mut RunState,
19836 ) -> Vec<(usize, SeatState, AgentOutcome)> {
19837 let prompts = Prompts::default();
19838 let ctx = WaveCtx {
19839 run: "r",
19840 node: "implement",
19841 prompts: &prompts,
19842 cache,
19843 round: None,
19844 carry_seats: false,
19845 pauses: None,
19846 };
19847 wave(jobs, sem, &ctx, st, 0).await
19848 }
19849
19850 #[tokio::test]
19851 async fn verification_cache_wait_preserves_the_command_budget() {
19852 crate::run::pin_test_home();
19853 let tmp = tempfile::tempdir().unwrap();
19854 let cache = tmp.path().join("target");
19855 let guard = hold(&cache);
19856 let release = tokio::spawn(async move {
19857 tokio::time::sleep(Duration::from_millis(100)).await;
19858 guard.release();
19859 });
19860 let mut st = state(10);
19861 let budget = Duration::from_millis(50);
19862 let wait = repository_wait(&st, None);
19863 let (outcomes, _) = with_cache_lease(
19864 &mut st,
19865 Some(&cache),
19866 "gate",
19867 "gate",
19868 tmp.path(),
19869 "abc1234",
19870 budget,
19871 "test gate",
19872 &wait,
19873 |_, available| async move {
19874 assert_eq!(
19875 available, budget,
19876 "contention must not shorten verification"
19877 );
19878 (Vec::new(), false, Vec::new())
19879 },
19880 )
19881 .await
19882 .unwrap();
19883 release.await.unwrap();
19884 assert!(outcomes.is_empty(), "{outcomes:?}");
19885 assert!(
19886 st.events
19887 .iter()
19888 .any(|event| event.message.contains("waiting for"))
19889 );
19890 assert!(st.resource_wait.is_none());
19891 hold(&cache).release();
19893 }
19894
19895 #[tokio::test]
19896 async fn verification_cache_wait_expires_without_starting_commands() {
19897 crate::run::pin_test_home();
19898 let tmp = tempfile::tempdir().unwrap();
19899 let cache = tmp.path().join("target");
19900 let _held = hold(&cache);
19901 let mut st = state(0);
19902 let wait = repository_wait(&st, None);
19903 let (outcomes, _) = with_cache_lease(
19904 &mut st,
19905 Some(&cache),
19906 "gate",
19907 "gate",
19908 tmp.path(),
19909 "abc1234",
19910 Duration::from_secs(60),
19911 "test gate",
19912 &wait,
19913 |_, _| async { panic!("commands must not start without the cache lease") },
19914 )
19915 .await
19916 .unwrap();
19917 assert_eq!(outcomes.len(), 1);
19918 assert!(outcomes[0].resource_blocked);
19919 assert!(st.resource_wait.is_none());
19920 assert!(matches!(
19921 st.resource_block.unwrap().cause,
19922 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19923 ));
19924 }
19925
19926 #[tokio::test]
19927 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19928 crate::run::pin_test_home();
19929 let tmp = tempfile::tempdir().unwrap();
19930 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19931 let cache = tmp.path().join("cache");
19932 let mut state = state(21600);
19933 let wait = repository_wait(&state, None);
19934 let (outcomes, _) = with_cache_lease(
19935 &mut state,
19936 Some(&cache),
19937 "gate",
19938 "gate",
19939 tmp.path(),
19940 "head",
19941 Duration::from_secs(60),
19942 "gate",
19943 &wait,
19944 |_, _| async { panic!("freshness failure must not start verification") },
19945 )
19946 .await
19947 .unwrap();
19948 assert_eq!(outcomes.len(), 1);
19949 assert!(outcomes[0].resource_blocked);
19950 let block = state.resource_block.unwrap();
19951 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19952 assert!(!block.description().contains("timed out"));
19953 }
19954
19955 #[tokio::test]
19956 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19957 crate::run::pin_test_home();
19958 let tmp = tempfile::tempdir().unwrap();
19959 let cache = tmp.path().join("cache");
19960 let _held = hold(&cache);
19961 let mut state = state(21600);
19962 let pause = Pause::new();
19963 let interrupt = Pause::new();
19964 let cause = ParkCause::Operator;
19965 pause.park_for(cause.clone());
19966 let prompts = Prompts::default();
19967 let ctx = WaveCtx {
19968 run: "test",
19969 node: "fix",
19970 prompts: &prompts,
19971 cache: Some(&cache),
19972 round: None,
19973 carry_seats: false,
19974 pauses: Some((&pause, &interrupt)),
19975 };
19976 let result = wave(
19977 vec![job(
19978 sh("writer", "exit 99"),
19979 tmp.path(),
19980 Duration::from_secs(1),
19981 )],
19982 Arc::new(Semaphore::new(1)),
19983 &ctx,
19984 &mut state,
19985 0,
19986 )
19987 .await;
19988 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19989 assert_eq!(result[0].1.turns, 0);
19990 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19991 assert_eq!(
19992 state.resource_block.unwrap().cause,
19993 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19994 );
19995 assert!(state.active.is_empty());
19996 }
19997
19998 #[tokio::test]
19999 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
20000 crate::run::pin_test_home();
20001 let tmp = tempfile::tempdir().unwrap();
20002 let cache = tmp.path().join("target-a");
20003 let guard = hold(&cache);
20004 tokio::spawn(async move {
20006 tokio::time::sleep(Duration::from_millis(1500)).await;
20007 guard.release();
20008 });
20009 let mut st = state(60);
20010 let results = run(
20011 vec![job(
20012 sh("alpha", "echo ok"),
20013 tmp.path(),
20014 Duration::from_secs(1),
20015 )],
20016 Arc::new(Semaphore::new(1)),
20017 Some(&cache),
20018 &mut st,
20019 )
20020 .await;
20021 assert!(
20022 matches!(results[0].2, AgentOutcome::Ok(_)),
20023 "the wait must not be charged to the agent"
20024 );
20025 assert!(st.resource_block.is_none());
20026 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20027 assert!(st.resource_wait.is_none() && st.active.is_empty());
20028 }
20029
20030 #[tokio::test]
20031 #[cfg(unix)]
20032 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20033 use std::os::fd::AsRawFd as _;
20034 crate::run::pin_test_home();
20035 let tmp = tempfile::tempdir().unwrap();
20036 crate::git::git(tmp.path(), &["init"]).await.unwrap();
20037 let common = tmp.path().join(".git");
20038 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20039 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20040 assert_eq!(
20041 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20042 0
20043 );
20044 let mut st = state(1);
20045 st.config.coordination = crate::config::test_coordination();
20046 let mut reader = job(
20047 sh("reader", "echo must-not-run"),
20048 tmp.path(),
20049 Duration::from_secs(30),
20050 );
20051 reader.allow_write = false;
20052 let results = run(
20053 vec![reader.clone()],
20054 Arc::new(Semaphore::new(1)),
20055 None,
20056 &mut st,
20057 )
20058 .await;
20059 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20060 assert_eq!(results[0].1.turns, 0);
20061 assert!(FailClass::of(&results[0].2).is_none());
20062 assert_eq!(
20063 st.resource_block.as_ref().unwrap().kind,
20064 "repository coordination"
20065 );
20066 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20067 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20068 drop(external);
20069 let editor = crate::repo_lock::SeatLease::for_checkout(
20070 tmp.path(),
20071 true,
20072 crate::config::test_coordination().as_ref(),
20073 )
20074 .await
20075 .unwrap();
20076 let pause = Pause::new();
20077 let interrupt = Pause::new();
20078 let prompts = Prompts::default();
20079 let ctx = WaveCtx {
20080 run: "r",
20081 node: "review",
20082 prompts: &prompts,
20083 cache: None,
20084 round: None,
20085 carry_seats: false,
20086 pauses: Some((&pause, &interrupt)),
20087 };
20088 let (results, ()) = tokio::join!(
20089 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20090 async {
20091 tokio::time::sleep(Duration::from_millis(50)).await;
20092 pause.park();
20093 }
20094 );
20095 assert!(
20096 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20097 );
20098 assert_eq!(results[0].1.turns, 0);
20099 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20100 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20101 drop(editor);
20102 }
20103
20104 #[tokio::test]
20105 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20106 crate::run::pin_test_home();
20107 let tmp = tempfile::tempdir().unwrap();
20108 let cache = tmp.path().join("target-b");
20109 let _held = hold(&cache);
20110 let mut st = state(1);
20111 let results = run(
20112 vec![job(
20113 sh("alpha", "echo ok"),
20114 tmp.path(),
20115 Duration::from_secs(1),
20116 )],
20117 Arc::new(Semaphore::new(1)),
20118 Some(&cache),
20119 &mut st,
20120 )
20121 .await;
20122 let (_, seat, out) = &results[0];
20123 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
20124 assert!(wave_blocked(&results));
20125 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
20126 assert_eq!(seat.turns, 0, "nothing was asked");
20127 assert!(
20128 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
20129 "no agent was spawned"
20130 );
20131 assert!(FailClass::of(out).is_none(), "not a failure class");
20132 let block = st.resource_block.as_ref().expect("recorded for the resume");
20133 assert_eq!(block.kind, "shared build cache");
20134 assert!(matches!(
20135 block.cause,
20136 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
20137 ));
20138 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20139 assert!(st.active.is_empty());
20140 }
20141
20142 #[tokio::test]
20143 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
20144 crate::run::pin_test_home();
20145 let tmp = tempfile::tempdir().unwrap();
20146 let mut st = state(1);
20147 let results = run(
20148 vec![job(
20149 sh("alpha", "echo ok"),
20150 tmp.path(),
20151 Duration::from_secs(1),
20152 )],
20153 Arc::new(Semaphore::new(0)),
20154 None,
20155 &mut st,
20156 )
20157 .await;
20158 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20159 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
20160 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
20161 }
20162
20163 #[tokio::test]
20164 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
20165 crate::run::pin_test_home();
20166 let tmp = tempfile::tempdir().unwrap();
20167 let mut st = state(60);
20168 let results = run(
20169 vec![job(
20170 sh("alpha", "sleep 30"),
20171 tmp.path(),
20172 Duration::from_secs(1),
20173 )],
20174 Arc::new(Semaphore::new(1)),
20175 None,
20176 &mut st,
20177 )
20178 .await;
20179 let out = &results[0].2;
20180 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
20181 let class = FailClass::of(out).expect("a failure");
20182 assert_eq!(class, FailClass::Timeout);
20183 assert!(should_hand_over(None, &class), "existing handover policy");
20184 assert!(st.resource_block.is_none());
20185 }
20186}