1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615impl Runner {
616 pub async fn start(
618 repo: &Path,
619 instruction: String,
620 config: Config,
621 origin: Origin,
622 ) -> Result<Self> {
623 Self::start_naming(repo, instruction, "", config, origin).await
624 }
625
626 pub async fn start_naming(
629 repo: &Path,
630 instruction: String,
631 _also_scan: &str,
632 config: Config,
633 origin: Origin,
634 ) -> Result<Self> {
635 let repo = git::toplevel(repo).await?;
636 let missing = agent::missing_programs(&config.agents);
637 if !missing.is_empty() {
638 bail!(
639 "these agent programs are not on PATH: {}. Fix the roster in \
640 magi.toml or install them.",
641 missing.join(", ")
642 );
643 }
644 let base_branch =
645 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646 .await?;
647 let fetched_base = resolve_base(
648 &repo,
649 &base_branch,
650 &config.merge.remote,
651 config.coordination.as_ref(),
652 )
653 .await?;
654 let base_commit = fetched_base.commit.clone();
655 let roles = config.resolve_roles()?;
656 let max_parallel = config.graph.max_parallel.max(1);
657 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658 state.fetched_base = Some(fetched_base);
662 state.origin = Some(origin);
663 state.reserve()?;
664 state.event("start", format!("run {} created", state.id));
665 state.save()?;
666 Ok(Self {
667 state,
668 roles,
669 sem: Arc::new(Semaphore::new(max_parallel)),
670 pause: Pause::new(),
671 interrupt: Pause::new(),
672 })
673 }
674
675 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689 Self::review_taking_over(repo, branch, config, None, origin).await
690 }
691
692 pub async fn review_taking_over(
699 repo: &Path,
700 branch: &str,
701 config: Config,
702 takeover: Option<crate::handover::Takeover>,
703 origin: Origin,
704 ) -> Result<Self> {
705 let repo = git::toplevel(repo).await?;
706 let missing = agent::missing_programs(&config.agents);
707 if !missing.is_empty() {
708 bail!(
709 "these agent programs are not on PATH: {}. Fix the roster in \
710 magi.toml or install them.",
711 missing.join(", ")
712 );
713 }
714 let base_branch =
715 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716 .await?;
717 if base_branch == branch {
718 bail!("`{branch}` is the base branch; there is nothing to review against");
719 }
720 let fetched_base = resolve_base(
721 &repo,
722 &base_branch,
723 &config.merge.remote,
724 config.coordination.as_ref(),
725 )
726 .await?;
727 let base_commit = fetched_base.commit.clone();
728
729 let roles = config.resolve_roles()?;
730 let max_parallel = config.graph.max_parallel.max(1);
731 let mut state = RunState::new(
732 repo.clone(),
733 base_branch,
734 base_commit.clone(),
735 String::new(),
736 config,
737 );
738 state.fetched_base = Some(fetched_base);
739 state.origin = Some(origin);
740 state.reserve()?;
741
742 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749 earlier: Vec::new(),
750 home: crate::run::home(),
751 choice: None,
752 task_instruction: None,
753 });
754 let wait = repository_wait(&state, None);
755 let coordination_config = state.config.coordination.clone();
756 let coordination =
757 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758 .await?;
759 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760 if let Some(released) = &released {
761 state.event(
762 "release",
763 format!(
764 "took `{branch}` over from run {}: its worktree was released: {}",
765 crate::run::short_of(&released.old_id),
766 released.audit
767 ),
768 );
769 }
770 if let Some(choice) = takeover.choice.as_ref()
774 && let Err(e) =
775 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
776 .await
777 {
778 if let Some(released) = &released {
779 released.restore(&repo, branch).await;
780 }
781 return Err(e.context("applying the owner's answer about the diverged branch"));
782 }
783 for id in takeover.earlier.iter().rev() {
789 if let Ok(old) = RunState::load_under(id, &takeover.home)
790 && let Some(head) = old.pr_head.clone()
791 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
792 {
793 state.pr_head = Some(head);
794 break;
795 }
796 }
797 drop(coordination);
798 let opened = Self::open_review(
799 &repo,
800 branch,
801 state,
802 roles,
803 max_parallel,
804 base_commit,
805 &takeover,
806 )
807 .await;
808 if opened.is_err()
809 && let Some(released) = &released
810 {
811 let _coordination = crate::repo_lock::SeatLease::operation_with(
812 &repo,
813 coordination_config.as_ref(),
814 &wait,
815 )
816 .await?;
817 released.restore(&repo, branch).await;
818 }
819 opened
820 }
821
822 pub async fn recover_pr(
824 repo: &Path,
825 task: &str,
826 selection: crate::pr_recovery::Selection,
827 mut config: Config,
828 ) -> Result<Self> {
829 selection.configure(&mut config)?;
830 selection.check_task(task, &selection.instruction)?;
831 let base_branch =
832 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
833 .await?;
834 let fetched_base = resolve_base(
835 repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842 selection
843 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
844 .await?;
845 Box::pin(selection.verify_source(repo)).await?;
846 if let Some(proof) = &selection.source_proof {
847 anyhow::ensure!(
848 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
849 "source rebase base is outside the fetched landing base"
850 );
851 }
852 let roles = config.resolve_roles()?;
853 let max_parallel = config.graph.max_parallel.max(1);
854 let mut state = RunState::new(
855 repo.to_path_buf(),
856 base_branch,
857 base_commit.clone(),
858 selection.instruction.clone(),
859 config,
860 );
861 state.fetched_base = Some(fetched_base);
862 state.origin = Some(crate::run::Origin::queue(task));
863 state.pr_head = Some(selection.branch.clone());
864 state.pr_recovery = Some(selection.clone());
865 state.reserve()?;
866 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
867 state.discard_reserved();
869 return Err(e);
870 }
871 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
872 }
873
874 async fn open_selected(
875 repo: &Path,
876 state: RunState,
877 roles: ResolvedRoles,
878 max_parallel: usize,
879 base_commit: String,
880 selection: crate::pr_recovery::Selection,
881 ) -> Result<Self> {
882 let mut state = state;
883 let branch = state.branch_for('A');
884 let wait = repository_wait(&state, None);
885 let prepared = async {
886 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
887 git::git(repo, &["branch", &branch, selection.source()]).await?;
888 drop(coordination);
889 anyhow::Ok(())
890 }
891 .await;
892 if let Err(e) = prepared {
893 state.discard_reserved();
896 return Err(e);
897 }
898 state.event(
899 "recovery",
900 format!(
901 "selected implementation {} for {} (public lease {})",
902 selection.source(),
903 selection.url,
904 selection.head
905 ),
906 );
907 let takeover = crate::handover::Takeover {
908 earlier: Vec::new(),
909 home: crate::run::home(),
910 choice: None,
911 task_instruction: Some(selection.instruction.clone()),
912 };
913 if let Err(e) = state.save() {
914 state.discard_reserved();
915 return Err(e);
916 }
917 let id = state.id.clone();
918 let opened = Self::open_review(
919 repo,
920 &branch,
921 state,
922 roles,
923 max_parallel,
924 base_commit,
925 &takeover,
926 )
927 .await;
928 match opened {
929 Ok(runner) => Ok(runner),
930 Err(e) => {
931 if let Ok(mut partial) = RunState::load(&id) {
932 partial.status = RunStatus::Blocked;
933 partial.driver_exited = true;
934 partial.event(
935 "recovery",
936 format!("startup failed; retained source and candidate {branch}: {e:#}"),
937 );
938 partial.save()?;
939 }
940 Err(crate::pr_recovery::StartupFailed {
941 run: id,
942 cause: format!("{e:#}"),
943 }
944 .into())
945 }
946 }
947 }
948
949 async fn inherit(
954 repo: &Path,
955 branch: &str,
956 base_commit: &str,
957 takeover: &crate::handover::Takeover,
958 ) -> Option<crate::recovery::RecoveryRecord> {
959 use crate::recovery::{Relation, assess};
960 let newest = takeover.earlier.last()?;
961 let mut chosen = None;
962 for id in takeover.earlier.iter().rev() {
963 if let Ok(old) = RunState::load_under(id, &takeover.home)
964 && !old.reviews.is_empty()
965 && (old.winner().is_some_and(|w| w.branch == branch)
966 || old.pr_head.as_deref() == Some(branch))
967 {
968 chosen = Some(old);
969 break;
970 }
971 }
972 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
973 .await
974 .unwrap_or_default();
975 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
976 Some(last) if last.head == tip => Relation::Same,
977 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
978 _ => Relation::Diverged,
979 };
980 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
981 Some(assess(from, chosen.as_ref(), base_commit, relation))
982 }
983
984 async fn open_review(
987 repo: &Path,
988 branch: &str,
989 mut state: RunState,
990 roles: ResolvedRoles,
991 max_parallel: usize,
992 base_commit: String,
993 takeover: &crate::handover::Takeover,
994 ) -> Result<Self> {
995 let wait = repository_wait(&state, None);
996 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
997 if state.pr_recovery.is_none() {
998 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
999 }
1000 let start = review_base(
1003 repo,
1004 &state.config.merge.remote,
1005 &state.base_branch,
1006 &base_commit,
1007 branch,
1008 )
1009 .await;
1010 let log = git::log_oneline(repo, &start, branch)
1011 .await
1012 .unwrap_or_default();
1013 let log_text = if log.trim().is_empty() {
1014 "(no commit messages)"
1015 } else {
1016 log.trim()
1017 };
1018 let mut instruction = format!(
1019 "Review the work already on branch `{branch}`. There is no task \
1020 statement: what the change claims to do is whatever its commits \
1021 say.\n\n{log_text}"
1022 );
1023 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1027 if let Some(task) = takeover
1028 .task_instruction
1029 .as_deref()
1030 .filter(|t| !t.trim().is_empty())
1031 && rec.reviewed_head.is_some()
1032 {
1033 instruction = format!(
1034 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1035 task.trim()
1036 );
1037 rec.criteria_inherited = true;
1038 }
1039 state.event("recovery", rec.summary());
1040 state.recovery = Some(rec);
1041 }
1042 if let Some(selected) = &state.pr_recovery {
1043 instruction = selected.instruction.clone();
1044 }
1045 state.task_instruction = takeover.task_instruction.clone();
1046 state.instruction = instruction;
1047 state.reviewed_commits = Some(
1048 git::subjects(repo, &start, branch)
1049 .await
1050 .unwrap_or_default(),
1051 );
1052
1053 for id in &takeover.earlier {
1056 if let Ok(old) = RunState::load_under(id, &takeover.home)
1057 && old.repo == state.repo
1058 && (old.pr_head.as_deref() == Some(branch)
1059 || old.winner().is_some_and(|w| w.branch == branch))
1060 {
1061 for hold in old.branch_owner_holds {
1062 if !state.branch_owner_holds.contains(&hold) {
1063 state.event(
1064 "approval-policy",
1065 format!(
1066 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1067 hold.control.key(),
1068 hold.file,
1069 hold.head
1070 ),
1071 );
1072 state.branch_owner_holds.push(hold);
1073 }
1074 }
1075 }
1076 }
1077 let reviewed_head = git::rev_parse(repo, branch).await?;
1078 state.observe_branch_owner_holds(&reviewed_head).await;
1079
1080 let worktree = state.worktree_root().join("under-review");
1083 if let Some(parent) = worktree.parent() {
1084 tokio::fs::create_dir_all(parent).await.ok();
1085 }
1086 let path = worktree.to_string_lossy().to_string();
1087 git::git(repo, &["worktree", "add", &path, branch])
1088 .await
1089 .with_context(|| {
1090 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1091 })?;
1092
1093 let setup = match ensure_setup_config(&mut state, repo).await {
1094 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1095 Err(e) => Err(e),
1096 };
1097 if let Err(e) = setup {
1098 git::worktree_remove_clean(repo, &worktree).await.ok();
1099 return Err(e);
1100 }
1101 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1102 .await
1103 .unwrap_or(0);
1104 if commits == 0 {
1105 git::worktree_remove_clean(repo, &worktree).await.ok();
1106 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1107 }
1108 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1109 .await
1110 .map(|f| f.len())
1111 .unwrap_or(0);
1112 if files == 0
1113 && let (Ok(head_tree), Ok(base_tree)) = (
1114 git::tree_of(&worktree, "HEAD").await,
1115 git::tree_of(&worktree, &base_commit).await,
1116 )
1117 && head_tree == base_tree
1118 {
1119 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1120 git::worktree_remove_clean(repo, &worktree).await.ok();
1121 bail!(
1122 "`{branch}` at {} has a tree identical to base {}; this usually means \
1123 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1124 against `{}/{branch}`) rather than an empty change",
1125 short(&head),
1126 short(&base_commit),
1127 state.config.merge.remote
1128 );
1129 }
1130 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1131 .await
1132 .unwrap_or_default();
1133
1134 state.candidates.push(Candidate {
1135 index: 0,
1136 label: 'A',
1137 agent: EXISTING_BRANCH.to_owned(),
1140 branch: branch.to_owned(),
1141 worktree,
1142 summary: String::new(),
1143 stat,
1144 files,
1145 commits,
1146 empty: false,
1147 failed: None,
1148 verified_noop: None,
1149 duration_ms: 0,
1150 folded: false,
1151 });
1152 state.tally = Some(Tally {
1153 first_choice: BTreeMap::from([('A', 0)]),
1154 borda: BTreeMap::new(),
1155 winner: 'A',
1156 rankings: 0,
1157 unanimous_initial: false,
1158 deliberated: false,
1159 changed_votes: 0,
1160 unanimous_final: false,
1161 tie_break: None,
1162 judges: 0,
1166 present: 0,
1167 quorum: 0,
1168 met_quorum: true,
1169 uncontested: Some("review-only run: nothing competed".to_owned()),
1170 });
1171 state.status = RunStatus::Reviewing;
1172 state.prep_complete = Some(true);
1173 state.event(
1174 "start",
1175 format!(
1176 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1177 state.id
1178 ),
1179 );
1180 state.save()?;
1181 Ok(Self {
1182 state,
1183 roles,
1184 sem: Arc::new(Semaphore::new(max_parallel)),
1185 pause: Pause::new(),
1186 interrupt: Pause::new(),
1187 })
1188 }
1189
1190 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1193 Self::authorize_base_recovery_request(home, id, task, false).await
1194 }
1195
1196 pub(crate) async fn authorize_base_recovery_from_resumption(
1198 home: &Path,
1199 id: &str,
1200 task: &str,
1201 ) -> Result<()> {
1202 Self::authorize_base_recovery_request(home, id, task, true).await
1203 }
1204
1205 async fn authorize_base_recovery_request(
1206 home: &Path,
1207 id: &str,
1208 task: &str,
1209 resumption: bool,
1210 ) -> Result<()> {
1211 let _owner = crate::pr_head::owner_lock(home, id)?;
1212 let mut state = RunState::load_under(id, home)?;
1213 crate::pr_head::ensure_owner(&state)?;
1214 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1215 anyhow::ensure!(
1216 queued.runs.last().is_some_and(|run| run == id)
1217 && queued.status != crate::queue::TaskStatus::Done
1218 && !queued.fresh_start
1219 && queued.review_branch.is_none(),
1220 "run task ownership changed"
1221 );
1222 anyhow::ensure!(
1223 state
1224 .origin
1225 .as_ref()
1226 .and_then(|o| o.task.as_deref())
1227 .is_none_or(|owner| owner == task),
1228 "run belongs to another task"
1229 );
1230 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1231 anyhow::ensure!(
1232 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1233 "run is live or its ownership is unknown"
1234 );
1235 let _execution = crate::driver::maintenance(home, id)?;
1238 anyhow::ensure!(
1239 state.status.resumable()
1240 && state.released_to.is_none()
1241 && state.retired_to.is_none()
1242 && state.pr_taken_over_by.is_none(),
1243 "run ownership is final"
1244 );
1245 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1246 anyhow::ensure!(
1247 !state.pr_publication_pending
1248 && !crate::handover::landing_checkpoint(&state)
1249 && state.merge_decision.is_none(),
1250 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1251 );
1252 let winner = state.winner().context("run has no retained winner")?;
1253 anyhow::ensure!(
1254 git::is_clean(&winner.worktree).await?,
1255 "retained worktree has uncommitted changes"
1256 );
1257 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1258 anyhow::ensure!(
1259 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1260 "retained worktree does not match the winner branch"
1261 );
1262 state.record_base_sync_stop(&head);
1263 state.save()?;
1264 if resumption {
1265 anyhow::ensure!(
1266 crate::daemon::compatible_run_instruction(&state, &queued),
1267 "resumption instruction does not match the retained run"
1268 );
1269 let approval = queued
1270 .resumption
1271 .as_ref()
1272 .context("no operator resumption")?;
1273 anyhow::ensure!(
1274 approval.instruction == queued.instruction,
1275 "resumption approval does not match the current task instruction"
1276 );
1277 state.authorize_base_sync_resumption(task, approval)?;
1278 } else {
1279 state.authorize_base_sync(task)?;
1280 }
1281 state.event(
1282 "base-recovery",
1283 "owner authorized one additional base rebase; fresh verification required",
1284 );
1285 state.save()?;
1286 Ok(())
1287 }
1288
1289 pub fn resume(id: &str) -> Result<Self> {
1291 let state = RunState::load(id)?;
1292 crate::pr_head::ensure_owner(&state)?;
1293 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1294 bail!(
1295 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1296 );
1297 }
1298 if let Some(to) = &state.released_to {
1299 bail!(
1300 "run {} cannot be resumed: its worktree was released to run {}",
1301 state.short(),
1302 crate::run::short_of(to)
1303 );
1304 }
1305 let roles = state.config.resolve_roles()?;
1306 if let Some(id) = &state.config.graph.selected_implementer
1307 && state.candidates.iter().any(|c| c.agent != *id)
1308 {
1309 bail!(
1310 "replacement implementer pin does not match the saved candidates; task must stay held"
1311 );
1312 }
1313 let max_parallel = state.config.graph.max_parallel.max(1);
1314 Ok(Self {
1315 state,
1316 roles,
1317 sem: Arc::new(Semaphore::new(max_parallel)),
1318 pause: Pause::new(),
1319 interrupt: Pause::new(),
1320 })
1321 }
1322
1323 pub async fn execute(&mut self) -> Result<()> {
1330 if let Some(selection) = &self.state.pr_recovery
1331 && (selection.source_proof.is_some() || selection.source_run.is_some())
1332 {
1333 Box::pin(selection.verify_source(&self.state.repo)).await?;
1334 }
1335 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1338 || self.state.land_approval.is_some()
1339 || self.state.merge_decision.is_some()
1340 || !self.state.merge_repairs.is_empty()
1341 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1342 {
1343 let claim = FixClaim::acquire(&self.state.dir())?;
1344 let disk = RunState::load(&self.state.id)?;
1345 let checkpoint = |s: &RunState| {
1346 (
1347 s.merge_repairs.len(),
1348 s.merge_repairs
1349 .last()
1350 .map(|r| (r.question.clone(), r.phase)),
1351 s.land_approval
1352 .as_ref()
1353 .map(|a| (a.question.clone(), a.head.clone())),
1354 )
1355 };
1356 if checkpoint(&disk) != checkpoint(&self.state) {
1357 bail!(
1358 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1359 );
1360 }
1361 Some(claim)
1362 } else {
1363 None
1364 };
1365 let mut execution = self.claim_driver().await?;
1366 let runner = &mut *execution.runner;
1367 let wait = runner.repository_wait();
1368 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1371 let result = match result {
1372 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1373 record_repository_block(&mut runner.state, &e);
1374 runner.park_here()?;
1375 Ok(())
1376 }
1377 other => other,
1378 };
1379 execution.driver.finish(&mut runner.state).await?;
1380 let ended = if result.is_err() {
1381 Some({
1382 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1383 if let Err(error) = &result {
1384 notice.message = format!(
1385 "Run {} stopped with an error: {}",
1386 runner.state.id,
1387 error.to_string().lines().next().unwrap_or("unknown error")
1388 );
1389 }
1390 notice
1391 })
1392 } else {
1393 crate::notices::run_ended(&runner.state)
1394 };
1395 if let Some(notice) = ended {
1396 crate::notices::raise(notice);
1397 }
1398 result
1399 }
1400
1401 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1402 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1403 }
1404
1405 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1406 let wait = self.repository_wait();
1407 coordinate_operation(&mut self.state, cwd, &wait).await
1408 }
1409
1410 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1412 let owner = match crate::pr_head::owner_lock_wait(
1413 &crate::run::home(),
1414 &self.state.id,
1415 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1416 )
1417 .await
1418 {
1419 Ok(owner) => owner,
1420 Err(error) => {
1421 let detail = format!("worktree maintenance or ownership update: {error:#}");
1422 self.state.resource_block = Some(crate::run::ResourceBlock {
1426 cause: crate::run::ResourceBlockCause::Other,
1427 node: "start".into(),
1428 kind: "worktree maintenance".into(),
1429 detail: detail.clone(),
1430 at: Timestamp::now(),
1431 });
1432 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1433 }
1434 };
1435 let disk = RunState::load(&self.state.id)?;
1436 crate::pr_head::ensure_owner(&disk)?;
1437 let claimed =
1438 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1439 anyhow::ensure!(
1444 disk.liveness(claimed) != crate::run::Liveness::Live,
1445 "run already has a live driver; wait for its execution to end"
1446 );
1447 if let Some(to) = &disk.released_to {
1448 bail!(
1449 "run {} cannot continue: its worktree was released to run {}",
1450 self.state.short(),
1451 crate::run::short_of(to)
1452 );
1453 }
1454 anyhow::ensure!(
1455 serde_json::to_value(&disk.base_sync_recovery)?
1456 == serde_json::to_value(&self.state.base_sync_recovery)?,
1457 "base recovery changed since this run was loaded; reload the saved run"
1458 );
1459 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1460 let mut execution = DriverExecution {
1461 runner: self,
1462 driver,
1463 };
1464 let _owner = owner;
1467 let state = &mut execution.runner.state;
1468 state.parked = false;
1473 state.resource_wait = None;
1474 state.resource_block = None;
1475 state.clear_active();
1482 let pid = std::process::id();
1497 state.driver_pid = Some(pid);
1498 state.driver_started_at = crate::proc::process_started_at(pid);
1499 state.driver_exited = false;
1500 state.driver_claimed = Some(true);
1501 state.begin_pass();
1502 execution.driver.bind(state);
1503 state.save()?;
1504 execution.driver.record(state)?;
1505 Ok(execution)
1506 }
1507
1508 async fn execute_graph(&mut self) -> Result<()> {
1509 self.state.configure_recovery_resume()?;
1510 self.persist_candidates().await?;
1513 if self.state.base_sync_blocked() {
1516 self.sync_to_base().await?;
1517 if !self.state.base_sync_blocked()
1518 && self.state.status != RunStatus::AlreadyInBase
1519 && !self.resource_halt()?
1520 {
1521 self.finish_after_tally().await?;
1522 }
1523 return Ok(());
1524 }
1525 if self
1526 .state
1527 .merge_decision
1528 .as_ref()
1529 .is_some_and(|decision| !decision.closed)
1530 && self.state.status != RunStatus::Landing
1531 {
1532 if crate::merge_decision::resumable_hold(&self.state) {
1533 self.state.merge = Some(MergeOutcome {
1534 mode: MergeMode::Pr,
1535 ok: false,
1536 detail: self
1537 .state
1538 .merge_decision
1539 .as_ref()
1540 .unwrap()
1541 .target
1542 .pr
1543 .clone(),
1544 empty: false,
1545 });
1546 self.run_land().await?;
1547 self.settle_questions();
1548 return Ok(());
1549 }
1550 bail!(
1551 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1552 );
1553 }
1554 if self.state.config.graph.selected_implementer.is_some()
1555 && self.state.status == RunStatus::Blocked
1556 && self.state.tally.is_none()
1557 && self.state.candidate_preservation.is_empty()
1558 && self.state.interrupted_implementations.is_empty()
1559 {
1560 bail!(
1561 "selected implementation is held; explicitly release the owning task to retry it"
1562 );
1563 }
1564
1565 if self.state.merge_repairs.last().is_some_and(|r| {
1566 matches!(
1567 r.phase,
1568 crate::run::MergeRepairPhase::Requested
1569 | crate::run::MergeRepairPhase::Fixing
1570 | crate::run::MergeRepairPhase::Reviewing
1571 )
1572 }) {
1573 if self.repair_merge_review().await? {
1574 self.run_land().await?;
1575 }
1576 self.settle_questions();
1577 return Ok(());
1578 }
1579 if self.state.pr_publication_pending {
1582 self.merge().await?;
1583 if self.resource_halt()? {
1584 return Ok(());
1585 }
1586 self.state.save()?;
1587 self.settle_questions();
1588 return Ok(());
1589 }
1590 if self.state.status == RunStatus::Stalled {
1603 if self.recover_stall().await? {
1604 self.finish_after_tally().await?;
1605 } else {
1606 self.state.save()?;
1608 }
1609 return Ok(());
1610 }
1611 if self.state.status == RunStatus::Landing
1621 || crate::handover::landing_checkpoint(&self.state)
1622 {
1623 self.state.land_sync_pending = true;
1624 self.state.save()?;
1625 self.run_land().await?;
1626 self.settle_questions();
1631 return Ok(());
1632 }
1633 if self.state.github_text.is_some()
1638 && self.state.merge.is_none()
1639 && self.state.status == RunStatus::Gating
1640 {
1641 self.merge().await?;
1642 if self.resource_halt()? {
1643 return Ok(());
1644 }
1645 self.state.save()?;
1646 return Ok(());
1647 }
1648 if let Err(e) = self.prep().await {
1649 if e.is::<crate::repo_lock::ResourceBlocked>() {
1650 return Err(e);
1651 }
1652 self.state.status = RunStatus::Blocked;
1653 self.state
1654 .prep_failure
1655 .get_or_insert_with(|| format!("{e:#}"));
1656 self.state
1657 .event("prep", format!("preparation stopped: {e:#}"));
1658 self.state.save()?;
1659 return Err(e);
1660 }
1661 if self.park_here()? {
1662 return Ok(());
1663 }
1664 self.advise().await?;
1665 if self.park_here()? {
1666 return Ok(());
1667 }
1668 self.implement().await?;
1669 if self.park_here()? {
1670 return Ok(());
1671 }
1672 if self.state.status == RunStatus::VerifiedNoop {
1676 return Ok(());
1677 }
1678 self.judge().await?;
1679 if self.park_here()? {
1680 return Ok(());
1681 }
1682 self.deliberate().await?;
1683 if self.park_here()? {
1684 return Ok(());
1685 }
1686 self.vote().await?;
1687 if self.park_here()? {
1688 return Ok(());
1689 }
1690 self.tally()?;
1691 if self.state.status == RunStatus::Stalled {
1696 self.state.save()?;
1700 return Ok(());
1701 }
1702 self.finish_after_tally().await?;
1703 Ok(())
1704 }
1705
1706 fn stop_for_resource(&mut self) -> Result<()> {
1711 if self.pause.parked() || self.interrupt.parked() {
1714 self.park_here()?;
1715 return Ok(());
1716 }
1717 if self.state.resource_block.is_some() {
1718 self.state.status = RunStatus::Blocked;
1719 }
1720 self.state.save()
1721 }
1722
1723 fn resource_halt(&mut self) -> Result<bool> {
1725 if self.state.resource_block.is_none() {
1726 return Ok(false);
1727 }
1728 self.stop_for_resource()?;
1729 Ok(true)
1730 }
1731
1732 fn park_here(&mut self) -> Result<bool> {
1739 let parking = self.pause.parked() || self.interrupt.parked();
1740 if !parking && self.state.resource_block.is_some() {
1741 self.stop_for_resource()?;
1742 return Ok(true);
1743 }
1744 if !self.pause.parked() && !self.interrupt.parked() {
1749 return Ok(false);
1750 }
1751 if self.state.parked {
1752 return Ok(true);
1753 }
1754 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1755 if let Some(block) = &mut self.state.resource_block
1756 && matches!(
1757 block.cause,
1758 crate::run::ResourceBlockCause::Cancelled { .. }
1759 )
1760 {
1761 block.cause = crate::run::ResourceBlockCause::Cancelled {
1762 park: cause.clone(),
1763 };
1764 }
1765 let why = match cause.as_ref().map(ParkCause::label) {
1766 Some(reason) => format!(
1767 "parked after `{}` ({reason}) — resume to carry on from here",
1768 self.state.status.as_str()
1769 ),
1770 None => format!(
1771 "parked after `{}` — resume to carry on from here",
1772 self.state.status.as_str()
1773 ),
1774 };
1775 self.state.event("park", why);
1776 self.state
1777 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1778 self.state.parked = true;
1779 self.state.save()?;
1780 Ok(true)
1781 }
1782
1783 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1785 if self.state.prep_failure.is_none() {
1786 return Ok(());
1787 }
1788 let failed_at = self
1789 .state
1790 .events
1791 .iter()
1792 .rev()
1793 .find(|event| event.node == "prep")
1794 .map_or(self.state.updated_at, |event| event.at);
1795 anyhow::ensure!(
1796 approval_at > failed_at,
1797 "preparation retry requires a new explicit release after the failure"
1798 );
1799 self.state.prep_failure = None;
1800 self.state.prep_complete = Some(false);
1801 self.state
1802 .event("prep-retry", "operator approved retry of preparation");
1803 Ok(())
1804 }
1805
1806 pub fn on_pause(&mut self, pause: Pause) {
1808 self.pause = pause;
1809 }
1810
1811 pub fn watch_interrupt(&mut self, pause: Pause) {
1817 self.interrupt = pause;
1818 }
1819
1820 fn settle_questions(&mut self) {
1840 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1841 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1842 }
1843 }
1844
1845 async fn finish_after_tally(&mut self) -> Result<()> {
1848 self.graft().await?;
1851 if self.resource_halt()? {
1852 return Ok(());
1853 }
1854 self.fold_losers().await?;
1855 self.sync_to_base().await?;
1860 if self.hold_base_sync().await? || self.resource_halt()? {
1861 return Ok(());
1862 }
1863 if self.state.status == RunStatus::AlreadyInBase {
1864 return Ok(());
1865 }
1866 self.review_loop().await?;
1867 if self.resource_halt()? {
1868 return Ok(());
1869 }
1870 self.sync_to_base().await?;
1871 if self.hold_base_sync().await? || self.resource_halt()? {
1872 return Ok(());
1873 }
1874 if self.state.status == RunStatus::AlreadyInBase {
1875 return Ok(());
1876 }
1877 self.gate().await?;
1878 self.merge().await?;
1879 if self.resource_halt()? {
1880 return Ok(());
1881 }
1882 self.state.save()?;
1883 Ok(())
1884 }
1885
1886 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1893 let result = match ensure_setup_config(&mut self.state, repo).await {
1894 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1895 Err(e) => Err(e),
1896 };
1897 if let Err(e) = result {
1898 git::worktree_remove_clean(repo, wt).await.ok();
1899 if in_prep {
1900 self.rollback_prep(repo).await;
1901 }
1902 self.state.status = RunStatus::Blocked;
1903 self.state.event(
1904 "setup",
1905 format!("worktree setup failed in {}: {e:#}", wt.display()),
1906 );
1907 self.state.save()?;
1908 return Err(e);
1909 }
1910 Ok(())
1911 }
1912
1913 async fn rollback_prep(&mut self, repo: &Path) {
1915 let root = self.state.worktree_root();
1916 for c in std::mem::take(&mut self.state.candidates) {
1917 if matches!(
1918 git::worktree_remove_clean(repo, &c.worktree).await,
1919 Ok(true)
1920 ) {
1921 git::branch_delete(repo, &c.branch).await.ok();
1922 } else {
1923 self.state.candidates.push(c);
1924 }
1925 }
1926 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1927 for j in 1..=seats {
1928 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1929 .await
1930 .ok();
1931 }
1932 for k in 1..=self.state.config.graph.advisors {
1933 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1934 .await
1935 .ok();
1936 }
1937 }
1938
1939 async fn prep(&mut self) -> Result<()> {
1940 if let Some(reason) = &self.state.prep_failure {
1941 bail!(
1942 "candidate preparation previously failed: {reason}; correct the source before retrying"
1943 );
1944 }
1945 if self
1946 .state
1947 .prep_complete
1948 .unwrap_or(!self.state.candidates.is_empty())
1949 {
1950 return Ok(());
1951 }
1952 let repo = self.state.repo.clone();
1953 let _coordination = self.coordinate(&repo).await?;
1956 let base = self.state.base_commit.clone();
1957 let plan = self.state.sources.plan(&repo, &base).await?;
1958 self.rollback_prep(&repo).await;
1959 if !self.state.candidates.is_empty() {
1960 bail!(
1961 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1962 );
1963 }
1964 self.state.prep_complete = Some(false);
1965 self.state.status = RunStatus::Prep;
1966 self.state.save()?;
1967 let repo = self.state.repo.clone();
1968 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1969 let root = self.state.worktree_root();
1970 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1971
1972 let hooks_dir = self.state.dir().join("hooks");
1975 if self.state.config.blind.commit_msg_hook {
1976 std::fs::create_dir_all(&hooks_dir)
1977 .with_context(|| format!("create {}", hooks_dir.display()))?;
1978 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1979 let path = hooks_dir.join("commit-msg");
1980 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1981 make_executable(&path)?;
1982 git::acquire_worktree_config(&repo).await?;
1990 self.state.enabled_worktree_config = true;
1991 }
1992
1993 ensure_setup_config(&mut self.state, &repo).await?;
1996
1997 for (index, (spec, label)) in self
1998 .roles
1999 .implementers
2000 .clone()
2001 .into_iter()
2002 .zip(labels)
2003 .enumerate()
2004 {
2005 let branch = self.state.branch_for(label);
2006 let worktree = root.join(format!("cand-{label}"));
2007 self.state.event(
2011 "prep-intent",
2012 serde_json::json!({
2013 "branch": branch,
2014 "worktree": worktree,
2015 "start": start,
2016 "picks": plan.picks,
2017 })
2018 .to_string(),
2019 );
2020 self.state.save()?;
2021 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2022 if self.state.config.blind.commit_msg_hook {
2023 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2024 }
2025 git::local_exclude(&worktree, "/.magi/").await?;
2026 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2027 git::worktree_remove_clean(&repo, &worktree).await.ok();
2030 git::branch_delete(&repo, &branch).await.ok();
2031 self.rollback_prep(&repo).await;
2032 self.state.status = RunStatus::Blocked;
2033 self.state
2034 .event("prep", format!("worktree setup failed: {e:#}"));
2035 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2036 self.state.save()?;
2037 return Err(e);
2038 }
2039 self.state.candidates.push(Candidate {
2040 index,
2041 label,
2042 agent: spec.id.clone(),
2043 branch,
2044 worktree,
2045 summary: String::new(),
2046 stat: String::new(),
2047 files: 0,
2048 commits: 0,
2049 empty: false,
2050 failed: None,
2051 verified_noop: None,
2052 duration_ms: 0,
2053 folded: false,
2054 });
2055 self.state.save()?;
2056 for pick in &plan.picks {
2057 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2058 {
2059 self.state.status = RunStatus::Blocked;
2060 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2061 self.state
2062 .event("prep", format!("cannot apply referenced commit: {e}"));
2063 self.state.save()?;
2064 return Err(e);
2065 }
2066 }
2067 }
2068
2069 for j in 1..=self.roles.judges.len() {
2070 let wt = root.join(format!("judge-{j}"));
2071 if !wt.exists() {
2072 git::worktree_add_detached(&repo, &wt, &base).await?;
2073 self.setup_seat_worktree(&repo, &wt, true).await?;
2074 }
2075 }
2076
2077 if self.state.config.graph.advise {
2085 for k in 1..=self.state.config.graph.advisors {
2086 let wt = root.join(format!("advisor-{k}"));
2087 if !wt.exists() {
2088 git::worktree_add_detached(&repo, &wt, &base).await?;
2089 self.setup_seat_worktree(&repo, &wt, true).await?;
2090 }
2091 }
2092 }
2093
2094 let authors: Vec<&str> = self
2099 .roles
2100 .implementers
2101 .iter()
2102 .map(|a| a.id.as_str())
2103 .collect();
2104 let overlap: Vec<String> = self
2105 .roles
2106 .judges
2107 .iter()
2108 .enumerate()
2109 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2110 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2111 .collect();
2112 if !overlap.is_empty() {
2113 let note = format!(
2114 "{} also authored a candidate; blind, but the panel is less \
2115 independent than {} distinct agents would be",
2116 overlap.join(", "),
2117 self.roles.judges.len()
2118 );
2119 self.state.event("prep", note);
2120 }
2121
2122 self.state.event(
2123 "prep",
2124 format!(
2125 "{} candidates, {} judges, base {} ({})",
2126 self.state.candidates.len(),
2127 self.roles.judges.len(),
2128 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2129 self.state.base_branch
2130 ),
2131 );
2132 self.state.prep_complete = Some(true);
2133 self.state.status = RunStatus::Implementing;
2134 self.state.save()?;
2135 Ok(())
2136 }
2137
2138 async fn advise(&mut self) -> Result<()> {
2171 let implement_untouched = self
2172 .state
2173 .candidates
2174 .iter()
2175 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2176 if !self.state.config.graph.advise || self.state.advise_attempted {
2177 return Ok(());
2178 }
2179 if !implement_untouched {
2180 self.state.event(
2181 "advise",
2182 "skipping the design-deliberation stage: at least one \
2183 candidate already shows implementation progress, so this \
2184 run is past the point the stage exists to run before"
2185 .to_owned(),
2186 );
2187 self.state.advise_attempted = true;
2188 self.state.save()?;
2189 return Ok(());
2190 }
2191 let run_id = self.state.id.clone();
2192 let prompts = self.state.config.prompts.clone();
2193 let instruction = self.state.instruction.clone();
2194 let language = self.state.config.graph.language.clone();
2195 let root = self.state.worktree_root();
2196 let n = self.state.config.graph.advisors;
2197 let where_recorded = self.state.dir().join("run.json");
2198
2199 let seats = match self.state.config.advisors() {
2200 Ok(seats) if !seats.is_empty() => seats,
2201 Ok(_) => {
2202 self.state.event(
2203 "advise",
2204 format!(
2205 "[graph] advisors is 0; skipping the design-deliberation \
2206 stage and continuing without a synthesis brief (see {})",
2207 where_recorded.display()
2208 ),
2209 );
2210 self.state.advise_attempted = true;
2211 self.state.save()?;
2212 return Ok(());
2213 }
2214 Err(e) => {
2215 self.state.event(
2216 "advise",
2217 format!(
2218 "could not resolve advisor seats ({e:#}); continuing \
2219 without a design-deliberation brief (see {})",
2220 where_recorded.display()
2221 ),
2222 );
2223 self.state.advise_attempted = true;
2224 self.state.save()?;
2225 return Ok(());
2226 }
2227 };
2228
2229 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2230 let artifacts = agent::artifacts_dir(&self.state.dir());
2231 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2232
2233 let mut jobs = Vec::new();
2234 for (i, spec) in seats.iter().cloned().enumerate() {
2235 let seat_key = format!("advisor-{}", i + 1);
2236 let seat = self.seat(&seat_key, &spec.id);
2237 jobs.push(SeatJob {
2238 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2239 spec,
2240 seat,
2241 cwd: worktrees[i % worktrees.len()].clone(),
2242 timeout,
2243 allow_write: false,
2244 sessions: false,
2245 artifacts: artifacts.clone(),
2246 stem: seat_key,
2247 handover: None,
2248 });
2249 }
2250
2251 self.state.event(
2252 "advise",
2253 format!(
2254 "{} advisor seat(s) sketching a design in parallel",
2255 jobs.len()
2256 ),
2257 );
2258 let mut quota_losses = Vec::new();
2259 let cache = self.state.config.cache_dir();
2260 let ctx = WaveCtx {
2261 carry_seats: false,
2262 pauses: Some((&self.pause, &self.interrupt)),
2263 run: &run_id,
2264 node: "advise",
2265 prompts: &prompts,
2266 cache: cache.as_deref(),
2267 round: None,
2268 };
2269 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2270 let results = ask_json_wave::<Proposal>(
2271 jobs,
2272 Arc::clone(&self.sem),
2273 self.state.config.graph.retries,
2274 &advisor_roster,
2275 &ctx,
2276 &mut quota_losses,
2277 &mut self.state,
2278 &|p: &Proposal| p.validate(),
2279 )
2280 .await;
2281 if self.state.resource_block.is_some() {
2282 return self.stop_for_resource();
2283 }
2284 self.state.quota.extend(quota_losses);
2285
2286 let mut records = Vec::with_capacity(results.len());
2287 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2288 let agent_id = seat.agent.clone();
2289 self.state.seats.insert(seat.key.clone(), seat);
2290 match res {
2291 Ok((proposal, out)) => {
2292 self.state
2293 .event("advise", format!("advisor-{} proposed a design", i + 1));
2294 records.push(advise::AdvisorRecord::proposed(
2295 i + 1,
2296 agent_id,
2297 proposal,
2298 out.duration_ms,
2299 ));
2300 }
2301 Err(e) => {
2302 self.state.event(
2303 "advise",
2304 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2305 );
2306 records.push(advise::AdvisorRecord::failed(
2307 i + 1,
2308 agent_id,
2309 e.to_string(),
2310 ));
2311 }
2312 }
2313 }
2314
2315 let mut advice = advise::Advice {
2316 records,
2317 synthesis: None,
2318 };
2319 if advice.proposals().is_empty() {
2320 self.state.event(
2321 "advise",
2322 "no advisor produced a usable proposal; continuing without a \
2323 synthesis brief"
2324 .to_owned(),
2325 );
2326 } else {
2327 match self
2328 .synthesize_brief(
2329 &advice,
2330 &instruction,
2331 &language,
2332 &worktrees[0],
2333 &artifacts,
2334 &run_id,
2335 &prompts,
2336 cache.as_deref(),
2337 )
2338 .await
2339 {
2340 Ok(Some(text)) => {
2341 self.state.event(
2342 "advise",
2343 "synthesized a design brief for the implementer".to_owned(),
2344 );
2345 advice.synthesis = Some(text);
2346 }
2347 Ok(None) => {
2348 self.state.event(
2349 "advise",
2350 "the synthesis seat produced nothing usable; continuing \
2351 without a design brief"
2352 .to_owned(),
2353 );
2354 }
2355 Err(e) => {
2356 if e.is::<crate::repo_lock::ResourceBlocked>() {
2357 return Err(e);
2358 }
2359 self.state.event(
2360 "advise",
2361 format!("could not synthesize a design brief: {e:#}"),
2362 );
2363 }
2364 }
2365 }
2366 advise::apply_reflection(&mut advice);
2367
2368 self.state.advice = Some(advice);
2369 self.state.advise_attempted = true;
2370 self.state.save()?;
2371 Ok(())
2372 }
2373
2374 #[allow(clippy::too_many_arguments)]
2386 async fn synthesize_brief(
2387 &mut self,
2388 advice: &advise::Advice,
2389 instruction: &str,
2390 language: &str,
2391 cwd: &Path,
2392 artifacts: &Path,
2393 run_id: &str,
2394 prompts: &Prompts,
2395 cache: Option<&Path>,
2396 ) -> Result<Option<String>> {
2397 let chain = agent::pick_chain(
2398 &self.state.config.agents,
2399 self.state.config.roles.synthesizer.as_ref(),
2400 &agent::installed,
2401 "synthesizer",
2402 )?;
2403 let proposals = advice.proposals();
2404 let mut prompt = prompt::with_overlay(
2405 prompt::synthesize_brief(instruction, &proposals, language),
2406 prompts.overlay("advise"),
2407 );
2408 if cache.is_some() {
2409 prompt.push('\n');
2414 prompt.push_str(&prompt::build_cache_note("advise", false));
2415 }
2416 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2417 let mut last = None;
2422 for (n, spec) in chain.iter().enumerate() {
2423 if n > 0 {
2424 self.state
2425 .event("advise", format!("synthesis falling back to {}", spec.id));
2426 }
2427 let mut seat = self.seat("advise-synthesis", &spec.id);
2428 let wait = self.repository_wait();
2429 let outcome = wait
2430 .scope(agent::invoke(
2431 spec,
2432 &mut seat,
2433 &Invocation {
2434 coordination: self.state.config.coordination.as_ref(),
2435 fetched_base: None,
2436 cwd,
2437 prompt: &prompt,
2438 timeout,
2439 allow_write: false,
2440 sessions: false,
2441 artifacts,
2442 stem: &if n == 0 {
2443 "advise-synthesis".to_owned()
2444 } else {
2445 format!("advise-synthesis-{}", spec.id)
2446 },
2447 run: run_id,
2448 node: "advise",
2449 cache_dir: None,
2450 attachments: &[],
2451 writable: &[],
2452 },
2453 ))
2454 .await;
2455 if outcome.is_ok() {
2456 self.state.seats.insert(seat.key.clone(), seat);
2457 }
2458 let advance = agent::chain_advances(&outcome);
2459 last = Some(outcome);
2460 if !advance {
2461 break;
2462 }
2463 }
2464 let out = last.expect("a chain holds at least one agent")?;
2467 if !out.usable() {
2468 return Ok(None);
2469 }
2470 let text =
2471 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2472 Ok((!text.trim().is_empty()).then_some(text))
2473 }
2474
2475 async fn implement(&mut self) -> Result<()> {
2478 self.persist_candidates().await?;
2479 let run_id = self.state.id.clone();
2484 let prompts = self.state.config.prompts.clone();
2485 let todo: Vec<usize> = self
2486 .state
2487 .candidates
2488 .iter()
2489 .enumerate()
2490 .filter(|(_, c)| {
2491 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2492 && c.failed.is_none()
2493 && !c.empty
2494 })
2495 .map(|(i, _)| i)
2496 .collect();
2497 if todo.is_empty() {
2498 return self.after_implement();
2499 }
2500 self.state.status = RunStatus::Implementing;
2501
2502 let language = self.state.config.graph.language.clone();
2503 let github_language = self.state.config.graph.github_language.clone();
2504 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2505 let sessions = self.state.config.graph.sessions;
2506 let artifacts = agent::artifacts_dir(&self.state.dir());
2507 let brief = self
2511 .state
2512 .advice
2513 .as_ref()
2514 .and_then(|a| a.synthesis.as_deref())
2515 .map(str::to_owned);
2516 let attachments = self.state.attachments.clone();
2517
2518 let mut jobs = Vec::new();
2519 for &i in &todo {
2520 let (index, label, worktree) = {
2521 let c = &self.state.candidates[i];
2522 (c.index, c.label, c.worktree.clone())
2523 };
2524 let spec = self.roles.implementers[index].clone();
2525 let seat_key = format!("impl-{label}");
2526 let seat = self.seat(&seat_key, &spec.id);
2527 let instruction = seeded_instruction(&self.state);
2528 jobs.push(SeatJob {
2529 spec,
2530 seat,
2531 prompt: prompt::implement_in(
2532 &instruction,
2533 &worktree.to_string_lossy(),
2534 &language,
2535 &github_language,
2536 brief.as_deref(),
2537 &attachments,
2538 ),
2539 cwd: worktree,
2540 timeout,
2541 allow_write: true,
2542 sessions,
2543 artifacts: artifacts.clone(),
2544 stem: format!("impl-{label}"),
2545 handover: None,
2546 });
2547 }
2548
2549 self.state.event(
2550 "implement",
2551 format!("{} candidates in parallel", jobs.len()),
2552 );
2553 let mut sent = jobs.clone();
2559 let cache = self.state.config.cache_dir();
2560 let ctx = WaveCtx {
2561 carry_seats: false,
2562 pauses: Some((&self.pause, &self.interrupt)),
2563 run: &run_id,
2564 node: "implement",
2565 prompts: &prompts,
2566 cache: cache.as_deref(),
2567 round: None,
2568 };
2569 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2570 if !wave_blocked(&results) {
2571 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2572 .await;
2573 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2574 .await?;
2575 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2576 .await;
2577 }
2578 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2579
2580 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2581 && results
2582 .iter()
2583 .any(|(_, _, out)| FailClass::of(out).is_some());
2584 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2585 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2586 if self
2590 .state
2591 .candidate_preservation
2592 .contains_key(&self.state.candidates[i].label)
2593 {
2594 self.state.seats.insert(seat.key.clone(), seat);
2595 }
2596 continue;
2597 }
2598 let seat_key = seat.key.clone();
2599 let agent = seat.agent.clone();
2608 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2609 self.state.seats.insert(seat.key.clone(), seat);
2610 let label = self.state.candidates[i].label;
2611 self.state.interrupted_implementations.remove(&label);
2612
2613 let (summary, duration, failed, verified_claim) = match out {
2614 AgentOutcome::Ok(o) => {
2615 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2616 let failed = (!o.usable()).then(|| {
2617 if o.timed_out {
2618 "agent timed out".to_owned()
2619 } else {
2620 format!(
2621 "agent exited with {}",
2622 crate::notices::exit_status(o.exit_code)
2623 )
2624 }
2625 });
2626 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2627 (text, o.duration_ms, failed, verified_claim)
2628 }
2629 AgentOutcome::Dropped(o) => {
2635 let why = o
2636 .dropped
2637 .as_ref()
2638 .map(|d| d.why.as_str())
2639 .unwrap_or("the CLI ended the stream without delivering its answer");
2640 (
2641 String::new(),
2642 o.duration_ms,
2643 Some(format!("the CLI dropped the stream ({why})")),
2644 None,
2645 )
2646 }
2647 AgentOutcome::Quota(o) => {
2648 self.state.quota.push(QuotaLoss {
2649 seat: seat_key,
2650 node: "implement".to_owned(),
2651 at: Timestamp::now(),
2652 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2653 });
2654 (
2655 String::new(),
2656 o.duration_ms,
2657 Some("rate limited (quota); produced no change".to_owned()),
2658 None,
2659 )
2660 }
2661 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2662 (String::new(), 0, Some(e), None)
2663 }
2664 };
2665
2666 let c = &mut self.state.candidates[i];
2669 if !exhausted_the_fallback_chain {
2670 c.agent = agent;
2671 }
2672 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2673 c.duration_ms = duration;
2674 c.failed = failed;
2675 c.verified_noop = verified_claim;
2676 self.state
2677 .candidate_preservation
2678 .insert(label, "awaiting persistence".to_owned());
2679 }
2680 self.state.save()?;
2681 self.persist_candidates().await?;
2682
2683 if resource_blocked {
2684 return self.stop_for_resource();
2685 }
2686 if selected_failed {
2687 self.state.status = RunStatus::Blocked;
2688 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2689 self.state.save()?;
2690 bail!(
2691 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2692 );
2693 }
2694 self.after_implement()
2695 }
2696
2697 async fn persist_candidates(&mut self) -> Result<()> {
2699 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2700 for label in labels {
2701 let i = self
2702 .state
2703 .candidates
2704 .iter()
2705 .position(|c| c.label == label)
2706 .context("pending candidate is missing")?;
2707 let c = self.state.candidates[i].clone();
2708 let interrupted = self.state.interrupted_implementations.contains(&label);
2709 let result = async {
2710 let wait = crate::repo_lock::WaitPolicy::new(
2711 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2712 || false,
2713 );
2714 let _coordination =
2715 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2716 if let Err(e) = git::snapshot_uncommitted(
2719 &c.worktree,
2720 &self
2721 .state
2722 .dir()
2723 .join("artifacts")
2724 .join(format!("candidate-{label}-recovery")),
2725 )
2726 .await
2727 {
2728 self.state.event(
2729 "preserve",
2730 format!("candidate {label} snapshot incomplete: {e:#}"),
2731 );
2732 }
2733 let rescued = git::rescue_commit(
2734 &c.worktree,
2735 &format!("magi: candidate {label} (uncommitted work)"),
2736 )
2737 .await?;
2738 self.state.note_withheld("implement", &rescued.withheld);
2739 let base = &self.state.base_commit;
2740 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2741 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2742 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2743 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2744 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2745 if c.empty && commits > 0 && !patch.trim().is_empty()
2746 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2747 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2748 || self.state.tally.is_some())
2749 {
2750 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2754 }
2755 let c = &mut self.state.candidates[i];
2756 c.commits = commits;
2757 c.stat = stat;
2758 c.files = files;
2759 let no_changes = commits == 0 || patch.trim().is_empty();
2760 c.empty = no_changes && !interrupted;
2761 if interrupted && no_changes {
2762 c.failed = None;
2763 c.verified_noop = None;
2764 }
2765 if !c.empty {
2766 c.failed = None;
2767 c.verified_noop = None;
2768 }
2769 self.state.event(
2770 "preserve",
2771 format!("candidate {label}: {files} files, {commits} commits"),
2772 );
2773 Ok::<_, anyhow::Error>(())
2774 }
2775 .await;
2776 match result {
2777 Ok(()) => {
2778 self.state.candidate_preservation.remove(&label);
2779 if !interrupted || self.state.candidates[i].files > 0 {
2780 self.state.interrupted_implementations.remove(&label);
2781 }
2782 }
2783 Err(e) => {
2784 let reason = format!("candidate {label} preservation failed: {e:#}");
2785 self.state.event("preserve", &reason);
2786 self.state.candidate_preservation.insert(label, reason);
2787 }
2788 }
2789 if self.state.candidate_preservation.is_empty()
2790 && self.state.status == RunStatus::Blocked
2791 && self.state.candidates.iter().all(|c| c.failed.is_none())
2792 {
2793 self.state.status = RunStatus::Implementing;
2797 self.state.event(
2798 "preserve",
2799 "candidate persistence recovered; implementation can continue",
2800 );
2801 }
2802 self.state.save()?;
2803 }
2804 if !self.state.candidate_preservation.is_empty() {
2805 self.state.status = RunStatus::Blocked;
2806 self.state.save()?;
2807 bail!(
2808 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2809 );
2810 }
2811 Ok(())
2812 }
2813
2814 async fn resume_undelivered(
2842 &mut self,
2843 results: &mut [(usize, SeatState, AgentOutcome)],
2844 sent: &[SeatJob],
2845 prompts: &Prompts,
2846 run_id: &str,
2847 ) {
2848 for (wi, seat, out) in results.iter_mut() {
2849 let Some(dropped) = (match &*out {
2850 AgentOutcome::Dropped(o) => o.dropped.clone(),
2851 _ => None,
2852 }) else {
2853 continue;
2854 };
2855 let Some(job) = sent.get(*wi) else { continue };
2856 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2858 self.state.event(
2859 "implement",
2860 format!(
2861 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2862 work is in the tree",
2863 seat.key, dropped.output_tokens, dropped.why
2864 ),
2865 );
2866 continue;
2867 }
2868 if !has_context(&job.spec, seat, job.sessions) {
2876 self.state.event(
2877 "implement",
2878 format!(
2879 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2880 is no session left to resume",
2881 seat.key, dropped.output_tokens, dropped.why
2882 ),
2883 );
2884 continue;
2885 }
2886 self.state.event(
2887 "implement",
2888 format!(
2889 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2890 conversation",
2891 seat.key, dropped.output_tokens, dropped.why
2892 ),
2893 );
2894 let mut retry = job.clone();
2895 retry.seat = seat.clone();
2896 retry.prompt = prompt::resume_after_drop(&dropped.why);
2897 retry.timeout = retry_budget(job.timeout, true);
2898 retry.stem = format!("{}-resume", job.stem);
2899 let cache = self.state.config.cache_dir();
2900 let ctx = WaveCtx {
2901 carry_seats: false,
2902 pauses: Some((&self.pause, &self.interrupt)),
2903 run: run_id,
2904 node: "implement",
2905 prompts,
2906 cache: cache.as_deref(),
2907 round: None,
2908 };
2909 let (resumed_seat, resumed) =
2910 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2911 *seat = resumed_seat;
2912 *out = resumed;
2913 }
2914 }
2915
2916 async fn resume_seat_handovers(
2981 &mut self,
2982 results: &mut [(usize, SeatState, AgentOutcome)],
2983 sent: &mut [SeatJob],
2984 prompts: &Prompts,
2985 run_id: &str,
2986 ) -> Result<()> {
2987 let instruction = seeded_instruction(&self.state);
2988 let language = self.state.config.graph.language.clone();
2989 let github_language = self.state.config.graph.github_language.clone();
2990 let brief = self
2991 .state
2992 .advice
2993 .as_ref()
2994 .and_then(|a| a.synthesis.as_deref())
2995 .map(str::to_owned);
2996 let attachments = self.state.attachments.clone();
2997 for (wi, seat, out) in results.iter_mut() {
2998 let others: BTreeSet<String> = sent
3001 .iter()
3002 .enumerate()
3003 .filter(|(j, _)| j != wi)
3004 .map(|(_, j)| j.spec.id.clone())
3005 .collect();
3006 let Some(job) = sent.get_mut(*wi) else {
3007 continue;
3008 };
3009 let start = self
3014 .roles
3015 .implementer_roster
3016 .iter()
3017 .position(|s| s.id == job.spec.id)
3018 .unwrap_or(0);
3019 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3020 let mut fallback_attempt = 0usize;
3021 let mut prev: Option<FailClass> = None;
3022 while let Some(cur) = FailClass::of(&*out) {
3023 if !should_hand_over(prev.as_ref(), &cur) {
3024 break;
3025 }
3026 let Some(next) =
3027 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3028 .cloned()
3029 else {
3030 break;
3031 };
3032 tried.insert(next.id.clone());
3033 fallback_attempt += 1;
3034
3035 let coordination = self.coordinate(&job.cwd).await?;
3036 if let Ok(r) = git::rescue_commit(
3037 &job.cwd,
3038 &format!(
3039 "magi: candidate {} (uncommitted work before {} fallback)",
3040 seat.key,
3041 if cur == FailClass::Quota {
3042 "quota"
3043 } else {
3044 "handover"
3045 }
3046 ),
3047 )
3048 .await
3049 {
3050 self.state.note_withheld("implement", &r.withheld);
3051 }
3052 drop(coordination);
3053
3054 record_handover(
3055 &mut self.state,
3056 "implement",
3057 &seat.key,
3058 &seat.agent,
3059 &next.id,
3060 &cur,
3061 &fail_reason(&*out),
3062 );
3063 prev = Some(cur.clone());
3064
3065 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3066 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3067 job.spec = next.clone();
3075 let mut retry = job.clone();
3076 retry.seat = new_seat;
3077 retry.prompt = prompt::implement_in(
3078 &instruction,
3079 &job.cwd.to_string_lossy(),
3080 &language,
3081 &github_language,
3082 brief.as_deref(),
3083 &attachments,
3084 );
3085 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3086 let cache = self.state.config.cache_dir();
3087 let ctx = WaveCtx {
3088 carry_seats: false,
3089 pauses: Some((&self.pause, &self.interrupt)),
3090 run: run_id,
3091 node: "implement",
3092 prompts,
3093 cache: cache.as_deref(),
3094 round: None,
3095 };
3096 let (fallback_seat, fallback_out) = run_one(
3097 retry,
3098 Arc::clone(&self.sem),
3099 &ctx,
3100 &mut self.state,
3101 fallback_attempt,
3102 )
3103 .await;
3104 *seat = fallback_seat;
3105 *out = fallback_out;
3106 }
3107 }
3108 Ok(())
3109 }
3110
3111 async fn resume_unconfirmed_commands(
3135 &mut self,
3136 results: &mut [(usize, SeatState, AgentOutcome)],
3137 sent: &[SeatJob],
3138 prompts: &Prompts,
3139 run_id: &str,
3140 ) {
3141 for (wi, seat, out) in results.iter_mut() {
3142 let AgentOutcome::Ok(o) = &*out else {
3143 continue;
3144 };
3145 if !has_unconfirmed_command(&o.commands) {
3146 continue;
3147 }
3148 let Some(job) = sent.get(*wi) else { continue };
3149 if !has_context(&job.spec, seat, job.sessions) {
3150 self.state.event(
3151 "implement",
3152 format!(
3153 "{}: the reply named a command whose own CLI never confirmed the exit \
3154 status of, but there is no session left to resume",
3155 seat.key
3156 ),
3157 );
3158 continue;
3159 }
3160 self.state.event(
3161 "implement",
3162 format!(
3163 "{}: the reply named a command whose own CLI never confirmed the exit \
3164 status of; resuming the conversation",
3165 seat.key
3166 ),
3167 );
3168 let mut retry = job.clone();
3169 retry.seat = seat.clone();
3170 retry.prompt = prompt::resume_incomplete(
3171 "a command in your last reply had no confirmed exit status",
3172 );
3173 retry.timeout = retry_budget(job.timeout, true);
3174 retry.stem = format!("{}-confirm", job.stem);
3175 let cache = self.state.config.cache_dir();
3176 let ctx = WaveCtx {
3177 carry_seats: false,
3178 pauses: Some((&self.pause, &self.interrupt)),
3179 run: run_id,
3180 node: "implement",
3181 prompts,
3182 cache: cache.as_deref(),
3183 round: None,
3184 };
3185 let (resumed_seat, resumed) =
3186 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3187 *seat = resumed_seat;
3188 *out = resumed;
3189 }
3190 }
3191
3192 async fn continue_fix_report(
3213 &mut self,
3214 mut seat: SeatState,
3215 parse_err: String,
3216 job: &SeatJob,
3217 prompts: &Prompts,
3218 run_id: &str,
3219 round: usize,
3220 ) -> (
3221 SeatState,
3222 Option<FixReport>,
3223 Option<String>,
3224 ContinuationRecord,
3225 ) {
3226 let mut last_err = parse_err;
3227 let mut cumulative_wait_ms = 0u64;
3228 let mut attempts = 0usize;
3229 loop {
3230 if !has_context(&job.spec, &seat, job.sessions) {
3231 self.state.event(
3232 "fix",
3233 format!(
3234 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3235 session left to resume into"
3236 ),
3237 );
3238 let outcome = if attempts == 0 {
3239 ContinuationOutcome::NoSession
3240 } else {
3241 ContinuationOutcome::Exhausted
3242 };
3243 return (
3244 seat,
3245 None,
3246 Some(format!("unparsable fix report: {last_err}")),
3247 ContinuationRecord {
3248 attempts,
3249 cumulative_wait_ms,
3250 outcome,
3251 },
3252 );
3253 }
3254 if attempts >= MAX_FIX_CONTINUATIONS {
3255 self.state.event(
3256 "fix",
3257 format!(
3258 "round {round}: fixer's reply still had no adoption report after \
3259 {attempts} continuation(s) ({last_err}); giving up"
3260 ),
3261 );
3262 return (
3263 seat,
3264 None,
3265 Some(format!(
3266 "unparsable fix report after {attempts} continuation(s): {last_err}"
3267 )),
3268 ContinuationRecord {
3269 attempts,
3270 cumulative_wait_ms,
3271 outcome: ContinuationOutcome::Exhausted,
3272 },
3273 );
3274 }
3275 attempts += 1;
3276 self.state.event(
3277 "fix",
3278 format!(
3279 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3280 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3281 ),
3282 );
3283 let mut retry = job.clone();
3284 retry.seat = seat.clone();
3285 retry.prompt = prompt::resume_incomplete(&last_err);
3286 retry.timeout = retry_budget(job.timeout, true);
3287 retry.stem = format!("{}-continue{attempts}", job.stem);
3288 let cache = self.state.config.cache_dir();
3289 let ctx = WaveCtx {
3290 carry_seats: false,
3291 pauses: Some((&self.pause, &self.interrupt)),
3292 run: run_id,
3293 node: "fix",
3294 prompts,
3295 cache: cache.as_deref(),
3296 round: Some(round),
3297 };
3298 let (resumed_seat, resumed_out) = run_one(
3299 retry,
3300 Arc::clone(&self.sem),
3301 &ctx,
3302 &mut self.state,
3303 attempts,
3304 )
3305 .await;
3306 seat = resumed_seat;
3307 match resumed_out {
3308 AgentOutcome::Ok(o) => {
3309 cumulative_wait_ms += o.duration_ms;
3310 match verdict::extract_json::<FixReport>(&o.text) {
3311 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3312 self.state.event(
3313 "fix",
3314 format!(
3315 "round {round}: fixer's adoption report recovered after \
3316 {attempts} continuation(s)"
3317 ),
3318 );
3319 return (
3320 seat,
3321 Some(report),
3322 None,
3323 ContinuationRecord {
3324 attempts,
3325 cumulative_wait_ms,
3326 outcome: ContinuationOutcome::Resumed,
3327 },
3328 );
3329 }
3330 Ok(_) => {
3338 last_err = "the reply parsed, but it reported a command whose own CLI \
3339 never confirmed an exit status"
3340 .to_owned();
3341 }
3342 Err(e) => last_err = e.to_string(),
3343 }
3344 }
3345 AgentOutcome::Quota(o) => {
3346 cumulative_wait_ms += o.duration_ms;
3347 self.state.quota.push(QuotaLoss {
3348 seat: seat.key.clone(),
3349 node: "fix".to_owned(),
3350 at: Timestamp::now(),
3351 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3352 });
3353 self.state.event(
3354 "fix",
3355 format!(
3356 "round {round}: continuation rate limited (quota); not retrying now"
3357 ),
3358 );
3359 return (
3360 seat,
3361 None,
3362 Some("rate limited (quota) while recovering the fix report".to_owned()),
3363 ContinuationRecord {
3364 attempts,
3365 cumulative_wait_ms,
3366 outcome: ContinuationOutcome::QuotaLost,
3367 },
3368 );
3369 }
3370 AgentOutcome::Dropped(o) => {
3371 cumulative_wait_ms += o.duration_ms;
3372 let why = o
3373 .dropped
3374 .as_ref()
3375 .map(|d| d.why.as_str())
3376 .unwrap_or("the CLI ended the stream without delivering its answer");
3377 last_err = format!("the CLI dropped the stream ({why})");
3378 }
3379 AgentOutcome::Failed(e) => last_err = e,
3380 AgentOutcome::ResourceBlocked(e) => {
3383 return (
3384 seat,
3385 None,
3386 Some(format!("the continuation was not started: {e}")),
3387 ContinuationRecord {
3388 attempts: attempts.saturating_sub(1),
3389 cumulative_wait_ms,
3390 outcome: ContinuationOutcome::NoSession,
3391 },
3392 );
3393 }
3394 }
3395 }
3396 }
3397
3398 fn after_implement(&mut self) -> Result<()> {
3399 if !self.state.candidate_preservation.is_empty() {
3400 bail!("candidate preservation incomplete; judging is unsafe");
3401 }
3402 if self.state.leaks.is_empty() {
3404 let cfg = self.state.config.blind.clone();
3405 let mut leaks = Vec::new();
3406 for c in &self.state.candidates {
3407 let Some(patch) =
3408 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3409 else {
3410 continue;
3411 };
3412 leaks.extend(blind::scan(
3413 &format!("candidate {} patch", c.label),
3414 &patch,
3415 &cfg.vendor_tokens,
3416 ));
3417 }
3418 if !leaks.is_empty() {
3419 let summary = leaks
3420 .iter()
3421 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3422 .collect::<Vec<_>>()
3423 .join(", ");
3424 match cfg.on_leak {
3425 LeakPolicy::Fail => {
3426 self.state.status = RunStatus::Failed;
3427 self.state
3428 .event("blind", format!("vendor text in a patch: {summary}"));
3429 self.state.leaks = leaks;
3430 self.state.save()?;
3431 self.settle_questions();
3432 bail!(
3433 "blind.on_leak = \"fail\" and vendor text reached a \
3434 judged patch: {summary}"
3435 );
3436 }
3437 LeakPolicy::Redact => self.state.event(
3438 "blind",
3439 format!("redacting vendor text for judging: {summary}"),
3440 ),
3441 LeakPolicy::Warn => self.state.event(
3442 "blind",
3443 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3444 ),
3445 }
3446 self.state.leaks = leaks;
3447 }
3448 }
3449
3450 if self.state.viable().is_empty() {
3451 if self.state.all_candidates_verified_noop() {
3452 self.state.status = RunStatus::VerifiedNoop;
3463 self.state.save()?;
3464 self.settle_questions();
3465 return Ok(());
3466 }
3467 self.state.status = RunStatus::Failed;
3468 self.state.save()?;
3469 self.settle_questions();
3470 bail!("no candidate produced a change; nothing to judge");
3471 }
3472 self.state.status = RunStatus::Judging;
3473 self.state.save()?;
3474 Ok(())
3475 }
3476
3477 async fn judge(&mut self) -> Result<()> {
3480 let run_id = self.state.id.clone();
3485 let prompts = self.state.config.prompts.clone();
3486 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3487 return Ok(());
3488 }
3489 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3490 if viable.len() == 1 {
3491 self.state.judge_skipped = true;
3498 self.state.event(
3499 "judge",
3500 format!(
3501 "only candidate {} produced a change; judging skipped",
3502 viable[0].label
3503 ),
3504 );
3505 self.state.save()?;
3506 return Ok(());
3507 }
3508 self.state.status = RunStatus::Judging;
3509
3510 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3511 let language = self.state.config.graph.language.clone();
3512 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3513 let sessions = self.state.config.graph.sessions;
3514 let artifacts = agent::artifacts_dir(&self.state.dir());
3515 let root = self.state.worktree_root();
3516 let base_short = short(&self.state.base_commit);
3517
3518 let mut jobs = Vec::new();
3519 let mut orders = Vec::new();
3520 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3521 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3522 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3523 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3524 let seat_key = format!("judge-{}", j + 1);
3525 let seat = self.seat(&seat_key, &spec.id);
3526 jobs.push(SeatJob {
3527 prompt: prompt::judge(
3528 &self.state.instruction,
3529 &views,
3530 self.roles.judges.len(),
3531 &base_short,
3532 &language,
3533 ),
3534 spec,
3535 seat,
3536 cwd: root.join(format!("judge-{}", j + 1)),
3537 timeout,
3538 allow_write: false,
3539 sessions,
3540 artifacts: artifacts.clone(),
3541 stem: format!("judge-{}", j + 1),
3542 handover: None,
3543 });
3544 }
3545
3546 self.state.event(
3547 "judge",
3548 format!(
3549 "{} judges ranking {} candidates blind",
3550 jobs.len(),
3551 viable.len()
3552 ),
3553 );
3554 let labels_for_check = labels.clone();
3555 let mut quota_losses = Vec::new();
3556 let cache = self.state.config.cache_dir();
3557 let ctx = WaveCtx {
3558 carry_seats: false,
3559 pauses: Some((&self.pause, &self.interrupt)),
3560 run: &run_id,
3561 node: "judge",
3562 prompts: &prompts,
3563 cache: cache.as_deref(),
3564 round: None,
3565 };
3566 let results = ask_json_wave::<Ranking>(
3567 jobs,
3568 Arc::clone(&self.sem),
3569 self.state.config.graph.retries,
3570 &self.roles.judge_roster,
3571 &ctx,
3572 &mut quota_losses,
3573 &mut self.state,
3574 &move |r: &Ranking| r.validate(&labels_for_check),
3575 )
3576 .await;
3577 if self.state.resource_block.is_some() {
3578 return self.stop_for_resource();
3579 }
3580 self.state.quota.extend(quota_losses);
3581
3582 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3583 let agent_id = seat.agent.clone();
3584 self.state.seats.insert(seat.key.clone(), seat);
3585 let mut record = Judgement {
3586 judge: j + 1,
3587 seat: format!("judge-{}", j + 1),
3588 agent: agent_id,
3589 ranking: Vec::new(),
3590 reasons: BTreeMap::new(),
3591 confidence: None,
3592 strengths: BTreeMap::new(),
3593 order: orders[j].clone(),
3594 failed: None,
3595 duration_ms: 0,
3596 };
3597 match res {
3598 Ok((ranking, out)) => {
3599 record.ranking = ranking.normalized();
3600 record.reasons = ranking.reasons;
3601 record.strengths = ranking.strengths;
3602 record.confidence = ranking.confidence;
3603 record.duration_ms = out.duration_ms;
3604 self.state.event(
3605 "judge",
3606 format!(
3607 "judge {} ranked {}",
3608 j + 1,
3609 record.ranking.iter().collect::<String>()
3610 ),
3611 );
3612 }
3613 Err(e) => {
3614 record.failed = Some(e.to_string());
3615 self.state
3616 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3617 }
3618 }
3619 self.state.judgements.push(record);
3620 self.state.save()?;
3621 }
3622 Ok(())
3623 }
3624
3625 async fn deliberate(&mut self) -> Result<()> {
3628 if self.hold_base_sync().await? {
3629 return Ok(());
3630 }
3631 let run_id = self.state.id.clone();
3636 let prompts = self.state.config.prompts.clone();
3637 if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3638 return Ok(());
3639 }
3640 let tops: Vec<char> = self
3641 .state
3642 .judgements
3643 .iter()
3644 .filter_map(|j| j.ranking.first().copied())
3645 .collect();
3646 let rounds = self.state.config.graph.deliberate_rounds;
3647 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3648 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3649 self.state.event(
3650 "deliberate",
3651 format!("judges agreed on {} outright; no deliberation", tops[0]),
3652 );
3653 }
3654 self.state.status = RunStatus::Voting;
3655 self.state.save()?;
3656 return Ok(());
3657 }
3658
3659 self.state.status = RunStatus::Deliberating;
3660 self.state.event(
3661 "deliberate",
3662 format!(
3663 "split: first choices were {} — opening {rounds} round(s)",
3664 tops.iter().collect::<String>()
3665 ),
3666 );
3667
3668 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3669 let language = self.state.config.graph.language.clone();
3670 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3671 let sessions = self.state.config.graph.sessions;
3672 let artifacts = agent::artifacts_dir(&self.state.dir());
3673 let root = self.state.worktree_root();
3674 let base_short = short(&self.state.base_commit);
3675
3676 for round in 1..=rounds {
3680 let mut turns: Vec<DeliberationTurn> = Vec::new();
3681 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3682 if self.state.judgements[j].failed.is_some() {
3683 continue;
3684 }
3685 let seat_key = format!("judge-{}", j + 1);
3686 let spec = self.occupant(&seat_key, spec);
3687 let mut seat = self.seat(&seat_key, &spec.id);
3688 let transcript = self.transcript(&turns, j);
3689 let build = |context: Option<&str>| {
3690 prompt::deliberate(
3691 &self.state.instruction,
3692 context,
3693 &transcript,
3694 round,
3695 rounds,
3696 &language,
3697 )
3698 };
3699 let block = self.candidate_block(&viable, &base_short);
3700 let full = build(Some(&block));
3701 let text = if has_context(&spec, &seat, sessions) {
3702 build(None)
3703 } else {
3704 full.clone()
3705 };
3706 let job = SeatJob {
3707 spec,
3708 seat: seat.clone(),
3709 prompt: text,
3710 cwd: root.join(format!("judge-{}", j + 1)),
3711 timeout,
3712 allow_write: false,
3713 sessions,
3714 artifacts: artifacts.clone(),
3715 stem: format!("delib-{round}-judge-{}", j + 1),
3716 handover: Some(full),
3717 };
3718 let cache = self.state.config.cache_dir();
3719 let ctx = WaveCtx {
3720 carry_seats: false,
3721 pauses: Some((&self.pause, &self.interrupt)),
3722 run: &run_id,
3723 node: "deliberate",
3724 prompts: &prompts,
3725 cache: cache.as_deref(),
3726 round: None,
3727 };
3728 let mut losses = Vec::new();
3732 let mut results = ask_wave_with::<String>(
3733 vec![job],
3734 Arc::clone(&self.sem),
3735 0,
3736 &self.roles.judge_roster,
3737 &ctx,
3738 &mut losses,
3739 &mut self.state,
3740 &|text: &str| {
3741 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3742 },
3743 )
3744 .await;
3745 self.state.quota.extend(losses);
3746 let (updated, res, _) = results.pop().expect("one job in, one result out");
3747 seat = updated;
3748 let agent_id = seat.agent.clone();
3749 self.state.seats.insert(seat.key.clone(), seat);
3750 let body = match res {
3751 Ok((body, _)) => body,
3752 Err(e) => {
3755 self.state
3756 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3757 continue;
3758 }
3759 };
3760 let tentative = verdict::extract_json::<Position>(&body)
3761 .ok()
3762 .and_then(|p| p.tentative)
3763 .and_then(|s| s.trim().chars().next())
3764 .map(|c| c.to_ascii_uppercase());
3765 self.state.event(
3766 "deliberate",
3767 format!(
3768 "round {round}: judge {} now favours {}",
3769 j + 1,
3770 tentative.map_or("—".to_owned(), |c| c.to_string())
3771 ),
3772 );
3773 turns.push(DeliberationTurn {
3774 judge: j + 1,
3775 agent: agent_id,
3776 body: blind::sanitize_prose(&body, &self.state.config.blind),
3777 tentative,
3778 });
3779 }
3780 self.state
3781 .deliberation
3782 .push(DeliberationRound { round, turns });
3783 self.state.save()?;
3784 }
3785
3786 self.state.status = RunStatus::Voting;
3787 self.state.save()?;
3788 Ok(())
3789 }
3790
3791 async fn vote(&mut self) -> Result<()> {
3794 if self.hold_base_sync().await? {
3795 return Ok(());
3796 }
3797 let run_id = self.state.id.clone();
3802 let prompts = self.state.config.prompts.clone();
3803 if !self.state.votes.is_empty() || self.state.tally.is_some() {
3804 return Ok(());
3805 }
3806 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3807 if viable.len() == 1 {
3808 return Ok(());
3809 }
3810 self.state.status = RunStatus::Voting;
3811
3812 let language = self.state.config.graph.language.clone();
3813 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3814 let sessions = self.state.config.graph.sessions;
3815 let artifacts = agent::artifacts_dir(&self.state.dir());
3816 let root = self.state.worktree_root();
3817 let base_short = short(&self.state.base_commit);
3818 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3819
3820 let mut jobs = Vec::new();
3821 let mut seats_at = Vec::new();
3822 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3823 if self
3824 .state
3825 .judgements
3826 .get(j)
3827 .is_some_and(|r| r.failed.is_some())
3828 {
3829 continue;
3830 }
3831 let seat_key = format!("judge-{}", j + 1);
3832 let spec = self.occupant(&seat_key, spec);
3833 let seat = self.seat(&seat_key, &spec.id);
3834 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3835 let text = if has_context(&spec, &seat, sessions) {
3836 prompt::final_vote(&viable, &language)
3837 } else {
3838 full.clone()
3839 };
3840 jobs.push(SeatJob {
3841 spec,
3842 seat,
3843 prompt: text,
3844 cwd: root.join(format!("judge-{}", j + 1)),
3845 timeout,
3846 allow_write: false,
3847 sessions,
3848 artifacts: artifacts.clone(),
3849 stem: format!("vote-judge-{}", j + 1),
3850 handover: Some(full),
3851 });
3852 seats_at.push(j);
3853 }
3854
3855 self.state.event(
3856 "vote",
3857 format!(
3858 "collecting {} final votes one by one, privately",
3859 jobs.len()
3860 ),
3861 );
3862 let allowed = viable.clone();
3863 let mut quota_losses = Vec::new();
3864 let cache = self.state.config.cache_dir();
3865 let ctx = WaveCtx {
3866 carry_seats: false,
3867 pauses: Some((&self.pause, &self.interrupt)),
3868 run: &run_id,
3869 node: "vote",
3870 prompts: &prompts,
3871 cache: cache.as_deref(),
3872 round: None,
3873 };
3874 let results = ask_json_wave::<FinalVote>(
3875 jobs,
3876 Arc::clone(&self.sem),
3877 self.state.config.graph.retries,
3878 &self.roles.judge_roster,
3879 &ctx,
3880 &mut quota_losses,
3881 &mut self.state,
3882 &move |v: &FinalVote| match v.label() {
3883 Some(c) if allowed.contains(&c) => Ok(()),
3884 other => bail!("vote {other:?} is not one of {allowed:?}"),
3885 },
3886 )
3887 .await;
3888 if self.state.resource_block.is_some() {
3889 return self.stop_for_resource();
3890 }
3891 self.state.quota.extend(quota_losses);
3892
3893 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3894 let agent_id = seat.agent.clone();
3895 self.state.seats.insert(seat.key.clone(), seat);
3896 let initial = self
3897 .state
3898 .judgements
3899 .get(j)
3900 .and_then(|r| r.ranking.first().copied());
3901 let mut record = VoteRecord {
3902 judge: j + 1,
3903 agent: agent_id,
3904 vote: None,
3905 reason: String::new(),
3906 changed: false,
3907 };
3908 match res {
3909 Ok((v, _)) => {
3910 record.vote = v.label();
3911 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3912 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3913 self.state.event(
3914 "vote",
3915 format!(
3916 "judge {} voted {}{}",
3917 j + 1,
3918 record.vote.unwrap_or('?'),
3919 if record.changed { " (changed)" } else { "" }
3920 ),
3921 );
3922 }
3923 Err(e) => {
3924 self.state
3925 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3926 }
3927 }
3928 self.state.votes.push(record);
3929 self.state.save()?;
3930 }
3931 Ok(())
3932 }
3933
3934 fn tally(&mut self) -> Result<()> {
3937 if self.state.tally.is_some() {
3938 return Ok(());
3939 }
3940 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3941 let tops: Vec<char> = self
3942 .state
3943 .judgements
3944 .iter()
3945 .filter_map(|j| j.ranking.first().copied())
3946 .collect();
3947 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3948
3949 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3952 let mut cast: Vec<char> = Vec::new();
3953 for (i, j) in self.state.judgements.iter().enumerate() {
3954 let vote = self
3955 .state
3956 .votes
3957 .iter()
3958 .find(|v| v.judge == i + 1)
3959 .and_then(|v| v.vote)
3960 .or_else(|| j.ranking.first().copied());
3961 if let Some(v) = vote {
3962 *first_choice.entry(v).or_insert(0) += 1;
3963 cast.push(v);
3964 }
3965 }
3966
3967 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3968 for j in &self.state.judgements {
3969 let n = j.ranking.len();
3970 for (pos, label) in j.ranking.iter().enumerate() {
3971 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3972 }
3973 }
3974
3975 let best = first_choice.values().copied().max().unwrap_or(0);
3976 let mut leaders: Vec<char> = first_choice
3977 .iter()
3978 .filter(|(_, v)| **v == best)
3979 .map(|(k, _)| *k)
3980 .collect();
3981 let mut tie_break = None;
3982 if leaders.len() > 1 {
3983 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3984 let borda_leaders: Vec<char> = leaders
3985 .iter()
3986 .copied()
3987 .filter(|l| borda[l] == top_borda)
3988 .collect();
3989 tie_break = Some(if borda_leaders.len() == 1 {
3990 format!(
3991 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3992 leaders.len()
3993 )
3994 } else {
3995 format!(
3996 "{} way tie on both first-choice votes and Borda points, broken by label order",
3997 leaders.len()
3998 )
3999 });
4000 leaders = borda_leaders;
4001 leaders.sort_unstable();
4002 }
4003 let winner = *leaders
4004 .first()
4005 .or(viable.first())
4006 .context("no candidate to declare a winner from")?;
4007
4008 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4009 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4010 let deliberated = !self.state.deliberation.is_empty();
4011
4012 let quota_seats: std::collections::BTreeSet<&str> =
4016 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4017 let mut present = 0usize;
4018 for (i, j) in self.state.judgements.iter().enumerate() {
4019 if quota_seats.contains(j.seat.as_str()) {
4020 continue;
4021 }
4022 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4023 let voted = self
4024 .state
4025 .votes
4026 .iter()
4027 .any(|v| v.judge == i + 1 && v.vote.is_some());
4028 if ranked || voted {
4029 present += 1;
4030 }
4031 }
4032 let needs_quorum = viable.len() > 1;
4038 let judges_total = if needs_quorum {
4039 self.roles.judges.len()
4040 } else {
4041 0
4042 };
4043 let quorum = if needs_quorum {
4044 judges_total / 2 + 1
4045 } else {
4046 0
4047 };
4048 let met_quorum = !needs_quorum || present >= quorum;
4049 let uncontested = (!needs_quorum).then(|| {
4050 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4051 });
4052
4053 self.state.event(
4054 "tally",
4055 match &uncontested {
4056 Some(reason) => format!("winner {winner} — {reason}"),
4057 None => format!(
4058 "winner {winner} — votes {} | initial {} | {} changed | \
4059 {present}/{judges_total} judges{}",
4060 first_choice
4061 .iter()
4062 .map(|(k, v)| format!("{k}:{v}"))
4063 .collect::<Vec<_>>()
4064 .join(" "),
4065 if unanimous_initial {
4066 "unanimous"
4067 } else {
4068 "split"
4069 },
4070 changed_votes,
4071 if met_quorum {
4072 String::new()
4073 } else {
4074 format!(" — below quorum ({quorum} required)")
4075 },
4076 ),
4077 },
4078 );
4079 if !met_quorum {
4080 self.state.event(
4081 "stall",
4082 format!(
4083 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4084 the run stops here, resumable"
4085 ),
4086 );
4087 }
4088 self.state.tally = Some(Tally {
4089 first_choice,
4090 borda,
4091 winner,
4092 rankings: tops.len(),
4093 unanimous_initial,
4094 deliberated,
4095 changed_votes,
4096 unanimous_final,
4097 tie_break,
4098 judges: judges_total,
4099 present,
4100 quorum,
4101 met_quorum,
4102 uncontested,
4103 });
4104 self.state.status = if met_quorum {
4105 RunStatus::Reviewing
4106 } else {
4107 RunStatus::Stalled
4108 };
4109 self.state.save()?;
4110 Ok(())
4111 }
4112
4113 #[allow(clippy::too_many_lines)]
4134 async fn recover_stall(&mut self) -> Result<bool> {
4135 let run_id = self.state.id.clone();
4140 let prompts = self.state.config.prompts.clone();
4141 let quota_seats: BTreeSet<&str> =
4146 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4147 let absent: Vec<String> = self
4148 .state
4149 .judgements
4150 .iter()
4151 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4152 .map(|j| j.seat.clone())
4153 .collect();
4154 if absent.is_empty() {
4155 return Ok(false);
4156 }
4157 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4158 if viable.len() <= 1 {
4159 return Ok(false);
4160 }
4161 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4162 let language = self.state.config.graph.language.clone();
4163 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4164 let sessions = self.state.config.graph.sessions;
4165 let artifacts = agent::artifacts_dir(&self.state.dir());
4166 let root = self.state.worktree_root();
4167 let base_short = short(&self.state.base_commit);
4168 let candidates: Vec<Candidate> = viable.clone();
4169
4170 let mut positions: Vec<usize> = absent
4172 .iter()
4173 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4174 .collect();
4175 if positions.is_empty() {
4176 return Ok(false);
4177 }
4178 positions.sort_unstable();
4179 positions.dedup();
4180
4181 let mut judge_jobs = Vec::new();
4183 for &j in &positions {
4184 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4185 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4186 let seat_key = format!("judge-{}", j + 1);
4187 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4188 let seat = self.seat(&seat_key, &spec.id);
4189 judge_jobs.push(SeatJob {
4190 spec,
4191 seat,
4192 prompt: prompt::judge(
4193 &self.state.instruction,
4194 &views,
4195 self.roles.judges.len(),
4196 &base_short,
4197 &language,
4198 ),
4199 cwd: root.join(seat_key),
4200 timeout,
4201 allow_write: false,
4202 sessions,
4203 artifacts: artifacts.clone(),
4204 stem: format!("judge-{}-recover", j + 1),
4205 handover: None,
4206 });
4207 }
4208
4209 let labels_for_check = labels.clone();
4210 let mut judge_losses = Vec::new();
4211 let retries = self.state.config.graph.retries;
4212 let cache = self.state.config.cache_dir();
4213 let ctx = WaveCtx {
4214 carry_seats: false,
4215 pauses: Some((&self.pause, &self.interrupt)),
4216 run: &run_id,
4217 node: "judge",
4218 prompts: &prompts,
4219 cache: cache.as_deref(),
4220 round: None,
4221 };
4222 let results = ask_json_wave::<Ranking>(
4223 judge_jobs,
4224 Arc::clone(&self.sem),
4225 retries,
4226 &self.roles.judge_roster,
4227 &ctx,
4228 &mut judge_losses,
4229 &mut self.state,
4230 &move |r: &Ranking| r.validate(&labels_for_check),
4231 )
4232 .await;
4233 if self.state.resource_block.is_some() {
4234 self.stop_for_resource()?;
4235 return Ok(false);
4236 }
4237
4238 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4240 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4241 let agent_id = seat.agent.clone();
4242 self.state.seats.insert(seat.key.clone(), seat);
4243 let record = &mut self.state.judgements[j];
4244 match res {
4245 Ok((ranking, out)) => {
4246 record.agent = agent_id;
4247 record.ranking = ranking.normalized();
4248 record.reasons = ranking.reasons;
4249 record.strengths = ranking.strengths;
4250 record.confidence = ranking.confidence;
4251 record.failed = None;
4252 record.duration_ms = out.duration_ms;
4253 recovered.insert(j);
4254 self.state.event(
4255 "recover",
4256 format!("judge {} ranked again after the limit", j + 1),
4257 );
4258 }
4259 Err(e) => {
4260 self.state
4261 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4262 }
4263 }
4264 }
4265
4266 let mut vote_jobs = Vec::new();
4268 let mut vote_pos: Vec<usize> = Vec::new();
4269 for &j in &recovered {
4270 let seat_key = format!("judge-{}", j + 1);
4271 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4272 let seat = self.seat(&seat_key, &spec.id);
4273 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4274 let text = if has_context(&spec, &seat, sessions) {
4275 prompt::final_vote(&labels, &language)
4276 } else {
4277 full.clone()
4278 };
4279 vote_jobs.push(SeatJob {
4280 spec,
4281 seat,
4282 prompt: text,
4283 cwd: root.join(seat_key),
4284 timeout,
4285 allow_write: false,
4286 sessions,
4287 artifacts: artifacts.clone(),
4288 stem: format!("vote-judge-{}-recover", j + 1),
4289 handover: Some(full),
4290 });
4291 vote_pos.push(j);
4292 }
4293 let allowed = labels.clone();
4294 let mut vote_losses = Vec::new();
4295 let vote_retries = self.state.config.graph.retries;
4296 let vote_cache = self.state.config.cache_dir();
4297 let ctx = WaveCtx {
4298 carry_seats: false,
4299 pauses: Some((&self.pause, &self.interrupt)),
4300 run: &run_id,
4301 node: "vote",
4302 prompts: &prompts,
4303 cache: vote_cache.as_deref(),
4304 round: None,
4305 };
4306 let votes = ask_json_wave::<FinalVote>(
4307 vote_jobs,
4308 Arc::clone(&self.sem),
4309 vote_retries,
4310 &self.roles.judge_roster,
4311 &ctx,
4312 &mut vote_losses,
4313 &mut self.state,
4314 &move |v: &FinalVote| match v.label() {
4315 Some(c) if allowed.contains(&c) => Ok(()),
4316 other => bail!("vote {other:?} is not one of {allowed:?}"),
4317 },
4318 )
4319 .await;
4320 if self.state.resource_block.is_some() {
4321 self.stop_for_resource()?;
4322 return Ok(false);
4323 }
4324 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4325 let agent_id = seat.agent.clone();
4326 self.state.seats.insert(seat.key.clone(), seat);
4327 match res {
4328 Ok((v, _)) => {
4329 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4330 rec.vote = v.label();
4331 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4332 } else {
4333 self.state.votes.push(VoteRecord {
4334 judge: j + 1,
4335 agent: agent_id,
4336 vote: v.label(),
4337 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4338 changed: false,
4339 });
4340 }
4341 self.state.event(
4342 "recover",
4343 format!("judge {} voted again after the limit", j + 1),
4344 );
4345 }
4346 Err(e) => {
4347 self.state
4348 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4349 }
4350 }
4351 }
4352
4353 let recovered_keys: BTreeSet<String> = recovered
4357 .iter()
4358 .map(|&j| format!("judge-{}", j + 1))
4359 .collect();
4360 self.state
4361 .quota
4362 .retain(|q| !recovered_keys.contains(&q.seat));
4363 for loss in judge_losses.into_iter().chain(vote_losses) {
4367 if recovered_keys.contains(&loss.seat) {
4368 continue;
4369 }
4370 self.state.quota.retain(|q| q.seat != loss.seat);
4371 self.state.quota.push(loss);
4372 }
4373
4374 self.state.tally = None;
4376 self.tally()?;
4377 Ok(self
4378 .state
4379 .tally
4380 .as_ref()
4381 .map(|t| t.met_quorum)
4382 .unwrap_or(false))
4383 }
4384
4385 async fn graft(&mut self) -> Result<()> {
4401 let Err(e) = self.graft_inner().await else {
4402 return Ok(());
4403 };
4404 if e.is::<crate::repo_lock::ResourceBlocked>() {
4405 return Err(e);
4406 }
4407 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4408 self.state.status = RunStatus::Blocked;
4409 self.state.event("graft", format!("blocked: {e:#}"));
4410 self.state.save()?;
4411 return Err(e);
4412 }
4413 if let Some(rec) = self.state.graft.clone()
4418 && !rec.outcome.settled()
4419 {
4420 if let Some(winner) = self.state.winner().cloned() {
4421 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4422 if re.is::<crate::repo_lock::ResourceBlocked>() {
4423 return Err(re);
4424 }
4425 self.state.status = RunStatus::Blocked;
4426 self.state.event("graft", format!("blocked: {re:#}"));
4427 self.state.save()?;
4428 return Err(re);
4429 }
4430 }
4431 if let Some(g) = self.state.graft.as_mut() {
4432 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4433 }
4434 }
4435 self.state.event("graft", format!("skipped: {e:#}"));
4436 if let Err(e) = self.state.save() {
4437 tracing::warn!("save after graft: {e:#}");
4438 }
4439 Ok(())
4440 }
4441
4442 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4445 let _coordination = self.coordinate(wt).await?;
4446 crate::recovery::restore_repair(wt, pre)
4447 .await
4448 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4449 }
4450
4451 fn graft_skipped(&mut self, why: &str) {
4453 let message = format!("skipped: {why}");
4454 if !self
4455 .state
4456 .events
4457 .iter()
4458 .any(|e| e.node == "graft" && e.message == message)
4459 {
4460 self.state.event("graft", message);
4461 }
4462 }
4463
4464 async fn graft_inner(&mut self) -> Result<()> {
4465 if let Some(rec) = self.state.graft.clone()
4468 && !rec.outcome.settled()
4469 {
4470 if self.state.reviews.is_empty()
4474 && let Some(winner) = self.state.winner().cloned()
4475 {
4476 self.graft_restore(&winner.worktree, &rec.pre).await?;
4477 }
4478 if rec.outcome == GraftOutcome::WaitingForRestore {
4479 self.state.graft = None;
4480 self.state.save()?;
4481 } else {
4482 if let Some(g) = self.state.graft.as_mut() {
4483 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4484 }
4485 self.state.event(
4486 "graft",
4487 "rolled back: the previous round was interrupted; the winner is untouched",
4488 );
4489 self.state.save()?;
4490 return Ok(());
4491 }
4492 }
4493 let plan = match graft::plan(&self.state) {
4494 Ok(plan) => plan,
4495 Err(skip) => {
4496 if skip != graft::Skip::AlreadyDone {
4497 self.graft_skipped(skip.reason());
4498 self.state.save()?;
4499 }
4500 return Ok(());
4501 }
4502 };
4503 let Some(winner) = self.state.winner().cloned() else {
4504 return Ok(());
4505 };
4506 let wt = winner.worktree.clone();
4507 if !git::is_clean(&wt).await? {
4508 self.graft_skipped("the winner's worktree has uncommitted changes");
4509 self.state.save()?;
4510 return Ok(());
4511 }
4512 let pre = git::rev_parse(&wt, "HEAD").await?;
4513 self.state.graft = Some(GraftRecord {
4516 pre: pre.clone(),
4517 post: None,
4518 lines: None,
4519 limit: None,
4520 agent: None,
4521 outcome: GraftOutcome::Running,
4522 requested: graft::requested(&plan),
4523 adoption: None,
4524 adoption_error: None,
4525 });
4526 self.state.event(
4527 "graft",
4528 format!(
4529 "porting strengths of candidate(s) {} onto {}",
4530 plan.strengths
4531 .iter()
4532 .map(|(l, _, _)| l.to_string())
4533 .collect::<Vec<_>>()
4534 .join(", "),
4535 plan.winner
4536 ),
4537 );
4538 self.state.save()?;
4539
4540 let fix_prompt = prompt::graft(
4541 &self.state.instruction,
4542 plan.winner,
4543 &plan.strengths,
4544 &self.state.config.graph.language,
4545 );
4546 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4547 let sessions = self.state.config.graph.sessions;
4548 let artifacts = agent::artifacts_dir(&self.state.dir());
4549 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4550 let (_, seat, out) = self
4551 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4552 SeatJob {
4553 prompt: fix_prompt.clone(),
4554 spec,
4555 seat,
4556 cwd: wt.clone(),
4557 timeout,
4558 allow_write: true,
4559 sessions,
4560 artifacts: artifacts.clone(),
4561 stem: "graft".to_owned(),
4562 handover: None,
4563 }
4564 })
4565 .await;
4566 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4567 self.state.graft.as_mut().expect("graft record").outcome =
4570 GraftOutcome::WaitingForRestore;
4571 return self.stop_for_resource();
4572 }
4573 let mut why: Option<String> = None;
4574 match &out {
4575 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4576 why = Some("the fixer produced nothing usable".to_owned());
4577 }
4578 AgentOutcome::Ok(_) => {}
4579 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4580 AgentOutcome::Quota(o) => {
4581 self.state.quota.push(QuotaLoss {
4582 seat: seat.key.clone(),
4583 node: "graft".to_owned(),
4584 at: Timestamp::now(),
4585 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4586 });
4587 why = Some("rate limited (quota)".to_owned());
4588 }
4589 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4590 why = Some(format!(
4591 "the fixer failed: {}",
4592 e.lines()
4593 .next()
4594 .unwrap_or("")
4595 .chars()
4596 .take(160)
4597 .collect::<String>()
4598 ));
4599 }
4600 }
4601 let agent_id = seat.agent.clone();
4602 self.state.seats.insert(seat.key.clone(), seat);
4603
4604 let mut post = None;
4605 let mut lines = None;
4606 let _coordination = self.coordinate(&wt).await?;
4607 if why.is_none() {
4608 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4609 if let Ok(r) = &rescued {
4610 self.state.note_withheld("graft", &r.withheld);
4611 }
4612 let head = git::rev_parse(&wt, "HEAD").await?;
4613 let dirty = match &rescued {
4617 Ok(r) => git::status_porcelain(&wt)
4618 .await
4619 .map(|st| {
4620 st.lines().any(|l| {
4621 !r.withheld
4622 .iter()
4623 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4624 })
4625 })
4626 .unwrap_or(true),
4627 Err(_) => true,
4628 };
4629 if dirty {
4630 why = Some("the fixer left changes that could not be committed".to_owned());
4631 } else if head == pre {
4632 why = Some("the fixer changed nothing".to_owned());
4633 } else {
4634 lines = git::changed_lines(&wt, &pre, &head).await;
4635 post = Some(head);
4636 }
4637 }
4638 let (adoption, adoption_error) = match &out {
4639 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4640 _ => (None, None),
4641 };
4642 let requested = graft::requested(&plan);
4643 let outcome = match why {
4644 None => {
4645 self.state.event(
4646 "graft",
4647 format!(
4648 "applied: {}",
4649 lines.map_or_else(
4650 || "changed lines unavailable".to_owned(),
4651 |n| format!("{n} changed lines")
4652 )
4653 ),
4654 );
4655 GraftOutcome::Applied
4656 }
4657 Some(why) => {
4658 crate::recovery::restore_repair(&wt, &pre)
4661 .await
4662 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4663 self.state.event(
4664 "graft",
4665 format!("rolled back: {why}; the winner is untouched"),
4666 );
4667 GraftOutcome::RolledBack(why)
4668 }
4669 };
4670 self.state.graft = Some(GraftRecord {
4671 pre,
4672 post,
4673 lines,
4674 limit: None,
4675 agent: Some(agent_id),
4676 outcome,
4677 requested,
4678 adoption,
4679 adoption_error,
4680 });
4681 self.state.save()?;
4682 Ok(())
4683 }
4684
4685 async fn fold_losers(&mut self) -> Result<()> {
4688 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4689 return Ok(());
4690 };
4691 if !self.state.candidate_preservation.is_empty() {
4692 bail!("candidate preservation incomplete; refusing cleanup");
4693 }
4694 let repo = self.state.repo.clone();
4695 let _coordination = self.coordinate(&repo).await?;
4696 let mut folded = Vec::new();
4697 for i in 0..self.state.candidates.len() {
4698 let c = &self.state.candidates[i];
4699 if c.label == winner || c.folded {
4700 continue;
4701 }
4702 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4703 remove_preserved_worktree(&repo, &wt).await?;
4704 if git::branch_exists(&repo, &branch).await?
4705 && !git::branch_delete(&repo, &branch).await?
4706 {
4707 bail!("candidate branch removal refused");
4708 }
4709 self.state.candidates[i].folded = true;
4710 self.state.save()?;
4711 folded.push(label.to_string());
4712 }
4713 let root = self.state.worktree_root();
4715 for j in 1..=self.roles.judges.len() {
4716 let wt = root.join(format!("judge-{j}"));
4717 if wt.exists() {
4718 remove_preserved_worktree(&repo, &wt).await?;
4719 }
4720 }
4721 if self.state.config.graph.advise {
4724 for k in 1..=self.state.config.graph.advisors {
4725 let wt = root.join(format!("advisor-{k}"));
4726 if wt.exists() {
4727 remove_preserved_worktree(&repo, &wt).await?;
4728 }
4729 }
4730 }
4731 if !folded.is_empty() {
4732 self.state
4733 .event("fold", format!("folded candidates {}", folded.join(", ")));
4734 self.state.save()?;
4735 }
4736 Ok(())
4737 }
4738
4739 async fn hold_base_sync(&mut self) -> Result<bool> {
4742 self.hold_base_sync_as(None).await
4743 }
4744
4745 async fn hold_base_sync_kind(
4746 &mut self,
4747 kind: crate::base_sync_recovery::StopKind,
4748 ) -> Result<()> {
4749 self.hold_base_sync_as(Some(kind)).await?;
4750 Ok(())
4751 }
4752
4753 async fn hold_base_sync_as(
4754 &mut self,
4755 kind: Option<crate::base_sync_recovery::StopKind>,
4756 ) -> Result<bool> {
4757 if !self.state.base_sync_blocked() {
4758 self.state.base_sync_recovery.stop = None;
4759 return Ok(false);
4760 }
4761 let head = match self.state.winner() {
4762 Some(w) => git::rev_parse(&w.worktree, "HEAD")
4763 .await
4764 .unwrap_or_default(),
4765 None => String::new(),
4766 };
4767 self.state.record_base_sync_stop(&head);
4768 if let Some(kind) = kind {
4769 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4770 }
4771 self.state.save()?;
4772 Ok(true)
4773 }
4774
4775 async fn sync_to_base(&mut self) -> Result<()> {
4813 if self.state.status == RunStatus::AlreadyInBase {
4814 return Ok(());
4815 }
4816 self.hold_base_sync().await?;
4817 let conflicted = self
4818 .state
4819 .base_sync
4820 .as_ref()
4821 .is_some_and(|s| s.conflict.is_some());
4822 let Some(winner) = self.state.winner().cloned() else {
4823 return Ok(());
4824 };
4825
4826 let repo = self.state.repo.clone();
4827 let remote = self.state.config.merge.remote.clone();
4828 let base_branch = self.state.base_branch.clone();
4829 let tracking = format!("{remote}/{base_branch}");
4830
4831 let fetched_base =
4832 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4833 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4838 return Ok(());
4839 };
4840
4841 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4842 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4843 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4844
4845 if (behind > 0 || conflicted || head == tip)
4854 && self
4855 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4856 .await?
4857 {
4858 return Ok(());
4859 }
4860 if conflicted && self.state.base_recovery_pending() {
4861 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4862 let owner =
4863 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4864 if !fetched_base
4865 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4866 || !git::is_clean(&winner.worktree).await?
4867 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4868 {
4869 return Ok(());
4870 }
4871 }
4872 let recovery = conflicted && self.state.consume_base_recovery(&head);
4873 if conflicted {
4874 self.state.save()?;
4876 if !recovery {
4877 return Ok(());
4878 }
4879 }
4880
4881 if behind == 0 {
4882 if let Some(from) = self
4891 .state
4892 .rebase_fixes
4893 .iter()
4894 .rev()
4895 .find_map(|r| r.from.clone())
4896 && from != head
4897 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4898 .await
4899 .is_ok_and(|o| o.ok())
4900 {
4901 git::sync_to_head(&winner.worktree).await?;
4902 }
4903 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4909 if let Some(why) = &conflict {
4910 self.state.status = RunStatus::Blocked;
4911 self.state.event("land", why.clone());
4912 }
4913 self.state.base_sync = Some(BaseSync {
4914 tip,
4915 behind: 0,
4916 attempts: attempts + usize::from(recovery),
4917 conflict,
4918 already_in: None,
4919 });
4920 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4921 .await?;
4922 self.state.save()?;
4923 return Ok(());
4924 }
4925
4926 if attempts >= BASE_SYNC_ROUNDS && !recovery {
4927 let why = format!(
4928 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4929 rebase(s); rebasing again would only race it",
4930 winner.branch
4931 );
4932 self.state.status = RunStatus::Blocked;
4933 self.state.base_sync = Some(BaseSync {
4934 tip,
4935 behind,
4936 attempts,
4937 conflict: Some(why.clone()),
4938 already_in: None,
4939 });
4940 self.state.event("land", why);
4941 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4942 .await?;
4943 self.state.save()?;
4944 return Ok(());
4945 }
4946
4947 self.state.event(
4948 "land",
4949 format!(
4950 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4951 winner.branch
4952 ),
4953 );
4954 self.state.save()?;
4955
4956 let branch_tracking = format!("{remote}/{}", winner.branch);
4962 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4963 let remote_tip = if self.state.task_pr_branch().is_none()
4964 && matches!(&fetched_branch, Ok(o) if o.ok())
4965 {
4966 git::rev_parse(&repo, &branch_tracking).await.ok()
4967 } else {
4968 None
4969 };
4970 if let Some(theirs) = &remote_tip
4974 && !git::is_ancestor(&repo, theirs, &head).await
4975 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4976 .await
4977 .is_ok_and(|missing| missing.is_empty())
4978 {
4979 let why = format!(
4980 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4981 them",
4982 short(theirs),
4983 winner.branch
4984 );
4985 self.state.status = RunStatus::Blocked;
4986 self.state.base_sync = Some(BaseSync {
4987 tip,
4988 behind,
4989 attempts: attempts + usize::from(recovery),
4990 conflict: Some(why.clone()),
4991 already_in: None,
4992 });
4993 self.state.event("land", why);
4994 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4995 .await?;
4996 self.state.save()?;
4997 return Ok(());
4998 }
4999
5000 let scratch = self.state.dir().join("base-sync");
5001 let rebased =
5002 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5003 .await
5004 {
5005 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5006 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5007 Err(e) => Err(e),
5008 };
5009 let attempts = attempts + 1;
5010 let stop_kind = match &rebased {
5011 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5012 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5013 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5014 };
5015 match rebased {
5016 Ok(None) => {
5017 git::sync_to_head(&winner.worktree).await?;
5021 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5022 let mut conflict = None;
5023 if let Some(pinned) = &remote_tip {
5024 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5025 match pushed {
5026 Ok(o) if o.ok() => self.state.event(
5027 "land",
5028 format!("pushed rebased {} to {remote}", winner.branch),
5029 ),
5030 Ok(o) => {
5031 conflict = Some(format!(
5032 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5033 winner.branch,
5034 short(pinned),
5035 o.stderr.chars().take(600).collect::<String>()
5036 ));
5037 }
5038 Err(e) => {
5039 conflict = Some(format!(
5040 "rebased {} locally but could not push it: {e:#}",
5041 winner.branch
5042 ));
5043 }
5044 }
5045 }
5046 if let Some(why) = &conflict {
5047 self.state.status = RunStatus::Blocked;
5048 self.state.event("land", why.clone());
5049 }
5050 self.state.base_sync = Some(BaseSync {
5051 tip: tip.clone(),
5052 behind: 0,
5053 attempts,
5054 conflict,
5055 already_in: None,
5056 });
5057 self.state
5058 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5059 }
5060 Ok(Some(conflict)) => {
5061 let why = format!(
5062 "{} conflicts with {tracking} and did not rebase: {}",
5063 winner.branch,
5064 conflict.chars().take(600).collect::<String>()
5065 );
5066 self.state.status = RunStatus::Blocked;
5067 self.state.base_sync = Some(BaseSync {
5068 tip,
5069 behind,
5070 attempts,
5071 conflict: Some(why.clone()),
5072 already_in: None,
5073 });
5074 self.state.event("land", why);
5075 }
5076 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5080 self.state.save()?;
5081 return Ok(());
5082 }
5083 Err(e) => {
5084 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5085 self.state.status = RunStatus::Blocked;
5086 self.state.base_sync = Some(BaseSync {
5087 tip,
5088 behind,
5089 attempts,
5090 conflict: Some(why.clone()),
5091 already_in: None,
5092 });
5093 self.state.event("land", why);
5094 }
5095 }
5096 if !self.state.base_sync_blocked() {
5097 self.state.base_sync_recovery.stop = None;
5098 }
5099 self.hold_base_sync_kind(stop_kind).await?;
5100 self.state.save()?;
5101 Ok(())
5102 }
5103
5104 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5110 if self.state.task_pr_branch().is_some() {
5113 return None;
5114 }
5115 let repo = self.state.repo.clone();
5116 let remote = self.state.config.merge.remote.clone();
5117 let fetched = git::fetch(&repo, &remote, branch).await;
5118 if !matches!(&fetched, Ok(o) if o.ok()) {
5119 return None;
5120 }
5121 let branch_tracking = format!("{remote}/{branch}");
5122 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5123 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5124 return None;
5125 }
5126 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5127 .await
5128 .is_ok_and(|missing| missing.is_empty())
5129 {
5130 return Some(format!(
5131 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5132 them",
5133 short(&theirs)
5134 ));
5135 }
5136 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5137 Ok(o) if o.ok() => {
5138 self.state
5139 .event("land", format!("pushed rebased {branch} to {remote}"));
5140 None
5141 }
5142 Ok(o) => Some(format!(
5143 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5144 someone may have pushed): {}",
5145 short(&theirs),
5146 o.stderr.chars().take(600).collect::<String>()
5147 )),
5148 Err(e) => Some(format!(
5149 "{branch} is rebased locally but could not be pushed: {e:#}"
5150 )),
5151 }
5152 }
5153
5154 async fn settle_already_in(
5171 &mut self,
5172 branch: &str,
5173 tip: &str,
5174 head: &str,
5175 attempts: usize,
5176 behind: usize,
5177 ) -> Result<bool> {
5178 let repo = self.state.repo.clone();
5179 let remote = self.state.config.merge.remote.clone();
5180 let start = self.state.base_commit.clone();
5181 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5182 Ok(Some(e)) => e,
5183 Ok(None) => return Ok(false),
5184 Err(e) => {
5185 tracing::warn!("already-in-base check for {branch}: {e:#}");
5186 return Ok(false);
5187 }
5188 };
5189 let mut verified = vec![head.to_owned()];
5190 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5191 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5192 if matches!(&fetched, Ok(o) if o.ok())
5193 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5194 && theirs != head
5195 {
5196 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5197 Ok(Some(_)) => verified.push(theirs),
5198 _ => return Ok(false),
5199 }
5200 }
5201 let base_branch = self.state.base_branch.clone();
5202 let message = format!(
5203 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5204 land",
5205 evidence.names(),
5206 evidence.proof.as_str()
5207 );
5208 let closed =
5209 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5210 .await;
5211 match closed {
5212 Ok(Ok(url)) => self
5213 .state
5214 .event("land", format!("closed {url}: superseded on {base_branch}")),
5215 Ok(Err(why)) => self
5216 .state
5217 .event("land", format!("did not close a pull request: {why}")),
5218 Err(e) => {
5219 let why = format!(
5220 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5221 not be closed ({e:#}); resume to retry"
5222 );
5223 self.state.status = RunStatus::Blocked;
5224 self.state.base_sync = Some(BaseSync {
5225 tip: tip.to_owned(),
5226 behind,
5227 attempts,
5228 conflict: Some(why.clone()),
5229 already_in: None,
5230 });
5231 self.state.event("land", why);
5232 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5233 .await?;
5234 self.state.save()?;
5235 return Ok(true);
5236 }
5237 }
5238 self.state.status = RunStatus::AlreadyInBase;
5239 self.state.base_sync_recovery.stop = None;
5240 self.state.base_sync = Some(BaseSync {
5241 tip: tip.to_owned(),
5242 behind,
5243 attempts,
5244 conflict: None,
5245 already_in: Some(evidence),
5246 });
5247 self.state.event("land", message);
5248 self.state.save()?;
5249 self.settle_questions();
5250 Ok(true)
5251 }
5252
5253 fn landing_base(&self) -> String {
5263 self.state
5264 .base_sync
5265 .as_ref()
5266 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5267 }
5268
5269 pub async fn fix_selected(
5302 &mut self,
5303 ids: &[String],
5304 reason: &str,
5305 allow_stale: bool,
5306 ) -> Result<()> {
5307 let reason = reason.trim();
5308 if reason.is_empty() {
5309 bail!("a fix request needs a reason — that is the operator's own record of why");
5310 }
5311 if ids.is_empty() {
5312 bail!("no finding id given");
5313 }
5314 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5315 bail!(
5316 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5317 has already concluded — can be given a targeted fix. A run still \
5318 in progress should simply be resumed; a `merged` run's branch has \
5319 already landed, so its answer is a fresh `magi review <branch>`, \
5320 not reopening this run's own record",
5321 self.state.id,
5322 self.state.status.as_str()
5323 );
5324 }
5325 let Some(winner) = self.state.winner().cloned() else {
5326 bail!("run {} has no winning candidate to fix", self.state.id);
5327 };
5328 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5329 bail!(
5330 "branch `{}` no longer exists; this run cannot be extended",
5331 winner.branch
5332 );
5333 }
5334 let home = crate::run::home();
5335 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5336 bail!(
5337 "run {} is currently being worked on by another magi process",
5338 self.state.id
5339 );
5340 }
5341 let _claim = FixClaim::acquire(&self.state.dir())?;
5347
5348 let mut seen = BTreeSet::new();
5352 let mut findings = Vec::new();
5353 let mut missing = Vec::new();
5354 for id in ids {
5355 if !seen.insert(id.clone()) {
5356 continue;
5357 }
5358 match self.state.finding(id) {
5359 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5360 id: f.id.clone(),
5361 severity: f.severity,
5362 reviewer_vote: rec.vote,
5363 round: round.round,
5364 round_head: round.head.clone(),
5365 reviewer: rec.reviewer,
5366 agent: rec.agent.clone(),
5367 file: f.file.clone(),
5368 line: f.line,
5369 title: f.title.clone(),
5370 detail: f.detail.clone(),
5371 outcome: OperatorFixOutcome::Pending,
5372 }),
5373 None => missing.push(id.clone()),
5374 }
5375 }
5376 if !missing.is_empty() {
5377 bail!(
5378 "unknown finding id(s): {}; nothing was changed",
5379 missing.join(", ")
5380 );
5381 }
5382
5383 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5384 let stale_details: Vec<(String, String)> = findings
5385 .iter()
5386 .filter(|f| f.round_head != head_at_request)
5387 .map(|f| (f.id.clone(), f.round_head.clone()))
5388 .collect();
5389 let stale = !stale_details.is_empty();
5390 if stale && !allow_stale {
5391 bail!(
5392 "the branch has moved since some finding(s) were raised — {} — now \
5393 at {}; pass --allow-stale to fix anyway, or re-run review first",
5394 stale_details
5395 .iter()
5396 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5397 .collect::<Vec<_>>()
5398 .join(", "),
5399 short(&head_at_request)
5400 );
5401 }
5402
5403 let request = OperatorFixRequest {
5404 requested_at: Timestamp::now(),
5405 reason: reason.to_owned(),
5406 findings,
5407 head_at_request: head_at_request.clone(),
5408 allow_stale,
5409 stale,
5410 fix: None,
5411 result_head: None,
5412 follow_up_review_run: None,
5413 };
5414 self.state.event(
5415 "fix",
5416 format!(
5417 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5418 request.findings.len(),
5419 request
5420 .findings
5421 .iter()
5422 .map(|f| f.id.as_str())
5423 .collect::<Vec<_>>()
5424 .join(", "),
5425 ),
5426 );
5427 self.state.operator_fixes.push(request);
5434 self.state.save()?;
5435 let request_index = self.state.operator_fixes.len() - 1;
5436
5437 let repo = self.state.repo.clone();
5446 let coordination = self.coordinate(&repo).await?;
5447 if winner.worktree.exists() {
5448 let dirty = git::git(
5451 &winner.worktree,
5452 &["status", "--porcelain", "--untracked-files=all"],
5453 )
5454 .await?;
5455 let only_withheld = dirty.lines().all(|l| {
5456 l.strip_prefix("?? ")
5457 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5458 });
5459 if !only_withheld {
5460 bail!(
5461 "`{}` has uncommitted changes; refusing to touch it — commit or \
5462 discard them first",
5463 winner.worktree.display()
5464 );
5465 }
5466 git::worktree_remove(&self.state.repo, &winner.worktree)
5467 .await
5468 .ok();
5469 }
5470 let fix_worktree = self.state.worktree_root().join("operator-fix");
5471 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5472 git::git(
5473 &self.state.repo,
5474 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5475 )
5476 .await
5477 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5478 if !git::is_clean(&fix_worktree).await? {
5479 git::worktree_remove(&self.state.repo, &fix_worktree)
5480 .await
5481 .ok();
5482 bail!(
5483 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5484 winner.branch
5485 );
5486 }
5487 let repo_for_setup = self.state.repo.clone();
5488 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5489 Ok(()) => {
5490 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5491 }
5492 Err(e) => Err(e),
5493 };
5494 if let Err(e) = setup {
5495 git::worktree_remove(&self.state.repo, &fix_worktree)
5496 .await
5497 .ok();
5498 return Err(e);
5499 }
5500
5501 let run_id = self.state.id.clone();
5502 let prompts = self.state.config.prompts.clone();
5503 let language = self.state.config.graph.language.clone();
5504 let sessions = self.state.config.graph.sessions;
5505 let artifacts = agent::artifacts_dir(&self.state.dir());
5506 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5507 .findings
5508 .iter()
5509 .map(|f| Finding {
5510 id: f.id.clone(),
5511 severity: f.severity,
5512 file: f.file.clone(),
5513 line: f.line,
5514 title: f.title.clone(),
5515 detail: f.detail.clone(),
5516 })
5517 .collect();
5518 let fix_prompt = prompt::operator_fix_in(
5519 &self.state.instruction,
5520 &finding_list,
5521 reason,
5522 &stale_details,
5523 &head_at_request,
5524 &language,
5525 &self.state.config.graph.github_language,
5526 );
5527 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5528 drop(coordination);
5529 let (job, seat, out) = self
5530 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5531 prompt: fix_prompt.clone(),
5532 spec,
5533 seat,
5534 cwd: fix_worktree.clone(),
5535 timeout,
5536 allow_write: true,
5537 sessions,
5538 artifacts: artifacts.clone(),
5539 stem: "operator-fix".to_owned(),
5540 handover: None,
5541 })
5542 .await;
5543 let agent_id = seat.agent.clone();
5544
5545 let mut fix = FixRecord {
5546 agent: agent_id,
5547 addressed: Vec::new(),
5548 rejected: Vec::new(),
5549 notes: String::new(),
5550 committed: false,
5551 failed: None,
5552 duration_ms: 0,
5553 continuation: None,
5554 };
5555 let mut final_seat = seat.clone();
5556 match out {
5557 AgentOutcome::Ok(o) => {
5558 fix.duration_ms = o.duration_ms;
5559 let parsed = verdict::extract_json::<FixReport>(&o.text);
5560 let incomplete_reason = match &parsed {
5561 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5562 "the reply parsed, but it reported a command whose own CLI \
5563 never confirmed an exit status"
5564 .to_owned(),
5565 ),
5566 Ok(_) => None,
5567 Err(e) => Some(e.to_string()),
5568 };
5569 match incomplete_reason {
5570 None => {
5571 let report = parsed.expect("checked Ok above");
5572 fix.addressed = report.addressed;
5573 fix.rejected = report.rejected;
5574 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5575 }
5576 Some(reason) => {
5577 let (resumed_seat, resolved, failure, cont) = self
5578 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5579 .await;
5580 fix.duration_ms += cont.cumulative_wait_ms;
5581 fix.continuation = Some(cont);
5582 final_seat = resumed_seat;
5583 match resolved {
5584 Some(report) => {
5585 fix.addressed = report.addressed;
5586 fix.rejected = report.rejected;
5587 fix.notes =
5588 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5589 }
5590 None => fix.failed = failure,
5591 }
5592 }
5593 }
5594 }
5595 AgentOutcome::Dropped(o) => {
5596 fix.duration_ms = o.duration_ms;
5597 let why = o
5598 .dropped
5599 .as_ref()
5600 .map(|d| d.why.as_str())
5601 .unwrap_or("the CLI ended the stream without delivering its answer");
5602 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5603 }
5604 AgentOutcome::Quota(o) => {
5605 self.state.quota.push(QuotaLoss {
5606 seat: final_seat.key.clone(),
5607 node: "fix".to_owned(),
5608 at: Timestamp::now(),
5609 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5610 });
5611 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5612 }
5613 AgentOutcome::Failed(e) => fix.failed = Some(e),
5614 AgentOutcome::ResourceBlocked(e) => {
5615 fix.failed = Some(format!("the fixer was not started: {e}"));
5616 }
5617 }
5618 if fix.continuation.is_none() {
5619 fix.continuation = Some(ContinuationRecord::not_needed());
5620 }
5621 self.state.seats.insert(final_seat.key.clone(), final_seat);
5622
5623 let rescue_message = format!(
5624 "magi: operator-selected fix ({}) (uncommitted work)",
5625 self.state.operator_fixes[request_index]
5626 .findings
5627 .iter()
5628 .map(|f| f.id.as_str())
5629 .collect::<Vec<_>>()
5630 .join(", ")
5631 );
5632 let repo = self.state.repo.clone();
5633 let coordination = self.coordinate(&repo).await?;
5634 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5635 self.state.note_withheld("fix", &r.withheld);
5636 }
5637 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5638 fix.committed = after != head_at_request;
5639 git::worktree_remove(&self.state.repo, &fix_worktree)
5640 .await
5641 .ok();
5642 drop(coordination);
5643
5644 self.state.event(
5645 "fix",
5646 match &fix.failed {
5647 Some(reason) => format!(
5648 "operator fix: adoption report was lost ({reason}); {}",
5649 if fix.committed {
5650 "committed"
5651 } else {
5652 "NO new commit"
5653 }
5654 ),
5655 None => format!(
5656 "operator fix: {} addressed, {} rejected, {}",
5657 fix.addressed.len(),
5658 fix.rejected.len(),
5659 if fix.committed {
5660 "committed"
5661 } else {
5662 "NO new commit"
5663 }
5664 ),
5665 },
5666 );
5667
5668 for f in &mut self.state.operator_fixes[request_index].findings {
5675 f.outcome = if fix.failed.is_some() {
5676 OperatorFixOutcome::Unreported
5677 } else if fix.addressed.contains(&f.id) {
5678 OperatorFixOutcome::Addressed
5679 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5680 OperatorFixOutcome::Rejected { why: r.why.clone() }
5681 } else {
5682 OperatorFixOutcome::Unreported
5683 };
5684 }
5685
5686 let committed = fix.committed;
5687 if committed {
5688 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5689 }
5690 self.state.operator_fixes[request_index].fix = Some(fix);
5691 self.state.save()?;
5694
5695 if committed {
5696 self.state.event(
5697 "fix",
5698 format!(
5699 "operator fix committed {}; opening a follow-up review-only run",
5700 short(&after)
5701 ),
5702 );
5703 let origin =
5706 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5707 match Self::review(
5708 &self.state.repo,
5709 &winner.branch,
5710 self.state.config.clone(),
5711 origin,
5712 )
5713 .await
5714 {
5715 Ok(mut follow_up) => {
5716 follow_up.state.event(
5717 "start",
5718 format!(
5719 "requested by an operator fix on run {} for finding(s) {}",
5720 self.state.id,
5721 self.state.operator_fixes[request_index]
5722 .findings
5723 .iter()
5724 .map(|f| f.id.as_str())
5725 .collect::<Vec<_>>()
5726 .join(", "),
5727 ),
5728 );
5729 follow_up.state.save()?;
5730 let follow_up_id = follow_up.state.id.clone();
5731 let adopted = match crate::direct::adopt(
5734 &follow_up.state,
5735 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5736 ) {
5737 Ok(a) => a,
5738 Err(e) => {
5739 self.state.event(
5742 "fix",
5743 format!(
5744 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5745 ),
5746 );
5747 self.state.save()?;
5748 return Ok(());
5749 }
5750 };
5751 let executed = follow_up.execute().await;
5752 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5753 if let Some(a) = adopted {
5754 a.finish(&follow_up.state, executed);
5755 }
5756 if let Some(e) = failure {
5757 self.state.event(
5758 "fix",
5759 format!(
5760 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5761 ),
5762 );
5763 }
5764 self.state.operator_fixes[request_index].follow_up_review_run =
5765 Some(follow_up_id);
5766 }
5767 Err(e) => {
5768 self.state.event(
5769 "fix",
5770 format!("committed the fix but could not open a follow-up review: {e:#}"),
5771 );
5772 }
5773 }
5774 self.state.save()?;
5775 }
5776
5777 Ok(())
5778 }
5779
5780 async fn ask_fixer(
5800 &mut self,
5801 winner: &Candidate,
5802 node: &'static str,
5803 round: Option<usize>,
5804 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5805 ) -> (SeatJob, SeatState, AgentOutcome) {
5806 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5807 self.ask_fixer_among(attempts, node, round, None, build)
5808 .await
5809 }
5810
5811 async fn ask_fixer_among(
5816 &mut self,
5817 attempts: Vec<(AgentSpec, String)>,
5818 node: &'static str,
5819 round: Option<usize>,
5820 restore: Option<(&Path, &str)>,
5821 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5822 ) -> (SeatJob, SeatState, AgentOutcome) {
5823 let run_id = self.state.id.clone();
5824 let prompts = self.state.config.prompts.clone();
5825 let cache = self.state.config.cache_dir();
5826 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5827 let mut last = None;
5828 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5829 let seat = self.seat(&key, &spec.id);
5830 let mut job = build(spec.clone(), seat);
5831 if i > 0 {
5832 job.stem = format!("{}-{}", job.stem, spec.id);
5833 }
5834 let ctx = WaveCtx {
5835 carry_seats: false,
5836 pauses: Some((&self.pause, &self.interrupt)),
5837 run: &run_id,
5838 node,
5839 prompts: &prompts,
5840 cache: cache.as_deref(),
5841 round,
5842 };
5843 let (seat, out) =
5844 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5845 let advances = match &out {
5848 AgentOutcome::Ok(o) => agent::output_advances(o),
5849 AgentOutcome::ResourceBlocked(_) => false,
5850 _ => true,
5851 };
5852 if let Some(next) = ids.get(i + 1)
5853 && advances
5854 {
5855 self.state.seats.insert(seat.key.clone(), seat.clone());
5856 let class =
5857 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5858 let reason = match &out {
5859 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5860 other => fail_reason(other),
5861 };
5862 if let Some((wt, rev)) = restore
5866 && let Err(e) = self.graft_restore(wt, rev).await
5867 {
5868 let why =
5869 format!("could not restore the worktree before the next fixer: {e:#}");
5870 tracing::warn!("{why}");
5871 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5872 AgentOutcome::ResourceBlocked(why)
5873 } else {
5874 AgentOutcome::Failed(why)
5875 };
5876 last = Some((job, seat, out));
5877 break;
5878 }
5879 record_handover(
5880 &mut self.state,
5881 node,
5882 &seat.key,
5883 &spec.id,
5884 next,
5885 &class,
5886 &reason,
5887 );
5888 continue;
5889 }
5890 last = Some((job, seat, out));
5891 break;
5892 }
5893 last.expect("the fixer chain always has an entry")
5894 }
5895
5896 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5907 use crate::recovery::{Mode, Repair};
5908 if !self.state.reviews.is_empty() {
5909 return Ok(());
5910 }
5911 let Some(mut rec) = self.state.recovery.clone() else {
5912 return Ok(());
5913 };
5914 if rec.mode != Mode::VerifyRepair {
5915 return Ok(());
5916 }
5917 let wt = winner.worktree.clone();
5918 let mut head = git::rev_parse(&wt, "HEAD").await?;
5919 match rec.repair.clone() {
5920 Repair::Done { .. } => return Ok(()),
5921 Repair::WaitingForRestore { before } => {
5922 self.graft_restore(&wt, &before).await?;
5923 head = before;
5924 rec.repair = Repair::NotStarted;
5925 self.state.recovery = Some(rec.clone());
5926 self.state.save()?;
5927 }
5928 Repair::Started { before } => {
5929 if let Err(e) = self.graft_restore(&wt, &before).await {
5933 if e.is::<crate::repo_lock::ResourceBlocked>() {
5934 return Err(e);
5935 }
5936 return self
5937 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5938 .await;
5939 }
5940 rec.repair = Repair::Done {
5941 before: before.clone(),
5942 after: before,
5943 committed: false,
5944 failed: Some("interrupted; rolled back".to_owned()),
5945 };
5946 self.state
5947 .event("recovery", "the repair was interrupted and rolled back");
5948 self.state.recovery = Some(rec);
5949 self.state.save()?;
5950 return Ok(());
5951 }
5952 Repair::NotStarted => {}
5953 }
5954 if crate::recovery::recheck(&mut rec, &head) {
5955 self.state.event("recovery", rec.summary());
5956 self.state.recovery = Some(rec);
5957 self.state.save()?;
5958 return Ok(());
5959 }
5960 rec.repair = Repair::Started {
5961 before: head.clone(),
5962 };
5963 self.state.recovery = Some(rec.clone());
5964 self.state.save()?;
5965
5966 let language = self.state.config.graph.language.clone();
5967 let sessions = self.state.config.graph.sessions;
5968 let artifacts = agent::artifacts_dir(&self.state.dir());
5969 let fix_prompt = prompt::recovery_fix(
5970 &self.state.instruction,
5971 &rec,
5972 self.state.config.graph.review_rounds,
5973 &language,
5974 );
5975 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5976 self.state.event(
5977 "recovery",
5978 format!(
5979 "repair: the fixer starts from the failed verification on {}",
5980 short(&head)
5981 ),
5982 );
5983 let (_job, seat, out) = self
5984 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5985 prompt: fix_prompt.clone(),
5986 spec,
5987 seat,
5988 cwd: wt.clone(),
5989 timeout,
5990 allow_write: true,
5991 sessions,
5992 artifacts: artifacts.clone(),
5993 stem: "fix-recovery".to_owned(),
5994 handover: None,
5995 })
5996 .await;
5997 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5998 rec.repair = Repair::WaitingForRestore { before: head };
5999 self.state.recovery = Some(rec);
6000 return self.stop_for_resource();
6001 }
6002 let failed = match &out {
6003 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6004 AgentOutcome::Quota(o) => {
6005 self.state.quota.push(QuotaLoss {
6006 seat: seat.key.clone(),
6007 node: "fix".to_owned(),
6008 at: Timestamp::now(),
6009 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6010 });
6011 Some("rate limited (quota); the fixer could not run".to_owned())
6012 }
6013 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6014 AgentOutcome::Failed(e) => Some(e.clone()),
6015 };
6016 self.state.seats.insert(seat.key.clone(), seat);
6017 let coordination = self.coordinate(&wt).await?;
6018 let failed = match failed {
6019 Some(why) => {
6021 if let Err(e) = git::restore_to(&wt, &head).await {
6022 return self
6023 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6024 .await;
6025 }
6026 Some(why)
6027 }
6028 None => {
6029 if let Ok(r) = git::rescue_commit(
6030 &wt,
6031 "magi: recovery repair of the failed verification (uncommitted work)",
6032 )
6033 .await
6034 {
6035 self.state.note_withheld("fix", &r.withheld);
6036 }
6037 None
6038 }
6039 };
6040 let after = git::rev_parse(&wt, "HEAD").await?;
6041 drop(coordination);
6042 self.state.event(
6043 "recovery",
6044 match &failed {
6045 Some(why) => format!("repair: nothing kept ({why})"),
6046 None if after != head => {
6047 format!("repair: committed {}", short(&after))
6048 }
6049 None => "repair: the fixer made no commit".to_owned(),
6050 },
6051 );
6052 rec.repair = Repair::Done {
6053 before: head.clone(),
6054 committed: after != head,
6055 after,
6056 failed,
6057 };
6058 self.state.recovery = Some(rec);
6059 self.state.save()?;
6060 Ok(())
6061 }
6062
6063 async fn stop_recovery(
6066 &mut self,
6067 mut rec: crate::recovery::RecoveryRecord,
6068 head: &str,
6069 why: String,
6070 ) -> Result<()> {
6071 rec.repair = crate::recovery::Repair::Started {
6073 before: head.to_owned(),
6074 };
6075 self.state.recovery = Some(rec);
6076 self.state
6077 .event("recovery", format!("repair could not be undone: {why}"));
6078 self.state.status = RunStatus::Blocked;
6079 self.state.save()?;
6080 bail!("recovery repair could not be rolled back: {why}")
6081 }
6082
6083 async fn review_loop(&mut self) -> Result<()> {
6084 if self.hold_base_sync().await? {
6085 return Ok(());
6086 }
6087 loop {
6088 if self
6089 .state
6090 .base_sync
6091 .as_ref()
6092 .is_some_and(|s| s.conflict.is_some())
6093 {
6094 return Ok(());
6095 }
6096 if self.catchup_repairable() {
6097 let record = self.state.reviews.last().expect("repairable round").clone();
6098 let winner = self
6099 .state
6100 .winner()
6101 .cloned()
6102 .context("catch-up repair has no winner")?;
6103 if !self.fix_review_round(&winner, record).await? {
6104 return Ok(());
6105 }
6106 }
6107 self.review_loop_pass().await?;
6108 if !self.catchup_repairable() {
6109 return Ok(());
6110 }
6111 }
6112 }
6113
6114 fn catchup_repairable(&self) -> bool {
6117 self.state.reviews.len() < self.state.config.graph.review_rounds
6118 && self.state.reviews.last().is_some_and(|r| {
6119 !r.clean
6120 && r.blocking == 0
6121 && r.fix.is_none()
6122 && r.e2e_status() == E2eStatus::Failed
6123 })
6124 }
6125
6126 async fn review_loop_pass(&mut self) -> Result<()> {
6127 if self
6132 .state
6133 .base_sync
6134 .as_ref()
6135 .is_some_and(|s| s.conflict.is_some())
6136 {
6137 return Ok(());
6138 }
6139 let run_id = self.state.id.clone();
6144 let prompts = self.state.config.prompts.clone();
6145 let Some(winner) = self.state.winner().cloned() else {
6146 return Ok(());
6147 };
6148 let max_rounds = self.state.config.graph.review_rounds;
6149 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6159 if status == RunStatus::Gating {
6162 self.record_contested_handoff();
6163 }
6164 self.state.status = status;
6165 self.state.save()?;
6166 return Ok(());
6167 }
6168 self.state.status = RunStatus::Reviewing;
6169 if self.state.reviews.last().is_some_and(|r| {
6179 r.e2e_status() == E2eStatus::ResourceBlocked
6180 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6181 }) {
6182 let shell = self.state.config.shell();
6183 return self
6184 .stop_unresolved_review(
6185 "the last round's own verification never resolved",
6186 &shell,
6187 &winner.worktree,
6188 )
6189 .await;
6190 }
6191
6192 self.recovery_repair(&winner).await?;
6193 if self.resource_halt()? {
6194 return Ok(());
6195 }
6196
6197 let repo = self.state.repo.clone();
6198 let root = self.state.worktree_root();
6199 let language = self.state.config.graph.language.clone();
6200 let sessions = self.state.config.graph.sessions;
6201 let artifacts = self.merge_repair_artifacts();
6202 let base = self.landing_base();
6203 let base_short = short(&base);
6204 let reviewers = self.roles.reviewers.clone();
6205 let shell = self.state.config.shell();
6206
6207 for round in (self.state.reviews.len() + 1)..=max_rounds {
6208 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6209 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6210 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6211 let prev_verification = self
6220 .state
6221 .reviews
6222 .last()
6223 .and_then(|r| r.verification_summary(&head));
6224 let prior_section = match (&self.state.recovery, round) {
6227 (Some(rec), 1) if self.state.reviews.is_empty() => {
6228 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6229 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6230 Some(crate::recovery::head_of(
6231 &git::diff(&winner.worktree, reviewed, "HEAD")
6232 .await
6233 .unwrap_or_default(),
6234 crate::recovery::SINCE_DIFF_MAX,
6235 ))
6236 }
6237 _ => None,
6238 };
6239 Some(prompt::recovery_review(rec, since.as_deref()))
6240 }
6241 _ => None,
6242 };
6243
6244 let mut jobs = Vec::new();
6248 let coordination = self.coordinate(&repo).await?;
6249 for (r, spec) in reviewers.iter().cloned().enumerate() {
6250 let wt = root.join(format!("review-{}", r + 1));
6251 if wt.exists() {
6254 git::reset_detached(&wt, &head).await?;
6255 } else {
6256 git::worktree_add_detached(&repo, &wt, &head).await?;
6257 }
6258 self.setup_seat_worktree(&repo, &wt, false).await?;
6259 let seat_key = format!("review-{}", r + 1);
6260 let spec = pick_start_spec(
6263 &self.roles.reviewer_roster,
6264 spec,
6265 self.state.seat_history.get(&seat_key),
6266 );
6267 let seat = self.seat(&seat_key, &spec.id);
6268 jobs.push(SeatJob {
6269 prompt: prompt::review(&prompt::ReviewCtx {
6270 instruction: &self.state.instruction,
6271 branch: &winner.branch,
6272 base_short: &base_short,
6273 stat: &stat,
6274 patch: &patch,
6275 verification: prev_verification.as_ref(),
6276 prior: prior_section.as_deref(),
6277 reviewers: reviewers.len(),
6278 round,
6279 rounds: max_rounds,
6280 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6283 lens: Lens::for_seat(r),
6284 language: &language,
6285 github_language: &self.state.config.graph.github_language,
6286 }),
6287 spec,
6288 seat,
6289 cwd: wt,
6290 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6291 allow_write: false,
6292 sessions,
6293 artifacts: artifacts.clone(),
6294 stem: format!("review-{round}-{}", r + 1),
6295 handover: None,
6296 });
6297 }
6298
6299 drop(coordination);
6300 self.state.event(
6301 "review",
6302 format!(
6303 "round {round}: {} reviewers on {}",
6304 jobs.len(),
6305 short(&head)
6306 ),
6307 );
6308 let mut quota_losses = Vec::new();
6309 let review_retries = self.state.config.graph.retries;
6310 let review_cache = self.state.config.cache_dir();
6311 let ctx = WaveCtx {
6312 carry_seats: true,
6313 pauses: Some((&self.pause, &self.interrupt)),
6314 run: &run_id,
6315 node: "review",
6316 prompts: &prompts,
6317 cache: review_cache.as_deref(),
6318 round: Some(round),
6319 };
6320 let results = ask_json_wave::<Review>(
6321 jobs,
6322 Arc::clone(&self.sem),
6323 review_retries,
6324 &self.roles.reviewer_roster,
6325 &ctx,
6326 &mut quota_losses,
6327 &mut self.state,
6328 &|_: &Review| Ok(()),
6329 )
6330 .await;
6331 if self.state.resource_block.is_some() {
6332 return self.stop_for_resource();
6333 }
6334 let round_quota_missing = quota_losses.len();
6338 self.state.quota.extend(quota_losses);
6339
6340 let mut records = Vec::new();
6341 let mut all_findings = Vec::new();
6342 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6343 let agent_id = seat.agent.clone();
6344 self.state.seats.insert(seat.key.clone(), seat);
6345 let mut record = ReviewRecord {
6346 reviewer: r + 1,
6347 agent: agent_id,
6348 summary: String::new(),
6349 findings: Vec::new(),
6350 vote: None,
6351 failed: None,
6352 duration_ms: 0,
6353 attempts,
6359 };
6360 match res {
6361 Ok((review, out)) => {
6362 record.summary =
6370 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6371 record.vote = Some(review.vote);
6372 record.duration_ms = out.duration_ms;
6373 for (n, mut f) in review.findings.into_iter().enumerate() {
6374 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6377 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6378 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6379 f.file = f
6385 .file
6386 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6387 all_findings.push(f.clone());
6388 record.findings.push(f);
6389 }
6390 self.state.event(
6391 "review",
6392 format!(
6393 "round {round}: reviewer {} voted {} with {} finding(s)",
6394 r + 1,
6395 review.vote.label(),
6396 record.findings.len()
6397 ),
6398 );
6399 }
6400 Err(e) => {
6401 record.failed = Some(e.to_string());
6402 self.state.event(
6403 "review",
6404 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6405 );
6406 }
6407 }
6408 records.push(record);
6409 }
6410
6411 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6418 let vote_split =
6419 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6420 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6421 if vote_split {
6422 self.state.event(
6423 "review",
6424 format!(
6425 "round {round}: votes split ({}) — one round of reconsideration",
6426 initial_votes
6427 .iter()
6428 .map(|v| v.label())
6429 .collect::<Vec<_>>()
6430 .join(", ")
6431 ),
6432 );
6433 let panel: Vec<ReviewSeatReport<'_>> = records
6436 .iter()
6437 .filter_map(|r| {
6438 r.vote.map(|vote| ReviewSeatReport {
6439 reviewer: r.reviewer,
6440 vote,
6441 summary: &r.summary,
6442 findings: &r.findings,
6443 })
6444 })
6445 .collect();
6446
6447 let mut jobs = Vec::new();
6448 let mut seats_at = Vec::new();
6449 for (r, spec) in reviewers.iter().cloned().enumerate() {
6450 if records[r].vote.is_none() {
6454 continue;
6455 }
6456 let wt = root.join(format!("review-{}", r + 1));
6457 let seat_key = format!("review-{}", r + 1);
6458 let spec = self.occupant(&seat_key, spec);
6459 let seat = self.seat(&seat_key, &spec.id);
6460 let build = |with_patch: bool| {
6468 prompt::review_reconsider(&ReviewReconsiderCtx {
6469 instruction: &self.state.instruction,
6470 reviewer: r + 1,
6471 lens: Lens::for_seat(r),
6472 panel: &panel,
6473 patch: with_patch.then_some(ReviewPatch {
6474 branch: &winner.branch,
6475 base_short: &base_short,
6476 stat: &stat,
6477 patch: &patch,
6478 }),
6479 round,
6480 rounds: max_rounds,
6481 language: &language,
6482 })
6483 };
6484 let full = build(true);
6485 let prompt = if has_context(&spec, &seat, sessions) {
6486 build(false)
6487 } else {
6488 full.clone()
6489 };
6490 jobs.push(SeatJob {
6491 prompt,
6492 spec,
6493 seat,
6494 cwd: wt,
6495 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6496 allow_write: false,
6497 sessions,
6498 artifacts: artifacts.clone(),
6499 stem: format!("review-{round}-reconsider-{}", r + 1),
6500 handover: Some(full),
6501 });
6502 seats_at.push(r);
6503 }
6504
6505 let mut recon_quota_losses = Vec::new();
6506 let recon_cache = self.state.config.cache_dir();
6507 let recon_ctx = WaveCtx {
6508 carry_seats: true,
6509 pauses: Some((&self.pause, &self.interrupt)),
6510 run: &run_id,
6511 node: "review",
6512 prompts: &prompts,
6513 cache: recon_cache.as_deref(),
6514 round: Some(round),
6515 };
6516 let recon_results = ask_json_wave::<ReviewRevote>(
6517 jobs,
6518 Arc::clone(&self.sem),
6519 review_retries,
6520 &self.roles.reviewer_roster,
6521 &recon_ctx,
6522 &mut recon_quota_losses,
6523 &mut self.state,
6524 &|_: &ReviewRevote| Ok(()),
6525 )
6526 .await;
6527 if self.state.resource_block.is_some() {
6528 return self.stop_for_resource();
6529 }
6530 self.state.quota.extend(recon_quota_losses);
6531
6532 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6533 let agent_id = seat.agent.clone();
6534 self.state.seats.insert(seat.key.clone(), seat);
6535 let mut rec = ReviewRevoteRecord {
6536 reviewer: r + 1,
6537 agent: agent_id,
6538 vote: None,
6539 reason: String::new(),
6540 failed: None,
6541 };
6542 match res {
6543 Ok((rv, _)) => {
6544 rec.vote = Some(rv.vote);
6545 rec.reason =
6546 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6547 self.state.event(
6548 "review",
6549 format!(
6550 "round {round}: reviewer {} revoted {}",
6551 r + 1,
6552 rv.vote.label()
6553 ),
6554 );
6555 }
6556 Err(e) => {
6557 rec.failed = Some(e.to_string());
6558 self.state.event(
6559 "review",
6560 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6561 );
6562 }
6563 }
6564 reconsideration.push(rec);
6565 }
6566 } else if initial_votes.len() > 1 {
6567 self.state.event(
6568 "review",
6569 format!(
6570 "round {round}: votes agreed ({}) — no reconsideration",
6571 initial_votes[0].label()
6572 ),
6573 );
6574 }
6575
6576 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6577 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6578 let defer_e2e =
6589 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6590 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6591 let reason =
6592 format!("{blocking} blocking finding(s) already required a fix this round");
6593 self.state.event(
6594 "verify",
6595 format!(
6596 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6597 {}); it will run once a round has none left",
6598 short(&head)
6599 ),
6600 );
6601 (Vec::new(), false, true, Some(reason))
6602 } else {
6603 let e2e_commands = self.state.config.verify.e2e.clone();
6604 let cache_dir = self.state.config.cache_dir();
6605 let context = format!("round {round}");
6606 let wait = self.repository_wait();
6607 let (e2e, verify_retried) = with_cache_lease(
6608 &mut self.state,
6609 cache_dir.as_deref(),
6610 "e2e",
6611 "e2e",
6612 &winner.worktree,
6613 &head,
6614 verify_timeout,
6615 &context,
6616 &wait,
6617 |state, budget| {
6618 let shell = shell.clone();
6619 let e2e_commands = e2e_commands.clone();
6620 let worktree = winner.worktree.clone();
6621 let context = context.clone();
6622 async move {
6623 run_e2e_with_retry(
6624 state,
6625 &shell,
6626 &e2e_commands,
6627 &worktree,
6628 budget,
6629 &context,
6630 )
6631 .await
6632 }
6633 },
6634 )
6635 .await?;
6636 (e2e, verify_retried, false, None)
6637 };
6638
6639 let expected = records.len();
6640 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6641 let incomplete = answered < expected;
6642 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6643 let policy = self.state.config.graph.incomplete_review;
6644 let clean = round_is_clean(
6645 blocking,
6646 e2e_ok,
6647 answered,
6648 expected,
6649 round_quota_missing,
6650 policy,
6651 );
6652
6653 let mut round_record = ReviewRound {
6654 round,
6655 head: head.clone(),
6656 verified_head: None,
6657 verified_at: None,
6658 reviews: records,
6659 e2e,
6660 verify_retried,
6661 e2e_deferred,
6662 e2e_defer_reason,
6663 fix: None,
6664 blocking,
6665 answered,
6666 expected,
6667 clean,
6668 progressed: false,
6669 vote_split,
6670 reconsideration,
6671 verdict: None,
6672 };
6673 round_record.verdict = ReviewVote::worst(
6677 round_record
6678 .final_votes()
6679 .into_iter()
6680 .map(|(_, _, vote)| vote),
6681 );
6682 if !matches!(
6693 round_record.e2e_status(),
6694 E2eStatus::Deferred | E2eStatus::NotConfigured
6695 ) {
6696 round_record.verified_head = Some(head.clone());
6697 round_record.verified_at = Some(Timestamp::now());
6698 }
6699
6700 if incomplete {
6701 let missing: Vec<String> = round_record
6702 .reviews
6703 .iter()
6704 .filter(|r| r.failed.is_some())
6705 .map(|r| format!("review-{}", r.reviewer))
6706 .collect();
6707 self.state.event(
6708 "review",
6709 format!(
6710 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6711 missing.join(", ")
6712 ),
6713 );
6714 }
6715
6716 if clean {
6717 self.state.event(
6718 "review",
6719 if incomplete && policy == IncompleteReviewPolicy::Warn {
6720 format!(
6721 "round {round}: clean (warn policy, incomplete panel) — no \
6722 blocking findings from the seats that answered, verification green"
6723 )
6724 } else if incomplete {
6725 format!(
6726 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6727 quorum) — no blocking findings from the seats that answered, \
6728 verification green",
6729 expected - answered
6730 )
6731 } else {
6732 format!("round {round}: clean — no blocking findings, verification green")
6733 },
6734 );
6735 self.state.reviews.push(round_record);
6736 self.state.status = RunStatus::Gating;
6737 self.state.save()?;
6738 return Ok(());
6739 }
6740
6741 if incomplete && blocking == 0 && e2e_ok {
6749 self.state.reviews.push(round_record);
6750 self.state.save()?;
6751 if round == max_rounds {
6752 self.state.status = RunStatus::Blocked;
6753 self.state.event(
6754 "review",
6755 format!(
6756 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6757 refusing to call it clean",
6758 expected - answered
6759 ),
6760 );
6761 return Ok(());
6762 }
6763 continue;
6764 }
6765
6766 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6778 self.state.reviews.push(round_record);
6779 return self
6780 .stop_unresolved_review(
6781 "the round's own verification could not run",
6782 &shell,
6783 &winner.worktree,
6784 )
6785 .await;
6786 }
6787
6788 if round == max_rounds {
6789 self.state.reviews.push(round_record);
6790 return self
6791 .stop_reviewing(
6792 &format!(
6793 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6794 ),
6795 &shell,
6796 &winner.worktree,
6797 )
6798 .await;
6799 }
6800
6801 if !self.fix_review_round(&winner, round_record).await? {
6802 return Ok(());
6803 }
6804 }
6805 Ok(())
6806 }
6807
6808 async fn fix_review_round(
6811 &mut self,
6812 winner: &Candidate,
6813 mut round_record: ReviewRound,
6814 ) -> Result<bool> {
6815 let round = round_record.round;
6816 let shell = self.state.config.shell();
6817 let step = self.run_fix_round(winner, &mut round_record).await?;
6818 if matches!(step, FixStep::Resource) {
6819 self.stop_for_resource()?;
6820 return Ok(false);
6821 }
6822 if self.state.reviews.len() < round {
6823 self.state.reviews.push(round_record);
6824 } else {
6825 self.state.reviews[round - 1] = round_record;
6826 }
6827 self.state.save()?;
6828
6829 if matches!(step, FixStep::Recorded { report_lost: true }) {
6842 self.stop_reviewing(
6843 "the fixer's adoption report never came back, even after resuming its \
6844 own seat; refusing to start another round against the same worktree \
6845 while that is unresolved",
6846 &shell,
6847 &winner.worktree,
6848 )
6849 .await?;
6850 return Ok(false);
6851 }
6852
6853 let streak = self
6854 .state
6855 .reviews
6856 .iter()
6857 .rev()
6858 .take_while(|r| !r.progressed)
6859 .count();
6860 if streak >= STAGNANT_LIMIT {
6861 self.stop_reviewing(
6862 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6863 &shell,
6864 &winner.worktree,
6865 )
6866 .await?;
6867 return Ok(false);
6868 }
6869 Ok(true)
6870 }
6871
6872 async fn run_fix_round(
6874 &mut self,
6875 winner: &Candidate,
6876 round_record: &mut ReviewRound,
6877 ) -> Result<FixStep> {
6878 let round = round_record.round;
6879 let max_rounds = self.state.config.graph.review_rounds;
6880 let run_id = self.state.id.clone();
6881 let prompts = self.state.config.prompts.clone();
6882 let language = self.state.config.graph.language.clone();
6883 let sessions = self.state.config.graph.sessions;
6884 let artifacts = self.merge_repair_artifacts();
6885 let base = self.landing_base();
6886 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6887 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6888 let this_round_verification = round_record.verification_summary(&head);
6889 let all_findings: Vec<_> = round_record
6890 .reviews
6891 .iter()
6892 .flat_map(|r| r.findings.iter().cloned())
6893 .collect();
6894 let blocking_findings: Vec<_> = all_findings
6897 .iter()
6898 .filter(|f| f.severity.blocks())
6899 .cloned()
6900 .collect();
6901 let fix_prompt = prompt::fix_in(
6902 &self.state.instruction,
6903 &blocking_findings,
6904 this_round_verification.as_ref(),
6905 round,
6906 max_rounds,
6907 &language,
6908 &self.state.config.graph.github_language,
6909 );
6910 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6911 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6912 let (job, seat, out) = self
6913 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6914 prompt: fix_prompt.clone(),
6915 spec,
6916 seat,
6917 cwd: winner.worktree.clone(),
6918 timeout,
6919 allow_write: true,
6920 sessions,
6921 artifacts: artifacts.clone(),
6922 stem: format!("fix-{round}"),
6923 handover: None,
6924 })
6925 .await;
6926 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6929 return Ok(FixStep::Resource);
6930 }
6931 let agent_id = seat.agent.clone();
6932
6933 let mut fix = FixRecord {
6934 agent: agent_id,
6935 addressed: Vec::new(),
6936 rejected: Vec::new(),
6937 notes: String::new(),
6938 committed: false,
6939 failed: None,
6940 duration_ms: 0,
6941 continuation: None,
6942 };
6943 let mut continuation = ContinuationRecord::not_needed();
6944 let mut final_seat = seat.clone();
6945 match out {
6946 AgentOutcome::Ok(o) => {
6947 fix.duration_ms = o.duration_ms;
6948 let parsed = verdict::extract_json::<FixReport>(&o.text);
6949 let incomplete_reason = match &parsed {
6956 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6957 "the reply parsed, but it reported a command whose own CLI never \
6958 confirmed an exit status"
6959 .to_owned(),
6960 ),
6961 Ok(_) => None,
6962 Err(e) => Some(e.to_string()),
6963 };
6964 match incomplete_reason {
6965 None => {
6966 let report = parsed.expect("checked Ok above");
6967 fix.addressed = report.addressed;
6968 fix.rejected = report.rejected;
6969 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6970 }
6971 Some(reason) => {
6972 let (resumed_seat, resolved, failure, cont) = self
6973 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6974 .await;
6975 fix.duration_ms += cont.cumulative_wait_ms;
6976 continuation = cont;
6977 final_seat = resumed_seat;
6978 match resolved {
6979 Some(report) => {
6980 fix.addressed = report.addressed;
6981 fix.rejected = report.rejected;
6982 fix.notes =
6983 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6984 }
6985 None => fix.failed = failure,
6986 }
6987 }
6988 }
6989 }
6990 AgentOutcome::Dropped(o) => {
6992 fix.duration_ms = o.duration_ms;
6993 let why = o
6994 .dropped
6995 .as_ref()
6996 .map(|d| d.why.as_str())
6997 .unwrap_or("the CLI ended the stream without delivering its answer");
6998 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6999 }
7000 AgentOutcome::Quota(o) => {
7001 self.state.quota.push(QuotaLoss {
7002 seat: final_seat.key.clone(),
7003 node: "fix".to_owned(),
7004 at: Timestamp::now(),
7005 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7006 });
7007 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7008 }
7009 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7010 }
7011 fix.continuation = Some(continuation);
7012 self.state.seats.insert(final_seat.key.clone(), final_seat);
7013 let coordination = self.coordinate(&winner.worktree).await?;
7014 if let Ok(r) = git::rescue_commit(
7015 &winner.worktree,
7016 &format!("magi: review round {round} fixes (uncommitted work)"),
7017 )
7018 .await
7019 {
7020 self.state.note_withheld("fix", &r.withheld);
7021 }
7022 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7023 fix.committed = after != before;
7024 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7032 drop(coordination);
7033 let progressed = diff_after != patch;
7034 let commit_note = if fix.committed {
7035 "committed"
7036 } else {
7037 "NO new commit"
7038 };
7039 let tree_note = if progressed {
7040 "changed vs base"
7041 } else {
7042 "unchanged vs base"
7043 };
7044 self.state.event(
7045 "fix",
7046 match &fix.failed {
7047 Some(reason) => {
7053 format!(
7054 "round {round}: fixer's adoption report was lost ({reason}); \
7055 {commit_note}, tree {tree_note}"
7056 )
7057 }
7058 None => format!(
7059 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7060 {tree_note}{}",
7061 fix.addressed.len(),
7062 fix.rejected.len(),
7063 if continuation.outcome == ContinuationOutcome::Resumed {
7064 format!(
7065 " (adoption report recovered after {} continuation(s))",
7066 continuation.attempts
7067 )
7068 } else {
7069 String::new()
7070 },
7071 ),
7072 },
7073 );
7074 round_record.fix = Some(fix);
7075 round_record.progressed = progressed;
7076 Ok(FixStep::Recorded {
7077 report_lost: matches!(
7078 continuation.outcome,
7079 ContinuationOutcome::Exhausted
7080 | ContinuationOutcome::QuotaLost
7081 | ContinuationOutcome::NoSession
7082 ),
7083 })
7084 }
7085
7086 fn record_contested_handoff(&mut self) {
7120 if self.state.contested_handoff.is_some() {
7121 return;
7122 }
7123 let Some(contested) = self
7124 .state
7125 .reviews
7126 .last()
7127 .and_then(ReviewRound::contested_handoff)
7128 else {
7129 return;
7130 };
7131 self.state.event(
7132 "review",
7133 format!(
7134 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7135 wait for the owner's approval",
7136 contested.findings.len(),
7137 contested.rejecters.len()
7138 ),
7139 );
7140 self.state.contested_handoff = Some(contested);
7141 }
7142
7143 async fn stop_unresolved_review(
7144 &mut self,
7145 why: &str,
7146 shell: &[String],
7147 worktree: &Path,
7148 ) -> Result<()> {
7149 self.stop_reviewing_inner(why, shell, worktree, true).await
7150 }
7151
7152 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7153 self.stop_reviewing_inner(why, shell, worktree, false).await
7154 }
7155
7156 async fn stop_reviewing_inner(
7157 &mut self,
7158 why: &str,
7159 shell: &[String],
7160 worktree: &Path,
7161 repairable: bool,
7162 ) -> Result<()> {
7163 let round_idx = self.state.reviews.len() - 1;
7164 let needs_catchup_run = matches!(
7172 self.state.reviews[round_idx].e2e_status(),
7173 E2eStatus::Deferred | E2eStatus::ResourceBlocked
7174 );
7175 if needs_catchup_run {
7176 let round = self.state.reviews[round_idx].round;
7177 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7178 let commands = self.state.config.verify.e2e.clone();
7179 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7180 let cache_dir = self.state.config.cache_dir();
7181 let context = format!(
7182 "round {round}: verification unresolved, catching up before the final decision"
7183 );
7184 let wait = self.repository_wait();
7185 let (outcomes, verify_retried) = with_cache_lease(
7186 &mut self.state,
7187 cache_dir.as_deref(),
7188 "e2e",
7189 "e2e",
7190 worktree,
7191 &attempted_head,
7192 timeout,
7193 &context,
7194 &wait,
7195 |state, budget| {
7196 let shell = shell.to_vec();
7197 let commands = commands.clone();
7198 let context = context.clone();
7199 async move {
7200 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7201 .await
7202 }
7203 },
7204 )
7205 .await?;
7206 let last = &mut self.state.reviews[round_idx];
7207 last.e2e = outcomes;
7208 last.verify_retried = verify_retried;
7209 last.verified_head = Some(attempted_head);
7216 last.verified_at = Some(Timestamp::now());
7217 if verify_inconclusive(&last.e2e) {
7218 self.state.save()?;
7225 return Ok(());
7226 }
7227 last.e2e_deferred = false;
7228 }
7229 let last = &self.state.reviews[round_idx];
7230 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7231
7232 match last.e2e_status() {
7233 E2eStatus::Failed => {
7234 let red: Vec<String> = last
7235 .e2e
7236 .iter()
7237 .filter(|o| !o.ok())
7238 .map(|o| {
7239 format!(
7240 "`{}` -> {:?}\n{}",
7241 o.command,
7242 o.code,
7243 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7244 )
7245 })
7246 .collect();
7247 let can_repair = last.blocking == 0 && last.fix.is_none();
7248 self.state.reviews[round_idx].clean = false;
7249 self.state
7250 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7251 self.state.status = RunStatus::Blocked;
7252 if repairable
7253 && can_repair
7254 && self.state.reviews.len() < self.state.config.graph.review_rounds
7255 {
7256 self.state.status = RunStatus::Reviewing;
7257 }
7258 }
7259 E2eStatus::ResourceBlocked => {
7264 self.state.event(
7265 "review",
7266 format!(
7267 "{why}; e2e could not run (resource unavailable); not \
7268 deciding yet"
7269 ),
7270 );
7271 }
7272 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7273 self.state.event(
7274 "review",
7275 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7276 );
7277 self.record_contested_handoff();
7278 self.state.status = RunStatus::Gating;
7279 }
7280 }
7281 self.state.save()?;
7282 Ok(())
7283 }
7284
7285 async fn gate(&mut self) -> Result<()> {
7288 if self.hold_base_sync().await? {
7289 return Ok(());
7290 }
7291 if self.state.status == RunStatus::Failed
7303 || self
7304 .state
7305 .base_sync
7306 .as_ref()
7307 .is_some_and(|s| s.conflict.is_some())
7308 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7309 != Some(RunStatus::Gating)
7310 {
7311 return Ok(());
7312 }
7313 if self.state.gate_ran {
7314 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7325 self.state.status = RunStatus::Blocked;
7326 self.state.save()?;
7327 }
7328 return Ok(());
7329 }
7330 let Some(winner) = self.state.winner().cloned() else {
7331 return Ok(());
7332 };
7333 self.state.status = RunStatus::Gating;
7334 let mut outcomes = self.run_gate(&winner).await?;
7335 loop {
7336 if verify_inconclusive(&outcomes) {
7347 self.state.save()?;
7348 return Ok(());
7349 }
7350 if outcomes.iter().all(CommandOutcome::ok) {
7351 break;
7352 }
7353 match self.gate_fix_round(&winner, &outcomes).await? {
7354 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7355 GateFix::Stop => break,
7356 GateFix::Defer => {
7357 self.state.save()?;
7358 return Ok(());
7359 }
7360 }
7361 }
7362 let passed = outcomes.iter().all(CommandOutcome::ok);
7363 self.state.gate = outcomes;
7364 self.state.gate_ran = true;
7365 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7366 if !passed {
7367 self.state.status = RunStatus::Blocked;
7368 let spent = self.state.gate_fixes.len();
7369 self.state.event(
7370 "gate",
7371 if spent == 0 {
7372 "gate failed; not merging".to_owned()
7373 } else {
7374 format!("gate failed after {spent} gate-fix round(s); not merging")
7375 },
7376 );
7377 }
7378 self.state.save()?;
7379 Ok(())
7380 }
7381
7382 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7392 let commands = state.config.verify.pre_gate.clone();
7393 if commands.is_empty() {
7394 return;
7395 }
7396 let shell = state.config.shell();
7397 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7398 let (outcomes, _) = run_commands(
7399 state,
7400 "pre_gate",
7401 "pre_gate",
7402 0,
7403 &shell,
7404 &commands,
7405 &winner.worktree,
7406 timeout,
7407 )
7408 .await;
7409 for o in &outcomes {
7410 if !o.ok() {
7411 tracing::warn!(
7412 "pre_gate `{}` failed ({:?}); the gate decides",
7413 o.command,
7414 o.code
7415 );
7416 }
7417 state.event(
7418 "pre_gate",
7419 format!(
7420 "`{}` -> {}",
7421 o.command,
7422 if o.ok() {
7423 "pass".to_owned()
7424 } else {
7425 format!(
7426 "FAIL ({:?})\n{}",
7427 o.code,
7428 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7429 )
7430 }
7431 ),
7432 );
7433 }
7434 state.pre_gate = outcomes;
7435 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7436 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7437 Ok(head) => {
7438 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7439 state.pre_gate_commit = Some(head);
7440 }
7441 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7442 },
7443 Ok(false) => {}
7444 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7445 }
7446 if let Err(e) = state.save() {
7447 tracing::warn!("could not persist the pre_gate record: {e:#}");
7448 }
7449 }
7450
7451 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7454 let wait = self.repository_wait();
7455 run_landing_gate(&mut self.state, winner, &wait).await
7456 }
7457
7458 async fn gate_fix_round(
7471 &mut self,
7472 winner: &Candidate,
7473 outcomes: &[CommandOutcome],
7474 ) -> Result<GateFix> {
7475 let cap = self.state.config.graph.gate_fix_rounds;
7476 let spent = self.state.gate_fixes.len();
7477 if spent >= cap {
7478 if cap > 0 {
7479 self.state.event(
7480 "gate",
7481 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7482 );
7483 }
7484 return Ok(GateFix::Stop);
7485 }
7486 if !gate_fixable(outcomes) {
7487 self.state.event(
7488 "gate",
7489 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7490 command or similar); not spending a fix round on it",
7491 );
7492 return Ok(GateFix::Stop);
7493 }
7494 let min_free = self.state.config.disk.min_free_bytes;
7495 if min_free > 0 {
7496 match crate::disk::free_bytes(&winner.worktree) {
7497 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7498 Ok(free) => {
7499 self.state.event(
7500 "gate",
7501 format!(
7502 "only {free} bytes free ({min_free} required by `[disk] \
7503 min_free_bytes`); not spending a fix round on a failure the disk \
7504 may explain"
7505 ),
7506 );
7507 return Ok(GateFix::Stop);
7508 }
7509 Err(e) => {
7510 self.state.event(
7511 "gate",
7512 format!("free disk space could not be measured ({e:#}); no fix round"),
7513 );
7514 return Ok(GateFix::Stop);
7515 }
7516 }
7517 }
7518
7519 let attempt = spent + 1;
7520 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7521 let base = self.landing_base();
7522 let fix_prompt = prompt::gate_fix_in(
7523 &self.state.instruction,
7524 &failed,
7525 attempt,
7526 cap,
7527 &self.state.config.graph.language,
7528 &self.state.config.graph.github_language,
7529 );
7530 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7531 let sessions = self.state.config.graph.sessions;
7532 let artifacts = self.merge_repair_artifacts();
7533 self.state.event(
7534 "gate",
7535 format!("gate failed; gate-fix round {attempt} of {cap}"),
7536 );
7537 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7538 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7539 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7542 head: before.clone(),
7543 passed: false,
7544 outcomes: Vec::new(),
7545 });
7546 self.state.save()?;
7547 let (_, seat, out) = self
7548 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7549 prompt: fix_prompt.clone(),
7550 spec,
7551 seat,
7552 cwd: winner.worktree.clone(),
7553 timeout,
7554 allow_write: true,
7555 sessions,
7556 artifacts: artifacts.clone(),
7557 stem: format!("gate-fix-{attempt}"),
7558 handover: None,
7559 })
7560 .await;
7561 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7564 return Ok(GateFix::Defer);
7565 }
7566 let mut record = GateFixRecord {
7567 agent: seat.agent.clone(),
7568 failed,
7569 notes: String::new(),
7570 committed: false,
7571 error: None,
7572 };
7573 match out {
7574 AgentOutcome::Ok(o) => {
7575 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7578 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7579 }
7580 }
7581 AgentOutcome::Dropped(_) => {
7582 record.error = Some("the CLI dropped the stream".to_owned());
7583 }
7584 AgentOutcome::Quota(o) => {
7585 self.state.quota.push(QuotaLoss {
7586 seat: seat.key.clone(),
7587 node: "gate-fix".to_owned(),
7588 at: Timestamp::now(),
7589 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7590 });
7591 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7592 }
7593 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7594 }
7595 self.state.seats.insert(seat.key.clone(), seat);
7596 let coordination = self.coordinate(&winner.worktree).await?;
7597 if let Ok(r) = git::rescue_commit(
7598 &winner.worktree,
7599 &format!("magi: gate fix {attempt} (uncommitted work)"),
7600 )
7601 .await
7602 {
7603 self.state.note_withheld("gate-fix", &r.withheld);
7604 }
7605 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7606 record.committed = after != before;
7607 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7608 let note = record.error.clone();
7609 self.state.gate_fixes.push(record);
7610 self.state.save()?;
7611 if !changed {
7612 self.state.event(
7613 "gate",
7614 match note {
7615 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7616 None => format!("gate-fix round {attempt}: the tree did not change"),
7617 },
7618 );
7619 return Ok(GateFix::Stop);
7620 }
7621 self.state.event(
7622 "gate",
7623 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7624 );
7625
7626 drop(coordination);
7627 let wait = self.repository_wait();
7628 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7629 }
7630
7631 async fn guarded_pr_message(
7638 &mut self,
7639 winner: &Candidate,
7640 facts: Option<&BranchFacts>,
7641 posting: bool,
7642 ) -> Result<Option<PrMessage>> {
7643 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7644 let head = match final_head.as_deref() {
7645 Some(h) => crate::pr_outcome::FinalHead::At(h),
7646 None => crate::pr_outcome::FinalHead::Unknown,
7647 };
7648 let summarized = if posting {
7650 Some(self.summarize_change(winner, final_head.as_deref()).await)
7651 } else {
7652 None
7653 };
7654 if self.state.resource_block.is_some() {
7659 self.state.event(
7660 "pr-summary",
7661 "a shared resource was busy; nothing was published, resume to summarize again",
7662 );
7663 return Ok(None);
7664 }
7665 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7666 anyhow::bail!(
7667 "the final commit moved during public description generation; retry before publishing"
7668 );
7669 }
7670 let note = match (&summarized, &self.state.change_summary) {
7671 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7672 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7673 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7674 (None, _) => ChangeNote::NotAsked,
7675 };
7676 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7677 if !posting {
7678 let identity = crate::scrub::Identity::current();
7679 return Ok(Some(PrMessage {
7680 title: crate::scrub::scrub(&pr.title, &identity),
7681 body: crate::scrub::scrub(&pr.body, &identity),
7682 }));
7683 }
7684 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7687 if let Some(known) = self
7688 .state
7689 .github_text
7690 .clone()
7691 .filter(|g| g.fingerprint == raw_fp)
7692 {
7693 return self.settle_withheld(winner, &pr, known).await;
7694 }
7695 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7696 let mut violations =
7697 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7698 let identity = crate::scrub::Identity::current();
7699 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7700 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7701 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7702 {
7703 violations.push(crate::github_text::Violation::SensitiveData);
7704 }
7705 let has_language = violations
7709 .iter()
7710 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7711 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7712 self.state.event(
7713 "github-text",
7714 if has_language {
7715 format!("description rejected: {violations:?}; requesting one rewrite")
7716 } else {
7717 format!("description rejected: {violations:?}; asking the owner")
7718 },
7719 );
7720 let mut rewritten = false;
7721 let mut rejected_rewrite = None;
7722 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7723 {
7724 let output = text;
7725 if let Some(title) = summary_title(&output) {
7726 let raw_body = summary_without_title(&output);
7729 let clean_title = crate::scrub::scrub(&title, &identity);
7730 let mut body = crate::scrub::scrub(&raw_body, &identity);
7731 let redacted = clean_title != title || body != raw_body;
7732 let title = clean_title;
7733 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7734 if !complete {
7735 self.state.event(
7736 "github-text",
7737 "description rewrite omitted required findings",
7738 );
7739 }
7740 if !body.trim().is_empty() {
7741 body.push_str("\n\n");
7742 body.push_str(&public_pr_evidence(&self.state, head));
7743 }
7744 let marker = format!("magi:run/{}", self.state.id);
7745 if !body.trim().is_empty() && !body.contains(&marker) {
7746 body.push_str(&format!("\n\n{marker}"));
7747 }
7748 let rewrite = PrMessage {
7749 title: title.clone(),
7750 body: body.clone(),
7751 };
7752 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7753 let mut failed = crate::github_text::check_for(
7754 &self.state.config.graph,
7755 &title,
7756 &body,
7757 redecision,
7758 );
7759 if (!crate::github_text::shareable(&title)
7760 || !crate::github_text::shareable(&body))
7761 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7762 {
7763 failed.push(crate::github_text::Violation::SensitiveData);
7764 }
7765 if complete
7766 && !title.trim().is_empty()
7767 && !body.trim().is_empty()
7768 && !failed.is_empty()
7769 {
7770 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7771 title: title.clone(),
7772 body: body.clone(),
7773 failed: failed
7774 .iter()
7775 .map(|v| crate::github_text::category(*v).to_owned())
7776 .collect(),
7777 });
7778 }
7779 if complete
7780 && !title.trim().is_empty()
7781 && !raw_body.trim().is_empty()
7782 && failed.is_empty()
7783 {
7784 decision = redecision;
7785 pr = PrMessage { title, body };
7786 rewritten = true;
7787 self.state
7788 .event("github-text", "description rewrite passed");
7789 if redacted {
7790 self.state.event(
7791 "github-text",
7792 "sensitive data removed from description rewrite",
7793 );
7794 }
7795 }
7796 }
7797 }
7798 if !rewritten {
7799 self.state.event(
7800 "github-text",
7801 "description rewrite unavailable or rejected; asking the owner",
7802 );
7803 if let Some(w) =
7804 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7805 {
7806 self.retire_github_text_question("the withheld text changed");
7809 let record = crate::run::GithubTextAsk {
7810 rewrite: rejected_rewrite,
7811 fingerprint: raw_fp,
7812 title: w.title,
7813 body: w.body,
7814 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7815 question: None,
7816 asks: 0,
7817 resolved: false,
7818 chosen_title: None,
7819 choice: None,
7820 };
7821 return self.settle_withheld(winner, &pr, record).await;
7822 }
7823 }
7824 }
7825 let (title, body) =
7826 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7827 Ok(Some(PrMessage { title, body }))
7828 }
7829
7830 async fn summarize_change(
7841 &mut self,
7842 winner: &Candidate,
7843 head: Option<&str>,
7844 ) -> std::result::Result<(), String> {
7845 let Some(head) = head else {
7846 return Err("the final commit could not be read".to_owned());
7847 };
7848 let remote = self.state.config.merge.remote.clone();
7849 let tracking = format!("{remote}/{}", self.state.base_branch);
7850 if !git::rev_exists(&winner.worktree, &tracking).await {
7851 return Err(format!("the remote base `{tracking}` could not be read"));
7852 }
7853 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7854 Ok(b) if !b.is_empty() => b,
7855 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7856 };
7857 let evidence = crate::pr_outcome::summary_evidence(
7858 &self.state,
7859 crate::pr_outcome::FinalHead::At(head),
7860 );
7861 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7862 if self
7863 .state
7864 .change_summary
7865 .as_ref()
7866 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7867 {
7868 return Ok(());
7869 }
7870 if self.state.change_summary.take().is_some() {
7871 self.state.event(
7872 "pr-summary",
7873 "the final diff or evidence changed; the earlier change summary was discarded",
7874 );
7875 }
7876 if let Err(e) = write_artifact(
7878 &self.state,
7879 &format!("public-description-{fingerprint}.md"),
7880 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7881 ) {
7882 tracing::warn!("could not save public description evidence: {e:#}");
7883 }
7884 let chain = match agent::pick_chain(
7885 &self.state.config.agents,
7886 self.state.config.roles.synthesizer.as_ref(),
7887 &agent::installed,
7888 "synthesizer",
7889 ) {
7890 Ok(chain) => chain,
7891 Err(e) => {
7892 tracing::warn!("no agent for the change summary: {e:#}");
7893 self.state.event("pr-summary", "no summary agent available");
7894 return Err("no agent was available to summarize it".to_owned());
7895 }
7896 };
7897 let subjects = git::subjects(&winner.worktree, &base, head)
7898 .await
7899 .unwrap_or_default();
7900 let stat = git::diff_stat(&winner.worktree, &base, head)
7901 .await
7902 .unwrap_or_default();
7903 let stat: String = stat.chars().take(6_000).collect();
7904 let mut prompt = crate::prompt::change_summary_in(
7905 &base,
7906 head,
7907 &subjects,
7908 stat.trim(),
7909 &self.state.config.graph.github_language,
7910 );
7911 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7912 let sessions = self.state.config.graph.sessions;
7913 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7914 let prompts = self.state.config.prompts.clone();
7915 let cache = self.state.config.cache_dir();
7916 let run = self.state.id.clone();
7917 for (i, spec) in chain.into_iter().enumerate() {
7918 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7919 let github_spec = match agent::for_github_text(
7920 &spec,
7921 &self.state.config.graph.github_language,
7922 &winner.worktree,
7923 ) {
7924 Ok(spec) => spec,
7925 Err(e) => {
7926 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7927 continue;
7928 }
7929 };
7930 let job = SeatJob {
7931 spec: github_spec,
7932 seat,
7933 cwd: winner.worktree.clone(),
7934 prompt: prompt.clone(),
7935 timeout,
7936 allow_write: false,
7937 sessions,
7938 artifacts: agent::artifacts_dir(&self.state.dir()),
7939 stem: if i == 0 {
7940 "change-summary".to_owned()
7941 } else {
7942 format!("change-summary-{}", spec.id)
7943 },
7944 handover: None,
7945 };
7946 let ctx = WaveCtx {
7947 run: &run,
7948 node: "pr-summary",
7949 prompts: &prompts,
7950 cache: cache.as_deref(),
7951 round: None,
7952 carry_seats: false,
7953 pauses: Some((&self.pause, &self.interrupt)),
7954 };
7955 let (seat, outcome) =
7956 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7957 self.state.seats.insert(seat.key.clone(), seat);
7958 let text = match outcome {
7959 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7960 AgentOutcome::ResourceBlocked(why) => {
7961 if self.state.resource_block.is_none() {
7964 self.state.resource_block = Some(crate::run::ResourceBlock {
7965 cause: crate::run::ResourceBlockCause::Other,
7966 node: "pr-summary".to_owned(),
7967 kind: "shared resource".to_owned(),
7968 detail: why,
7969 at: Timestamp::now(),
7970 });
7971 }
7972 return Err("a shared resource was busy".to_owned());
7973 }
7974 _ => continue,
7975 };
7976 let blind = &self.state.config.blind;
7977 let Some(title) = summary_title(&text) else {
7978 continue;
7979 };
7980 let body: String = summary_without_title(&text)
7981 .lines()
7982 .filter(|l| !l.contains("magi:run/"))
7983 .collect::<Vec<_>>()
7984 .join("\n");
7985 let body = blind::sanitize_prose(body.trim(), blind);
7986 let title = blind::sanitize_prose(&title, blind);
7987 if body.trim().is_empty() || title.trim().is_empty() {
7988 continue;
7989 }
7990 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7991 continue;
7992 }
7993 if git::rev_parse(&winner.worktree, "HEAD")
7995 .await
7996 .ok()
7997 .as_deref()
7998 != Some(head)
7999 {
8000 return Err("the branch moved while the summary was written".to_owned());
8001 }
8002 if git::merge_base(&winner.worktree, &tracking, head)
8003 .await
8004 .ok()
8005 .as_deref()
8006 != Some(base.as_str())
8007 || crate::pr_outcome::summary_evidence(
8008 &self.state,
8009 crate::pr_outcome::FinalHead::At(head),
8010 ) != evidence
8011 {
8012 return Err("the summary evidence changed while it was written".to_owned());
8013 }
8014 self.state.change_summary = Some(crate::run::ChangeSummary {
8015 head: head.to_owned(),
8016 base,
8017 evidence: fingerprint,
8018 title,
8019 body,
8020 });
8021 self.state.event(
8022 "pr-summary",
8023 format!("change summary written by {} for the final diff", spec.id),
8024 );
8025 if let Err(e) = self.state.save() {
8026 tracing::warn!("could not save the change summary: {e:#}");
8027 }
8028 return Ok(());
8029 }
8030 self.state
8031 .event("pr-summary", "no agent produced a change summary");
8032 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8033 }
8034
8035 async fn rewrite_pr_text(
8045 &mut self,
8046 winner: &Candidate,
8047 pr: &PrMessage,
8048 violations: &[crate::github_text::Violation],
8049 ) -> Option<String> {
8050 let prompt = format!(
8051 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8052 pr.title, pr.body
8053 );
8054 let sessions = self.state.config.graph.sessions;
8055 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8056 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8059 let key = format!("impl-{}", winner.label);
8060 if let Some(spec) = self
8061 .state
8062 .config
8063 .agents
8064 .iter()
8065 .find(|a| a.id == winner.agent)
8066 .cloned()
8067 && let Some(existing) = self.state.seats.get(&key)
8068 && existing.agent == spec.id
8069 && agent::has_session(spec.kind, existing, sessions)
8070 {
8071 attempts.push((spec, Some(existing.clone())));
8072 }
8073 match agent::pick_chain(
8074 &self.state.config.agents,
8075 self.state.config.roles.language_judge.as_ref(),
8076 &agent::installed,
8077 "language_judge",
8078 ) {
8079 Ok(chain) => {
8080 for spec in chain {
8081 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8082 continue;
8083 }
8084 attempts.push((spec, None));
8085 }
8086 }
8087 Err(e) if attempts.is_empty() => {
8088 tracing::warn!("no agent for the description rewrite: {e:#}");
8089 self.state
8090 .event("github-text", "no rewrite agent available");
8091 return None;
8092 }
8093 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8094 }
8095 let prompts = self.state.config.prompts.clone();
8096 let cache = self.state.config.cache_dir();
8097 let run = self.state.id.clone();
8098 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8099 let resumed = resumed_seat.is_some();
8100 let seat = resumed_seat.unwrap_or_else(|| {
8101 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8102 });
8103 let github_spec = match agent::for_github_text(
8104 &spec,
8105 &self.state.config.graph.github_language,
8106 &winner.worktree,
8107 ) {
8108 Ok(spec) => spec,
8109 Err(e) => {
8110 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8111 continue;
8112 }
8113 };
8114 let job = SeatJob {
8115 spec: github_spec,
8116 seat,
8117 cwd: winner.worktree.clone(),
8118 prompt: prompt.clone(),
8119 timeout: retry_budget(timeout_full, resumed),
8120 allow_write: false,
8121 sessions,
8122 artifacts: agent::artifacts_dir(&self.state.dir()),
8123 stem: if i == 0 {
8124 "github-text-rewrite".to_owned()
8125 } else {
8126 format!("github-text-rewrite-{}", spec.id)
8127 },
8128 handover: None,
8129 };
8130 let ctx = WaveCtx {
8131 run: &run,
8132 node: "github-text",
8133 prompts: &prompts,
8134 cache: cache.as_deref(),
8135 round: None,
8136 carry_seats: false,
8137 pauses: Some((&self.pause, &self.interrupt)),
8138 };
8139 let (seat, outcome) =
8140 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8141 self.state.seats.insert(seat.key.clone(), seat);
8142 match outcome {
8143 AgentOutcome::Ok(o)
8144 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8145 {
8146 return Some(o.text);
8147 }
8148 AgentOutcome::ResourceBlocked(_) => return None,
8149 _ => {}
8150 }
8151 }
8152 None
8153 }
8154
8155 fn retire_github_text_question(&mut self, why: &str) {
8157 let Some(id) = self
8158 .state
8159 .github_text
8160 .as_ref()
8161 .and_then(|g| g.question.clone())
8162 else {
8163 return;
8164 };
8165 let _ = ask::Questions::open().update(&id, |q| {
8166 q.abandon(why);
8167 Ok(())
8168 });
8169 }
8170
8171 async fn settle_withheld(
8175 &mut self,
8176 winner: &Candidate,
8177 pr: &PrMessage,
8178 mut g: crate::run::GithubTextAsk,
8179 ) -> Result<Option<PrMessage>> {
8180 use crate::github_text as gt;
8181 let store = ask::Questions::open();
8182 let timeout = self.state.config.graph.answer_timeout;
8183 if !g.resolved {
8184 let mut standing = g
8185 .question
8186 .as_ref()
8187 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8188 if let Some(q) = standing.as_ref()
8189 && q.status.open()
8190 && gt::expired(q, timeout)
8191 {
8192 let why = format!("no answer within {}s of asking", timeout.max(1));
8193 standing = store
8194 .update(&q.id, |q| {
8195 q.abandon(&why);
8196 Ok(())
8197 })
8198 .ok()
8199 .map(|(q, ())| q);
8200 }
8201 let mut ask_again: Option<Option<Vec<String>>> = None;
8202 match standing {
8203 None if g.asks <= 1 => {
8206 g.asks = 0;
8207 ask_again = Some(None);
8208 }
8209 None => {
8210 g.resolved = true;
8211 g.chosen_title = None;
8212 g.choice = Some(GithubTextChoice::Fallback);
8213 }
8214 Some(q) if q.status.open() => {
8215 self.park_on_github_text(&g)?;
8216 return Ok(None);
8217 }
8218 Some(q) => match gt::read_reply(&q) {
8219 gt::Reply::Fallback => {
8220 g.resolved = true;
8221 g.chosen_title = None;
8222 g.choice = Some(GithubTextChoice::Fallback);
8223 }
8224 gt::Reply::Original => {
8225 g.resolved = true;
8226 g.chosen_title = None;
8227 g.choice = Some(GithubTextChoice::Original);
8228 }
8229 gt::Reply::Rewrite => {
8230 g.resolved = true;
8231 g.chosen_title = None;
8232 g.choice = Some(if g.rewrite.is_some() {
8233 GithubTextChoice::Rewrite
8234 } else {
8235 GithubTextChoice::Fallback
8236 });
8237 }
8238 gt::Reply::Title(text) => {
8239 let first = text
8240 .lines()
8241 .map(str::trim)
8242 .find(|l| !l.is_empty())
8243 .unwrap_or("");
8244 let decision = if first.is_empty() {
8245 None
8246 } else {
8247 let candidate = PrMessage {
8248 title: first.to_owned(),
8249 body: String::new(),
8250 };
8251 self.judge_pr_language(&winner.worktree, &candidate).await
8252 };
8253 match gt::vet_title(
8254 &text,
8255 decision,
8256 self.state.config.graph.github_english(),
8257 ) {
8258 Ok(title) => {
8259 g.resolved = true;
8260 g.chosen_title = Some(title);
8261 g.choice = Some(GithubTextChoice::MyTitle);
8262 }
8263 Err(cats) if g.asks < 2 => {
8264 ask_again = Some(Some(
8265 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8266 ))
8267 }
8268 Err(_) => {
8269 g.resolved = true;
8270 g.chosen_title = None;
8271 g.choice = Some(GithubTextChoice::Fallback);
8272 }
8273 }
8274 }
8275 },
8276 }
8277 if let Some(rejected) = ask_again {
8278 let w = gt::Withheld {
8281 title: g.title,
8282 body: g.body,
8283 categories: g.categories.clone(),
8284 };
8285 let artifact = match crate::run::write_artifact(
8288 &self.state,
8289 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8290 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8291 ) {
8292 Ok(p) => Some(p),
8293 Err(e) => {
8294 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8295 None
8296 }
8297 };
8298 let (detail, comparison) = gt::question_content(
8299 &w,
8300 &pr.title,
8301 &pr.body,
8302 &self.state.id,
8303 rejected.as_deref(),
8304 artifact.as_deref(),
8305 g.rewrite.as_ref(),
8306 );
8307 let mut q = ask::Question::new(
8308 self.state.id.clone(),
8309 gt::ASK_NODE.to_owned(),
8310 gt::ASK_SEAT.to_owned(),
8311 gt::question_summary(&self.state.config.graph.language, &w),
8312 detail,
8313 gt::question_choices(&w, g.rewrite.as_ref()),
8314 );
8315 q.posting_comparison = Some(comparison);
8316 q.answer_timeout = timeout;
8317 g.question = Some(q.id.clone());
8320 g.asks += 1;
8321 self.state.github_text = Some(g.clone());
8322 self.state.event(
8323 "github-text",
8324 format!("asking the owner about the withheld text ({})", q.short()),
8325 );
8326 self.state.save()?;
8327 store
8328 .put_executable(&mut q)
8329 .context("file the posting-gate question")?;
8330 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8331 tracing::warn!(
8332 "could not notify about posting-gate question {}: {e:#}",
8333 q.short()
8334 );
8335 }
8336 self.park_on_github_text(&g)?;
8337 return Ok(None);
8338 }
8339 }
8340 let resolved = g.chosen_title.clone();
8341 let original = g.choice == Some(GithubTextChoice::Original);
8342 self.state.event(
8344 "github-text",
8345 if original {
8346 format!(
8347 "posting the original text for: {}",
8348 match (g.title, g.body) {
8349 (true, true) => "title, description",
8350 (true, false) => "title",
8351 _ => "description",
8352 }
8353 )
8354 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8355 "posting the saved rewrite title and description".to_owned()
8356 } else if resolved.is_some() {
8357 "posting the owner's replacement title".to_owned()
8358 } else {
8359 "posting the neutral text for the withheld fields".to_owned()
8360 },
8361 );
8362 self.state.github_text = Some(g.clone());
8363 self.state.save()?;
8364 if g.choice == Some(GithubTextChoice::Rewrite)
8365 && let Some(r) = g.rewrite.as_ref()
8366 {
8367 return Ok(Some(PrMessage {
8368 title: r.title.clone(),
8369 body: r.body.clone(),
8370 }));
8371 }
8372 let w = gt::Withheld {
8373 title: g.title,
8374 body: g.body,
8375 categories: g.categories.clone(),
8376 };
8377 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8378 if let Some(t) = resolved {
8379 title = t;
8380 }
8381 let english = gt::LanguageDecision {
8383 title_english: true,
8384 body_english: true,
8385 body_complete: true,
8386 };
8387 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8388 if original {
8389 if g.title {
8392 title = pr.title.clone();
8393 }
8394 if g.body {
8395 body = pr.body.clone();
8396 }
8397 }
8398 Ok(Some(PrMessage { title, body }))
8399 }
8400
8401 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8402 self.state.github_text = Some(g.clone());
8403 self.state.end_pass_parked(ParkCause::Question);
8404 self.state.parked = true;
8405 self.state.event(
8406 "github-text",
8407 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8408 );
8409 self.state.save()
8410 }
8411
8412 async fn judge_pr_language(
8415 &mut self,
8416 cwd: &Path,
8417 pr: &PrMessage,
8418 ) -> Option<crate::github_text::LanguageDecision> {
8419 self.state.config.roles.language_judge.as_ref()?;
8420 if !self.state.config.graph.language_gate() {
8421 return None;
8422 }
8423 let decision =
8424 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8425 self.state.event(
8426 "github-text",
8427 if decision.is_some() {
8428 "language decided by the language judge"
8429 } else {
8430 "language judge unavailable; using built-in heuristics"
8431 },
8432 );
8433 decision
8434 }
8435
8436 async fn merge(&mut self) -> Result<()> {
8437 if self.hold_base_sync().await? {
8438 return Ok(());
8439 }
8440 if self
8455 .state
8456 .base_sync
8457 .as_ref()
8458 .is_some_and(|s| s.conflict.is_some())
8459 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8460 != Some(RunStatus::Gating)
8461 || !self.state.gate_allows_progress()
8469 {
8470 return Ok(());
8471 }
8472 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8481 return Ok(());
8482 }
8483 let Some(winner) = self.state.winner().cloned() else {
8484 return Ok(());
8485 };
8486 let repo = self.state.repo.clone();
8487 let base = self.state.base_branch.clone();
8488 let mode = self.state.config.merge.mode;
8489 let style = self.state.config.merge.style;
8490 let facts = if is_review_run(&self.state) {
8491 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8492 let start = review_base(
8493 &repo,
8494 &self.state.config.merge.remote,
8495 &base,
8496 &self.state.base_commit,
8497 &winner.branch,
8498 )
8499 .await;
8500 branch_facts(&repo, &start, &winner.branch).await
8501 } else {
8502 None
8503 };
8504 let leaked = if is_review_run(&self.state) {
8507 leaked_subjects(&self.state, &winner.branch).await
8508 } else {
8509 Some(Vec::new())
8510 };
8511 let Some(pr) = self
8515 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8516 .await?
8517 else {
8518 return Ok(());
8519 };
8520 let message = pr.commit_message();
8521
8522 let outcome = match mode {
8523 MergeMode::None => MergeOutcome {
8524 mode,
8525 ok: true,
8526 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8527 empty: false,
8528 },
8529 MergeMode::Pr | MergeMode::Local
8530 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8531 {
8532 MergeOutcome {
8533 mode,
8534 ok: false,
8535 detail: empty_candidate_detail(&self.state, &base),
8536 empty: true,
8537 }
8538 }
8539 MergeMode::Local => {
8540 let on = git::current_branch(&repo).await?;
8541 if on.as_deref() != Some(base.as_str()) {
8542 MergeOutcome {
8543 mode,
8544 ok: false,
8545 detail: format!(
8546 "{} has {} checked out, not the base branch {base}",
8547 repo.display(),
8548 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8549 ),
8550 empty: false,
8551 }
8552 } else if !git::is_clean(&repo).await? {
8553 MergeOutcome {
8554 mode,
8555 ok: false,
8556 detail: format!("{} is dirty; refusing to merge", repo.display()),
8557 empty: false,
8558 }
8559 } else {
8560 let out = match style {
8561 MergeStyle::Merge => {
8562 git::merge_no_ff(&repo, &winner.branch, &message).await?
8563 }
8564 MergeStyle::Squash => {
8565 git::merge_squash(&repo, &winner.branch, &message).await?
8566 }
8567 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8568 };
8569 MergeOutcome {
8570 mode,
8571 ok: out.ok(),
8572 detail: if out.ok() { out.stdout } else { out.stderr },
8573 empty: false,
8574 }
8575 }
8576 }
8577 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8578 self.state.pr_publication_pending = true;
8579 self.state.save()?;
8580 let out =
8581 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8582 self.state.pr_publication_pending = out.is_err();
8583 MergeOutcome {
8584 mode,
8585 ok: out.is_ok(),
8586 detail: match out {
8587 Ok(url) => url,
8588 Err(e) => format!("{e:#}"),
8589 },
8590 empty: false,
8591 }
8592 }
8593 MergeMode::Pr => {
8594 let remote = self.state.config.merge.remote.clone();
8595 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8596 if !pushed.ok() {
8597 MergeOutcome {
8598 mode,
8599 ok: false,
8600 detail: pushed.stderr,
8601 empty: false,
8602 }
8603 } else {
8604 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8610 let out = match pr_merge_plan(found) {
8611 PrPlan::Create => {
8612 gh_pr_create(
8613 &winner.worktree,
8614 &base,
8615 &winner.branch,
8616 &pr.title,
8617 &pr.body,
8618 )
8619 .await
8620 }
8621 PrPlan::Adopt { url, title } => {
8622 self.state
8623 .event("merge", format!("Pr: adopted open pull request {url}"));
8624 if self.state.github_text.as_ref().is_some_and(|g| {
8625 g.resolved
8626 && g.choice == Some(GithubTextChoice::Rewrite)
8627 && g.rewrite.is_some()
8628 }) {
8629 land::set_pr_text_verbatim(
8632 &winner.worktree,
8633 &url,
8634 &pr.title,
8635 &pr.body,
8636 )
8637 .await
8638 .map(|()| url)
8639 } else {
8640 if title != pr.title
8641 && (!is_review_run(&self.state)
8642 || leaked
8643 .as_deref()
8644 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8645 && let Err(e) =
8646 if self.state.github_text.as_ref().is_some_and(|g| {
8647 g.resolved
8648 && g.title
8649 && g.choice == Some(GithubTextChoice::Original)
8650 }) {
8651 land::set_pr_title_verbatim(
8653 &winner.worktree,
8654 &url,
8655 &pr.title,
8656 )
8657 .await
8658 } else {
8659 land::set_pr_title(
8660 &mut self.state,
8661 &winner.worktree,
8662 &url,
8663 &pr.title,
8664 )
8665 .await
8666 }
8667 {
8668 tracing::warn!("could not refresh title of {url}: {e:#}");
8669 self.state
8670 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8671 }
8672 Ok(url)
8673 }
8674 }
8675 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8676 };
8677 match out {
8678 Ok(url) => MergeOutcome {
8679 mode,
8680 ok: true,
8681 detail: url,
8682 empty: false,
8683 },
8684 Err(e) => MergeOutcome {
8685 mode,
8686 ok: false,
8687 detail: e.to_string(),
8688 empty: false,
8689 },
8690 }
8691 }
8692 }
8693 };
8694
8695 self.state.status = match (mode, outcome.ok) {
8696 (MergeMode::None, _) => RunStatus::Ready,
8697 (_, true) if outcome.empty => RunStatus::Ready,
8698 (MergeMode::Pr, true) => RunStatus::Ready,
8699 (_, true) => RunStatus::Merged,
8700 (_, false) => RunStatus::Blocked,
8701 };
8702 self.state.event(
8703 "merge",
8704 format!(
8705 "{:?}: {}",
8706 mode,
8707 outcome.detail.lines().next().unwrap_or("")
8708 ),
8709 );
8710 self.state.merge = Some(outcome);
8711 self.state.save()?;
8712
8713 if self.state.config.graph.land
8719 && mode == MergeMode::Pr
8720 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8721 {
8722 self.run_land().await?;
8723 }
8724 self.settle_questions();
8729 Ok(())
8730 }
8731
8732 async fn run_land(&mut self) -> Result<()> {
8743 crate::pr_head::ensure_owner(&self.state)?;
8744 let url = self
8745 .state
8746 .pr
8747 .as_ref()
8748 .map(|pr| pr.url.clone())
8749 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8750 .unwrap_or_default();
8751 let url = url.lines().next().unwrap_or("").trim().to_owned();
8752 if !url.starts_with("http") {
8753 return Ok(());
8754 }
8755 loop {
8758 match land::land(&mut self.state, &url).await {
8759 Ok(_)
8760 if self
8761 .state
8762 .merge_repairs
8763 .last()
8764 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8765 {
8766 if self.repair_merge_review().await? {
8767 continue;
8768 }
8769 }
8770 Ok(pr) if self.state.parked => {
8771 let _ = pr;
8775 }
8776 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8777 self.state.status = RunStatus::Superseded;
8778 self.state.save()?;
8779 }
8780 Ok(pr) => {
8781 self.state.status = match pr.state {
8782 land::PrLifecycle::Merged => RunStatus::Merged,
8783 _ => RunStatus::Blocked,
8784 };
8785 if bump::should_release_bump(self.state.status)
8792 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8793 {
8794 self.state
8800 .event("bump", format!("release bump skipped: {e:#}"));
8801 }
8802 if self.state.status == RunStatus::Merged {
8805 crate::followup::after_merge(&mut self.state, &pr.url).await;
8806 }
8807 self.state.save()?;
8808 }
8809 Err(e) => {
8810 if e.is::<crate::repo_lock::ResourceBlocked>() {
8811 return Err(e);
8812 }
8813 self.state.status = RunStatus::Blocked;
8814 self.state.event("land", format!("gave up: {e}"));
8815 self.state.save()?;
8816 }
8817 }
8818 break;
8819 }
8820 Ok(())
8821 }
8822
8823 async fn repair_merge_review(&mut self) -> Result<bool> {
8826 use crate::run::MergeRepairPhase as Phase;
8827 crate::pr_head::ensure_owner(&self.state)?;
8828 let i = self.state.merge_repairs.len() - 1;
8829 let request = self.state.merge_repairs[i].clone();
8830 if request.phase == Phase::Fixing {
8831 let restored = match self.state.winner().cloned() {
8832 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8833 None => Err(anyhow::anyhow!("repair has no winning branch")),
8834 };
8835 let why = match restored {
8836 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8837 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8838 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8839 };
8840 self.hold_merge_repair(i, why)?;
8841 return Ok(false);
8842 }
8843 let failure = if i >= self.state.config.graph.land_rounds
8844 || self.state.config.graph.review_rounds == 0
8845 {
8846 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8847 } else {
8848 None
8849 };
8850 if let Some(why) = failure {
8851 self.hold_merge_repair(i, why)?;
8852 return Ok(false);
8853 }
8854 if request.phase == Phase::Requested {
8855 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8856 Ok(pr) => pr,
8857 Err(e) => {
8858 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8859 return Ok(false);
8860 }
8861 };
8862 self.state.merge_repairs[i].phase = Phase::Fixing;
8863 self.state.land_approval = None;
8864 self.state.land_armed_head = None;
8865 self.state.save()?;
8866 let evidence = serde_json::to_string_pretty(&request)?;
8869 let reason = format!(
8870 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8871 );
8872 let budget = self.state.config.graph.land_rounds;
8873 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8874 match result {
8875 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8876 Ok(land::Fixed::Failed(why)) => {
8877 self.hold_merge_repair(
8878 i,
8879 format!(
8880 "repair failed: {why}; inspect the branch and run an explicit review"
8881 ),
8882 )?;
8883 return Ok(false);
8884 }
8885 Err(e) => {
8886 self.hold_merge_repair(
8887 i,
8888 format!(
8889 "repair failed: {e:#}; inspect the branch and run an explicit review"
8890 ),
8891 )?;
8892 return Ok(false);
8893 }
8894 }
8895 self.begin_merge_repair_review(i)?;
8899 }
8900 self.review_loop().await?;
8901 self.gate().await?;
8902 if self.resource_halt()? || self.state.parked {
8903 return Ok(false);
8904 }
8905 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8906 != Some(RunStatus::Gating)
8907 || !self.state.gate_allows_progress()
8908 {
8909 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8910 return Ok(false);
8911 }
8912 let winner = self
8915 .state
8916 .winner()
8917 .cloned()
8918 .context("repair has no winning branch")?;
8919 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8920 if self.state.gate_head.as_deref() != Some(after.as_str())
8921 || !git::is_clean(&winner.worktree).await?
8922 {
8923 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8924 return Ok(false);
8925 }
8926 let expected = self
8927 .state
8928 .pr_pushed_head
8929 .as_deref()
8930 .unwrap_or(&request.head)
8931 .to_owned();
8932 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8933 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8936 let refspec = format!("{after}:refs/heads/{public}");
8937 let push = git::git_raw(
8938 &winner.worktree,
8939 &["push", &lease, &self.state.config.merge.remote, &refspec],
8940 )
8941 .await;
8942 match push {
8943 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8944 Ok(out) => {
8945 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8946 return Ok(false);
8947 }
8948 Err(e) => {
8949 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8950 return Ok(false);
8951 }
8952 }
8953 self.state.merge_repairs[i].phase = Phase::Complete;
8954 self.state.status = RunStatus::Landing;
8955 self.state.event(
8956 "repair",
8957 "new review and gate completed; fresh owner merge decision required",
8958 );
8959 self.state.save()?;
8960 Ok(true)
8961 }
8962
8963 fn merge_repair_artifacts(&self) -> PathBuf {
8964 let artifacts = agent::artifacts_dir(&self.state.dir());
8965 match self
8966 .state
8967 .merge_repairs
8968 .last()
8969 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8970 {
8971 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8972 None => artifacts,
8973 }
8974 }
8975
8976 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8977 self.state.reviews.clear();
8978 self.state.gate.clear();
8979 self.state.gate_ran = false;
8980 self.state.gate_head = None;
8981 self.state.gate_fixes.clear();
8982 self.state.gate_repair_e2e = None;
8983 self.state.pre_gate.clear();
8984 self.state.pre_gate_commit = None;
8985 self.state.contested_handoff = None;
8986 self.state.land_reruns = None;
8987 self.state.land_approval = None;
8988 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8989 self.state.status = RunStatus::Reviewing;
8990 self.state.save()
8991 }
8992
8993 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8994 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8995 self.state.merge_repairs[index].failure = Some(why.clone());
8996 self.state.status = RunStatus::Blocked;
8997 self.state.event("repair", why);
8998 self.state.save()
8999 }
9000
9001 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9005 if let Some(existing) = self.state.seats.get(key)
9006 && existing.agent == agent
9007 {
9008 return existing.clone();
9009 }
9010 let fresh = if self.state.seats.contains_key(key) {
9013 handover_seat(key, agent, self.state.next_seat_seed())
9014 } else {
9015 SeatState::new(key, agent, self.state.seed)
9016 };
9017 self.state.seats.insert(key.to_owned(), fresh.clone());
9018 fresh
9019 }
9020
9021 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9027 match self.state.seats.get(key) {
9028 Some(s) if s.agent != spec.id => {
9029 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9030 }
9031 _ => spec,
9032 }
9033 }
9034
9035 fn view(&self, c: &Candidate) -> CandidateView {
9037 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9038 .unwrap_or_default();
9039 let (patch, _) = blind::sanitize_patch(
9040 &format!("candidate {} patch", c.label),
9041 &raw,
9042 &self.state.config.blind,
9043 );
9044 CandidateView {
9045 label: c.label,
9046 branch: c.branch.clone(),
9047 summary: c.summary.clone(),
9048 stat: c.stat.clone(),
9049 patch,
9050 }
9051 }
9052
9053 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9055 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9056 prompt::judge(
9057 "(see above)",
9058 &views,
9059 self.roles.judges.len(),
9060 base_short,
9061 "en",
9062 )
9063 }
9064
9065 fn vote_prompt_full(
9071 &self,
9072 j: usize,
9073 labels: &[char],
9074 language: &str,
9075 candidates: &[Candidate],
9076 base_short: &str,
9077 ) -> String {
9078 let mut text = format!(
9079 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9080 prompt::final_vote(labels, language),
9081 self.state.instruction,
9082 self.candidate_block(candidates, base_short)
9083 );
9084 if let Some(own) = self
9085 .state
9086 .judgements
9087 .get(j)
9088 .filter(|r| !r.ranking.is_empty())
9089 {
9090 let reasons = own
9091 .reasons
9092 .iter()
9093 .map(|(k, v)| format!("- {k}: {v}"))
9094 .collect::<Vec<_>>()
9095 .join("\n");
9096 text.push_str(&format!(
9097 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9098 own.ranking.iter().collect::<String>(),
9099 if reasons.is_empty() {
9100 ""
9101 } else {
9102 ", because:\n"
9103 }
9104 ));
9105 }
9106 if !self.state.deliberation.is_empty() {
9107 text.push_str("\n# What was argued before this vote\n");
9108 for t in self.transcript(&[], j) {
9109 text.push_str(&format!(
9110 "\n## {}{}\n\n{}\n",
9111 t.who,
9112 if t.is_self { " (you)" } else { "" },
9113 t.body.trim()
9114 ));
9115 }
9116 }
9117 text
9118 }
9119
9120 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9127 let mut turns = Vec::new();
9128 for j in &self.state.judgements {
9129 if j.ranking.is_empty() {
9130 continue;
9131 }
9132 let reasons = j
9133 .reasons
9134 .iter()
9135 .map(|(k, v)| format!("- {k}: {v}"))
9136 .collect::<Vec<_>>()
9137 .join("\n");
9138 turns.push(Turn {
9139 who: format!("Judge {} (opening ranking)", j.judge),
9140 is_self: j.judge == self_idx + 1,
9141 body: format!(
9142 "Ranked {}{}{reasons}",
9143 j.ranking.iter().collect::<String>(),
9144 if reasons.is_empty() {
9145 ""
9146 } else {
9147 ", because:\n"
9148 }
9149 ),
9150 });
9151 }
9152 for t in self
9153 .state
9154 .deliberation
9155 .iter()
9156 .flat_map(|r| r.turns.iter())
9157 .chain(current)
9158 {
9159 turns.push(Turn {
9160 who: format!("Judge {}", t.judge),
9161 is_self: t.judge == self_idx + 1,
9162 body: t.body.clone(),
9163 });
9164 }
9165 turns
9166 }
9167}
9168
9169fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9171 agent::has_session(spec.kind, seat, sessions)
9172}
9173
9174fn next_untried_in_roster<'a>(
9195 roster: &'a [AgentSpec],
9196 start: usize,
9197 tried: &BTreeSet<String>,
9198) -> Option<&'a AgentSpec> {
9199 roster
9200 .get(start + 1..)?
9201 .iter()
9202 .find(|s| !tried.contains(&s.id))
9203}
9204
9205fn pick_successor<'a>(
9226 roster: &'a [AgentSpec],
9227 start: usize,
9228 tried: &BTreeSet<String>,
9229 carried: Option<&BTreeSet<String>>,
9230 others: &BTreeSet<String>,
9231) -> Option<&'a AgentSpec> {
9232 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9233 roster
9234 .get(start + 1..)
9235 .and_then(|tail| {
9236 tail.iter()
9237 .filter(free)
9238 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9239 })
9240 .or_else(|| {
9241 carried
9242 .is_some()
9243 .then(|| roster.iter().find(free))
9244 .flatten()
9245 })
9246 .or_else(|| match carried {
9247 Some(c) => next_for_seat(roster, start, tried, c),
9248 None => next_untried_in_roster(roster, start, tried),
9249 })
9250}
9251
9252fn next_for_seat<'a>(
9262 roster: &'a [AgentSpec],
9263 start: usize,
9264 round_tried: &BTreeSet<String>,
9265 carried_failed: &BTreeSet<String>,
9266) -> Option<&'a AgentSpec> {
9267 roster
9268 .get(start + 1..)?
9269 .iter()
9270 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9271 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9272}
9273
9274fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9280 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9281 return spec;
9282 };
9283 let ok = |id: &str| !h.failed.contains(id);
9284 if let Some(last) = h.last_ok.as_deref()
9285 && ok(last)
9286 && let Some(s) = roster.iter().find(|s| s.id == last)
9287 {
9288 return s.clone();
9289 }
9290 if ok(&spec.id) {
9291 return spec;
9292 }
9293 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9294 roster
9295 .get(from + 1..)
9296 .into_iter()
9297 .flatten()
9298 .chain(roster.iter())
9299 .find(|s| ok(&s.id))
9300 .cloned()
9301 .unwrap_or(spec)
9302}
9303
9304pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9308 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9309}
9310
9311const TIMED_OUT: &str = "timed out";
9314
9315impl FailClass {
9316 fn of(out: &AgentOutcome) -> Option<Self> {
9318 match out {
9319 AgentOutcome::Ok(_) => None,
9320 AgentOutcome::Quota(_) => Some(Self::Quota),
9321 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9322 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9323 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9324 AgentOutcome::ResourceBlocked(_) => None,
9327 }
9328 }
9329
9330 fn stem_word(&self) -> &'static str {
9332 match self {
9333 Self::Quota => "quota",
9334 _ => "handover",
9335 }
9336 }
9337}
9338
9339fn failure_signature(msg: &str) -> String {
9343 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9344 if let Some(code) = line
9347 .strip_prefix("process exited with no usable output (exit code ")
9348 .and_then(|s| s.strip_suffix(')'))
9349 .and_then(|s| s.parse::<i32>().ok())
9350 {
9351 line = format!("exited with some({code}) and no usable output");
9352 } else if matches!(
9353 line.as_str(),
9354 "process exited with no usable output (no exit code reported)"
9355 | "process exited with no usable output (exit code unavailable)"
9356 ) {
9357 line = "exited with none and no usable output".to_owned();
9358 }
9359 let mut out = Vec::new();
9360 for word in line.split_whitespace() {
9361 if word.contains('/') || word.contains('\\') {
9362 out.push("<path>".to_owned());
9363 continue;
9364 }
9365 let mut w = String::new();
9366 let mut in_digits = false;
9367 for c in word.chars() {
9368 if c.is_ascii_digit() {
9369 if !in_digits {
9370 w.push('#');
9371 }
9372 in_digits = true;
9373 } else {
9374 in_digits = false;
9375 w.push(c);
9376 }
9377 }
9378 out.push(w);
9379 }
9380 out.join(" ").chars().take(120).collect()
9381}
9382
9383fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9390 match cur {
9391 FailClass::Quota | FailClass::Timeout => true,
9392 FailClass::Other(_) => prev != Some(cur),
9393 }
9394}
9395
9396fn fail_reason(out: &AgentOutcome) -> String {
9398 match out {
9399 AgentOutcome::Ok(_) => String::new(),
9400 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9401 AgentOutcome::Dropped(o) => format!(
9402 "the CLI dropped the stream ({})",
9403 o.dropped
9404 .as_ref()
9405 .map(|d| d.why.as_str())
9406 .unwrap_or("it ended without delivering its answer")
9407 ),
9408 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9409 e.lines().next().unwrap_or("").chars().take(160).collect()
9410 }
9411 }
9412}
9413
9414pub(crate) fn record_handover(
9417 state: &mut RunState,
9418 node: &str,
9419 seat: &str,
9420 from: &str,
9421 to: &str,
9422 class: &FailClass,
9423 reason: &str,
9424) {
9425 let message = if *class == FailClass::Quota {
9426 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9427 } else {
9428 format!("{seat}: handed over {from} -> {to} ({reason})")
9429 };
9430 state.event(node, message);
9431 state.handovers.push(Handover {
9432 at: Timestamp::now(),
9433 node: node.to_owned(),
9434 seat: seat.to_owned(),
9435 from: from.to_owned(),
9436 to: to.to_owned(),
9437 reason: reason.to_owned(),
9438 });
9439}
9440
9441fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9455 commands.iter().any(|c| c.exit_code.is_none())
9456}
9457
9458fn verified_noop_claim(
9471 usable: bool,
9472 commands: &[agent::CommandEvidence],
9473 text: &str,
9474) -> Option<String> {
9475 (usable && !has_unconfirmed_command(commands))
9476 .then(|| verdict::verified_noop(text))
9477 .flatten()
9478}
9479
9480fn short(commit: &str) -> String {
9481 commit.chars().take(7).collect()
9482}
9483
9484fn make_executable(path: &Path) -> Result<()> {
9485 #[cfg(unix)]
9486 {
9487 use std::os::unix::fs::PermissionsExt as _;
9488 let mut perms = std::fs::metadata(path)?.permissions();
9489 perms.set_mode(0o755);
9490 std::fs::set_permissions(path, perms)?;
9491 }
9492 #[cfg(not(unix))]
9493 {
9494 let _ = path;
9495 }
9496 Ok(())
9497}
9498
9499struct WaveCtx<'a> {
9506 run: &'a str,
9509 node: &'a str,
9511 prompts: &'a Prompts,
9512 cache: Option<&'a Path>,
9514 round: Option<usize>,
9517 carry_seats: bool,
9521 pauses: Option<(&'a Pause, &'a Pause)>,
9524}
9525
9526fn repository_wait(
9528 state: &RunState,
9529 pauses: Option<(&Pause, &Pause)>,
9530) -> crate::repo_lock::WaitPolicy {
9531 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9532 let inherited = crate::repo_lock::WaitPolicy::current();
9533 crate::repo_lock::WaitPolicy::new(
9534 Duration::from_secs(state.config.graph.timeout_resource_wait),
9535 move || {
9536 inherited.cancelled()
9537 || pauses
9538 .as_ref()
9539 .is_some_and(|(a, b)| a.parked() || b.parked())
9540 },
9541 )
9542}
9543
9544fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9545 if error.is::<crate::repo_lock::ResourceBlocked>() {
9546 state.resource_block = Some(crate::run::ResourceBlock {
9547 cause: crate::run::ResourceBlockCause::Other,
9548 node: "repository".into(),
9549 kind: if error.to_string().contains("disk pressure") {
9550 "disk pressure"
9551 } else {
9552 "repository coordination"
9553 }
9554 .into(),
9555 detail: error.to_string(),
9556 at: Timestamp::now(),
9557 });
9558 state.event("resource", error.to_string());
9559 }
9560}
9561
9562async fn coordinate_operation(
9563 state: &mut RunState,
9564 cwd: &Path,
9565 wait: &crate::repo_lock::WaitPolicy,
9566) -> Result<Option<crate::repo_lock::SeatLease>> {
9567 let result =
9568 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9569 .await;
9570 if let Err(e) = &result {
9571 record_repository_block(state, e);
9572 }
9573 result
9574}
9575
9576async fn run_one(
9578 job: SeatJob,
9579 sem: Arc<Semaphore>,
9580 ctx: &WaveCtx<'_>,
9581 state: &mut RunState,
9582 attempt: usize,
9583) -> (SeatState, AgentOutcome) {
9584 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9585 .await
9586 .pop()
9587 .expect("one job in, one result out");
9588 (seat, out)
9589}
9590
9591async fn wave(
9597 jobs: Vec<SeatJob>,
9598 sem: Arc<Semaphore>,
9599 ctx: &WaveCtx<'_>,
9600 state: &mut RunState,
9601 attempt: usize,
9602) -> Vec<(usize, SeatState, AgentOutcome)> {
9603 let WaveCtx {
9604 run,
9605 node,
9606 prompts,
9607 cache,
9608 round,
9609 carry_seats: _,
9610 pauses,
9611 } = *ctx;
9612 for job in &jobs {
9613 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9614 }
9615 if let Err(e) = state.save() {
9616 tracing::warn!("could not persist in-progress seats: {e:#}");
9621 }
9622 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9638 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9642 let cache_guard = if let Some(cache_dir) = cache {
9643 if jobs_had_a_writer {
9644 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9645 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9646 match acquire_cache_lease(
9647 state,
9648 cache_dir,
9649 &owner,
9650 resource_limit,
9651 node,
9652 node,
9653 &cancel,
9654 )
9655 .await
9656 {
9657 Ok(Some(g)) => Some(g),
9658 Ok(None) => {
9661 if let Some(block) = &mut state.resource_block {
9662 block.cause = crate::run::ResourceBlockCause::Cancelled {
9663 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9664 };
9665 }
9666 return block_wave(
9667 jobs,
9668 state,
9669 node,
9670 attempt,
9671 "parked while waiting for the shared build cache",
9672 );
9673 }
9674 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9677 Err(e) => {
9678 let why = format!("{e:#}");
9682 state.event(
9683 "resource",
9684 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9685 );
9686 return block_wave(jobs, state, node, attempt, &why);
9687 }
9688 }
9689 } else {
9690 None
9691 }
9692 } else {
9693 None
9694 };
9695 state.seats_spawning(node, attempt);
9698 if let Err(e) = state.save() {
9699 tracing::warn!("could not persist the start of a wave: {e:#}");
9700 }
9701 let slot_limit = resource_limit
9705 + jobs
9706 .iter()
9707 .map(|j| j.timeout)
9708 .max()
9709 .unwrap_or_default()
9710 .saturating_mul(jobs.len() as u32);
9711 let mut set = tokio::task::JoinSet::new();
9712 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9715 let overlay = prompts.overlay(node);
9716 let repository_wait = repository_wait(state, pauses);
9717 for (i, mut job) in jobs.into_iter().enumerate() {
9718 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9719 if cache.is_some() {
9720 job.prompt.push('\n');
9721 job.prompt
9722 .push_str(&prompt::build_cache_note(node, job.allow_write));
9723 }
9724 let sem = Arc::clone(&sem);
9725 let run = run.to_owned();
9726 let node = node.to_owned();
9727 let attachments = if node == "implement" {
9730 state.attachments.clone()
9731 } else {
9732 Vec::new()
9733 };
9734 let cache = cache
9745 .filter(|_| job.allow_write && cache_guard.is_some())
9746 .map(Path::to_path_buf);
9747 let spawned_tx = spawned_tx.clone();
9748 let repository_wait = repository_wait.clone();
9749 let coordination = state.config.coordination.clone();
9750 let fetched_base = state.fetched_base.clone();
9751 set.spawn(async move {
9752 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9756 Ok(p) => p,
9757 Err(_) => {
9758 return (
9759 i,
9760 job.seat,
9761 AgentOutcome::ResourceBlocked(format!(
9762 "no parallel slot became free within {}s",
9763 slot_limit.as_secs()
9764 )),
9765 false,
9766 );
9767 }
9768 };
9769 spawned_tx.send(job.seat.key.clone()).ok();
9770 let mut seat = job.seat;
9771 let out = repository_wait
9772 .scope(agent::invoke(
9773 &job.spec,
9774 &mut seat,
9775 &Invocation {
9776 coordination: coordination.as_ref(),
9777 fetched_base: fetched_base.as_ref(),
9778 cwd: &job.cwd,
9779 prompt: &job.prompt,
9780 timeout: job.timeout,
9781 allow_write: job.allow_write,
9782 sessions: job.sessions,
9783 artifacts: &job.artifacts,
9784 stem: &job.stem,
9785 run: &run,
9786 node: &node,
9787 cache_dir: cache.as_deref(),
9788 attachments: &attachments,
9789 writable: &[],
9790 },
9791 ))
9792 .await;
9793 let interrupted = out
9794 .as_ref()
9795 .err()
9796 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9797 let out = match out {
9798 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9799 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9800 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9808 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9809 Ok(o) => AgentOutcome::Failed(format!(
9810 "process exited with no usable output ({})",
9811 crate::notices::exit_status(o.exit_code)
9812 )),
9813 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9814 AgentOutcome::ResourceBlocked(e.to_string())
9815 }
9816 Err(e) => AgentOutcome::Failed(e.to_string()),
9817 };
9818 (i, seat, out, interrupted)
9819 });
9820 }
9821 drop(spawned_tx);
9822 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9823 loop {
9824 let joined = tokio::select! {
9825 Some(key) = spawned_rx.recv() => {
9826 state.seat_spawned(&key);
9828 if let Err(e) = state.save() {
9829 tracing::warn!("could not persist a seat's start: {e:#}");
9830 }
9831 continue;
9832 }
9833 joined = set.join_next() => joined,
9834 };
9835 let Some(joined) = joined else {
9836 break;
9837 };
9838 let (i, seat, out, interrupted) = match joined {
9839 Ok(v) => v,
9840 Err(e) => {
9844 tracing::error!("agent task panicked: {e}");
9845 continue;
9846 }
9847 };
9848 state.seat_finished(&seat.key);
9849 if interrupted
9850 && node == "implement"
9851 && let Some(label) = state
9852 .candidates
9853 .iter()
9854 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9855 .map(|candidate| candidate.label)
9856 {
9857 state.candidate_preservation.insert(
9858 label,
9859 "interrupted implementation awaiting persistence".into(),
9860 );
9861 state.interrupted_implementations.insert(label);
9862 }
9863 if let AgentOutcome::ResourceBlocked(why) = &out
9864 && state.resource_block.is_none()
9865 {
9866 state.resource_block = Some(crate::run::ResourceBlock {
9867 cause: crate::run::ResourceBlockCause::Other,
9868 node: node.to_owned(),
9869 kind: if why.contains("disk pressure") {
9870 "disk pressure"
9871 } else if why.contains("repository coordination") {
9872 "repository coordination"
9873 } else {
9874 "parallel slot"
9875 }
9876 .to_owned(),
9877 detail: why.clone(),
9878 at: Timestamp::now(),
9879 });
9880 let activity = if interrupted {
9881 "agent execution interrupted"
9882 } else {
9883 "no agent was started"
9884 };
9885 state.event("resource", format!("{node}: {activity} - {why}"));
9886 }
9887 record_jobs(state, node, round, &seat.key, &out);
9888 if let Err(e) = state.save() {
9889 tracing::warn!("could not persist a seat's completion: {e:#}");
9890 }
9891 if collected.len() <= i {
9892 collected.resize_with(i + 1, || None);
9893 }
9894 collected[i] = Some((i, seat, out));
9895 }
9896 if state
9902 .active
9903 .values()
9904 .any(|a| a.node == node && a.attempt == attempt)
9905 {
9906 state
9907 .active
9908 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9909 if let Err(e) = state.save() {
9910 tracing::warn!("could not persist the end of a wave: {e:#}");
9911 }
9912 }
9913 if let Some(cache_dir) = cache
9920 && jobs_had_a_writer
9921 {
9922 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9923 }
9924 if let Some(guard) = cache_guard {
9925 guard.release();
9926 }
9927 collected.into_iter().flatten().collect()
9928}
9929
9930fn record_jobs(
9941 state: &mut RunState,
9942 node: &str,
9943 round: Option<usize>,
9944 seat: &str,
9945 out: &AgentOutcome,
9946) {
9947 let commands: &[agent::CommandEvidence] = match out {
9948 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9949 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9950 };
9951 let checked_at = Timestamp::now();
9952 for c in commands {
9953 state.jobs.push(JobRecord {
9954 node: node.to_owned(),
9955 round,
9956 seat: seat.to_owned(),
9957 id: c.id.clone(),
9958 description: c.description.clone(),
9959 checked_at,
9960 status: match c.exit_code {
9961 Some(0) => JobStatus::Completed,
9962 Some(_) => JobStatus::Failed,
9963 None => JobStatus::Unknown,
9964 },
9965 exit_code: c.exit_code,
9966 result_summary: c.result_summary.clone(),
9967 source: c.source.clone(),
9968 });
9969 }
9970}
9971
9972fn round_is_clean(
9993 blocking: usize,
9994 e2e_ok: bool,
9995 answered: usize,
9996 expected: usize,
9997 quota_missing: usize,
9998 policy: IncompleteReviewPolicy,
9999) -> bool {
10000 if blocking != 0 || !e2e_ok {
10001 return false;
10002 }
10003 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10004 return true;
10005 }
10006 answered > 0 && expected - answered <= quota_missing
10007}
10008
10009pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10033 if max_rounds == 0
10034 || reviews.last().is_some_and(|r| {
10035 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10036 })
10037 {
10038 return Some(RunStatus::Gating);
10039 }
10040 let last = reviews.last()?;
10041 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10042 if reviews.len() < max_rounds && !stagnant {
10043 return None;
10044 }
10045 if last.incomplete() && last.blocking == 0 {
10046 return Some(RunStatus::Blocked);
10047 }
10048 if last.e2e_status() == E2eStatus::ResourceBlocked
10049 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10050 {
10051 return None;
10052 }
10053 Some(if last.e2e.iter().all(CommandOutcome::ok) {
10054 RunStatus::Gating
10055 } else {
10056 RunStatus::Blocked
10057 })
10058}
10059
10060pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10075 if nudged {
10076 (full / 4).max(Duration::from_secs(120)).min(full)
10077 } else {
10078 full
10079 }
10080}
10081
10082#[allow(clippy::too_many_arguments)]
10111async fn ask_json_wave<T>(
10112 jobs: Vec<SeatJob>,
10113 sem: Arc<Semaphore>,
10114 retries: usize,
10115 roster: &[AgentSpec],
10116 ctx: &WaveCtx<'_>,
10117 losses: &mut Vec<QuotaLoss>,
10118 state: &mut RunState,
10119 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10120) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10121where
10122 T: serde::de::DeserializeOwned + Send + 'static,
10123{
10124 ask_wave_with(
10125 jobs,
10126 sem,
10127 retries,
10128 roster,
10129 ctx,
10130 losses,
10131 state,
10132 &|text: &str| {
10133 let v = verdict::extract_json::<T>(text)?;
10134 validate(&v)?;
10135 Ok(v)
10136 },
10137 )
10138 .await
10139}
10140
10141#[allow(clippy::too_many_arguments)]
10150async fn ask_wave_with<T>(
10151 jobs: Vec<SeatJob>,
10152 sem: Arc<Semaphore>,
10153 retries: usize,
10154 roster: &[AgentSpec],
10155 ctx: &WaveCtx<'_>,
10156 losses: &mut Vec<QuotaLoss>,
10157 state: &mut RunState,
10158 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10159) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10160where
10161 T: Send + 'static,
10162{
10163 let n = jobs.len();
10164 let originals: Vec<SeatJob> = jobs;
10165 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10166 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10167 let mut nudges: Vec<usize> = vec![0; n];
10174 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10178 let mut tried: Vec<BTreeSet<String>> = specs
10179 .iter()
10180 .map(|s| BTreeSet::from([s.id.clone()]))
10181 .collect();
10182 let starts: Vec<usize> = specs
10183 .iter()
10184 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10185 .collect();
10186 let carry = ctx.carry_seats && !roster.is_empty();
10187 let carried: Vec<BTreeSet<String>> = originals
10190 .iter()
10191 .map(|j| {
10192 state
10193 .seat_history
10194 .get(&j.seat.key)
10195 .filter(|_| carry)
10196 .map(|h| h.failed.clone())
10197 .unwrap_or_default()
10198 })
10199 .collect();
10200 let mut prev: Vec<Option<FailClass>> = originals
10201 .iter()
10202 .map(|j| {
10203 state
10204 .seat_history
10205 .get(&j.seat.key)
10206 .filter(|_| carry)
10207 .and_then(|h| h.last_fail.clone())
10208 })
10209 .collect();
10210 let next_agent =
10211 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10212 let others: BTreeSet<String> = specs
10213 .iter()
10214 .enumerate()
10215 .filter(|(j, _)| *j != i)
10216 .map(|(_, s)| s.id.clone())
10217 .collect();
10218 pick_successor(
10219 roster,
10220 starts[i],
10221 tried,
10222 carry.then(|| &carried[i]),
10223 &others,
10224 )
10225 .cloned()
10226 };
10227 let mut fresh: Vec<Option<String>> = vec![None; n];
10228 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10229 let mut pending: Vec<usize> = (0..n).collect();
10230
10231 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10236 for round in 0..max_rounds {
10237 if pending.is_empty() {
10238 break;
10239 }
10240 let mut batch = Vec::with_capacity(pending.len());
10241 let mut renudged: Vec<&str> = Vec::new();
10242 for &i in &pending {
10243 let src = &originals[i];
10244 let full: &str = match &src.handover {
10249 Some(h) if specs[i].id != src.spec.id => h,
10250 _ => &src.prompt,
10251 };
10252 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10255 (
10256 full.to_owned(),
10257 src.timeout,
10258 format!("{}-{word}-{}", src.stem, specs[i].id),
10259 )
10260 } else if nudges[i] == 0 {
10261 (full.to_owned(), src.timeout, src.stem.clone())
10262 } else {
10263 renudged.push(src.seat.key.as_str());
10264 let why = done[i]
10265 .as_ref()
10266 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10267 .unwrap_or_else(|| "no parsable answer".to_owned());
10268 let nudge = prompt::nudge(&why);
10269 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10270 let prompt = if nudged {
10271 nudge
10272 } else {
10273 format!("{full}\n\n---\n\n{nudge}")
10274 };
10275 (
10276 prompt,
10277 retry_budget(src.timeout, nudged),
10278 format!("{}-retry{}", src.stem, nudges[i]),
10279 )
10280 };
10281 batch.push(SeatJob {
10282 spec: specs[i].clone(),
10283 seat: seats[i].clone(),
10284 cwd: src.cwd.clone(),
10285 prompt,
10286 timeout,
10287 allow_write: src.allow_write,
10288 sessions: src.sessions,
10289 artifacts: src.artifacts.clone(),
10290 stem,
10291 handover: None,
10292 });
10293 }
10294
10295 if !renudged.is_empty() {
10296 state.event(
10297 ctx.node,
10298 format!("retry {round}: re-asking {}", renudged.join(", ")),
10299 );
10300 }
10301 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10302 let mut still = Vec::new();
10303 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10304 seats[i] = seat;
10305 if let AgentOutcome::ResourceBlocked(why) = &out {
10308 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10309 continue;
10310 }
10311 let class = FailClass::of(&out);
10312 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10315 && nudges[i] < retries
10316 && !roster.is_empty();
10317 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10318 let next = should_hand_over(prev[i].as_ref(), &cur)
10319 .then(|| next_agent(i, &tried[i], &specs))
10320 .flatten();
10321 if carry {
10322 let h = state
10323 .seat_history
10324 .entry(originals[i].seat.key.clone())
10325 .or_default();
10326 h.failed.insert(specs[i].id.clone());
10327 h.last_fail = Some(cur.clone());
10328 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10329 h.last_ok = None;
10330 }
10331 if let Err(e) = state.save() {
10334 tracing::warn!("could not persist a seat's failure history: {e:#}");
10335 }
10336 }
10337 if let Some(next) = next {
10338 record_handover(
10339 state,
10340 ctx.node,
10341 &originals[i].seat.key,
10342 &specs[i].id,
10343 &next.id,
10344 &cur,
10345 &fail_reason(&out),
10346 );
10347 tried[i].insert(next.id.clone());
10348 prev[i] = Some(cur.clone());
10349 seats[i] =
10350 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10351 specs[i] = next;
10352 fresh[i] = Some(cur.stem_word().to_owned());
10353 nudges[i] = 0;
10354 done[i] = Some(Err(anyhow::anyhow!(
10355 "handed over after: {}",
10356 fail_reason(&out)
10357 )));
10358 still.push(i);
10359 continue;
10360 }
10361 }
10362 if carry
10363 && !nudge_first
10364 && let Some(cur) = class.clone()
10365 {
10366 let h = state
10368 .seat_history
10369 .entry(originals[i].seat.key.clone())
10370 .or_default();
10371 h.failed.insert(specs[i].id.clone());
10372 h.last_fail = Some(cur);
10373 }
10374 let parsed = match out {
10375 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10376 AgentOutcome::Quota(o) => {
10377 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10378 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10379 }
10380 AgentOutcome::Dropped(o) => {
10384 let why = o
10385 .dropped
10386 .as_ref()
10387 .map(|d| d.why.as_str())
10388 .unwrap_or("the CLI ended the stream without delivering its answer");
10389 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10390 }
10391 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10392 Err(anyhow::anyhow!(e))
10393 }
10394 };
10395 let quota = class == Some(FailClass::Quota);
10396 let failed = parsed.is_err();
10397 done[i] = Some(parsed);
10398 if carry && !failed {
10399 let h = state
10400 .seat_history
10401 .entry(originals[i].seat.key.clone())
10402 .or_default();
10403 h.failed.remove(&specs[i].id);
10404 h.last_ok = Some(specs[i].id.clone());
10405 h.last_fail = None;
10406 }
10407 let agent_failure = class.is_some()
10415 && !nudge_first
10416 && !roster.is_empty()
10417 && next_agent(i, &tried[i], &specs).is_some();
10418 if failed && !quota && !agent_failure && nudges[i] < retries {
10419 nudges[i] += 1;
10420 still.push(i);
10421 }
10422 }
10423 pending = still;
10424 }
10425
10426 for (i, q) in last_quota.into_iter().enumerate() {
10429 if let Some(reset) = q {
10430 losses.push(QuotaLoss {
10431 seat: originals[i].seat.key.clone(),
10432 node: ctx.node.to_owned(),
10433 at: Timestamp::now(),
10434 reset,
10435 });
10436 }
10437 }
10438
10439 seats
10440 .into_iter()
10441 .zip(done)
10442 .zip(nudges)
10443 .map(|((seat, res), attempts)| {
10444 (
10445 seat,
10446 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10447 attempts,
10448 )
10449 })
10450 .collect()
10451}
10452
10453fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10455 results
10456 .iter()
10457 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10458}
10459
10460fn block_wave(
10464 jobs: Vec<SeatJob>,
10465 state: &mut RunState,
10466 node: &str,
10467 attempt: usize,
10468 why: &str,
10469) -> Vec<(usize, SeatState, AgentOutcome)> {
10470 state.resource_wait = None;
10471 state
10472 .active
10473 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10474 if let Err(e) = state.save() {
10475 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10476 }
10477 jobs.into_iter()
10478 .enumerate()
10479 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10480 .collect()
10481}
10482
10483async fn acquire_cache_lease(
10496 state: &mut RunState,
10497 cache_dir: &Path,
10498 owner: &crate::cache::Owner,
10499 budget: Duration,
10500 node: &str,
10501 context: &str,
10502 cancel: &(dyn Fn() -> bool + Sync),
10503) -> Result<Option<crate::cache::Guard>> {
10504 let home = crate::run::home();
10505 let started = Instant::now();
10506 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10507 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10508 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10509 Err(e) => {
10510 state.event(
10511 "verify",
10512 format!("{context}: could not check the shared build cache: {e:#}"),
10513 );
10514 if let Err(e2) = state.save() {
10515 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10516 }
10517 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10518 }
10519 };
10520 state.event(
10521 "verify",
10522 format!(
10523 "{context}: waiting for the shared build cache at {} ({})",
10524 cache_dir.display(),
10525 busy.describe()
10526 ),
10527 );
10528 state.resource_wait = Some(crate::run::ResourceWait {
10531 node: context.to_owned(),
10532 kind: "shared build cache".to_owned(),
10533 holder: busy.describe(),
10534 since: Timestamp::now(),
10535 limit_secs: budget.as_secs(),
10536 });
10537 if let Err(e) = state.save() {
10538 tracing::warn!("could not persist a cache wait: {e:#}");
10539 }
10540 let remaining = budget.saturating_sub(started.elapsed());
10541 let waited = crate::cache::wait_for_unless(
10542 &home,
10543 cache_dir,
10544 owner,
10545 remaining,
10546 Duration::from_secs(5),
10547 cancel,
10548 )
10549 .await;
10550 state.resource_wait = None;
10551 let cause = match &waited {
10552 Ok(Some(_)) => None,
10553 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10554 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10555 Some(crate::run::ResourceBlockCause::Timeout {
10556 waited_secs: started.elapsed().as_secs(),
10557 limit_secs: budget.as_secs(),
10558 })
10559 }
10560 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10561 };
10562 if let Some(cause) = cause {
10563 let detail = match &waited {
10564 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10565 Err(error) => format!("{error:#}"),
10566 };
10567 state.resource_block = Some(crate::run::ResourceBlock {
10568 cause,
10569 node: node.to_owned(),
10570 kind: "shared build cache".into(),
10571 detail,
10572 at: Timestamp::now(),
10573 });
10574 state.event(
10575 "resource",
10576 state.resource_block.as_ref().unwrap().description(),
10577 );
10578 }
10579 if let Err(error) = state.save() {
10580 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10581 }
10582 waited
10583}
10584
10585#[derive(Debug)]
10590struct CacheCheckError(String);
10591
10592impl std::fmt::Display for CacheCheckError {
10593 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10594 f.write_str(&self.0)
10595 }
10596}
10597
10598impl std::error::Error for CacheCheckError {}
10599
10600fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10602 state
10603 .resource_block
10604 .get_or_insert_with(|| crate::run::ResourceBlock {
10605 cause: crate::run::ResourceBlockCause::Other,
10606 node: node.to_owned(),
10607 kind: "shared build cache".into(),
10608 detail: format!("{error:#}"),
10609 at: Timestamp::now(),
10610 });
10611 if let Err(error) = state.save() {
10612 tracing::warn!("could not persist cache resource block: {error:#}");
10613 }
10614}
10615
10616#[allow(clippy::too_many_arguments)]
10637async fn with_cache_lease<'s, F, Fut>(
10638 state: &'s mut RunState,
10639 cache_dir: Option<&Path>,
10640 node: &str,
10641 seat: &str,
10642 worktree: &Path,
10643 head: &str,
10644 budget: Duration,
10645 context: &str,
10646 wait: &crate::repo_lock::WaitPolicy,
10647 body: F,
10648) -> Result<(Vec<CommandOutcome>, bool)>
10649where
10650 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10651 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10652{
10653 let started = Instant::now();
10656 let home = crate::run::home();
10657 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10658 let guard = if let Some(cache_dir) = cache_dir {
10659 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10660 match acquire_cache_lease(
10661 state,
10662 cache_dir,
10663 &owner,
10664 resource_limit,
10665 node,
10666 context,
10667 &|| wait.cancelled(),
10668 )
10669 .await
10670 .and_then(|g| {
10671 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10672 }) {
10673 Ok(g) => Some(g),
10674 Err(e) => {
10675 record_cache_error(state, node, &e);
10676 return Ok((
10677 vec![CommandOutcome {
10678 command: "(waiting for the shared build cache)".into(),
10679 code: None,
10680 output_tail: e.to_string(),
10681 duration_ms: started.elapsed().as_millis() as u64,
10682 resource_blocked: true,
10683 }],
10684 false,
10685 ));
10686 }
10687 }
10688 } else {
10689 None
10690 };
10691 let _coordination = coordinate_operation(state, worktree, wait).await?;
10692 if let Some(cache_dir) = cache_dir {
10693 let identity = crate::cache::Identity::new(worktree, head);
10694 if let Err(e) = crate::cache::ensure_fresh(
10695 &home,
10696 cache_dir,
10697 &identity,
10698 &state.config.verify.cargo_manifests(),
10699 ) {
10700 record_cache_error(state, node, &e);
10701 state.event(
10702 "verify",
10703 format!(
10704 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10705 worktree.display(),
10706 short(head)
10707 ),
10708 );
10709 if let Some(guard) = guard {
10710 guard.release();
10711 }
10712 return Ok((
10713 vec![CommandOutcome {
10714 command: "(confirming the shared build cache is fresh)".into(),
10715 code: None,
10716 output_tail: e.to_string(),
10717 duration_ms: started.elapsed().as_millis() as u64,
10718 resource_blocked: true,
10719 }],
10720 false,
10721 ));
10722 }
10723 }
10724 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10725 if !timed_out_pids.is_empty() {
10726 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10727 }
10728 drop(_coordination);
10731 if let Some(guard) = guard {
10732 guard.release();
10733 }
10734 Ok((outcomes, retried))
10735}
10736
10737async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10749 wait_for_pids_with(
10750 pids,
10751 crate::proc::pid_alive,
10752 LEASE_RELEASE_POLL,
10753 LEASE_RELEASE_MAX_WAIT,
10754 )
10755 .await;
10756}
10757
10758async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10764 pids: &[u32],
10765 alive: F,
10766 poll: Duration,
10767 max_wait: Duration,
10768) {
10769 let deadline = Instant::now() + max_wait;
10770 loop {
10771 if pids.iter().all(|&pid| !alive(pid)) {
10772 return;
10773 }
10774 if Instant::now() >= deadline {
10775 return;
10776 }
10777 tokio::time::sleep(poll).await;
10778 }
10779}
10780
10781fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10788 outcomes.iter().any(|o| o.resource_blocked)
10789}
10790
10791pub(crate) enum GateFix {
10793 Retry,
10795 Stop,
10798 Defer,
10801}
10802
10803fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10811 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10812 red.peek().is_some()
10813 && red.all(|o| {
10814 !o.resource_blocked
10815 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10816 && !o.output_tail.trim().is_empty()
10817 })
10818}
10819
10820fn e2e_outcome_label(o: &CommandOutcome) -> String {
10824 if o.ok() {
10825 return "pass".to_owned();
10826 }
10827 let reason = if o.build_failed() {
10828 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10829 } else {
10830 format!("FAIL ({:?})", o.code)
10831 };
10832 format!(
10833 "{reason}\n{}",
10834 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
10835 )
10836}
10837
10838async fn run_e2e_with_retry(
10846 state: &mut RunState,
10847 shell: &[String],
10848 commands: &[String],
10849 worktree: &Path,
10850 timeout: Duration,
10851 context: &str,
10852) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10853 let (mut e2e, mut timed_out_pids) = run_commands(
10854 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10855 )
10856 .await;
10857 for o in &e2e {
10858 state.event(
10859 "verify",
10860 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10861 );
10862 }
10863 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10867 if verify_retried {
10868 state.event(
10869 "verify",
10870 format!(
10871 "{context}: verify could not build/link, not a test result — retrying once \
10872 before concluding"
10873 ),
10874 );
10875 let retried = run_commands(
10876 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10877 )
10878 .await;
10879 e2e = retried.0;
10880 timed_out_pids.extend(retried.1);
10883 for o in &e2e {
10884 state.event(
10885 "verify",
10886 format!(
10887 "{context}: retry `{}` -> {}",
10888 o.command,
10889 e2e_outcome_label(o)
10890 ),
10891 );
10892 }
10893 }
10894 (e2e, verify_retried, timed_out_pids)
10895}
10896
10897#[allow(clippy::too_many_arguments)]
10913async fn run_commands(
10914 state: &mut RunState,
10915 node: &str,
10916 task: &str,
10917 attempt: usize,
10918 shell: &[String],
10919 commands: &[String],
10920 cwd: &Path,
10921 timeout: Duration,
10922) -> (Vec<CommandOutcome>, Vec<u32>) {
10923 if commands.is_empty() {
10924 return (Vec::new(), Vec::new());
10929 }
10930 let mut out = Vec::new();
10931 let mut timed_out_pids = Vec::new();
10932 let total = commands.len();
10933 for (idx, command) in commands.iter().enumerate() {
10934 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10935 if let Err(e) = state.save() {
10936 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10937 }
10938 let started = Instant::now();
10939 let mut cmd = tokio::process::Command::new(&shell[0]);
10940 cmd.quiet();
10941 for key in crate::worker::INVOCATION_ENV {
10945 cmd.env_remove(key);
10946 }
10947 cmd.args(&shell[1..])
10948 .arg(command)
10949 .current_dir(cwd)
10950 .stdin(std::process::Stdio::null())
10951 .stdout(std::process::Stdio::piped())
10952 .stderr(std::process::Stdio::piped())
10953 .kill_on_drop(true);
10954 let mut outputs = Vec::new();
10955 let protected = (|| -> Result<()> {
10956 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10957 outputs.push(guard);
10958 }
10959 if let Some(cache) = state.config.cache_dir() {
10960 outputs.push(crate::retention::OutputLease::cache(
10961 &crate::run::home(),
10962 &cache,
10963 false,
10964 )?);
10965 }
10966 Ok(())
10967 })();
10968 if let Err(e) = protected {
10969 out.push(CommandOutcome {
10970 command: command.clone(),
10971 code: None,
10972 output_tail: format!("output ownership unavailable: {e:#}"),
10973 duration_ms: 0,
10974 resource_blocked: true,
10975 });
10976 continue;
10977 }
10978 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10979 let (code, body) = match spawned {
10980 Ok(mut running) => {
10981 let pid = running.child().id();
10982 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10983 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10984 let mut paths = vec![cwd.to_owned()];
10985 if let Some(cache) = state.config.cache_dir() {
10986 paths.push(cache);
10987 }
10988 let mut pressure = None;
10989 let status = tokio::select! {
10990 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10991 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10992 pressure = Some(reason); None
10993 }
10994 };
10995 let (code, failure) = match status {
10996 Some(Ok(Ok(status))) => (
10997 status.code(),
10998 status
10999 .code()
11000 .is_none()
11001 .then(|| format!("command terminated: {status}")),
11002 ),
11003 Some(Ok(Err(error))) => {
11004 (None, Some(format!("failed waiting for command: {error}")))
11005 }
11006 Some(Err(_)) => {
11007 if !running.coordinated()
11011 && let Some(pid) = pid
11012 {
11013 timed_out_pids.push(pid);
11014 }
11015 (
11016 None,
11017 Some(format!("timed out after {}s", timeout.as_secs())),
11018 )
11019 }
11020 None => (None, None),
11021 };
11022 running.finish().await;
11023 if let Some(reason) = pressure {
11024 state.resource_block = Some(crate::run::ResourceBlock {
11025 cause: crate::run::ResourceBlockCause::Other,
11026 node: node.to_owned(),
11027 kind: "disk pressure".to_owned(),
11028 detail: reason.clone(),
11029 at: Timestamp::now(),
11030 });
11031 out.push(CommandOutcome {
11032 command: command.clone(),
11033 code: None,
11034 output_tail: reason,
11035 duration_ms: started.elapsed().as_millis() as u64,
11036 resource_blocked: true,
11037 });
11038 break;
11039 }
11040 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11041 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11042 if let Some(failure) = failure {
11043 body.push('\n');
11044 body.push_str(&failure);
11045 }
11046 (code, body)
11047 }
11048 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11049 };
11050 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11051 if !body.is_empty() {
11052 match crate::run::write_verification_artifact(state, task, &body) {
11056 Ok(name) if summary != body => {
11057 summary = format!("[full output: artifacts/{name}]\n{summary}");
11058 }
11059 Ok(_) => {}
11060 Err(e) => {
11061 let warning = format!("could not save the full {task} output: {e:#}");
11062 tracing::warn!("{warning}");
11063 state.event(node, warning);
11064 if summary != body {
11065 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11066 }
11067 }
11068 }
11069 }
11070 out.push(CommandOutcome {
11071 command: command.clone(),
11072 code,
11073 output_tail: summary,
11074 duration_ms: started.elapsed().as_millis() as u64,
11075 resource_blocked: false,
11076 });
11077 }
11078 state.task_finished(task);
11079 if let Err(e) = state.save() {
11080 tracing::warn!("could not persist the end of {task}: {e:#}");
11081 }
11082 (out, timed_out_pids)
11083}
11084
11085fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11097 let repo = repo.display();
11098 match style {
11099 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11100 MergeStyle::Squash => {
11101 let subject = message
11104 .lines()
11105 .next()
11106 .unwrap_or(branch)
11107 .replace(['\\', '"', '$', '`'], "");
11108 format!(
11109 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11110 )
11111 }
11112 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11113 }
11114}
11115
11116const PR_TITLE_MAX: usize = 240;
11127
11128fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11139 let line = queue::first_line(text)?;
11140 let chars: Vec<char> = line.chars().collect();
11141 let (kept, mark) = if chars.len() <= max {
11142 (line.to_owned(), "")
11143 } else if let Some(end) = sentence_end(&chars, max) {
11144 (chars[..end].iter().collect::<String>(), "")
11145 } else {
11146 let room = max.saturating_sub(3);
11147 let cut = if chars[room].is_whitespace() {
11150 room
11151 } else {
11152 chars[..room]
11153 .iter()
11154 .rposition(|c| c.is_whitespace())
11155 .unwrap_or(room)
11156 };
11157 let head: String = chars[..cut].iter().collect();
11158 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11159 (head.to_owned(), "...")
11160 };
11161 if kept.is_empty() {
11162 return None;
11163 }
11164 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11165 return None;
11166 }
11167 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11168 return None;
11169 }
11170 Some(format!("{kept}{mark}"))
11171}
11172
11173fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11177 const MIN: usize = 20;
11178 for i in MIN..max.min(chars.len()) {
11179 if !matches!(chars[i], '.' | '!' | '?') {
11180 continue;
11181 }
11182 let Some(&next) = chars.get(i + 1) else {
11183 continue;
11184 };
11185 if !next.is_whitespace() {
11186 continue;
11187 }
11188 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11189 if after.is_some_and(|c| c.is_ascii_lowercase()) {
11190 continue;
11191 }
11192 let word: String = chars[..i]
11193 .iter()
11194 .rev()
11195 .take_while(|c| !c.is_whitespace())
11196 .collect::<Vec<_>>()
11197 .into_iter()
11198 .rev()
11199 .collect();
11200 let word = word.to_ascii_lowercase();
11201 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11202 continue;
11203 }
11204 let end = chars[..i]
11205 .iter()
11206 .rposition(|c| !c.is_whitespace())
11207 .map_or(i, |p| p + 1);
11208 return Some(end);
11209 }
11210 None
11211}
11212
11213struct PrMessage {
11218 title: String,
11219 body: String,
11220}
11221
11222impl PrMessage {
11223 fn commit_message(&self) -> String {
11227 format!("{}\n\n{}", self.title, self.body)
11228 }
11229}
11230
11231fn title_marker(line: &str) -> Option<&str> {
11233 let line = line.trim();
11234 let head = line.get(..6)?;
11235 head.eq_ignore_ascii_case("title:")
11236 .then(|| line[6..].trim())
11237}
11238
11239fn summary_title(summary: &str) -> Option<String> {
11244 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11245 let raw = title_marker(first)?;
11246 if raw.is_empty() {
11247 return None;
11248 }
11249 let title = queue::title_from(raw, PR_TITLE_MAX);
11250 let lower = title.to_ascii_lowercase();
11251 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11252 return None;
11253 }
11254 Some(title)
11255}
11256
11257const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11259
11260pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11262
11263pub(crate) fn is_review_run(state: &RunState) -> bool {
11267 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11268}
11269
11270fn review_title(state: &RunState) -> Option<String> {
11275 english_subject(
11276 state.reviewed_commits.as_ref()?.first()?,
11277 state.config.graph.github_english(),
11278 )
11279}
11280
11281fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11284 let raw = raw.trim();
11285 if raw.is_empty() {
11286 return None;
11287 }
11288 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11289 return None;
11290 }
11291 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11292 return None;
11293 }
11294 let title = queue::title_from(raw, PR_TITLE_MAX);
11295 let lower = title.to_ascii_lowercase();
11296 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11297 return None;
11298 }
11299 Some(title)
11300}
11301
11302#[derive(Debug, Clone, PartialEq, Eq)]
11305struct BranchFacts {
11306 commits: Vec<(String, String)>,
11308 stat: String,
11310}
11311
11312const STAT_FILE_LINES: usize = 25;
11314const COMMITS_MAX_CHARS: usize = 20_000;
11316
11317async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11320 let commits = git::commit_log(repo, base, branch).await.ok()?;
11321 if commits.is_empty() {
11322 return None;
11323 }
11324 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11325 let lines: Vec<&str> = stat.lines().collect();
11326 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11327 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11328 let more = format!(" ... {omitted} more file(s)");
11329 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11330 kept.push(&more);
11331 kept.push(lines[lines.len() - 1]);
11332 kept.join("\n")
11333 } else {
11334 lines.join("\n")
11335 };
11336 Some(BranchFacts { commits, stat })
11337}
11338
11339fn markdown_safe(text: &str) -> String {
11342 text.replace("</details>", "</details>")
11343 .replace("\x60\x60\x60", "~~~")
11344}
11345
11346fn neutral_title(state: &RunState, winner: char) -> String {
11347 format!(
11348 "chore: land candidate {} of run {}",
11349 winner.to_ascii_uppercase(),
11350 state.id
11351 )
11352}
11353
11354pub(crate) async fn repair_landing_gate(
11357 state: &mut RunState,
11358 winner: &Candidate,
11359 outcomes: &[CommandOutcome],
11360) -> Result<GateFix> {
11361 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11362 return Ok(GateFix::Stop);
11363 }
11364 let mut runner = Runner {
11365 roles: state.config.resolve_roles()?,
11366 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11367 state: state.clone(),
11368 pause: Pause::new(),
11369 interrupt: Pause::new(),
11370 };
11371 let result = runner.gate_fix_round(winner, outcomes).await;
11372 *state = runner.state;
11373 result
11374}
11375
11376pub(crate) async fn check_gate_repair_e2e(
11379 state: &mut RunState,
11380 winner: &Candidate,
11381 wait: &crate::repo_lock::WaitPolicy,
11382) -> Result<GateFix> {
11383 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11384 return Ok(GateFix::Retry);
11385 }
11386 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11387 if state
11388 .gate_repair_e2e
11389 .as_ref()
11390 .is_some_and(|r| r.head == head && r.passed)
11391 {
11392 return Ok(GateFix::Retry);
11393 }
11394 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11395 head: head.clone(),
11396 passed: false,
11397 outcomes: Vec::new(),
11398 });
11399 state.save()?;
11400 let commands = state.config.verify.e2e.clone();
11401 let outcomes = if commands.is_empty() {
11402 Vec::new()
11403 } else {
11404 let shell = state.config.shell();
11405 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11406 let cache_dir = state.config.cache_dir();
11407 let (outcomes, _) = with_cache_lease(
11408 state,
11409 cache_dir.as_deref(),
11410 "e2e",
11411 "e2e",
11412 &winner.worktree,
11413 &head,
11414 timeout,
11415 "gate repair e2e",
11416 wait,
11417 |state, budget| async move {
11418 run_e2e_with_retry(
11419 state,
11420 &shell,
11421 &commands,
11422 &winner.worktree,
11423 budget,
11424 "gate repair e2e",
11425 )
11426 .await
11427 },
11428 )
11429 .await?;
11430 outcomes
11431 };
11432 let deferred = verify_inconclusive(&outcomes);
11433 let passed = outcomes.iter().all(CommandOutcome::ok)
11434 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11435 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11436 head,
11437 passed,
11438 outcomes,
11439 });
11440 state.save()?;
11441 if deferred {
11442 Ok(GateFix::Defer)
11443 } else if !passed {
11444 state.event(
11445 "gate",
11446 "verify.e2e failed after gate repair; nothing published",
11447 );
11448 Ok(GateFix::Stop)
11449 } else {
11450 Ok(GateFix::Retry)
11451 }
11452}
11453
11454pub(crate) async fn run_landing_gate(
11456 state: &mut RunState,
11457 winner: &Candidate,
11458 wait: &crate::repo_lock::WaitPolicy,
11459) -> Result<Vec<CommandOutcome>> {
11460 let shell = state.config.shell();
11461 let gate_commands = state.config.verify.gate.clone();
11462 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11463 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11464 Vec::new()
11465 } else {
11466 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11467 let cache_dir = state.config.cache_dir();
11468 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11469 let (outcomes, _) = with_cache_lease(
11470 state,
11471 cache_dir.as_deref(),
11472 "gate",
11473 "gate",
11474 &winner.worktree,
11475 &head,
11476 timeout,
11477 "final gate",
11478 wait,
11479 |state, budget| {
11480 let shell = shell.clone();
11481 let gate_commands = gate_commands.clone();
11482 let winner = winner.clone();
11483 async move {
11484 Runner::run_pre_gate(state, &winner).await;
11485 let (outcomes, timed_out_pids) = run_commands(
11486 state,
11487 "gate",
11488 "gate",
11489 0,
11490 &shell,
11491 &gate_commands,
11492 &winner.worktree,
11493 budget,
11494 )
11495 .await;
11496 (outcomes, false, timed_out_pids)
11497 }
11498 },
11499 )
11500 .await?;
11501 outcomes
11502 };
11503 if outcomes.is_empty() {
11504 state.event("gate", "no gate commands configured; not tested");
11509 }
11510 for o in &outcomes {
11511 state.event(
11512 "gate",
11513 format!(
11514 "`{}` -> {}",
11515 o.command,
11516 if o.ok() {
11517 "pass".to_owned()
11518 } else {
11519 format!(
11520 "FAIL ({:?})\n{}",
11521 o.code,
11522 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11523 )
11524 }
11525 ),
11526 );
11527 }
11528 Ok(outcomes)
11529}
11530
11531pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11537 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11538 ""
11539 } else {
11540 pr_title
11541 }
11542}
11543
11544pub fn landing_subject_source(state: &RunState) -> String {
11548 if is_review_run(state) {
11549 let winner = state.candidates.first().map_or('A', |c| c.label);
11550 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11551 }
11552 state.instruction.clone()
11553}
11554
11555fn summary_without_title(summary: &str) -> String {
11558 let mut lines = summary.trim().lines().peekable();
11559 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11560 lines.next();
11561 }
11562 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11563}
11564
11565#[cfg(test)]
11577fn pr_message(state: &RunState, winner: char) -> PrMessage {
11578 pr_message_with(state, winner, None)
11579}
11580
11581#[cfg(test)]
11584fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11585 let raw = pr_message_raw(
11586 state,
11587 winner,
11588 facts,
11589 crate::pr_outcome::FinalHead::Unreported,
11590 ChangeNote::NotAsked,
11591 );
11592 let id = crate::scrub::Identity::current();
11593 PrMessage {
11594 title: crate::scrub::scrub(&raw.title, &id),
11595 body: crate::scrub::scrub(&raw.body, &id),
11596 }
11597}
11598
11599#[derive(Debug, Clone, Copy)]
11601enum ChangeNote<'a> {
11602 NotAsked,
11604 Summary(&'a crate::run::ChangeSummary),
11606 Unavailable(&'a str),
11608}
11609
11610fn pr_message_raw(
11611 state: &RunState,
11612 winner: char,
11613 facts: Option<&BranchFacts>,
11614 head: crate::pr_outcome::FinalHead<'_>,
11615 change: ChangeNote<'_>,
11616) -> PrMessage {
11617 if !matches!(change, ChangeNote::NotAsked) {
11618 let (title, mut body) = match change {
11619 ChangeNote::Summary(c) => (
11620 c.title.clone(),
11621 format!("## Summary\n\n{}\n\n", c.body.trim()),
11622 ),
11623 ChangeNote::Unavailable(why) => (
11624 neutral_title(state, winner),
11625 format!(
11626 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11627 ),
11628 ),
11629 ChangeNote::NotAsked => unreachable!(),
11630 };
11631 body.push_str(&public_pr_evidence(state, head));
11632 body.push_str(&format!(
11633 "\n---\nmagi:run/{} magi:candidate-{}\n",
11634 state.id,
11635 winner.to_ascii_lowercase()
11636 ));
11637 return PrMessage { title, body };
11638 }
11639 pr_preview(state, winner, facts, head)
11640}
11641
11642fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11645 let mut body = crate::pr_outcome::review_rounds(state);
11646 body.push_str(&crate::pr_outcome::final_verification(state, head));
11647 let open = state.last_round_findings();
11648 if !open.is_empty() {
11649 body.push_str("## Open review findings\n\n");
11650 for f in open {
11651 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11652 }
11653 body.push('\n');
11654 }
11655 if let Some(c) = state.change_summary.as_ref()
11656 && head == crate::pr_outcome::FinalHead::At(&c.head)
11657 {
11658 body.push_str(&format!(
11659 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11660 c.base, c.head
11661 ));
11662 }
11663 body
11664}
11665
11666fn pr_preview(
11668 state: &RunState,
11669 winner: char,
11670 facts: Option<&BranchFacts>,
11671 head: crate::pr_outcome::FinalHead<'_>,
11672) -> PrMessage {
11673 let summary = state
11674 .candidates
11675 .iter()
11676 .find(|c| c.label == winner)
11677 .map(|c| c.summary.as_str())
11678 .unwrap_or_default();
11679 let review = is_review_run(state);
11682 let title = if review {
11683 facts
11684 .and_then(|f| {
11685 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11686 })
11687 .or_else(|| review_title(state))
11688 .or_else(|| {
11689 state
11690 .candidates
11691 .iter()
11692 .find(|c| c.label == winner)
11693 .filter(|c| !c.branch.starts_with("magi/"))
11694 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11695 })
11696 .unwrap_or_else(|| neutral_title(state, winner))
11697 } else {
11698 summary_title(summary).unwrap_or_else(|| {
11699 english_title(
11700 &state.instruction,
11701 PR_TITLE_MAX,
11702 state.config.graph.github_english(),
11703 )
11704 .unwrap_or_else(|| neutral_title(state, winner))
11705 })
11706 };
11707
11708 let mut body = String::new();
11709 let what = summary_without_title(summary);
11710 if !what.is_empty() {
11711 body.push_str("## Summary\n\n");
11712 body.push_str(&what);
11713 body.push_str("\n\n");
11714 }
11715
11716 body.push_str(&crate::pr_outcome::review_rounds(state));
11718 body.push_str(&crate::pr_outcome::final_verification(state, head));
11719 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11720
11721 let open = state.open_findings();
11722 if !open.is_empty() {
11723 body.push_str("## Open review findings\n\n");
11724 for f in &open {
11725 body.push_str(&format!(
11726 "- `{}` [{:?}] {}\n",
11727 f.id, f.severity, "Context retained in local review records."
11728 ));
11729 }
11730 body.push('\n');
11731 }
11732
11733 if let Some(fix) = fix
11734 && !fix.rejected.is_empty()
11735 {
11736 body.push_str("## Declined by the fixer\n\n");
11737 for r in &fix.rejected {
11738 body.push_str(&format!(
11739 "- `{}`: {}\n",
11740 r.id, "Reason retained in local review records."
11741 ));
11742 }
11743 body.push('\n');
11744 }
11745
11746 if review {
11747 body.push_str("## Commits under review\n\n");
11749 if let Some(facts) = facts {
11750 let mut left = COMMITS_MAX_CHARS;
11751 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11752 let mut entry = format!("- {}\n", markdown_safe(subject));
11753 for l in markdown_safe(text).lines() {
11754 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11755 }
11756 if left == 0 {
11757 body.push_str(&format!(
11758 "- ... {} more commit(s)\n",
11759 facts.commits.len() - i
11760 ));
11761 break;
11762 }
11763 if entry.len() > left {
11764 let mut end = left;
11767 while !entry.is_char_boundary(end) {
11768 end -= 1;
11769 }
11770 entry.truncate(end);
11771 entry.push_str("\n ... (truncated)\n");
11772 left = 0;
11773 } else {
11774 left -= entry.len();
11775 }
11776 body.push_str(&entry);
11777 }
11778 if !facts.stat.trim().is_empty() {
11779 body.push_str(&format!(
11780 "\n## Diff stat\n\n```\n{}\n```\n",
11781 markdown_safe(facts.stat.trim())
11782 ));
11783 }
11784 } else {
11785 match &state.reviewed_commits {
11786 Some(subjects) => {
11787 for s in subjects {
11788 body.push_str(&format!("- {}\n", s.trim()));
11789 }
11790 }
11791 None => {
11792 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11795 body.push_str(rest.trim());
11796 body.push('\n');
11797 }
11798 }
11799 }
11800 } else {
11801 let task = state.instruction.trim();
11802 let task = if task.is_empty() {
11803 "(empty task)"
11804 } else {
11805 task
11806 };
11807 body.push_str(&format!(
11808 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11809 task.replace("</details>", "</details>")
11810 ));
11811 }
11812
11813 body.push_str(&format!(
11814 "\n---\nmagi:run/{} magi:candidate-{}\n",
11815 state.id,
11816 winner.to_ascii_lowercase()
11817 ));
11818
11819 PrMessage { title, body }
11820}
11821
11822fn seeded_instruction(state: &RunState) -> String {
11825 let mut inputs = Vec::new();
11826 if let Some(start) = &state.sources.start_from {
11827 inputs.push(format!(
11828 "Start from `{}` at `{}` (including ancestry).",
11829 start.token, start.sha
11830 ));
11831 }
11832 for pick in &state.sources.pick_commits {
11833 inputs.push(format!(
11834 "Pick `{}` at `{}` (single commit).",
11835 pick.token, pick.sha
11836 ));
11837 }
11838 if inputs.is_empty() {
11839 state.instruction.clone()
11840 } else {
11841 format!(
11842 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11843 state.instruction,
11844 inputs.join("\n")
11845 )
11846 }
11847}
11848
11849async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11853 let base = &state.base_branch;
11854 let mut against = base.clone();
11855 if mode == MergeMode::Pr {
11856 let remote = &state.config.merge.remote;
11860 let tracking = format!("{remote}/{base}");
11861 let fetched = git::fetch(repo, remote, base).await;
11862 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11863 against = tracking;
11864 } else {
11865 return false;
11866 }
11867 }
11868 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11869}
11870
11871fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11874 let mut detail = format!(
11875 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11876 nothing to open a pull request for"
11877 );
11878 detail.push_str(
11879 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11880 );
11881 detail
11882}
11883
11884#[derive(Debug, PartialEq, Eq)]
11887enum PrPlan {
11888 Create,
11889 Adopt { url: String, title: String },
11890 Stop(String),
11891}
11892
11893fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11896 match found {
11897 Ok(land::OpenPr::None) => PrPlan::Create,
11898 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11899 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11900 "several open pull requests exist for this branch, not picking one: {}",
11901 urls.join(" ")
11902 )),
11903 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11904 }
11905}
11906
11907pub(crate) async fn gh_pr_create(
11909 cwd: &Path,
11910 base: &str,
11911 head: &str,
11912 title: &str,
11913 body: &str,
11914) -> Result<String> {
11915 let out = tokio::process::Command::new("gh")
11916 .args([
11917 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11918 ])
11919 .current_dir(cwd)
11920 .quiet()
11921 .stdin(std::process::Stdio::null())
11922 .output()
11923 .await
11924 .context("spawn gh")?;
11925 if out.status.success() {
11926 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11927 } else {
11928 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11929 }
11930}
11931
11932async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11934 match std::fs::symlink_metadata(path) {
11935 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11936 git::worktree_remove_clean(repo, path).await?;
11939 return Ok(());
11940 }
11941 Err(e) => return Err(e.into()),
11942 Ok(_) => {}
11943 }
11944 if !git::is_clean(path).await? {
11945 bail!(
11946 "worktree has uncommitted work; refusing removal: {}",
11947 path.display()
11948 );
11949 }
11950 if !git::worktree_remove_clean(repo, path).await? {
11951 bail!("worktree removal refused: {}", path.display());
11952 }
11953 Ok(())
11954}
11955
11956pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11965 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11966 bail!("refusing fold: {reason}");
11967 }
11968 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11969 bail!("candidate preservation incomplete; refusing cleanup");
11970 }
11971 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11975 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11976 let _driver = crate::driver::maintenance(home, &state.id)?;
11977 let mut output_guards = Vec::new();
11978 for entry in std::fs::read_dir(state.worktree_root())
11979 .into_iter()
11980 .flatten()
11981 .flatten()
11982 {
11983 if entry.path().join(".git").is_file() {
11984 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11985 }
11986 }
11987 let repo = state.repo.clone();
11988 let wait = repository_wait(state, None);
11991 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11992 Ok(lease) => lease,
11993 Err(e) => {
11994 state.save_under(home)?;
11995 return Err(e);
11996 }
11997 };
11998 let _source_outputs = if state.status == RunStatus::Merged
12001 && state.merge.as_ref().is_some_and(|merge| merge.ok)
12002 {
12003 verify_merged_fold_sources(state, drop_winner, home).await?
12004 } else {
12005 Vec::new()
12006 };
12007 let root = state.worktree_root();
12008 let winner = state.tally.as_ref().map(|t| t.winner);
12009 let mut removed = Vec::new();
12010
12011 for i in 0..state.candidates.len() {
12012 let c = state.candidates[i].clone();
12013 let is_winner = Some(c.label) == winner;
12014 if is_winner && !drop_winner {
12015 continue;
12016 }
12017 let existed = c.worktree.exists();
12018 remove_preserved_worktree(&repo, &c.worktree).await?;
12021 if existed {
12022 removed.push(c.worktree.to_string_lossy().into_owned());
12023 }
12024 let handed_over = state.released_branches.contains(&c.branch);
12027 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12028 if !git::branch_delete(&repo, &c.branch).await? {
12029 bail!("candidate branch removal refused");
12030 }
12031 removed.push(c.branch.clone());
12032 }
12033 state.candidates[i].folded = true;
12034 }
12035
12036 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12037 let path = name.path();
12038 let keep = !drop_winner
12039 && winner.is_some_and(|w| {
12040 path.file_name()
12041 .is_some_and(|n| n == format!("cand-{w}").as_str())
12042 });
12043 if keep {
12044 continue;
12045 }
12046 remove_preserved_worktree(&repo, &path).await?;
12047 removed.push(path.to_string_lossy().into_owned());
12048 }
12049
12050 remove_if_empty(&root);
12059
12060 if state.enabled_worktree_config && drop_winner {
12061 git::release_worktree_config(&repo).await.ok();
12065 state.enabled_worktree_config = false;
12066 }
12067 state.save_under(home)?;
12068 Ok(removed)
12069}
12070
12071async fn verify_merged_fold_sources(
12072 state: &RunState,
12073 drop_winner: bool,
12074 home: &Path,
12075) -> Result<Vec<std::fs::File>> {
12076 let winner = state.tally.as_ref().map(|t| t.winner);
12077 let mut heads = BTreeSet::new();
12078 for candidate in &state.candidates {
12079 if Some(candidate.label) == winner && !drop_winner {
12080 continue;
12081 }
12082 anyhow::ensure!(
12083 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12084 "candidate recovery source is outside the inventoried run root: {}",
12085 candidate.worktree.display()
12086 );
12087 match std::fs::symlink_metadata(&candidate.worktree) {
12088 Ok(_) => {}
12089 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12090 Err(error) => return Err(error.into()),
12091 }
12092 if !state.released_branches.contains(&candidate.branch)
12093 && git::branch_exists(&state.repo, &candidate.branch).await?
12094 {
12095 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12096 }
12097 }
12098 let mut cargo_guards = Vec::new();
12099 let entries = match std::fs::read_dir(state.worktree_root()) {
12100 Ok(entries) => Some(entries),
12101 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12102 Err(error) => return Err(error.into()),
12103 };
12104 for entry in entries.into_iter().flatten() {
12105 let path = entry?.path();
12106 let metadata = std::fs::symlink_metadata(&path)?;
12107 anyhow::ensure!(
12108 metadata.is_dir() && !metadata.file_type().is_symlink(),
12109 "non-directory or symlink recovery source retained: {}",
12110 path.display()
12111 );
12112 if !drop_winner
12113 && winner.is_some_and(|w| {
12114 path.file_name()
12115 .is_some_and(|name| name == format!("cand-{w}").as_str())
12116 })
12117 {
12118 continue;
12119 }
12120 anyhow::ensure!(
12121 path.join(".git").is_file(),
12122 "unregistered recovery source retained: {}",
12123 path.display()
12124 );
12125 anyhow::ensure!(
12126 git::git(
12127 &path,
12128 &[
12129 "status",
12130 "--porcelain",
12131 "--untracked-files=all",
12132 "--ignored",
12133 "--",
12134 ".",
12135 ":(exclude)target"
12136 ]
12137 )
12138 .await?
12139 .is_empty(),
12140 "source changes or ignored recovery files retained: {}",
12141 path.display()
12142 );
12143 match std::fs::symlink_metadata(path.join("target")) {
12144 Ok(_) => {
12145 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12146 }
12147 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12148 Err(error) => return Err(error.into()),
12149 }
12150 for checkpoint in [
12151 "rebase-merge",
12152 "rebase-apply",
12153 "sequencer",
12154 "CHERRY_PICK_HEAD",
12155 "MERGE_HEAD",
12156 "REBASE_HEAD",
12157 "REVERT_HEAD",
12158 "BISECT_LOG",
12159 "AUTO_MERGE",
12160 "MERGE_MSG",
12161 ] {
12162 let location = git::git(
12163 &path,
12164 &[
12165 "rev-parse",
12166 "--path-format=absolute",
12167 "--git-path",
12168 checkpoint,
12169 ],
12170 )
12171 .await?;
12172 anyhow::ensure!(
12173 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12174 if e.kind() == std::io::ErrorKind::NotFound),
12175 "Git recovery checkpoint retained: {checkpoint}"
12176 );
12177 }
12178 heads.insert(git::rev_parse(&path, "HEAD").await?);
12179 }
12180 crate::fold_evidence::preserve(state, home, &heads).await?;
12181
12182 Ok(cargo_guards)
12183}
12184
12185fn remove_if_empty(dir: &Path) {
12196 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12197 std::fs::remove_dir(dir).ok();
12198 }
12199}
12200
12201pub fn worst_open(state: &RunState) -> Option<Severity> {
12203 state
12204 .reviews
12205 .last()?
12206 .reviews
12207 .iter()
12208 .flat_map(|r| r.findings.iter())
12209 .map(|f| f.severity)
12210 .max()
12211}
12212
12213async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12217 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12218 git::acquire_worktree_config(repo).await?;
12219 state.enabled_worktree_config = true;
12220 }
12221 Ok(())
12222}
12223
12224#[cfg(test)]
12225mod tests {
12226 #[tokio::test]
12227 async fn launcher_base_fetch_requires_remote_success() {
12228 let tmp = tempfile::tempdir().unwrap();
12229 let repo = tmp.path().join("repo");
12230 let remote = tmp.path().join("remote");
12231 std::fs::create_dir(&repo).unwrap();
12232 std::fs::create_dir(&remote).unwrap();
12233 git::git(&remote, &["init", "--bare", "-b", "main"])
12234 .await
12235 .unwrap();
12236 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12237 git::git(
12238 &repo,
12239 &[
12240 "-c",
12241 "user.name=Fixture",
12242 "-c",
12243 "user.email=fixture@example.invalid",
12244 "commit",
12245 "--allow-empty",
12246 "-m",
12247 "fixture",
12248 ],
12249 )
12250 .await
12251 .unwrap();
12252 git::git(
12253 &repo,
12254 &["remote", "add", "origin", remote.to_str().unwrap()],
12255 )
12256 .await
12257 .unwrap();
12258 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12259 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12260 assert_eq!(
12261 resolve_base(&repo, "main", "origin", None)
12262 .await
12263 .unwrap()
12264 .commit,
12265 head
12266 );
12267 #[cfg(unix)]
12268 {
12269 let policy: crate::config::Coordination = toml::from_str(
12270 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12271 ).unwrap();
12272 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12273 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12274 let held = crate::repo_lock::SeatLease::operation_with(
12275 &repo,
12276 Some(&policy),
12277 &crate::repo_lock::WaitPolicy::default(),
12278 )
12279 .await
12280 .unwrap();
12281 assert!(held.is_some());
12282 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12283 tokio::pin!(fetch);
12284 assert!(
12285 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12286 .await
12287 .is_err()
12288 );
12289 assert!(
12290 !repo.join(".git/FETCH_HEAD").exists(),
12291 "no fetch before admission"
12292 );
12293 drop(held);
12294 let fetched = fetch.await.unwrap();
12295 assert_eq!(fetched.commit, head);
12296 assert_eq!(fetched.remote, "origin");
12297 assert_eq!(fetched.branch, "main");
12298 }
12299 git::git(
12301 &repo,
12302 &[
12303 "remote",
12304 "set-url",
12305 "origin",
12306 tmp.path().join("missing").to_str().unwrap(),
12307 ],
12308 )
12309 .await
12310 .unwrap();
12311 let error = resolve_base(&repo, "main", "origin", None)
12312 .await
12313 .unwrap_err()
12314 .to_string();
12315 assert!(error.contains("cannot read origin/main"));
12316 assert!(error.contains("refusing to branch off"));
12317 }
12318
12319 #[test]
12320 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12321 let leaked = vec!["chore(deps): update a crate".to_owned()];
12322 let own = "fix(daemon): apply a chosen action";
12323 assert!(should_retitle("", own, &leaked));
12324 assert!(should_retitle(
12325 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12326 own,
12327 &leaked
12328 ));
12329 assert!(should_retitle(
12330 "chore: land candidate A of run 1",
12331 own,
12332 &leaked
12333 ));
12334 assert!(should_retitle(
12335 "magi: candidate A (uncommitted work)",
12336 own,
12337 &leaked
12338 ));
12339 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12340 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12341 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12342 }
12343
12344 #[test]
12345 fn pr_merge_plan_creates_adopts_or_stops() {
12346 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12347 assert_eq!(
12348 pr_merge_plan(Ok(land::OpenPr::One {
12349 url: "u".into(),
12350 title: "t".into()
12351 })),
12352 PrPlan::Adopt {
12353 url: "u".into(),
12354 title: "t".into()
12355 }
12356 );
12357 let PrPlan::Stop(many) =
12358 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12359 else {
12360 panic!("many must stop");
12361 };
12362 assert!(many.contains('a') && many.contains('b'));
12363 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12364 panic!("a failed lookup must stop");
12365 };
12366 assert!(err.contains("bad token"));
12367 }
12368
12369 use super::*;
12370 use crate::run::GateStatus;
12371 use std::collections::BTreeMap;
12372 use std::time::Duration;
12373
12374 fn conductor() -> AgentSpec {
12375 AgentSpec {
12376 id: "conductor".to_owned(),
12377 kind: crate::config::AgentKind::Command,
12378 model: None,
12379 command: vec!["true".to_owned()],
12380 extra_args: Vec::new(),
12381 env: BTreeMap::new(),
12382 prompt_delivery: None,
12383 max_turn_secs: None,
12384 }
12385 }
12386
12387 fn spec(id: &str) -> AgentSpec {
12388 AgentSpec {
12389 id: id.to_owned(),
12390 kind: crate::config::AgentKind::Command,
12391 model: None,
12392 command: vec!["true".to_owned()],
12393 extra_args: Vec::new(),
12394 env: BTreeMap::new(),
12395 prompt_delivery: None,
12396 max_turn_secs: None,
12397 }
12398 }
12399
12400 fn ids(xs: &[&str]) -> BTreeSet<String> {
12401 xs.iter().map(|s| (*s).to_owned()).collect()
12402 }
12403
12404 #[test]
12405 fn pick_successor_skips_an_agent_another_seat_holds() {
12406 let roster = [spec("a"), spec("b"), spec("c")];
12408 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12409 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12410 }
12411
12412 #[test]
12413 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12414 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12416 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12417 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12418 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12419 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12420 }
12421
12422 #[test]
12423 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12424 let roster = [spec("a"), spec("b")];
12425 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12426 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12427 let carried = ids(&["b"]);
12428 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12429 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12430 }
12431
12432 #[test]
12433 fn pick_successor_returns_none_without_any_untried_successor() {
12434 let roster = [spec("a"), spec("b")];
12435 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12436 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12437 let carried = ids(&["a"]);
12438 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12439 }
12440
12441 #[test]
12442 fn pick_successor_rescue_avoids_another_seats_occupant() {
12443 let roster = [spec("a"), spec("b"), spec("c")];
12445 let carried = ids(&["b", "c"]);
12446 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12447 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12448 }
12449
12450 #[test]
12451 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12452 let roster = [spec("a"), spec("b"), spec("c")];
12453 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12454 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12455 }
12456
12457 #[test]
12458 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12459 let roster = [spec("a"), spec("b"), spec("c")];
12460 let failed = ids(&["a", "b", "c"]);
12461 let mut tried = ids(&["b"]);
12463 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12464 assert_eq!(first.id, "a");
12465 tried.insert(first.id.clone());
12466 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12467 assert_eq!(second.id, "c");
12468 tried.insert(second.id.clone());
12469 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12470 }
12471
12472 #[test]
12473 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12474 let roster = [spec("a"), spec("b")];
12475 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12476 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12477 }
12478
12479 #[test]
12480 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12481 let roster = [spec("a"), spec("b"), spec("c")];
12482 let h = SeatHistory {
12483 failed: ids(&["a"]),
12484 last_ok: Some("b".to_owned()),
12485 last_fail: None,
12486 };
12487 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12488 let gone = SeatHistory {
12490 last_ok: Some("zzz".to_owned()),
12491 ..h.clone()
12492 };
12493 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12494 let failed = SeatHistory {
12495 failed: ids(&["a", "b"]),
12496 last_ok: Some("b".to_owned()),
12497 last_fail: None,
12498 };
12499 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12500 }
12501
12502 #[test]
12503 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12504 let first = SeatState::new("review-1", "alpha", 7);
12505 let next = handover_seat("review-1", "gamma", 7);
12506 assert_ne!(first.claude_session, next.claude_session);
12507 }
12508
12509 #[test]
12510 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12511 let mut state = state_with_summary("x", "y");
12512 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12513 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12514 assert_ne!(a.claude_session, b.claude_session);
12515 }
12516
12517 #[test]
12518 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12519 let roster = [spec("a"), spec("b")];
12520 let h = SeatHistory {
12521 failed: ids(&["a", "b"]),
12522 ..SeatHistory::default()
12523 };
12524 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12525 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12526 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12527 }
12528
12529 #[test]
12536 fn failure_signature_ignores_numbers_and_paths() {
12537 assert_eq!(
12538 failure_signature("exited with Some(2) and no usable output"),
12539 failure_signature("exited with Some(137) and no usable output")
12540 );
12541 assert_eq!(
12542 failure_signature("process exited with no usable output (exit code 2)"),
12543 failure_signature("process exited with no usable output (exit code 137)")
12544 );
12545 for (old, new) in [
12546 (
12547 "exited with Some(2) and no usable output",
12548 "process exited with no usable output (exit code 2)",
12549 ),
12550 (
12551 "exited with None and no usable output",
12552 "process exited with no usable output (exit code unavailable)",
12553 ),
12554 ] {
12555 let previous = FailClass::Other(failure_signature(old));
12556 let current = FailClass::Other(failure_signature(new));
12557 assert_eq!(previous, current, "resumption retains the failure class");
12558 assert!(!should_hand_over(Some(&previous), ¤t));
12559 }
12560 assert_eq!(
12561 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12562 failure_signature("cannot open /var/x.txt: denied")
12563 );
12564 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12565 }
12566
12567 #[test]
12568 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12569 let other = FailClass::Other("x".into());
12570 assert!(should_hand_over(None, &FailClass::Quota));
12571 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12572 assert!(should_hand_over(
12573 Some(&FailClass::Timeout),
12574 &FailClass::Timeout
12575 ));
12576 assert!(should_hand_over(None, &other));
12577 assert!(!should_hand_over(Some(&other), &other));
12578 assert!(should_hand_over(
12579 Some(&other),
12580 &FailClass::Other("y".into())
12581 ));
12582 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12584 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12585 }
12586
12587 #[test]
12588 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12589 let a = SeatState::new("judge-1", "alpha", 7);
12590 let b = handover_seat("judge-1", "beta", 7);
12591 assert_ne!(a.claude_session, b.claude_session);
12592 assert_eq!(b.turns, 0);
12593 }
12594
12595 #[test]
12596 fn a_timeout_is_classified_apart_from_other_failures() {
12597 assert_eq!(
12598 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12599 Some(FailClass::Timeout)
12600 );
12601 assert!(matches!(
12602 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12603 Some(FailClass::Other(_))
12604 ));
12605 }
12606
12607 #[test]
12608 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12609 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12610 let tried = BTreeSet::from(["beta".to_owned()]);
12611 let next = next_untried_in_roster(&roster, 1, &tried);
12614 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12615 }
12616
12617 #[test]
12618 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12619 let roster = vec![spec("alpha"), spec("beta")];
12620 let tried = BTreeSet::from(["beta".to_owned()]);
12621 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12625 }
12626
12627 #[test]
12628 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12629 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12630 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12631 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12635 }
12636
12637 #[test]
12638 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12639 let roster = vec![spec("a"), spec("a"), spec("b")];
12640 let tried = BTreeSet::from(["a".to_owned()]);
12641 let next = next_untried_in_roster(&roster, 0, &tried);
12642 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12643 }
12644
12645 #[test]
12646 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12647 let roster = vec![spec("a"), spec("b")];
12648 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12649 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12650 }
12651
12652 #[test]
12653 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12654 let dir = tempfile::tempdir().unwrap();
12655 let bay = dir.path().join("ffff");
12656
12657 remove_if_empty(&bay);
12659 assert!(!bay.exists());
12660
12661 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12664 remove_if_empty(&bay);
12665 assert!(bay.exists(), "non-empty directory must survive");
12666
12667 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12669 remove_if_empty(&bay);
12670 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12671 }
12672
12673 #[test]
12682 fn a_full_panel_that_found_nothing_is_clean() {
12683 assert!(round_is_clean(
12684 0,
12685 true,
12686 2,
12687 2,
12688 0,
12689 IncompleteReviewPolicy::Block
12690 ));
12691 }
12692
12693 #[test]
12694 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12695 assert!(!round_is_clean(
12696 0,
12697 true,
12698 1,
12699 2,
12700 0,
12701 IncompleteReviewPolicy::Block
12702 ));
12703 }
12704
12705 #[test]
12706 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12707 assert!(!round_is_clean(
12708 1,
12709 true,
12710 1,
12711 2,
12712 0,
12713 IncompleteReviewPolicy::Warn
12714 ));
12715 }
12716
12717 #[test]
12718 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12719 assert!(round_is_clean(
12720 0,
12721 true,
12722 1,
12723 2,
12724 0,
12725 IncompleteReviewPolicy::Warn
12726 ));
12727 }
12728
12729 #[test]
12730 fn a_full_panel_with_an_open_finding_is_not_clean() {
12731 assert!(!round_is_clean(
12732 1,
12733 true,
12734 2,
12735 2,
12736 0,
12737 IncompleteReviewPolicy::Block
12738 ));
12739 }
12740
12741 #[test]
12742 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12743 assert!(!round_is_clean(
12744 0,
12745 false,
12746 2,
12747 2,
12748 0,
12749 IncompleteReviewPolicy::Block
12750 ));
12751 }
12752
12753 #[test]
12760 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12761 assert!(round_is_clean(
12764 0,
12765 true,
12766 1,
12767 2,
12768 1,
12769 IncompleteReviewPolicy::Block
12770 ));
12771 }
12772
12773 #[test]
12774 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12775 assert!(!round_is_clean(
12778 0,
12779 true,
12780 1,
12781 2,
12782 0,
12783 IncompleteReviewPolicy::Block
12784 ));
12785 }
12786
12787 #[test]
12788 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12789 assert!(!round_is_clean(
12790 1,
12791 true,
12792 1,
12793 2,
12794 1,
12795 IncompleteReviewPolicy::Block
12796 ));
12797 assert!(!round_is_clean(
12798 0,
12799 false,
12800 1,
12801 2,
12802 1,
12803 IncompleteReviewPolicy::Block
12804 ));
12805 }
12806
12807 #[test]
12808 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12809 assert!(!round_is_clean(
12813 0,
12814 true,
12815 0,
12816 2,
12817 2,
12818 IncompleteReviewPolicy::Block
12819 ));
12820 }
12821
12822 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12823 CommandOutcome {
12824 command: "test".to_owned(),
12825 code,
12826 output_tail: String::new(),
12827 duration_ms: 0,
12828 resource_blocked,
12829 }
12830 }
12831
12832 #[test]
12833 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12834 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12835 assert!(
12836 !verify_inconclusive(&[outcome(Some(1), false)]),
12837 "an ordinary failure is still evidence about the patch"
12838 );
12839 assert!(verify_inconclusive(&[outcome(None, true)]));
12840 assert!(
12841 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12842 "one inconclusive outcome taints the whole batch"
12843 );
12844 assert!(!verify_inconclusive(&[]));
12845 }
12846
12847 #[tokio::test]
12848 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12849 let calls = std::sync::atomic::AtomicUsize::new(0);
12853 let started = Instant::now();
12854 wait_for_pids_with(
12855 &[123],
12856 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12857 Duration::from_millis(5),
12858 Duration::from_secs(5),
12859 )
12860 .await;
12861 assert!(
12862 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12863 "must keep checking rather than deciding on the first answer"
12864 );
12865 assert!(
12866 started.elapsed() < Duration::from_secs(1),
12867 "must return the moment it is confirmed dead, not wait out the ceiling"
12868 );
12869 }
12870
12871 #[tokio::test]
12872 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12873 let started = Instant::now();
12874 wait_for_pids_with(
12875 &[123],
12876 |_| true, Duration::from_millis(5),
12878 Duration::from_millis(30),
12879 )
12880 .await;
12881 let elapsed = started.elapsed();
12882 assert!(
12883 elapsed >= Duration::from_millis(30),
12884 "must not give up before its own ceiling: {elapsed:?}"
12885 );
12886 assert!(
12887 elapsed < Duration::from_secs(1),
12888 "must not wait past its own ceiling either: {elapsed:?}"
12889 );
12890 }
12891
12892 #[tokio::test]
12893 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12894 let started = Instant::now();
12895 wait_for_pids_with(
12896 &[],
12897 |_| true,
12898 Duration::from_secs(5),
12899 Duration::from_secs(5),
12900 )
12901 .await;
12902 assert!(
12903 started.elapsed() < Duration::from_millis(200),
12904 "an empty pid list has nothing to confirm"
12905 );
12906 }
12907
12908 fn review_round(
12914 clean: bool,
12915 blocking: usize,
12916 answered: usize,
12917 expected: usize,
12918 progressed: bool,
12919 e2e_ok: bool,
12920 ) -> ReviewRound {
12921 ReviewRound {
12922 round: 1,
12923 head: "h".to_owned(),
12924 verified_head: None,
12925 verified_at: None,
12926 reviews: Vec::new(),
12927 e2e: vec![CommandOutcome {
12928 command: "test".to_owned(),
12929 code: Some(if e2e_ok { 0 } else { 1 }),
12930 output_tail: String::new(),
12931 duration_ms: 0,
12932 resource_blocked: false,
12933 }],
12934 verify_retried: false,
12935 e2e_deferred: false,
12936 e2e_defer_reason: None,
12937 fix: None,
12938 blocking,
12939 answered,
12940 expected,
12941 clean,
12942 progressed,
12943 vote_split: false,
12944 reconsideration: Vec::new(),
12945 verdict: None,
12946 }
12947 }
12948
12949 #[test]
12950 fn review_conclusion_is_none_when_nothing_has_run() {
12951 assert_eq!(review_conclusion(&[], 3), None);
12952 }
12953
12954 #[test]
12955 fn review_conclusion_is_none_while_rounds_remain() {
12956 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12957 assert_eq!(review_conclusion(&rounds, 3), None);
12958 }
12959
12960 #[test]
12961 fn review_conclusion_is_gating_once_a_round_is_clean() {
12962 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12963 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12964 }
12965
12966 #[test]
12967 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12968 let rounds = vec![
12969 review_round(false, 1, 2, 2, true, true),
12970 review_round(false, 1, 2, 2, true, true),
12971 ];
12972 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12973 }
12974
12975 #[test]
12976 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12977 let rounds = vec![
12978 review_round(false, 1, 2, 2, true, true),
12979 review_round(false, 1, 2, 2, true, false),
12980 ];
12981 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12982 }
12983
12984 #[test]
12985 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12986 let mut blocked = review_round(false, 1, 2, 2, true, false);
12993 blocked.e2e[0].resource_blocked = true;
12994 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12995 assert_eq!(review_conclusion(&rounds, 2), None);
12996 }
12997
12998 #[test]
12999 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13000 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13002 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13003 }
13004
13005 #[test]
13006 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13007 let rounds = vec![
13008 review_round(false, 1, 2, 2, false, true),
13009 review_round(false, 1, 2, 2, false, true),
13010 ];
13011 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13012 }
13013
13014 fn secs(n: u64) -> Duration {
13015 Duration::from_secs(n)
13016 }
13017
13018 #[tokio::test]
13021 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13022 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13023 if std::env::var_os(CHILD).is_none() {
13024 let output = std::process::Command::new(std::env::current_exe().unwrap())
13028 .args([
13029 "--exact",
13030 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13031 "--nocapture",
13032 ])
13033 .env(CHILD, "1")
13034 .env_remove("MAGI_WORKER_HOME")
13035 .env_remove("MAGI_WORKER_ID")
13036 .output()
13037 .unwrap();
13038 assert!(
13039 output.status.success()
13040 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13041 "{}\n{}",
13042 String::from_utf8_lossy(&output.stdout),
13043 String::from_utf8_lossy(&output.stderr)
13044 );
13045 return;
13046 }
13047 ask_test_home();
13048 let tmp = tempfile::tempdir().unwrap();
13049 let repo = tmp.path().join("repo");
13050 std::fs::create_dir_all(&repo).unwrap();
13051 init_repo(&repo);
13052 let g = |args: &[&str]| {
13053 let out = std::process::Command::new("git")
13054 .args(args)
13055 .current_dir(&repo)
13056 .quiet()
13057 .output()
13058 .unwrap();
13059 assert!(
13060 out.status.success(),
13061 "{}",
13062 String::from_utf8_lossy(&out.stderr)
13063 );
13064 };
13065 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13066 g(&["checkout", "-b", "historical"]);
13067 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13068 g(&["add", "."]);
13069 g(&["commit", "-m", "old alternative"]);
13070 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13071 g(&["checkout", "-b", "magi/task-selected", &base]);
13072 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13073 g(&["add", "."]);
13074 g(&["commit", "-m", "measurement infrastructure"]);
13075 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13076 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13077 g(&["add", "."]);
13078 g(&["commit", "-m", "dependent implementation"]);
13079 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13080 g(&["checkout", "main"]);
13081 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13082 g(&["add", "."]);
13083 g(&["commit", "-m", "unrelated base change"]);
13084 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13085 let origin = tmp.path().join("origin.git");
13086 g(&[
13087 "clone",
13088 "--bare",
13089 repo.to_str().unwrap(),
13090 origin.to_str().unwrap(),
13091 ]);
13092 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13093 g(&["fetch", "origin"]);
13094 let selection = crate::pr_recovery::Selection {
13095 url: "https://example.test/project/pull/1".into(),
13096 branch: "magi/task-selected".into(),
13097 head: head.clone(),
13098 source_head: None,
13099 source_run: None,
13100 source_proof: None,
13101 disposition: Some(crate::pr_recovery::Disposition::Review),
13102 instruction: "preserve measured behavior".into(),
13103 approved_at: jiff::Timestamp::now(),
13104 };
13105 let mut template = runner_at(RunStatus::Reviewing);
13106 template.state.repo = repo.clone();
13107 template.state.base_commit = current.clone();
13108 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13109 template.state.origin = Some(crate::run::Origin::queue("selected"));
13110 template.state.pr_head = Some(selection.branch.clone());
13111 template.state.pr_recovery = Some(selection.clone());
13112 let mut runner =
13113 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13114 .await
13115 .unwrap();
13116 let wt = runner.state.winner().unwrap().worktree.clone();
13117 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13118 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13119 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13120 assert!(runner.state.seeds.is_empty());
13121 assert_eq!(runner.state.instruction, "preserve measured behavior");
13122 runner.sync_to_base().await.unwrap();
13123 assert_eq!(
13124 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13125 "prerequisite"
13126 );
13127 assert_eq!(
13128 std::fs::read_to_string(wt.join("implementation")).unwrap(),
13129 "measured implementation"
13130 );
13131 assert_eq!(
13132 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13133 "current main change"
13134 );
13135 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13136 assert_eq!(
13137 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13138 head
13139 );
13140
13141 let mut partial = runner_at(RunStatus::Reviewing);
13143 partial.state.repo = repo.clone();
13144 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13145 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13146 run: Some("exit 1".into()),
13147 ..Default::default()
13148 }];
13149 let selection = runner.state.pr_recovery.clone().unwrap();
13150 partial.state.pr_recovery = Some(selection.clone());
13151 partial.state.pr_head = Some(selection.branch.clone());
13152 partial.state.origin = Some(crate::run::Origin::queue("selected"));
13153 let id = partial.state.id.clone();
13154 let error = match Runner::open_selected(
13155 &repo,
13156 partial.state,
13157 partial.roles,
13158 1,
13159 runner.state.base_commit.clone(),
13160 selection,
13161 )
13162 .await
13163 {
13164 Ok(_) => panic!("setup must fail"),
13165 Err(e) => e,
13166 };
13167 assert_eq!(
13168 error
13169 .downcast_ref::<crate::pr_recovery::StartupFailed>()
13170 .unwrap()
13171 .run,
13172 id
13173 );
13174 let failed = RunState::load(&id).unwrap();
13175 assert_eq!(failed.status, RunStatus::Blocked);
13176 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13177 assert!(Runner::resume(&id).is_err());
13178 runner.state.status = RunStatus::Ready;
13179 runner.state.driver_exited = true;
13180 runner.state.save().unwrap();
13181 let mut retry = runner_at(RunStatus::Reviewing);
13182 retry.state.repo = repo.clone();
13183 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13184 let mut reselected = runner.state.pr_recovery.clone().unwrap();
13185 reselected.approved_at = jiff::Timestamp::now();
13186 retry.state.origin = Some(crate::run::Origin::queue("selected"));
13187 retry.state.pr_head = Some(reselected.branch.clone());
13188 retry.state.pr_recovery = Some(reselected.clone());
13189 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13190 .await
13191 .unwrap();
13192 let restarted = Runner::open_selected(
13193 &repo,
13194 retry.state,
13195 retry.roles,
13196 1,
13197 runner.state.base_commit.clone(),
13198 reselected,
13199 )
13200 .await
13201 .unwrap();
13202 assert_eq!(
13203 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13204 .await
13205 .unwrap(),
13206 head
13207 );
13208 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13209 assert_eq!(
13210 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13211 head
13212 );
13213 }
13214
13215 #[tokio::test]
13216 #[cfg(unix)]
13217 async fn fallback_and_interrupted_restoration_require_coordination() {
13218 use std::os::fd::AsRawFd as _;
13219 ask_test_home();
13220 let tmp = tempfile::tempdir().unwrap();
13221 let mut runner = registered_runner(tmp.path()).await;
13222 runner.state.config.graph.timeout_resource_wait = 1;
13223 let winner = runner.state.candidates[0].clone();
13224 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13225 let leftover = winner.worktree.join("leftover");
13226 std::fs::write(&leftover, "untrusted edits").unwrap();
13227 let next = tmp.path().join("next-fixer");
13228 let mut failed = spec("first");
13229 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13230 let mut successor = spec("second");
13231 successor.command = vec![
13232 "sh".into(),
13233 "-c".into(),
13234 format!("touch '{}'; echo done", next.display()),
13235 ];
13236 let attempts = vec![
13237 (failed.clone(), "first".into()),
13238 (successor.clone(), "second".into()),
13239 ];
13240 let lock_path = runner.state.repo.join(".git/updater.lock");
13241 let external = std::fs::File::create(&lock_path).unwrap();
13242 assert_eq!(
13243 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13244 0
13245 );
13246 let artifacts = tmp.path().join("artifacts");
13247 let build = |spec, seat| SeatJob {
13248 spec,
13249 seat,
13250 cwd: winner.worktree.clone(),
13251 prompt: "fixture".into(),
13252 timeout: Duration::from_secs(10),
13253 allow_write: true,
13254 sessions: false,
13255 artifacts: artifacts.clone(),
13256 stem: "fix".into(),
13257 handover: None,
13258 };
13259 let (_, _, out) = runner
13260 .ask_fixer_among(
13261 attempts.clone(),
13262 "graft",
13263 None,
13264 Some((&winner.worktree, &head)),
13265 build,
13266 )
13267 .await;
13268 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13269 assert!(leftover.exists(), "contention must prevent clean");
13270 assert!(!next.exists());
13271 assert!(runner.state.handovers.is_empty());
13272 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13274 from_run: "earlier".into(),
13275 generation: 1,
13276 mode: crate::recovery::Mode::VerifyRepair,
13277 reasons: Vec::new(),
13278 reviewed_head: None,
13279 rounds: Vec::new(),
13280 open_findings: Vec::new(),
13281 evidence: None,
13282 criteria_inherited: true,
13283 repair: crate::recovery::Repair::Started {
13284 before: head.clone(),
13285 },
13286 });
13287 assert!(
13288 runner
13289 .recovery_repair(&winner)
13290 .await
13291 .unwrap_err()
13292 .is::<crate::repo_lock::ResourceBlocked>()
13293 );
13294 assert!(matches!(
13295 runner.state.recovery.as_ref().unwrap().repair,
13296 crate::recovery::Repair::Started { .. }
13297 ));
13298 assert!(leftover.exists());
13299 drop(external);
13300 runner.recovery_repair(&winner).await.unwrap();
13301 assert!(!leftover.exists());
13302 assert!(matches!(
13303 runner.state.recovery.as_ref().unwrap().repair,
13304 crate::recovery::Repair::Done { .. }
13305 ));
13306 std::fs::remove_file(&lock_path).unwrap();
13308 std::fs::create_dir(&lock_path).unwrap();
13309 std::fs::write(&leftover, "untrusted edits").unwrap();
13310 let (_, _, out) = runner
13311 .ask_fixer_among(
13312 attempts,
13313 "graft",
13314 None,
13315 Some((&winner.worktree, &head)),
13316 build,
13317 )
13318 .await;
13319 assert!(matches!(out, AgentOutcome::Failed(_)));
13320 assert!(leftover.exists());
13321 assert!(!next.exists());
13322 assert_eq!(
13323 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13324 head
13325 );
13326 }
13327
13328 #[cfg(unix)]
13329 async fn registered_runner(root: &Path) -> Runner {
13330 ask_test_home();
13331 let repo = root.join("repo");
13332 std::fs::create_dir_all(&repo).unwrap();
13333 init_repo(&repo);
13334 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13335 let mut runner = runner_at(RunStatus::Prep);
13336 runner.state.repo = repo.clone();
13337 runner.state.config.coordination = crate::config::test_coordination();
13338 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13339 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13340 runner.state.config.graph.advise = false;
13341 runner.roles.implementers = vec![spec("implement")];
13342 runner.roles.judges = vec![spec("judge")];
13343 runner.prep().await.unwrap();
13344 runner
13345 }
13346
13347 #[tokio::test]
13348 #[cfg(unix)]
13349 async fn parked_implementer_persists_completed_work() {
13350 for registered in [false, true] {
13351 let tmp = tempfile::tempdir().unwrap();
13352 let mut runner = registered_runner(tmp.path()).await;
13353 if !registered {
13354 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13355 }
13356 let marker = tmp.path().join("started");
13357 let release = tmp.path().join("release");
13358 let mut implementer = spec("implement");
13359 implementer.command = vec![
13360 "sh".into(),
13361 "-c".into(),
13362 format!(
13363 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13364 marker.display(),
13365 release.display()
13366 ),
13367 ];
13368 runner.roles.implementers = vec![implementer.clone()];
13369 runner.state.config.agents = vec![implementer];
13370 let pause = runner.pause.clone();
13371 let (result, ()) = tokio::join!(runner.implement(), async {
13372 tokio::time::timeout(Duration::from_secs(10), async {
13373 while !marker.exists() {
13374 tokio::time::sleep(Duration::from_millis(10)).await;
13375 }
13376 })
13377 .await
13378 .unwrap();
13379 pause.park();
13380 std::fs::write(&release, "go").unwrap();
13381 });
13382 result.unwrap();
13383 let candidate = &runner.state.candidates[0];
13384 assert!(candidate.commits > 0, "{candidate:?}");
13385 assert!(candidate.summary.contains("saved-result"));
13386 assert_eq!(candidate.files, 1);
13387 let before = candidate.commits;
13388 runner.implement().await.unwrap();
13389 assert_eq!(runner.state.candidates[0].commits, before);
13390 }
13391 }
13392
13393 #[tokio::test]
13394 #[cfg(unix)]
13395 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13396 let tmp = tempfile::tempdir().unwrap();
13397 let mut runner = registered_runner(tmp.path()).await;
13398 let winner = runner.state.candidates[0].clone();
13399 let cache = tmp.path().join("cache");
13400 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13401 runner.state.config.graph.timeout_resource_wait = 0;
13402 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13403 let crate::cache::AcquireOutcome::Acquired(held) =
13404 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13405 else {
13406 panic!("cache must be free")
13407 };
13408 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13409 from_run: "earlier".into(),
13410 generation: 1,
13411 mode: crate::recovery::Mode::VerifyRepair,
13412 reasons: Vec::new(),
13413 reviewed_head: None,
13414 rounds: Vec::new(),
13415 open_findings: Vec::new(),
13416 evidence: None,
13417 criteria_inherited: true,
13418 repair: crate::recovery::Repair::NotStarted,
13419 });
13420 runner.recovery_repair(&winner).await.unwrap();
13421 assert!(matches!(
13422 runner.state.recovery.as_ref().unwrap().repair,
13423 crate::recovery::Repair::WaitingForRestore { .. }
13424 ));
13425 drop(held);
13426 runner.state.resource_block = None;
13427 runner.state.config.graph.timeout_resource_wait = 1;
13428 runner.recovery_repair(&winner).await.unwrap();
13429 assert!(matches!(
13430 runner.state.recovery.as_ref().unwrap().repair,
13431 crate::recovery::Repair::Done { .. }
13432 ));
13433 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13434 }
13435
13436 #[tokio::test]
13437 #[cfg(unix)]
13438 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13439 use std::os::fd::AsRawFd as _;
13440 for graft in [false, true] {
13441 let tmp = tempfile::tempdir().unwrap();
13442 let mut runner = registered_runner(tmp.path()).await;
13443 runner.state.config.graph.timeout_resource_wait = 1;
13444 let winner = runner.state.candidates[0].clone();
13445 let marker = tmp.path().join("fixer-started");
13446 let mut fixer = spec("fixer");
13447 fixer.command = vec![
13448 "sh".into(),
13449 "-c".into(),
13450 format!("echo started >> '{}'; echo done", marker.display()),
13451 ];
13452 runner.roles.fixer = Some(vec![fixer]);
13453 if graft {
13454 let mut loser = winner.clone();
13455 loser.label = 'B';
13456 loser.index = 1;
13457 runner.state.candidates.push(loser);
13458 runner.state.config.graph.graft_runner_up = true;
13459 runner.state.tally = Some(Tally {
13460 first_choice: BTreeMap::new(),
13461 borda: BTreeMap::new(),
13462 winner: winner.label,
13463 rankings: 1,
13464 unanimous_initial: true,
13465 deliberated: false,
13466 changed_votes: 0,
13467 unanimous_final: true,
13468 tie_break: None,
13469 judges: 1,
13470 present: 1,
13471 quorum: 1,
13472 met_quorum: true,
13473 uncontested: None,
13474 });
13475 runner.state.judgements.push(crate::run::Judgement {
13476 judge: 1,
13477 seat: "judge-1".into(),
13478 agent: "judge".into(),
13479 ranking: vec![winner.label, 'B'],
13480 reasons: BTreeMap::new(),
13481 confidence: None,
13482 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13483 order: vec![0, 1],
13484 failed: None,
13485 duration_ms: 0,
13486 });
13487 } else {
13488 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13489 from_run: "earlier".into(),
13490 generation: 1,
13491 mode: crate::recovery::Mode::VerifyRepair,
13492 reasons: Vec::new(),
13493 reviewed_head: None,
13494 rounds: Vec::new(),
13495 open_findings: Vec::new(),
13496 evidence: None,
13497 criteria_inherited: true,
13498 repair: crate::recovery::Repair::NotStarted,
13499 });
13500 }
13501 let reader = crate::repo_lock::SeatLease::for_checkout(
13502 &winner.worktree,
13503 false,
13504 crate::config::test_coordination().as_ref(),
13505 )
13506 .await
13507 .unwrap();
13508 let pause = runner.pause.clone();
13509 let (result, ()) = tokio::join!(
13510 async {
13511 if graft {
13512 runner.graft().await
13513 } else {
13514 runner.recovery_repair(&winner).await
13515 }
13516 },
13517 async {
13518 tokio::time::sleep(Duration::from_millis(100)).await;
13519 pause.park();
13520 }
13521 );
13522 result.unwrap();
13523 assert!(!marker.exists(), "a blocked seat must not spawn");
13524 runner.state = RunState::load(&runner.state.id).unwrap();
13526 if graft {
13527 assert_eq!(
13528 runner.state.graft.as_ref().unwrap().outcome,
13529 GraftOutcome::WaitingForRestore
13530 );
13531 } else {
13532 assert!(matches!(
13533 runner.state.recovery.as_ref().unwrap().repair,
13534 crate::recovery::Repair::WaitingForRestore { .. }
13535 ));
13536 }
13537 drop(reader);
13538 runner.pause = Pause::new();
13539 runner.state.resource_block = None;
13540 let external =
13542 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13543 assert_eq!(
13544 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13545 0
13546 );
13547 let leftover = winner.worktree.join("leftover");
13548 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13549 let result = if graft {
13550 runner.graft().await
13551 } else {
13552 runner.recovery_repair(&winner).await
13553 };
13554 assert!(
13555 result
13556 .unwrap_err()
13557 .is::<crate::repo_lock::ResourceBlocked>()
13558 );
13559 assert!(leftover.exists());
13560 assert!(!marker.exists());
13561 drop(external);
13562 runner.state.resource_block = None;
13563 let result = if graft {
13564 runner.graft().await
13565 } else {
13566 runner.recovery_repair(&winner).await
13567 };
13568 result.unwrap();
13569 assert!(!leftover.exists(), "restore must precede retry");
13570 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13571 if graft {
13572 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13573 } else {
13574 assert!(matches!(
13575 runner.state.recovery.as_ref().unwrap().repair,
13576 crate::recovery::Repair::Done { .. }
13577 ));
13578 }
13579 }
13580 }
13581
13582 #[tokio::test]
13583 #[cfg(unix)]
13584 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13585 use std::os::fd::AsRawFd as _;
13586 let tmp = tempfile::tempdir().unwrap();
13587 let mut runner = registered_runner(tmp.path()).await;
13588 runner.state.config.graph.timeout_resource_wait = 1;
13589 let winner = runner.state.candidates[0].clone();
13590 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13591 runner.state.tally = Some(Tally {
13592 first_choice: BTreeMap::new(),
13593 borda: BTreeMap::new(),
13594 winner: winner.label,
13595 rankings: 0,
13596 unanimous_initial: false,
13597 deliberated: false,
13598 changed_votes: 0,
13599 unanimous_final: false,
13600 tie_break: None,
13601 judges: 0,
13602 present: 0,
13603 quorum: 0,
13604 met_quorum: true,
13605 uncontested: Some("fixture".into()),
13606 });
13607 runner.state.graft = Some(GraftRecord {
13608 pre: pre.clone(),
13609 post: None,
13610 lines: None,
13611 limit: None,
13612 agent: None,
13613 outcome: GraftOutcome::Running,
13614 requested: Vec::new(),
13615 adoption: None,
13616 adoption_error: None,
13617 });
13618 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13619 git::commit_all(&winner.worktree, "unfinished graft")
13620 .await
13621 .unwrap();
13622 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13623 let leftover = winner.worktree.join("leftover");
13624 std::fs::write(&leftover, "untrusted edits").unwrap();
13625 let path = runner.state.repo.join(".git/updater.lock");
13626 let external = std::fs::File::create(&path).unwrap();
13627 assert_eq!(
13628 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13629 0
13630 );
13631 assert!(
13632 runner
13633 .graft()
13634 .await
13635 .unwrap_err()
13636 .is::<crate::repo_lock::ResourceBlocked>()
13637 );
13638 assert!(matches!(
13639 runner.state.graft.as_ref().unwrap().outcome,
13640 GraftOutcome::Running
13641 ));
13642 assert_eq!(
13643 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13644 unfinished
13645 );
13646 assert!(leftover.exists());
13647 drop(external);
13648 std::fs::remove_file(&path).unwrap();
13649 std::fs::create_dir(&path).unwrap();
13650 assert!(runner.graft().await.is_err());
13651 assert!(matches!(
13652 runner.state.graft.as_ref().unwrap().outcome,
13653 GraftOutcome::Running
13654 ));
13655 assert_eq!(
13656 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13657 unfinished
13658 );
13659 assert!(leftover.exists());
13660 std::fs::remove_dir(&path).unwrap();
13661 runner.graft().await.unwrap();
13662 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13663 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13664 assert!(!leftover.exists());
13665 }
13666
13667 #[tokio::test]
13668 #[cfg(unix)]
13669 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13670 use std::os::fd::AsRawFd as _;
13671 let tmp = tempfile::tempdir().unwrap();
13672 let mut runner = registered_runner(tmp.path()).await;
13673 runner.state.prep_complete = Some(false);
13674 runner.state.config.graph.timeout_resource_wait = 1;
13675 runner.state.save().unwrap();
13676 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13677 assert_eq!(
13678 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13679 0
13680 );
13681 runner.execute().await.unwrap();
13682 let disk = RunState::load(&runner.state.id).unwrap();
13683 assert!(disk.driver_exited);
13684 assert_eq!(disk.driver_pid, None);
13685 assert_eq!(runner.state.status, RunStatus::Blocked);
13686 assert_eq!(
13687 runner.state.resource_block.as_ref().unwrap().kind,
13688 "repository coordination"
13689 );
13690 assert!(runner.state.prep_failure.is_none());
13691 assert!(runner.state.seats.is_empty());
13692 runner.pause.park();
13693 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13696 .await
13697 .unwrap()
13698 .unwrap();
13699 assert!(runner.state.parked);
13700 assert!(runner.state.prep_failure.is_none());
13701 let disk = RunState::load(&runner.state.id).unwrap();
13702 assert!(disk.driver_exited);
13703 assert_eq!(disk.driver_pid, None);
13704 }
13705
13706 #[tokio::test]
13707 #[cfg(unix)]
13708 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13709 use std::os::fd::AsRawFd as _;
13710 let tmp = tempfile::tempdir().unwrap();
13711 let mut runner = registered_runner(tmp.path()).await;
13712 runner.state.prep_complete = Some(false);
13713 runner.state.config.graph.timeout_resource_wait = 30;
13714 runner.state.save().unwrap();
13715 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13716 assert_eq!(
13717 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13718 0
13719 );
13720 assert!(
13721 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13722 .await
13723 .is_err()
13724 );
13725 let disk = RunState::load(&runner.state.id).unwrap();
13726 assert!(disk.driver_exited);
13727 assert_eq!(disk.driver_pid, None);
13728 assert!(disk.status.resumable());
13729 assert!(runner.state.driver_exited);
13730 assert_eq!(runner.state.driver_pid, None);
13731 runner.state.save().unwrap();
13732 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13733 }
13734
13735 #[tokio::test]
13736 #[cfg(unix)]
13737 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13738 let tmp = tempfile::tempdir().unwrap();
13739 let mut runner = registered_runner(tmp.path()).await;
13740 let winner = runner.state.candidates[0].clone();
13741 let cache = tmp.path().join("cache");
13742 let started = tmp.path().join("pre-gate-started");
13743 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13744 runner.state.config.verify.gate =
13745 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13746 let home = crate::run::home();
13747 let owner =
13748 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13749 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13750 crate::cache::AcquireOutcome::Acquired(g) => g,
13751 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13752 };
13753 let mut gate = Box::pin(runner.run_gate(&winner));
13754 assert!(
13755 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13756 .await
13757 .is_err()
13758 );
13759 assert!(!started.exists(), "pre-gate must wait for the cache too");
13760 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13764 let editor = crate::repo_lock::SeatLease::checkout_with(
13765 &winner.worktree,
13766 true,
13767 crate::config::test_coordination().as_ref(),
13768 &wait,
13769 )
13770 .await
13771 .unwrap();
13772 drop(editor);
13773 guard.release();
13774 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13775 .await
13776 .unwrap()
13777 .unwrap();
13778 drop(gate);
13779 assert!(outcomes.iter().all(CommandOutcome::ok));
13780 assert!(started.exists());
13781 }
13782
13783 #[tokio::test]
13784 #[cfg(unix)]
13785 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13786 use std::os::fd::AsRawFd as _;
13787
13788 ask_test_home();
13789 let tmp = tempfile::tempdir().unwrap();
13790 let repo = tmp.path().join("repo");
13791 std::fs::create_dir_all(&repo).unwrap();
13792 init_repo(&repo);
13793 let common = repo.join(".git");
13794 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13795 let mut runner = runner_at(RunStatus::Prep);
13796 runner.state.repo = repo.clone();
13797 runner.state.config.coordination = crate::config::test_coordination();
13798 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13799 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13800 runner.state.config.graph.advise = false;
13801 runner.roles.implementers = vec![spec("implement")];
13802 runner.roles.judges = vec![spec("judge")];
13803 runner.prep().await.unwrap();
13804 let winner = runner.state.candidates[0].clone();
13805 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13808 let completed = tmp.path().join("seat-completed");
13809 runner.roles.implementers[0].command = vec![
13810 "sh".into(),
13811 "-c".into(),
13812 format!("touch '{}'; echo completed", completed.display()),
13813 ];
13814 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13815 assert_eq!(
13816 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13817 0
13818 );
13819 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13820 let mut implement = Box::pin(runner.implement());
13821 tokio::select! {
13822 result = &mut implement => panic!("rescue must wait: {result:?}"),
13823 result = tokio::time::timeout(Duration::from_secs(10), async {
13824 while !completed.exists() {
13825 tokio::time::sleep(Duration::from_millis(20)).await;
13826 }
13827 }) => result.unwrap(),
13828 }
13829 assert!(
13830 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13831 .await
13832 .is_err()
13833 );
13834 assert_eq!(
13835 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13836 before
13837 );
13838 assert_eq!(
13839 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13840 0
13841 );
13842 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13843 .await
13844 .unwrap()
13845 .unwrap();
13846 drop(implement);
13847 assert_ne!(
13848 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13849 before
13850 );
13851 assert!(git::is_clean(&winner.worktree).await.unwrap());
13852
13853 let started = tmp.path().join("pre-gate-started");
13854 let proceed = tmp.path().join("pre-gate-proceed");
13855 let checked = tmp.path().join("gate-ran");
13856 runner.state.config.verify.pre_gate = vec![format!(
13857 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13858 started.display(),
13859 proceed.display()
13860 )];
13861 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13862 assert_eq!(
13863 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13864 0
13865 );
13866 let mut gate = Box::pin(runner.run_gate(&winner));
13867 assert!(
13868 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13869 .await
13870 .is_err()
13871 );
13872 assert!(!started.exists());
13873 assert!(!checked.exists());
13874 assert!(git::is_clean(&winner.worktree).await.unwrap());
13875 assert_eq!(
13876 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13877 0
13878 );
13879 tokio::select! {
13880 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13881 result = tokio::time::timeout(Duration::from_secs(10), async {
13882 while !started.exists() {
13883 tokio::time::sleep(Duration::from_millis(20)).await;
13884 }
13885 }) => result.unwrap(),
13886 }
13887 assert_eq!(
13888 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13889 -1,
13890 "mechanical edits must hold the real lock"
13891 );
13892 std::fs::write(proceed, "ready").unwrap();
13893 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13894 .await
13895 .unwrap()
13896 .unwrap();
13897 drop(gate);
13898 assert!(outcomes.iter().all(CommandOutcome::ok));
13899 assert!(checked.exists());
13900 assert!(runner.state.pre_gate_commit.is_some());
13901 assert!(git::is_clean(&winner.worktree).await.unwrap());
13902 assert_eq!(
13903 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13904 0,
13905 "gate completion must release coordination"
13906 );
13907 drop(outsider);
13908 std::fs::remove_file(common.join("updater.lock")).unwrap();
13909 std::fs::create_dir(common.join("updater.lock")).unwrap();
13910 std::fs::remove_file(&started).unwrap();
13911 std::fs::remove_file(&checked).unwrap();
13912 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13913 let error = runner.run_gate(&winner).await.unwrap_err();
13914 assert!(
13915 error
13916 .to_string()
13917 .contains("acquire mandatory repository lock")
13918 );
13919 assert!(
13920 !started.exists(),
13921 "failed acquisition must prevent pre-gate"
13922 );
13923 assert!(!checked.exists(), "failed acquisition must prevent gate");
13924 assert!(git::is_clean(&winner.worktree).await.unwrap());
13925 assert_eq!(
13926 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13927 head
13928 );
13929 }
13930
13931 #[tokio::test]
13932 #[cfg(unix)]
13933 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13934 use std::os::fd::AsRawFd as _;
13935 use std::time::Duration;
13936
13937 ask_test_home();
13938 let tmp = tempfile::tempdir().unwrap();
13939 let repo = tmp.path().join("repo");
13940 std::fs::create_dir_all(&repo).unwrap();
13941 init_repo(&repo);
13942 let common = repo.join(".git");
13943 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13944 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13945 assert_eq!(
13946 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13947 0
13948 );
13949 let mut runner = runner_at(RunStatus::Prep);
13950 runner.state.repo = repo.clone();
13951 runner.state.config.coordination = crate::config::test_coordination();
13952 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13953 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13954 runner.state.config.graph.advise = true;
13955 runner.state.config.graph.advisors = 1;
13956 runner.roles.implementers = vec![spec("implement")];
13957 runner.roles.judges = vec![spec("judge")];
13958 let root = runner.state.worktree_root();
13959 let started = tmp.path().join("setup-started");
13960 let proceed = tmp.path().join("setup-proceed");
13961 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13962 run: Some(format!(
13963 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13964 started.display(),
13965 proceed.display()
13966 )),
13967 ..Default::default()
13968 }];
13969 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13970 let config = git::git(&repo, &["config", "--local", "--list"])
13971 .await
13972 .unwrap();
13973 let mut prep = Box::pin(runner.prep());
13974 assert!(
13975 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13976 .await
13977 .is_err()
13978 );
13979 assert!(
13980 !root.exists(),
13981 "contention must prevent even checkout creation"
13982 );
13983 assert!(!started.exists(), "contention must prevent setup");
13984 assert_eq!(
13985 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13986 branches
13987 );
13988 assert_eq!(
13989 git::git(&repo, &["config", "--local", "--list"])
13990 .await
13991 .unwrap(),
13992 config
13993 );
13994 drop(lock);
13995 tokio::select! {
13996 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13997 result = tokio::time::timeout(Duration::from_secs(10), async {
13998 while !started.exists() {
13999 tokio::time::sleep(Duration::from_millis(20)).await;
14000 }
14001 }) => result.unwrap(),
14002 }
14003 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14004 assert_eq!(
14005 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14006 -1
14007 );
14008 assert!(
14009 tokio::time::timeout(
14010 Duration::from_millis(100),
14011 crate::repo_lock::SeatLease::for_checkout(
14012 &repo,
14013 true,
14014 crate::config::test_coordination().as_ref()
14015 )
14016 )
14017 .await
14018 .is_err(),
14019 "preparation must also exclude local editors"
14020 );
14021 std::fs::write(proceed, "ready").unwrap();
14022 tokio::time::timeout(Duration::from_secs(10), &mut prep)
14023 .await
14024 .unwrap()
14025 .unwrap();
14026 drop(prep);
14027 assert_eq!(runner.state.prep_complete, Some(true));
14028 assert!(root.join("judge-1").exists());
14029 assert!(root.join("advisor-1").exists());
14030 assert_eq!(
14031 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14032 0
14033 );
14034 }
14035
14036 #[tokio::test]
14037 #[cfg(unix)]
14038 async fn preparation_lock_failure_prevents_repository_changes() {
14039 ask_test_home();
14040 let tmp = tempfile::tempdir().unwrap();
14041 let repo = tmp.path().join("repo");
14042 std::fs::create_dir_all(&repo).unwrap();
14043 init_repo(&repo);
14044 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14045 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14048 let mut runner = runner_at(RunStatus::Prep);
14049 runner.state.repo = repo.clone();
14050 runner.state.config.coordination = crate::config::test_coordination();
14051 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14052 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14053 runner.roles.implementers = vec![spec("implement")];
14054 let error = runner.prep().await.unwrap_err();
14055 assert!(
14056 error
14057 .to_string()
14058 .contains("acquire mandatory repository lock")
14059 );
14060 assert!(!runner.state.worktree_root().exists());
14061 assert!(runner.state.candidates.is_empty());
14062 assert!(!repo.join(".git/config.worktree").exists());
14063 assert_eq!(
14064 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14065 "* main"
14066 );
14067 }
14068
14069 #[tokio::test]
14070 async fn preparation_records_creation_intent_before_git_can_fail() {
14071 ask_test_home();
14072 let tmp = tempfile::tempdir().unwrap();
14073 let repo = tmp.path().join("repo");
14074 std::fs::create_dir_all(&repo).unwrap();
14075 init_repo(&repo);
14076 let mut runner = runner_at(RunStatus::Prep);
14077 runner.state.repo = repo.clone();
14078 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14079 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14080 runner.state.config.blind.commit_msg_hook = false;
14081 runner.roles.implementers = vec![spec("one")];
14082 let label = blind::assign_labels(1, runner.state.seed)[0];
14083 let branch = runner.state.branch_for(label);
14084 git::git(&repo, &["branch", &branch]).await.unwrap();
14087 assert!(runner.prep().await.is_err());
14088 let saved = RunState::load(&runner.state.id).unwrap();
14089 assert!(saved.candidates.is_empty());
14090 let event = saved
14091 .events
14092 .iter()
14093 .find(|e| e.node == "prep-intent")
14094 .unwrap();
14095 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14096 assert_eq!(intent["branch"], branch);
14097 assert_eq!(intent["start"], runner.state.base_commit);
14098 assert_eq!(
14099 intent["worktree"],
14100 runner
14101 .state
14102 .worktree_root()
14103 .join(format!("cand-{label}"))
14104 .to_string_lossy()
14105 .as_ref()
14106 );
14107 assert_eq!(intent["picks"], serde_json::json!([]));
14108 }
14109
14110 #[tokio::test]
14111 async fn invalid_pinned_sources_create_no_partial_worktrees() {
14112 ask_test_home();
14113 let tmp = tempfile::tempdir().unwrap();
14114 let repo = tmp.path().join("repo");
14115 std::fs::create_dir_all(&repo).unwrap();
14116 init_repo(&repo);
14117 let mut runner = runner_at(RunStatus::Prep);
14118 runner.state.repo = repo.clone();
14119 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14120 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14121 runner.state.sources.pick_commits.push(refs::Input {
14122 token: "missing".into(),
14123 sha: "0".repeat(40),
14124 provenance: refs::Provenance::ExplicitCli,
14125 });
14126 assert!(runner.prep().await.is_err());
14127 assert!(runner.state.candidates.is_empty());
14128 assert!(!runner.state.worktree_root().exists());
14129 }
14130
14131 #[tokio::test]
14132 async fn prose_and_legacy_inference_never_import_into_candidates() {
14133 ask_test_home();
14134 let tmp = tempfile::tempdir().unwrap();
14135 let repo = tmp.path().join("repo");
14136 std::fs::create_dir_all(&repo).unwrap();
14137 init_repo(&repo);
14138 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14139 git::git(&repo, &["checkout", "-b", "magi/history/A"])
14140 .await
14141 .unwrap();
14142 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14143 git::git(&repo, &["add", "."]).await.unwrap();
14144 git::git(&repo, &["commit", "-m", "historical implementation"])
14145 .await
14146 .unwrap();
14147 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14148 git::git(&repo, &["checkout", "main"]).await.unwrap();
14149 for instruction in [
14150 format!("Example: magi/history/A at {source}"),
14151 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14152 format!("Never import magi/history/A or {source}"),
14153 format!("Conversation answer: I mentioned {source} only as background"),
14154 ] {
14155 let mut runner = runner_at(RunStatus::Prep);
14156 runner.state.repo = repo.clone();
14157 runner.state.base_commit = base.clone();
14158 runner.state.instruction = instruction.clone();
14159 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14160 runner.state.config.blind.commit_msg_hook = false;
14161 runner.state.seeds = vec![refs::Seed {
14162 token: "magi/history/A".into(),
14163 sha: source.clone(),
14164 kind: refs::SeedKind::Unmerged,
14165 branch: true,
14166 detail: String::new(),
14167 }];
14168 runner.roles.implementers = vec![spec("one")];
14169 assert_eq!(seeded_instruction(&runner.state), instruction);
14170 let mut legacy = serde_json::to_value(&runner.state).unwrap();
14172 legacy.as_object_mut().unwrap().remove("sources");
14173 runner.state = serde_json::from_value(legacy).unwrap();
14174 assert_eq!(runner.state.sources, refs::Sources::default());
14175 runner.state.save().unwrap();
14176 runner.state = RunState::load(&runner.state.id).unwrap();
14177 runner.prep().await.unwrap();
14178 for candidate in &runner.state.candidates {
14179 assert_eq!(
14180 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14181 base
14182 );
14183 assert!(!candidate.worktree.join("unwanted").exists());
14184 }
14185 }
14186 }
14187
14188 #[tokio::test]
14189 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14190 ask_test_home();
14191 let tmp = tempfile::tempdir().unwrap();
14192 let repo = tmp.path().join("repo");
14193 std::fs::create_dir_all(&repo).unwrap();
14194 init_repo(&repo);
14195 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14196 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14197 std::fs::write(repo.join("referenced"), "required commit").unwrap();
14198 git::git(&repo, &["add", "."]).await.unwrap();
14199 git::git(&repo, &["commit", "-m", "required seed"])
14200 .await
14201 .unwrap();
14202 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14203 git::git(&repo, &["checkout", "main"]).await.unwrap();
14204 let mut runner = runner_at(RunStatus::Prep);
14205 runner.state.repo = repo.clone();
14206 runner.state.base_commit = base.clone();
14207 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14208 runner.state.config.graph.advise = false;
14209 runner.state.config.blind.commit_msg_hook = false;
14210 runner.state.sources =
14211 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14212 .await
14213 .unwrap();
14214 runner.roles.implementers = vec![spec("one"), spec("two")];
14215 runner.roles.judges = vec![spec("judge")];
14216 let label = blind::assign_labels(2, runner.state.seed)[0];
14217 let branch = runner.state.branch_for(label);
14218 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14219 git::worktree_add_branch(&repo, &worktree, &branch, &base)
14220 .await
14221 .unwrap();
14222 runner.state.candidates.push(Candidate {
14223 index: 0,
14224 label,
14225 agent: "one".into(),
14226 branch,
14227 worktree,
14228 summary: String::new(),
14229 stat: String::new(),
14230 files: 0,
14231 commits: 0,
14232 empty: false,
14233 failed: None,
14234 verified_noop: None,
14235 duration_ms: 0,
14236 folded: false,
14237 });
14238 runner.state.save().unwrap();
14239 runner.state = RunState::load(&runner.state.id).unwrap();
14241 assert_eq!(runner.state.prep_complete, Some(false));
14242 runner.prep().await.unwrap();
14243 assert_eq!(runner.state.prep_complete, Some(true));
14244 assert_eq!(runner.state.candidates.len(), 2);
14245 for candidate in &runner.state.candidates {
14246 assert_eq!(
14247 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14248 git::tree_of(&repo, &seed).await.unwrap()
14249 );
14250 assert_eq!(
14251 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14252 "required commit"
14253 );
14254 }
14255 assert!(runner.state.worktree_root().join("judge-1").exists());
14256 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14257 runner.prep().await.unwrap();
14258 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14259
14260 runner.state.prep_complete = Some(false);
14261 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14262 run: Some(format!(
14263 "test -f '{}'",
14264 tmp.path().join("dependency-ready").display()
14265 )),
14266 ..Default::default()
14267 }];
14268 assert!(runner.prep().await.is_err());
14269 assert!(runner.state.prep_failure.is_some());
14270 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14271 runner.state = RunState::load(&runner.state.id).unwrap();
14272 assert!(
14273 runner.prep().await.is_err(),
14274 "correcting the cause alone must not auto-retry"
14275 );
14276 let failed_at = runner.state.events.last().unwrap().at;
14277 assert!(
14278 runner.retry_preparation(failed_at).is_err(),
14279 "old approvals cannot unlock the hold"
14280 );
14281 let mut task = crate::queue::Task::new(
14282 "retry".into(),
14283 "retry setup".into(),
14284 repo.clone(),
14285 crate::queue::Source::Human,
14286 );
14287 task.start(runner.state.id.clone());
14288 task.hold_machine(Some("setup failed".into()));
14289 task.release_operator("cli", None);
14290 runner
14291 .retry_preparation(task.resumption.as_ref().unwrap().at)
14292 .unwrap();
14293 runner.prep().await.unwrap();
14294 assert!(runner.state.prep_failure.is_none());
14295 assert_eq!(runner.state.prep_complete, Some(true));
14296 assert_eq!(runner.state.candidates.len(), 2);
14297 assert!(runner.state.worktree_root().join("judge-1").exists());
14298 for candidate in &runner.state.candidates {
14299 assert_eq!(
14300 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14301 git::tree_of(&repo, &seed).await.unwrap()
14302 );
14303 }
14304 }
14305
14306 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14307 let mut heads = Vec::new();
14308 for candidate in candidates {
14309 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14310 }
14311 heads
14312 }
14313
14314 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14315 ask_test_home();
14316 let tmp = tempfile::tempdir().unwrap();
14317 let repo = tmp.path().join("repo");
14318 std::fs::create_dir(&repo).unwrap();
14319 init_repo(&repo);
14320 let mut runner = runner_at(RunStatus::Reviewing);
14321 runner.state.repo = repo.clone();
14322 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14323 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14324 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14325 let branch = format!("candidate-{label}");
14326 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14327 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14328 .await
14329 .unwrap();
14330 if label == 'B' {
14331 std::fs::write(worktree.join("change"), "winner").unwrap();
14332 git::rescue_commit(&worktree, "candidate change")
14333 .await
14334 .unwrap();
14335 }
14336 runner.state.candidates.push(Candidate {
14337 index,
14338 label,
14339 agent: "alpha".into(),
14340 branch,
14341 worktree,
14342 summary: "completed reply".into(),
14343 stat: String::new(),
14344 files: usize::from(label == 'B'),
14345 commits: usize::from(label == 'B'),
14346 empty: label == 'A',
14347 failed: None,
14348 verified_noop: None,
14349 duration_ms: 1,
14350 folded: false,
14351 });
14352 }
14353 runner.state.judge_skipped = true;
14354 runner.state.tally = Some(Tally {
14355 first_choice: BTreeMap::new(),
14356 borda: BTreeMap::new(),
14357 winner: 'B',
14358 rankings: 0,
14359 unanimous_initial: false,
14360 deliberated: false,
14361 changed_votes: 0,
14362 unanimous_final: false,
14363 tie_break: None,
14364 judges: 0,
14365 present: 0,
14366 quorum: 0,
14367 met_quorum: true,
14368 uncontested: Some("only B produced a change".into()),
14369 });
14370 runner.state.save().unwrap();
14371 (tmp, runner)
14372 }
14373
14374 #[tokio::test]
14375 #[cfg(unix)]
14376 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14377 for (solo, failed_checkpoint) in
14378 [(true, false), (false, false), (true, true), (false, true)]
14379 {
14380 let (_tmp, mut runner) = preservation_resume_fixture().await;
14381 if solo {
14382 runner.state.candidates.retain(|c| c.label == 'A');
14383 }
14384 runner.state.candidates[0].empty = false;
14385 runner.state.candidates[0].summary.clear();
14386 runner.state.tally = None;
14387 runner.state.judge_skipped = false;
14388 runner.state.status = RunStatus::Implementing;
14389 runner.state.config.disk.min_free_bytes = u64::MAX;
14390 let mut implementer = spec("alpha");
14391 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14392 runner.roles.implementers = vec![implementer];
14393 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14394 if failed_checkpoint {
14395 std::fs::create_dir_all(&patch).unwrap();
14396 }
14397 runner.state.save().unwrap();
14398 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14399 runner.state = RunState::load(&runner.state.id).unwrap();
14400 assert_eq!(runner.state.status, RunStatus::Blocked);
14401 assert!(runner.state.interrupted_implementations.contains(&'A'));
14402 assert_eq!(runner.state.candidates[0].commits, 0);
14403 assert!(!runner.state.candidates[0].empty);
14404 assert!(runner.state.candidates[0].failed.is_none());
14405 if failed_checkpoint {
14406 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14407 std::fs::remove_dir(patch).unwrap();
14408 }
14409 runner.state.resource_block = None;
14410 runner.state.config.disk.min_free_bytes = 0;
14411 runner.roles.implementers[0].command = vec![
14412 "sh".into(),
14413 "-c".into(),
14414 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14415 ];
14416 runner.implement().await.unwrap();
14417 assert!(runner.state.interrupted_implementations.is_empty());
14418 assert!(runner.state.candidate_preservation.is_empty());
14419 assert_eq!(runner.state.candidates[0].commits, 1);
14420 assert!(!runner.state.candidates[0].empty);
14421 assert!(
14422 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14423 .unwrap()
14424 .contains("retried empty interrupted seat")
14425 );
14426 if !solo {
14427 assert_eq!(runner.state.candidates[1].commits, 1);
14428 }
14429 }
14430 }
14431
14432 #[tokio::test]
14433 #[cfg(unix)]
14434 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14435 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14436 let (_tmp, mut runner) = preservation_resume_fixture().await;
14437 runner.state.candidates.retain(|c| c.label == 'A');
14438 runner.state.candidates[0].empty = false;
14439 runner.state.candidates[0].summary.clear();
14440 runner.state.tally = None;
14441 runner.state.judge_skipped = false;
14442 runner.state.status = RunStatus::Implementing;
14443 runner.state.config.disk.min_free_bytes = u64::MAX;
14446 let wt = runner.state.candidates[0].worktree.clone();
14447 std::fs::write(
14448 wt.join("interrupted-source"),
14449 "retain implementation work\n",
14450 )
14451 .unwrap();
14452 if committed {
14453 git::rescue_commit(&wt, "implementation checkpoint")
14454 .await
14455 .unwrap();
14456 }
14457 let lock = git::git(
14458 &wt,
14459 &[
14460 "rev-parse",
14461 "--path-format=absolute",
14462 "--git-path",
14463 "index.lock",
14464 ],
14465 )
14466 .await
14467 .unwrap();
14468 if block_rescue {
14469 std::fs::write(&lock, "fixture-owned lock").unwrap();
14470 }
14471 let mut implementer = spec("alpha");
14472 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14473 runner.roles.implementers = vec![implementer];
14474 runner.state.save().unwrap();
14475 let result = runner.implement().await;
14476 assert_eq!(result.is_err(), block_rescue);
14477 assert_eq!(
14478 runner.state.resource_block.as_ref().unwrap().kind,
14479 "disk pressure"
14480 );
14481 assert!(
14482 runner
14483 .state
14484 .events
14485 .iter()
14486 .any(|e| e.message.contains("agent execution interrupted"))
14487 );
14488 assert!(
14489 runner
14490 .state
14491 .dir()
14492 .join("artifacts/candidate-A-recovery/staged.patch")
14493 .exists()
14494 );
14495 if block_rescue {
14496 let saved = RunState::load(&runner.state.id).unwrap();
14497 assert!(saved.candidate_preservation.contains_key(&'A'));
14498 assert!(!saved.candidates[0].empty);
14499 assert_eq!(
14500 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14501 "retain implementation work\n"
14502 );
14503 std::fs::remove_file(lock).unwrap();
14504 runner.state = saved;
14505 } else {
14506 assert_eq!(runner.state.status, RunStatus::Blocked);
14507 assert_eq!(runner.state.candidates[0].commits, 1);
14508 }
14509 runner.roles.implementers.clear();
14512 runner.state.config.disk.min_free_bytes = 0;
14513 runner.state.resource_block = None;
14514 runner.implement().await.unwrap();
14515 assert!(runner.state.candidate_preservation.is_empty());
14516 assert_eq!(runner.state.candidates[0].commits, 1);
14517 assert!(!runner.state.candidates[0].empty);
14518 assert!(git::is_clean(&wt).await.unwrap());
14519 assert!(
14520 crate::run::read_artifact(&runner.state, "cand-A.patch")
14521 .unwrap()
14522 .contains("retain implementation work")
14523 );
14524 }
14525 }
14526
14527 #[tokio::test]
14528 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14529 let (_tmp, mut runner) = preservation_resume_fixture().await;
14530 runner.implement().await.unwrap();
14532 assert!(runner.state.candidate_preservation.is_empty());
14533 assert!(
14534 !runner
14535 .state
14536 .dir()
14537 .join("artifacts/candidate-A-recovery")
14538 .exists()
14539 );
14540 runner.fold_losers().await.unwrap();
14541 assert!(!runner.state.candidates[0].worktree.exists());
14542 for schema in [14, crate::run::SCHEMA] {
14543 runner.state.schema = schema;
14544 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14545 runner.state.save().unwrap();
14546 runner.state = RunState::load(&runner.state.id).unwrap();
14547 runner.implement().await.unwrap();
14548 assert!(runner.state.candidate_preservation.is_empty());
14549 assert!(runner.state.candidates[0].folded);
14550 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14551 }
14552 }
14553
14554 #[tokio::test]
14555 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14556 let (_tmp, mut runner) = preservation_resume_fixture().await;
14557 runner.state.candidates.retain(|c| c.label == 'A');
14558 runner.state.tally = None;
14559 runner.state.judge_skipped = false;
14560 let candidate = &mut runner.state.candidates[0];
14561 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14562 candidate.failed = Some("deployment validation unavailable".into());
14563 std::fs::write(
14564 candidate.worktree.join("repair"),
14565 "future launcher policy\n",
14566 )
14567 .unwrap();
14568 runner
14569 .state
14570 .candidate_preservation
14571 .insert('A', "awaiting persistence".into());
14572 runner.persist_candidates().await.unwrap();
14573 let candidate = &runner.state.candidates[0];
14574 assert!(!candidate.empty);
14575 assert_eq!(candidate.commits, 1);
14576 assert_eq!(candidate.files, 1);
14577 assert!(candidate.failed.is_none());
14578 assert!(candidate.summary.contains("EPERM"));
14579 assert!(
14580 crate::run::read_artifact(&runner.state, "cand-A.patch")
14581 .unwrap()
14582 .contains("future launcher policy")
14583 );
14584 assert!(runner.state.candidate_preservation.is_empty());
14585 }
14586
14587 #[tokio::test]
14588 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14589 let (_tmp, mut runner) = preservation_resume_fixture().await;
14590 let wt = runner.state.candidates[0].worktree.clone();
14591 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14592 runner.state.schema = 14;
14593 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14594 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14595 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14596 for _ in 0..2 {
14597 assert!(runner.implement().await.is_err());
14598 assert_eq!(runner.state.status, RunStatus::Blocked);
14599 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14600 assert!(
14601 runner.state.candidates[0].empty,
14602 "old verdict must remain unusable"
14603 );
14604 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14605 assert!(runner.state.judge_skipped);
14606 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14607 assert!(
14608 crate::run::read_artifact(&runner.state, "cand-A.patch")
14609 .unwrap()
14610 .contains("owned edits")
14611 );
14612 assert!(wt.exists());
14613 assert_eq!(
14614 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14615 .await
14616 .unwrap(),
14617 1
14618 );
14619 runner.state = RunState::load(&runner.state.id).unwrap();
14620 }
14621 }
14622
14623 #[tokio::test]
14624 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14625 let (_tmp, mut runner) = preservation_resume_fixture().await;
14626 let wt = runner.state.candidates[0].worktree.clone();
14627 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14628 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14629 let index_lock = PathBuf::from(
14630 git::git(
14631 &wt,
14632 &[
14633 "rev-parse",
14634 "--path-format=absolute",
14635 "--git-path",
14636 "index.lock",
14637 ],
14638 )
14639 .await
14640 .unwrap(),
14641 );
14642 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14643 runner.state.schema = 14;
14644 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14645 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14646 assert!(runner.implement().await.is_err());
14647 assert!(runner.after_implement().is_err());
14648 assert!(runner.fold_losers().await.is_err());
14649 assert_eq!(
14650 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14651 "failed seat edits"
14652 );
14653 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14654 runner.state = RunState::load(&runner.state.id).unwrap();
14655 std::fs::remove_file(index_lock).unwrap();
14656 assert!(runner.implement().await.is_err());
14658 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14659 assert!(runner.state.candidates[0].empty);
14660 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14661 assert!(
14662 crate::run::read_artifact(&runner.state, "cand-A.patch")
14663 .unwrap()
14664 .contains("failed seat edits")
14665 );
14666 assert!(wt.exists());
14667 }
14668
14669 #[tokio::test]
14670 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14671 let (_tmp, mut runner) = preservation_resume_fixture().await;
14672 let repo = runner.state.repo.clone();
14673 let common = git::git(
14674 &repo,
14675 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14676 )
14677 .await
14678 .unwrap();
14679 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14680 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14681 assert!(runner.fold_losers().await.is_err());
14682 assert!(!runner.state.candidates[0].worktree.exists());
14683 assert!(!runner.state.candidates[0].folded);
14684 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14685 runner.state = RunState::load(&runner.state.id).unwrap();
14686 std::fs::remove_file(branch_lock).unwrap();
14687 let judge = runner.state.worktree_root().join("judge-1");
14688 git::worktree_add_detached(&repo, &judge, "main")
14689 .await
14690 .unwrap();
14691 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14692 runner.roles.judges = vec![spec("judge")];
14693 assert!(runner.fold_losers().await.is_err());
14694 runner.state = RunState::load(&runner.state.id).unwrap();
14695 assert!(
14696 runner.state.candidates[0].folded,
14697 "candidate progress saved before judge failure"
14698 );
14699 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14700 assert!(judge.join("uncommitted").exists());
14701 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14702 runner.fold_losers().await.unwrap();
14703 assert!(!judge.exists());
14704 runner.fold_losers().await.unwrap();
14705 }
14706
14707 #[tokio::test]
14708 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14709 let (_tmp, mut runner) = preservation_resume_fixture().await;
14710 runner.state.candidates.retain(|c| c.label == 'B');
14711 runner.state.tally = None;
14712 runner.state.judge_skipped = false;
14713 runner.state.prep_complete = Some(true);
14714 runner.state.config.graph.advise = false;
14715 runner.state.config.graph.selected_implementer = Some("alpha".into());
14716 let wt = runner.state.candidates[0].worktree.clone();
14717 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14718 runner
14719 .state
14720 .candidate_preservation
14721 .insert('B', "awaiting persistence".into());
14722 let index_lock = git::git(
14723 &wt,
14724 &[
14725 "rev-parse",
14726 "--path-format=absolute",
14727 "--git-path",
14728 "index.lock",
14729 ],
14730 )
14731 .await
14732 .unwrap();
14733 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14734 runner.state.save().unwrap();
14735 assert!(runner.execute_graph().await.is_err());
14736 assert_eq!(runner.state.status, RunStatus::Blocked);
14737 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14738 std::fs::remove_file(index_lock).unwrap();
14739 for _ in 0..2 {
14742 runner.state = RunState::load(&runner.state.id).unwrap();
14743 runner.pause.park();
14744 runner.execute_graph().await.unwrap();
14745 assert_eq!(runner.state.status, RunStatus::Implementing);
14746 assert!(runner.state.candidate_preservation.is_empty());
14747 assert_eq!(runner.state.candidates[0].commits, 2);
14748 assert!(runner.state.tally.is_none());
14749 }
14750 runner.implement().await.unwrap();
14751 assert_eq!(runner.state.status, RunStatus::Judging);
14752 runner.state.status = RunStatus::Blocked;
14754 runner.state.candidates[0].failed = Some("implementation failed".into());
14755 runner.state.save().unwrap();
14756 let error = runner.execute_graph().await.unwrap_err();
14757 assert!(
14758 error
14759 .to_string()
14760 .contains("selected implementation is held")
14761 );
14762 }
14763
14764 #[tokio::test]
14765 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14766 let (_tmp, mut runner) = preservation_resume_fixture().await;
14767 let repo = runner.state.repo.clone();
14768 let candidate = runner.state.candidates[0].clone();
14769 let registered = git::worktree_holding(&repo, &candidate.branch)
14772 .await
14773 .unwrap()
14774 .unwrap();
14775 assert_eq!(
14776 registered.canonicalize().unwrap(),
14777 candidate.worktree.canonicalize().unwrap()
14778 );
14779 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14780 assert_eq!(
14781 git::worktree_holding(&repo, &candidate.branch)
14782 .await
14783 .unwrap(),
14784 Some(registered)
14785 );
14786 assert!(
14787 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14788 "the stale registration initially keeps the branch checked out"
14789 );
14790 runner.fold_losers().await.unwrap();
14791 assert!(runner.state.candidates[0].folded);
14792 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14793 assert_eq!(
14794 git::worktree_holding(&repo, &candidate.branch)
14795 .await
14796 .unwrap(),
14797 None
14798 );
14799 runner.state = RunState::load(&runner.state.id).unwrap();
14800 runner.fold_losers().await.unwrap();
14801 }
14802
14803 #[tokio::test]
14804 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14805 let (_tmp, mut runner) = preservation_resume_fixture().await;
14806 let repo = runner.state.repo.clone();
14807 let candidate = runner.state.candidates[0].clone();
14808 let winner = runner.state.candidates[1].clone();
14809 let registered = git::worktree_holding(&repo, &candidate.branch)
14810 .await
14811 .unwrap()
14812 .unwrap();
14813 assert_eq!(
14814 registered.canonicalize().unwrap(),
14815 candidate.worktree.canonicalize().unwrap()
14816 );
14817 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14818 assert_eq!(
14819 git::worktree_holding(&repo, &candidate.branch)
14820 .await
14821 .unwrap(),
14822 Some(registered)
14823 );
14824 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14825
14826 for _ in 0..2 {
14827 fold_run(&mut runner.state, false, &crate::run::home())
14828 .await
14829 .unwrap();
14830 runner.state = RunState::load(&runner.state.id).unwrap();
14831 assert!(runner.state.candidates[0].folded);
14832 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14833 assert_eq!(
14834 git::worktree_holding(&repo, &candidate.branch)
14835 .await
14836 .unwrap(),
14837 None
14838 );
14839 assert!(!runner.state.candidates[1].folded);
14840 assert!(winner.worktree.exists());
14841 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14842 }
14843 }
14844
14845 #[tokio::test]
14846 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14847 ask_test_home();
14848 let tmp = tempfile::tempdir().unwrap();
14849 let repo = tmp.path().join("repo");
14850 std::fs::create_dir(&repo).unwrap();
14851 init_repo(&repo);
14852 let mut runner = runner_at(RunStatus::Implementing);
14853 runner.state.repo = repo.clone();
14854 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14855 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14856 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14857 let branch = format!("candidate-{label}");
14858 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14859 git::worktree_add_branch(&repo, &wt, &branch, "main")
14860 .await
14861 .unwrap();
14862 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14863 git::git(&wt, &["add", "README.md"]).await.unwrap();
14864 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14865 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14866 runner.state.candidates.push(Candidate {
14867 index,
14868 label,
14869 agent: "alpha".to_owned(),
14870 branch,
14871 worktree: wt,
14872 summary: "completed implementation".to_owned(),
14873 stat: String::new(),
14874 files: 0,
14875 commits: 0,
14876 empty: false,
14877 failed: None,
14878 verified_noop: None,
14879 duration_ms: 42,
14880 folded: false,
14881 });
14882 runner
14883 .state
14884 .candidate_preservation
14885 .insert(label, "awaiting persistence".to_owned());
14886 }
14887 let wt = runner.state.candidates[0].worktree.clone();
14888 let index_lock = git::git(
14889 &wt,
14890 &[
14891 "rev-parse",
14892 "--path-format=absolute",
14893 "--git-path",
14894 "index.lock",
14895 ],
14896 )
14897 .await
14898 .unwrap();
14899 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14900 runner.state.save().unwrap();
14901 assert!(runner.persist_candidates().await.is_err());
14902 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14903 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14904 assert_eq!(
14905 runner.state.candidates[1].commits, 1,
14906 "other results are still collected"
14907 );
14908 assert!(
14909 !runner.state.candidates[0].empty,
14910 "failed rescue is not no change"
14911 );
14912 assert_eq!(
14913 runner.state.candidates[0].summary,
14914 "completed implementation"
14915 );
14916 assert!(runner.after_implement().is_err());
14917 let home = tmp.path().join("fold-home");
14918 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14919 assert!(wt.join("new-file").exists());
14920 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14921 assert!(recovery.join("staged.patch").exists());
14922 assert!(recovery.join("unstaged.patch").exists());
14923 assert_eq!(
14924 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14925 "untracked edits\n"
14926 );
14927 runner.state.begin_pass();
14928 runner.state.end_pass_parked(ParkCause::Upgrade {
14929 from: "1.0".into(),
14930 to: Some("1.1".into()),
14931 });
14932 runner.state.save().unwrap();
14933 runner.state = RunState::load(&runner.state.id).unwrap();
14934 assert!(matches!(
14935 runner.state.passes.last().unwrap().end,
14936 Some(crate::run::PassEnd::Parked {
14937 cause: ParkCause::Upgrade { .. }
14938 })
14939 ));
14940 std::fs::remove_file(index_lock).unwrap();
14941 runner.implement().await.unwrap();
14943 assert!(runner.state.candidate_preservation.is_empty());
14944 assert!(
14945 runner
14946 .state
14947 .candidates
14948 .iter()
14949 .all(|c| c.commits == 1 && !c.empty)
14950 );
14951 assert!(git::is_clean(&wt).await.unwrap());
14952 assert!(
14953 crate::run::read_artifact(&runner.state, "cand-A.patch")
14954 .unwrap()
14955 .contains("later tracked edits")
14956 );
14957 }
14958
14959 #[tokio::test]
14960 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14961 ask_test_home();
14962 let tmp = tempfile::tempdir().unwrap();
14963 let repo = tmp.path().join("repo");
14964 std::fs::create_dir(&repo).unwrap();
14965 init_repo(&repo);
14966 let wt = tmp.path().join("candidate");
14967 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14968 .await
14969 .unwrap();
14970 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14971 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14972 assert!(wt.join("uncommitted").exists());
14973 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14974 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14975 let mut runner = runner_at(RunStatus::Implementing);
14976 runner.state.repo = repo.clone();
14977 runner.state.base_commit = "missing-base".to_owned();
14978 runner.state.candidates.push(Candidate {
14979 index: 0,
14980 label: 'A',
14981 agent: "alpha".into(),
14982 branch: "candidate".into(),
14983 worktree: wt.clone(),
14984 summary: "completed".into(),
14985 stat: String::new(),
14986 files: 0,
14987 commits: 0,
14988 empty: false,
14989 failed: None,
14990 verified_noop: None,
14991 duration_ms: 1,
14992 folded: false,
14993 });
14994 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14995 assert!(
14996 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14997 .await
14998 .is_err()
14999 );
15000 assert!(!runner.state.candidates[0].folded);
15001 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15002 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15003 runner
15004 .state
15005 .candidate_preservation
15006 .insert('A', "pending".into());
15007 assert!(runner.persist_candidates().await.is_err());
15008 assert!(!runner.state.candidates[0].empty);
15009 assert!(!runner.state.candidates[0].folded);
15010 assert!(wt.exists());
15011 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15012 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15013 std::fs::create_dir_all(&patch_path).unwrap();
15014 assert!(
15015 runner.persist_candidates().await.is_err(),
15016 "artifact failure stays pending"
15017 );
15018 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15019 assert!(!runner.state.candidates[0].empty);
15020 std::fs::remove_dir(patch_path).unwrap();
15021 runner.persist_candidates().await.unwrap();
15022 assert!(
15023 runner.state.candidates[0].empty,
15024 "only successful inspection confirms no change"
15025 );
15026 }
15027
15028 fn init_repo(dir: &Path) {
15029 let run = |args: &[&str]| {
15030 let out = std::process::Command::new("git")
15031 .args(args)
15032 .current_dir(dir)
15033 .quiet()
15034 .output()
15035 .expect("spawn git");
15036 assert!(
15037 out.status.success(),
15038 "git {args:?} failed: {}",
15039 String::from_utf8_lossy(&out.stderr)
15040 );
15041 };
15042 run(&["init", "-b", "main"]);
15043 run(&["config", "user.name", "magi test"]);
15044 run(&["config", "user.email", "magi@example.com"]);
15045 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15046 run(&["add", "-A"]);
15047 run(&["commit", "-m", "init"]);
15048 }
15049
15050 fn ask_test_home() {
15058 crate::run::pin_test_home();
15059 }
15060
15061 #[cfg(unix)]
15062 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15063 ask_test_home();
15064 let tmp = tempfile::tempdir().unwrap();
15065 let repo = tmp.path().join("repo");
15066 std::fs::create_dir(&repo).unwrap();
15067 init_repo(&repo);
15068 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15069 let origin = tmp.path().join("origin.git");
15070 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15071 .await
15072 .unwrap();
15073 git::git(
15074 &repo,
15075 &["remote", "add", "origin", origin.to_str().unwrap()],
15076 )
15077 .await
15078 .unwrap();
15079 git::git(&repo, &["checkout", "-b", "fix/retained"])
15080 .await
15081 .unwrap();
15082 std::fs::write(repo.join("retained"), "retained change").unwrap();
15083 git::git(&repo, &["add", "."]).await.unwrap();
15084 git::git(&repo, &["commit", "-m", "retained implementation"])
15085 .await
15086 .unwrap();
15087 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15088 git::git(&repo, &["checkout", "main"]).await.unwrap();
15089 std::fs::write(repo.join("base-change"), "new base").unwrap();
15090 git::git(&repo, &["add", "."]).await.unwrap();
15091 git::git(&repo, &["commit", "-m", "base moved"])
15092 .await
15093 .unwrap();
15094 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15095 git::git(&repo, &["checkout", "fix/retained"])
15096 .await
15097 .unwrap();
15098 let mut runner = runner_at(RunStatus::Voting);
15099 runner.state.repo = repo.clone();
15100 runner.state.base_commit = base;
15101 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15102 runner.state.config.disk.min_free_bytes = 0;
15103 runner.state.config.graph.review_rounds = 1;
15104 runner.state.config.verify.e2e = vec!["true".into()];
15105 runner.state.config.verify.gate = vec!["true".into()];
15106 runner.state.candidates = vec![Candidate {
15107 index: 0,
15108 label: 'A',
15109 agent: "fixture".into(),
15110 branch: "fix/retained".into(),
15111 worktree: repo.clone(),
15112 summary: String::new(),
15113 stat: String::new(),
15114 files: 1,
15115 commits: 1,
15116 empty: false,
15117 failed: None,
15118 verified_noop: None,
15119 duration_ms: 0,
15120 folded: false,
15121 }];
15122 runner.state.tally = Some(Tally {
15123 first_choice: BTreeMap::new(),
15124 borda: BTreeMap::new(),
15125 winner: 'A',
15126 rankings: 0,
15127 unanimous_initial: true,
15128 deliberated: false,
15129 changed_votes: 0,
15130 unanimous_final: true,
15131 tie_break: None,
15132 judges: 0,
15133 present: 0,
15134 quorum: 0,
15135 met_quorum: true,
15136 uncontested: Some("review-only".into()),
15137 });
15138 let mut task = crate::queue::Task::new(
15139 "fixture".into(),
15140 "fixture".into(),
15141 repo,
15142 crate::queue::Source::Human,
15143 );
15144 task.link_run(&runner.state.id);
15145 crate::queue::Queue::open().put(&mut task).unwrap();
15146 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15147 runner.state.base_sync = Some(BaseSync {
15148 tip: String::new(),
15149 behind: 1,
15150 attempts: BASE_SYNC_ROUNDS,
15151 conflict: None,
15152 already_in: None,
15153 });
15154 runner.sync_to_base().await.unwrap();
15155 assert_eq!(runner.state.status, RunStatus::Blocked);
15156 (tmp, runner, head, task.id)
15157 }
15158
15159 #[cfg(unix)]
15160 #[tokio::test]
15161 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15162 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15163 if std::env::var_os(CHILD).is_none() {
15164 let home = tempfile::tempdir().unwrap();
15165 let output = std::process::Command::new(std::env::current_exe().unwrap())
15166 .args([
15167 "--exact",
15168 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15169 "--nocapture",
15170 ])
15171 .env(CHILD, "1")
15172 .env("MAGI_HOME", home.path())
15173 .env("MAGI_WORKER_HOME", home.path())
15174 .env("MAGI_WORKER_ID", "abcdef")
15175 .output()
15176 .unwrap();
15177 assert!(
15178 output.status.success()
15179 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15180 "{}\n{}",
15181 String::from_utf8_lossy(&output.stdout),
15182 String::from_utf8_lossy(&output.stderr)
15183 );
15184 return;
15185 }
15186 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15187 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15188 let home = crate::run::home();
15189 let queue = crate::queue::Queue::open();
15190 let mut owner = queue.get(&task).unwrap();
15191 owner.release_operator("test-release", None);
15192 owner.start(runner.state.id.clone());
15193 queue.put(&mut owner).unwrap();
15194 runner.state.driver_exited = true;
15195 runner.state.task_instruction = Some(owner.instruction.clone());
15196 runner.state.save().unwrap();
15197 let _claim = queue.claim(&task).unwrap();
15198 let reserve = |id: &str| {
15199 let directory = home.join("workers/jobs").join(id);
15200 std::fs::create_dir_all(&directory).unwrap();
15201 let job = crate::worker::Job {
15202 id: id.into(),
15203 sequence: 1,
15204 kind: "task".into(),
15205 key: format!("task:{task}"),
15206 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15207 generation: "fixture-generation".into(),
15208 version: None,
15209 status: crate::worker::Status::Running,
15210 result: None,
15211 pid: Some(std::process::id()),
15212 created_at: 0,
15213 finished_at: None,
15214 };
15215 std::fs::write(
15216 directory.join("job.json"),
15217 serde_json::to_vec(&job).unwrap(),
15218 )
15219 .unwrap();
15220 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15221 };
15222 let _own = reserve("abcdef");
15223 let other = reserve("fedcba");
15224 assert!(
15225 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15226 .await
15227 .is_err()
15228 );
15229 assert!(
15230 RunState::load(&runner.state.id)
15231 .unwrap()
15232 .base_sync_recovery
15233 .authorizations
15234 .is_empty()
15235 );
15236 drop(other);
15237 assert!(crate::daemon::is_working_on(
15238 &home,
15239 &runner.state.id,
15240 Timestamp::now()
15241 ));
15242 assert!(
15243 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15244 );
15245 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15246 .await
15247 .unwrap();
15248 assert!(
15249 RunState::load(&runner.state.id)
15250 .unwrap()
15251 .base_recovery_pending()
15252 );
15253 }
15254
15255 #[cfg(unix)]
15256 #[tokio::test]
15257 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15258 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15259 let home = crate::run::home();
15260 runner.state.driver_pid = Some(std::process::id());
15261 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15262 runner.state.driver_exited = false;
15263 runner.state.begin_pass();
15264 runner.state.save().unwrap();
15265 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15266 driver.record(&runner.state).unwrap();
15267 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15268 drop(driver);
15270 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15271 assert_eq!(
15272 runner.state.liveness_under(&home, false),
15273 crate::run::Liveness::Dead
15274 );
15275 let queue = crate::queue::Queue::open();
15276 let _claim = queue.claim(&task).unwrap();
15277 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15278 .await
15279 .unwrap();
15280 let disk = RunState::load(&runner.state.id).unwrap();
15281 assert!(!disk.driver_exited);
15282 assert!(disk.base_recovery_pending());
15283 }
15284
15285 #[cfg(unix)]
15286 #[tokio::test]
15287 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15288 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15289 if std::env::var_os(CHILD).is_none() {
15290 use std::os::unix::fs::PermissionsExt;
15291 let tmp = tempfile::tempdir().unwrap();
15292 let gh = tmp.path().join("gh");
15293 std::fs::write(&gh, r#"#!/bin/sh
15294case "$1 $2" in
15295 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15296 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15297 *) exit 2 ;;
15298esac
15299"#).unwrap();
15300 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15301 let mut paths = vec![tmp.path().to_path_buf()];
15302 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15303 let output = std::process::Command::new(std::env::current_exe().unwrap())
15304 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15305 .env(CHILD, "1")
15306 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15307 .env("PATH", std::env::join_paths(paths).unwrap())
15308 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15309 .output().unwrap();
15310 assert!(
15311 output.status.success()
15312 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15313 "{}\n{}",
15314 String::from_utf8_lossy(&output.stdout),
15315 String::from_utf8_lossy(&output.stderr)
15316 );
15317 return;
15318 }
15319 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15320 let repo = runner.state.repo.clone();
15321 git::git(&repo, &["checkout", "main"]).await.unwrap();
15322 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15323 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15324 git::git(&repo, &["checkout", "fix/retained"])
15325 .await
15326 .unwrap();
15327 runner.state.pr = Some(crate::run::PrRecord {
15328 url: "https://example.test/project/pull/1".into(),
15329 number: 1,
15330 state: "open".into(),
15331 checks: "unknown".into(),
15332 round: 0,
15333 rounds: 1,
15334 red_at_merge: Vec::new(),
15335 });
15336 runner.sync_to_base().await.unwrap();
15337 assert_eq!(runner.state.status, RunStatus::Blocked);
15338 assert_eq!(
15339 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15340 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15341 );
15342 assert!(runner.state.base_sync_closure_pending());
15343 assert!(crate::handover::can_resume(&runner.state));
15344 let queue = crate::queue::Queue::open();
15345 assert!(!crate::base_sync_recovery::held_task(
15346 &queue,
15347 &queue.get(&task).unwrap()
15348 ));
15349 assert!(runner.state.authorize_base_sync(&task).is_err());
15350 runner.state.base_sync_recovery.stop = None;
15352 runner.state.status = RunStatus::Voting;
15353 runner.state.gate_ran = true;
15354 runner.state.config.graph.review_rounds = 0;
15355 runner.state.save().unwrap();
15356 assert!(crate::handover::can_resume(&runner.state));
15357 runner.state = RunState::load(&runner.state.id).unwrap();
15358 runner.execute_graph().await.unwrap();
15359 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15360 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15361 assert_eq!(
15362 runner.state.base_sync.as_ref().unwrap().attempts,
15363 BASE_SYNC_ROUNDS
15364 );
15365 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15366 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15367 assert!(runner.state.reviews.is_empty());
15368 assert!(runner.state.gate.is_empty());
15369 }
15370
15371 #[cfg(unix)]
15372 #[tokio::test]
15373 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15374 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15375 runner.state.driver_exited = true;
15376 runner.state.save().unwrap();
15377 let home = crate::run::home();
15378 let queue = crate::queue::Queue::open();
15379 let _claim = queue.claim(&task).unwrap();
15380 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15381 assert!(
15382 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15383 .await
15384 .is_err()
15385 );
15386 drop(lease);
15387 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15388 assert!(
15389 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15390 .await
15391 .is_err()
15392 );
15393 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15394 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15395 .await
15396 .unwrap();
15397 let disk = RunState::load(&runner.state.id).unwrap();
15398 assert!(disk.base_recovery_pending());
15399 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15400 assert!(
15401 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15402 .await
15403 .is_err()
15404 );
15405 assert_eq!(
15406 RunState::load(&runner.state.id)
15407 .unwrap()
15408 .base_sync_recovery
15409 .authorizations
15410 .len(),
15411 1
15412 );
15413 let mut owner = queue.get(&task).unwrap();
15414 owner.runs.push("20260101-000000-ffff".into());
15415 queue.put(&mut owner).unwrap();
15416 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15417 .await
15418 .unwrap_err();
15419 assert!(error.to_string().contains("run task ownership changed"));
15420 }
15421
15422 #[cfg(unix)]
15423 #[tokio::test]
15424 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15425 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15426 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15427 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15428 legacy["status"] = serde_json::json!("voting");
15429 runner.state = serde_json::from_value(legacy).unwrap();
15430 runner.state.save().unwrap();
15431 for _ in 0..3 {
15432 runner.execute_graph().await.unwrap();
15433 assert_eq!(runner.state.status, RunStatus::Blocked);
15434 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15435 assert!(!crate::handover::can_resume(&runner.state));
15436 assert_eq!(
15437 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15438 head
15439 );
15440 assert!(runner.state.reviews.is_empty());
15441 assert!(!runner.state.gate_ran);
15442 }
15443 let notice = crate::notices::run_ended(&runner.state).unwrap();
15444 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15445 runner.state.begin_pass();
15446 runner.state.status = RunStatus::Voting;
15447 runner.deliberate().await.unwrap();
15448 runner.vote().await.unwrap();
15449 runner.finish_after_tally().await.unwrap();
15450 assert_eq!(runner.state.status, RunStatus::Blocked);
15451 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15452 assert_eq!(
15453 crate::notices::run_ended(&runner.state).unwrap().since,
15454 notice.since
15455 );
15456 }
15457
15458 #[cfg(unix)]
15459 #[tokio::test]
15460 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15461 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15462 for detail in [
15463 "fix/retained conflicts with origin/main and did not rebase: conflict",
15464 "rebased fix/retained locally but origin refused the push",
15465 "unknown failure",
15466 ] {
15467 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15468 runner.state.record_base_sync_stop(&head);
15469 assert!(runner.state.authorize_base_sync(&task).is_err());
15470 runner.state.status = RunStatus::Voting;
15471 runner.execute_graph().await.unwrap();
15472 assert_eq!(runner.state.status, RunStatus::Blocked);
15473 assert!(!crate::handover::can_resume(&runner.state));
15474 assert_eq!(
15475 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15476 head
15477 );
15478 }
15479 }
15480
15481 #[cfg(unix)]
15482 #[tokio::test]
15483 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15484 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15485 runner.state.authorize_base_sync(&task).unwrap();
15486 assert!(runner.state.consume_base_recovery(&head));
15487 runner.state.save().unwrap();
15488 runner.state = RunState::load(&runner.state.id).unwrap();
15489 runner.execute_graph().await.unwrap();
15490 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15491 assert_eq!(runner.state.status, RunStatus::Blocked);
15492 assert!(!runner.state.consume_base_recovery(&head));
15493 assert!(!crate::handover::can_resume(&runner.state));
15494 runner.state.authorize_base_sync(&task).unwrap();
15495 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15496 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15497 assert_eq!(
15498 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15499 head
15500 );
15501 }
15502
15503 #[cfg(unix)]
15504 #[tokio::test]
15505 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
15506 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15507 let mut old_review = review_round(true, 0, 1, 1, false, true);
15508 old_review.head = head.clone();
15509 old_review.verified_head = Some(head.clone());
15510 runner.state.reviews = vec![old_review];
15511 runner.state.gate = vec![CommandOutcome {
15512 command: "old gate".into(),
15513 code: None,
15514 output_tail: "timed out after 2400s".into(),
15515 duration_ms: 2_400_000,
15516 resource_blocked: false,
15517 }];
15518 runner.state.gate_ran = true;
15519 runner.state.gate_head = Some(head.clone());
15520 runner.state.land_approval = Some(crate::run::LandApproval {
15521 question: "old-head-approval".into(),
15522 head: head.clone(),
15523 });
15524 runner.state.authorize_base_sync(&task).unwrap();
15525 runner.sync_to_base().await.unwrap();
15526 assert!(!runner.state.base_sync_blocked());
15527 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15528 assert_ne!(head, fresh);
15529 assert!(runner.state.reviews.is_empty());
15530 assert!(!runner.state.gate_ran);
15531 let saved = &runner.state.base_sync_recovery.authorizations[0];
15532 assert_eq!(saved.reviews[0].head, head);
15533 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
15534 assert!(saved.gate_ran);
15535 assert!(!saved.gate[0].ok());
15536 runner.review_loop().await.unwrap();
15537 runner.gate().await.unwrap();
15538 assert_eq!(runner.state.reviews[0].head, fresh);
15539 assert_eq!(
15540 runner.state.reviews[0].verified_head.as_deref(),
15541 Some(fresh.as_str())
15542 );
15543 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
15544 assert!(runner.state.gate_ran);
15545 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15546 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
15547 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
15548 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15549 assert_eq!(
15550 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
15551 "retained change"
15552 );
15553 }
15554
15555 #[cfg(unix)]
15556 #[tokio::test]
15557 async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
15558 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15559 let queue = crate::queue::Queue::open();
15560 let mut owner = queue.get(&task).unwrap();
15561 owner.release_operator("test-resume", None);
15562 queue.put(&mut owner).unwrap();
15563 let generation = owner.resumption.as_ref().unwrap().generation;
15564 runner.state.task_instruction = Some(owner.instruction.clone());
15565 runner.state.driver_exited = true;
15566 runner.state.save().unwrap();
15567 let home = crate::run::home();
15568 let _claim = queue.claim(&task).unwrap();
15569 owner.resumption.as_mut().unwrap().at =
15570 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
15571 queue.put(&mut owner).unwrap();
15572 assert!(
15573 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15574 .await
15575 .is_err(),
15576 "an approval from before the stop grants nothing"
15577 );
15578 owner.resumption.as_mut().unwrap().at = Timestamp::now();
15579 queue.put(&mut owner).unwrap();
15580 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15581 .await
15582 .unwrap();
15583 assert!(
15584 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15585 .await
15586 .is_err()
15587 );
15588 runner.state = RunState::load(&runner.state.id).unwrap();
15589 assert_eq!(
15590 runner.state.base_sync_recovery.authorizations[0].generation,
15591 Some(generation)
15592 );
15593 runner.sync_to_base().await.unwrap();
15594 let repo = runner.state.repo.clone();
15595 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15596 git::git(&repo, &["checkout", "main"]).await.unwrap();
15597 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
15598 git::git(&repo, &["add", "."]).await.unwrap();
15599 git::git(&repo, &["commit", "-m", "move base again"])
15600 .await
15601 .unwrap();
15602 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15603 git::git(&repo, &["checkout", "fix/retained"])
15604 .await
15605 .unwrap();
15606 runner.finish_after_tally().await.unwrap();
15607 assert_eq!(runner.state.status, RunStatus::Blocked);
15608 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15609 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15610 assert!(!runner.state.base_recovery_pending());
15611 assert!(!crate::handover::can_resume(&runner.state));
15612 assert!(!runner.state.gate_ran);
15613 owner.resumption.as_mut().unwrap().at = Timestamp::now();
15615 queue.put(&mut owner).unwrap();
15616 assert!(
15617 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15618 .await
15619 .is_err()
15620 );
15621 runner.state = RunState::load(&runner.state.id).unwrap();
15622 runner.sync_to_base().await.unwrap();
15623 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15624 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
15625 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15626 }
15627
15628 #[cfg(unix)]
15629 #[tokio::test]
15630 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
15631 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
15632 runner.state.base_sync = None;
15633 runner.state.status = RunStatus::Blocked;
15634 let mut second = runner.state.candidates[0].clone();
15635 second.label = 'B';
15636 runner.state.candidates.push(second);
15637 assert!(runner.state.votes.is_empty());
15638 assert!(runner.state.deliberation.is_empty());
15639 runner.deliberate().await.unwrap();
15640 runner.vote().await.unwrap();
15641 assert_eq!(runner.state.status, RunStatus::Blocked);
15642 assert!(runner.state.votes.is_empty());
15643 assert!(runner.state.deliberation.is_empty());
15644 }
15645
15646 fn runner_at(status: RunStatus) -> Runner {
15649 let mut state = RunState::new(
15650 PathBuf::from("/nonexistent/repo"),
15651 "main".to_owned(),
15652 "deadbeef".to_owned(),
15653 "task".to_owned(),
15654 Config::default(),
15655 );
15656 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
15659 state.id = format!(
15660 "20000101-000000-{:04x}",
15661 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
15662 );
15663 state.status = status;
15664 Runner {
15665 state,
15666 roles: ResolvedRoles {
15667 implementers: Vec::new(),
15668 judges: Vec::new(),
15669 reviewers: Vec::new(),
15670 fixer: None,
15671 conductor: conductor(),
15672 implementer_roster: Vec::new(),
15673 judge_roster: Vec::new(),
15674 reviewer_roster: Vec::new(),
15675 },
15676 sem: Arc::new(Semaphore::new(1)),
15677 pause: Pause::new(),
15678 interrupt: Pause::new(),
15679 }
15680 }
15681
15682 #[tokio::test]
15683 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
15684 crate::run::pin_test_home();
15685 let mut runner = runner_at(RunStatus::Gating);
15686 runner.state.save().unwrap();
15687 let id = runner.state.id.clone();
15688 let passes = runner.state.passes.len();
15689 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15690 let mut claiming = Box::pin(runner.claim_driver());
15691 std::future::poll_fn(|cx| {
15692 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
15693 std::task::Poll::Ready(())
15694 })
15695 .await;
15696 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
15697 drop(owner);
15698 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
15699 .await
15700 .unwrap()
15701 .unwrap();
15702 assert_eq!(execution.runner.state.passes.len(), passes + 1);
15703 assert!(execution.runner.state.resource_block.is_none());
15704 assert_eq!(
15705 execution.runner.state.liveness(false),
15706 crate::run::Liveness::Live
15707 );
15708 drop(execution);
15709 assert!(RunState::load(&id).unwrap().driver_exited);
15710 }
15711
15712 #[tokio::test]
15713 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
15714 crate::run::pin_test_home();
15715 let mut runner = runner_at(RunStatus::Gating);
15716 runner.state.save().unwrap();
15717 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
15720 assert!(runner.claim_driver().await.is_err());
15721 assert!(runner.state.driver_exited);
15722 assert_eq!(runner.state.driver_pid, None);
15723 let disk = RunState::load(&runner.state.id).unwrap();
15724 assert!(disk.driver_exited);
15725 assert_eq!(disk.driver_pid, None);
15726 assert_eq!(disk.status, RunStatus::Gating);
15727 }
15728
15729 #[tokio::test]
15730 async fn an_early_graph_error_releases_its_driver() {
15731 crate::run::pin_test_home();
15732 let mut runner = runner_at(RunStatus::Blocked);
15733 runner.state.config.graph.selected_implementer = Some("selected".into());
15734 runner.state.save().unwrap();
15735 assert!(runner.execute().await.is_err());
15736 let disk = RunState::load(&runner.state.id).unwrap();
15737 assert!(disk.driver_exited);
15738 assert_eq!(disk.driver_pid, None);
15739 assert_eq!(disk.status, RunStatus::Blocked);
15740 }
15741
15742 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
15743 crate::run::MergeRepair {
15744 question: "repair-question".into(),
15745 head: "old-head".into(),
15746 pr: "https://github.com/o/r/pull/42".into(),
15747 reviews: vec![review_round(false, 1, 1, 1, false, true)],
15748 gate: vec![outcome(Some(0), false)],
15749 gate_head: Some("old-head".into()),
15750 gate_fixes: Vec::new(),
15751 pre_gate: Vec::new(),
15752 phase,
15753 failure: None,
15754 }
15755 }
15756
15757 #[tokio::test]
15758 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
15759 crate::run::pin_test_home();
15760 let mut runner = runner_at(RunStatus::Landing);
15761 runner.state.merge_repairs.push(merge_repair_request(
15762 crate::run::MergeRepairPhase::Requested,
15763 ));
15764 runner.state.save().unwrap();
15765 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
15766 assert!(
15767 runner
15768 .execute()
15769 .await
15770 .unwrap_err()
15771 .to_string()
15772 .contains("already running")
15773 );
15774 assert_eq!(
15775 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
15776 crate::run::MergeRepairPhase::Requested
15777 );
15778 drop(claim);
15779 let mut disk = runner.state.clone();
15780 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
15781 disk.save().unwrap();
15782 assert!(
15783 runner
15784 .execute()
15785 .await
15786 .unwrap_err()
15787 .to_string()
15788 .contains("decision changed")
15789 );
15790 assert_eq!(
15791 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
15792 crate::run::MergeRepairPhase::Complete
15793 );
15794 }
15795
15796 #[test]
15797 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
15798 crate::run::pin_test_home();
15799 let mut runner = runner_at(RunStatus::Landing);
15800 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
15801 runner.state.reviews = request.reviews.clone();
15802 runner.state.gate = request.gate.clone();
15803 runner.state.gate_ran = true;
15804 runner.state.land_approval = Some(crate::run::LandApproval {
15805 question: "old".into(),
15806 head: "old-head".into(),
15807 });
15808 runner.state.merge_repairs.push(request);
15809 runner.begin_merge_repair_review(0).unwrap();
15810 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
15811 assert!(!runner.state.gate_status().ok());
15812 assert!(runner.state.land_approval.is_none());
15813 let disk = RunState::load(&runner.state.id).unwrap();
15814 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
15815 assert!(disk.merge_repairs[0].gate[0].ok());
15816 assert_eq!(
15817 disk.merge_repairs[0].phase,
15818 crate::run::MergeRepairPhase::Reviewing
15819 );
15820 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
15821 assert!(disk.reviews.is_empty());
15822 }
15823
15824 #[tokio::test]
15825 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
15826 crate::run::pin_test_home();
15827 for phase in [
15828 crate::run::MergeRepairPhase::Fixing,
15829 crate::run::MergeRepairPhase::Requested,
15830 ] {
15831 let mut runner = runner_at(RunStatus::Landing);
15832 runner.state.config.graph.land_rounds = 0;
15833 runner.state.merge_repairs.push(merge_repair_request(phase));
15834 runner.state.save().unwrap();
15835 assert!(!runner.repair_merge_review().await.unwrap());
15836 assert_eq!(runner.state.status, RunStatus::Blocked);
15837 assert_eq!(
15838 runner.state.merge_repairs[0].phase,
15839 crate::run::MergeRepairPhase::Failed
15840 );
15841 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
15842 assert!(
15843 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
15844 "interrupted"
15845 } else {
15846 "budget exhausted"
15847 })
15848 );
15849 assert!(runner.state.merge.is_none());
15850 }
15851 }
15852
15853 #[test]
15857 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15858 crate::run::pin_test_home();
15859 let mut runner = runner_at(RunStatus::Implementing);
15860 let interrupt = Pause::new();
15861 runner.watch_interrupt(interrupt.clone());
15862
15863 interrupt.park_because("task a1b2 asked to run first");
15864
15865 assert!(runner.park_here().expect("park_here"));
15866 assert!(runner.state.parked);
15867 let last = runner.state.events.last().expect("a park event");
15868 assert_eq!(last.node, "park");
15869 assert!(
15870 last.message.contains("task a1b2 asked to run first"),
15871 "expected the interrupt reason in {:?}",
15872 last.message
15873 );
15874 }
15875
15876 #[test]
15879 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15880 crate::run::pin_test_home();
15881 let mut runner = runner_at(RunStatus::Implementing);
15882 runner.state.begin_pass();
15883 let shutdown = Pause::new();
15884 runner.on_pause(shutdown.clone());
15885 let upgrade = ParkCause::Upgrade {
15886 from: "0.121.0".to_owned(),
15887 to: Some("0.121.1".to_owned()),
15888 };
15889 shutdown.park_for(upgrade.clone());
15890 shutdown.park_for(ParkCause::Operator);
15891
15892 assert!(runner.park_here().expect("park_here"));
15893 assert_eq!(
15894 runner.state.passes[0].end,
15895 Some(crate::run::PassEnd::Parked { cause: upgrade })
15896 );
15897 let last = runner.state.events.last().expect("a park event");
15898 assert!(last.message.contains("0.121.0"), "{}", last.message);
15899 }
15900
15901 #[tokio::test]
15902 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15903 crate::run::pin_test_home();
15904 let mut runner = runner_at(RunStatus::Reviewing);
15905 runner.state.save().unwrap();
15906 let mut execution = runner.claim_driver().await.expect("first execution");
15907 assert_eq!(execution.runner.state.passes.len(), 1);
15908 execution
15909 .driver
15910 .finish(&mut execution.runner.state)
15911 .await
15912 .unwrap();
15913 }
15914
15915 #[tokio::test]
15916 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15917 crate::run::pin_test_home();
15918 let mut runner = runner_at(RunStatus::Reviewing);
15919 runner.state.config.agents = vec![conductor()];
15920 runner.state.config.graph.timeout_resource_wait = 0;
15921 runner.state.save().unwrap();
15922 let id = runner.state.id.clone();
15923 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15924 assert!(runner.claim_driver().await.is_err());
15925 drop(owner);
15926 drop(runner);
15927 let mut resumed = Runner::resume(&id).unwrap();
15928 assert_eq!(resumed.state.driver_claimed, Some(false));
15929 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15930 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15931 let mut execution = resumed
15932 .claim_driver()
15933 .await
15934 .expect("retry initial claim after reload");
15935 assert_eq!(execution.runner.state.driver_claimed, Some(true));
15936 assert_eq!(execution.runner.state.passes.len(), 1);
15937 execution
15938 .driver
15939 .finish(&mut execution.runner.state)
15940 .await
15941 .unwrap();
15942 }
15943
15944 #[tokio::test]
15945 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15946 crate::run::pin_test_home();
15947 let mut runner = runner_at(RunStatus::Reviewing);
15948 runner.state.config.agents = vec![conductor()];
15949 runner.state.save().unwrap();
15950 let id = runner.state.id.clone();
15951 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15952 let mut resumed = Runner::resume(&id).unwrap();
15953 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15954 assert!(resumed.claim_driver().await.is_err());
15955 assert!(RunState::load(&id).unwrap().passes.is_empty());
15956 drop(lease);
15957 let mut execution = resumed.claim_driver().await.unwrap();
15958 execution
15959 .driver
15960 .finish(&mut execution.runner.state)
15961 .await
15962 .unwrap();
15963 }
15964
15965 #[tokio::test]
15966 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15967 crate::run::pin_test_home();
15968 let mut runner = runner_at(RunStatus::Reviewing);
15969 let state = &mut runner.state;
15970 state.config.agents = vec![conductor()];
15971 state.driver_claimed = None;
15972 state.driver_pid = Some(std::process::id());
15973 state.driver_started_at = None;
15974 state.begin_pass();
15975 state.save().unwrap();
15976 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15977 std::fs::write(
15978 state.dir().join("driver.json"),
15979 serde_json::to_vec(&serde_json::json!({
15980 "pid": state.driver_pid, "started_at": null,
15981 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15982 }))
15983 .unwrap(),
15984 )
15985 .unwrap();
15986 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15987 drop(lease);
15988 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15989 let mut resumed = Runner::resume(&state.id).unwrap();
15990 assert!(!resumed.state.driver_exited);
15991 let mut execution = resumed
15992 .claim_driver()
15993 .await
15994 .expect("lease proves previous execution ended");
15995 assert_eq!(execution.runner.state.passes.len(), 2);
15996 execution
15997 .driver
15998 .finish(&mut execution.runner.state)
15999 .await
16000 .unwrap();
16001 }
16002
16003 #[tokio::test]
16004 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16005 crate::run::pin_test_home();
16006 let mut runner = runner_at(RunStatus::Reviewing);
16007 let mut json = serde_json::to_value(&runner.state).unwrap();
16008 json.as_object_mut().unwrap().remove("driver_claimed");
16009 runner.state = serde_json::from_value(json).unwrap();
16010 assert_eq!(runner.state.driver_claimed, None);
16011 runner.state.save().unwrap();
16012 let mut execution = runner
16013 .claim_driver()
16014 .await
16015 .expect("the execution lease, not a missing marker, excludes a second driver");
16016 assert_eq!(execution.runner.state.passes.len(), 1);
16017 execution
16018 .driver
16019 .finish(&mut execution.runner.state)
16020 .await
16021 .unwrap();
16022 }
16023
16024 #[tokio::test]
16025 async fn driver_admission_refuses_a_proven_live_driver() {
16026 crate::run::pin_test_home();
16027 let mut runner = runner_at(RunStatus::Reviewing);
16028 let pid = std::process::id();
16029 runner.state.driver_pid = Some(pid);
16030 runner.state.driver_started_at = crate::proc::process_started_at(pid);
16031 assert!(runner.state.driver_started_at.is_some());
16032 runner.state.save().unwrap();
16033 let error = runner
16034 .claim_driver()
16035 .await
16036 .err()
16037 .expect("a live driver must refuse");
16038 assert!(error.to_string().contains("live driver"), "{error:#}");
16039 let saved = RunState::load(&runner.state.id).unwrap();
16040 assert_eq!(saved.driver_pid, Some(pid));
16041 assert_eq!(saved.status, RunStatus::Reviewing);
16042 assert!(saved.passes.is_empty());
16043 }
16044
16045 #[test]
16046 fn parking_resource_halt_persists_the_pass_once() {
16047 crate::run::pin_test_home();
16048 let mut runner = runner_at(RunStatus::Reviewing);
16049 runner.state.begin_pass();
16050 runner.state.resource_block = Some(crate::run::ResourceBlock {
16051 node: "fix".into(),
16052 kind: "shared build cache".into(),
16053 detail: "cancelled".into(),
16054 at: Timestamp::now(),
16055 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16056 });
16057 let cause = ParkCause::Upgrade {
16058 from: "old".into(),
16059 to: Some("new".into()),
16060 };
16061 runner.pause.park_for(cause.clone());
16062 runner.stop_for_resource().unwrap();
16065 assert!(runner.resource_halt().unwrap());
16066 assert!(runner.park_here().unwrap());
16067 let saved = RunState::load(&runner.state.id).unwrap();
16068 assert!(saved.parked);
16069 assert_eq!(saved.status, RunStatus::Reviewing);
16070 assert!(saved.passes[0].ended_at.is_some());
16071 assert_eq!(
16072 saved.passes[0].end,
16073 Some(crate::run::PassEnd::Parked {
16074 cause: cause.clone()
16075 })
16076 );
16077 assert_eq!(
16078 saved.resource_block.unwrap().cause,
16079 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16080 );
16081 assert_eq!(
16082 saved
16083 .events
16084 .iter()
16085 .filter(|event| event.node == "park")
16086 .count(),
16087 1
16088 );
16089 }
16090
16091 #[test]
16099 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16100 crate::run::pin_test_home();
16101 let mut runner = runner_at(RunStatus::Implementing);
16102 let shutdown = Pause::new();
16103 runner.on_pause(shutdown.clone());
16104 let interrupt = Pause::new();
16105 runner.watch_interrupt(interrupt.clone());
16106
16107 assert!(!runner.park_here().expect("park_here"));
16109 assert!(!runner.state.parked);
16110
16111 interrupt.park_because("test");
16113 assert!(!shutdown.parked());
16114 assert!(runner.park_here().expect("park_here"));
16115 }
16116
16117 #[tokio::test]
16131 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16132 crate::run::pin_test_home();
16133 let mut runner = runner_at(RunStatus::Implementing);
16134 let interrupt = Pause::new();
16135 runner.watch_interrupt(interrupt.clone());
16136
16137 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16138 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16139
16140 let node = async move {
16144 started_tx.send(()).expect("send started");
16145 finish_rx.await.expect("recv finish");
16146 "node finished"
16147 };
16148
16149 let interrupter = async move {
16150 started_rx.await.expect("recv started");
16151 interrupt.park_because("higher-priority task waiting");
16153 tokio::task::yield_now().await;
16157 finish_tx.send(()).expect("send finish");
16158 };
16159
16160 let (node_result, ()) = tokio::join!(node, interrupter);
16161 assert_eq!(
16162 node_result, "node finished",
16163 "the in-flight call ran to completion"
16164 );
16165
16166 assert!(runner.park_here().expect("park_here"));
16169 assert!(runner.state.parked);
16170 }
16171
16172 #[test]
16178 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16179 crate::run::pin_test_home();
16180 let mut runner = runner_at(RunStatus::Judging);
16181 runner.state.config.agents = vec![conductor()];
16185 runner.state.candidates = vec![Candidate {
16186 index: 0,
16187 label: 'A',
16188 agent: "alpha".to_owned(),
16189 branch: "magi/x/A".to_owned(),
16190 worktree: PathBuf::from("/nonexistent/worktree"),
16191 summary: "did the thing".to_owned(),
16192 stat: "1 file changed".to_owned(),
16193 files: 1,
16194 commits: 1,
16195 empty: false,
16196 failed: None,
16197 verified_noop: None,
16198 duration_ms: 1234,
16199 folded: false,
16200 }];
16201 let run_id = runner.state.id.clone();
16202
16203 let interrupt = Pause::new();
16204 runner.watch_interrupt(interrupt.clone());
16205 interrupt.park_because("task c3d4 asked to run first");
16206 assert!(runner.park_here().expect("park_here"));
16207
16208 let resumed = Runner::resume(&run_id).expect("resume");
16209 assert_eq!(resumed.state.candidates.len(), 1);
16210 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16211 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16212 assert_eq!(resumed.state.status, runner.state.status);
16213 assert!(
16214 resumed.state.parked,
16215 "still parked until `execute` actually walks the graph again"
16216 );
16217 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16218 }
16219
16220 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16222 let mut q = ask::Question::new(
16223 run.to_owned(),
16224 "implement".to_owned(),
16225 "impl-A".to_owned(),
16226 "Which storage backend should the cache use?".to_owned(),
16227 String::new(),
16228 vec!["SQLite".to_owned(), "Redis".to_owned()],
16229 );
16230 store.put(&mut q).unwrap();
16231 q
16232 }
16233
16234 #[test]
16235 fn a_failed_runs_open_question_is_abandoned() {
16236 ask_test_home();
16237 let store = ask::Questions::open();
16238 let mut runner = runner_at(RunStatus::Failed);
16239 let run = runner.state.id.clone();
16240 let q = ask_open_question(&store, &run);
16241
16242 runner.settle_questions();
16243
16244 let back = store.get(&q.id).unwrap();
16245 assert!(
16246 !back.status.open(),
16247 "the seat that asked died with the run; nobody is left to read an answer"
16248 );
16249 assert!(
16250 back.detail.contains(&run) && back.detail.contains("failed"),
16251 "the reason names what the run became, not just that it is gone: {}",
16252 back.detail
16253 );
16254 }
16255
16256 #[test]
16257 fn a_merged_runs_open_question_is_abandoned_too() {
16258 ask_test_home();
16259 let store = ask::Questions::open();
16260 for status in [RunStatus::Merged, RunStatus::Ready] {
16263 let mut runner = runner_at(status);
16264 let run = runner.state.id.clone();
16265 let q = ask_open_question(&store, &run);
16266
16267 runner.settle_questions();
16268
16269 let back = store.get(&q.id).unwrap();
16270 assert!(
16271 !back.status.open(),
16272 "{status:?} run's question must not outlive the run"
16273 );
16274 }
16275 }
16276
16277 #[test]
16278 fn a_still_resumable_runs_open_question_is_left_alone() {
16279 ask_test_home();
16280 let store = ask::Questions::open();
16281 for status in [RunStatus::Blocked, RunStatus::Stalled] {
16287 let mut runner = runner_at(status);
16288 let run = runner.state.id.clone();
16289 let q = ask_open_question(&store, &run);
16290
16291 runner.settle_questions();
16292
16293 let back = store.get(&q.id).unwrap();
16294 assert!(
16295 back.status.open(),
16296 "{status:?} is still alive; the question must still be waiting"
16297 );
16298 }
16299 }
16300
16301 #[test]
16302 fn settle_questions_never_touches_an_already_answered_question() {
16303 ask_test_home();
16304 let store = ask::Questions::open();
16305 let mut runner = runner_at(RunStatus::Failed);
16306 let run = runner.state.id.clone();
16307 let mut q = ask_open_question(&store, &run);
16308 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16309 .unwrap();
16310 store.put(&mut q).unwrap();
16311
16312 runner.settle_questions();
16317 runner.settle_questions();
16318
16319 let back = store.get(&q.id).unwrap();
16320 assert_eq!(
16321 back.status,
16322 ask::QuestionStatus::Answered,
16323 "a real answer is a decision on record, never overwritten by a sweep"
16324 );
16325 }
16326
16327 #[tokio::test]
16338 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16339 crate::run::pin_test_home();
16340 let tmp = tempfile::tempdir().expect("tempdir");
16341 let repo = tmp.path().join("repo");
16342 std::fs::create_dir_all(&repo).unwrap();
16343 init_repo(&repo);
16344
16345 let mut config = Config::default();
16346 config.graph.worktree_root = Some(tmp.path().join("wt"));
16347
16348 let mut state = RunState::new(
16349 repo.clone(),
16350 "main".to_owned(),
16351 "deadbeef".to_owned(),
16352 "task".to_owned(),
16353 config,
16354 );
16355 let root = state.worktree_root();
16356 let wt_a = root.join("cand-A");
16357 let wt_b = root.join("cand-B");
16358 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16359 .await
16360 .expect("worktree A");
16361 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16362 .await
16363 .expect("worktree B");
16364
16365 state.candidates = vec![
16366 Candidate {
16367 index: 0,
16368 label: 'A',
16369 agent: "alpha".to_owned(),
16370 branch: "magi/x/A".to_owned(),
16371 worktree: wt_a.clone(),
16372 summary: String::new(),
16373 stat: String::new(),
16374 files: 0,
16375 commits: 0,
16376 empty: false,
16377 failed: None,
16378 verified_noop: None,
16379 duration_ms: 0,
16380 folded: false,
16381 },
16382 Candidate {
16383 index: 1,
16384 label: 'B',
16385 agent: "beta".to_owned(),
16386 branch: "magi/x/B".to_owned(),
16387 worktree: wt_b.clone(),
16388 summary: String::new(),
16389 stat: String::new(),
16390 files: 0,
16391 commits: 0,
16392 empty: false,
16393 failed: None,
16394 verified_noop: None,
16395 duration_ms: 0,
16396 folded: false,
16397 },
16398 ];
16399 state.tally = Some(Tally {
16400 first_choice: BTreeMap::from([('A', 1)]),
16401 borda: BTreeMap::new(),
16402 winner: 'A',
16403 rankings: 1,
16404 unanimous_initial: true,
16405 deliberated: false,
16406 changed_votes: 0,
16407 unanimous_final: true,
16408 tie_break: None,
16409 judges: 1,
16410 present: 1,
16411 quorum: 1,
16412 met_quorum: true,
16413 uncontested: None,
16414 });
16415 state.status = RunStatus::Ready;
16416
16417 fold_run(&mut state, false, &crate::run::home())
16418 .await
16419 .expect("fold_run");
16420
16421 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
16422 assert!(
16423 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16424 "the unmerged winner's branch survives"
16425 );
16426 assert!(
16427 !state.candidates[0].folded,
16428 "the winner is not marked folded"
16429 );
16430
16431 assert!(!wt_b.exists(), "the loser's worktree is removed");
16432 assert!(
16433 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
16434 "the loser's branch is removed"
16435 );
16436 assert!(state.candidates[1].folded, "the loser is marked folded");
16437 }
16438
16439 #[tokio::test]
16440 #[cfg(unix)]
16441 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
16442 use std::os::fd::AsRawFd as _;
16443 use std::time::Duration;
16444
16445 let tmp = tempfile::tempdir().unwrap();
16446 let repo = tmp.path().join("repo");
16447 std::fs::create_dir_all(&repo).unwrap();
16448 init_repo(&repo);
16449 let common = repo.join(".git");
16450 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
16451 let home = tmp.path().join("home");
16452 let mut config = Config::default();
16453 config.coordination = crate::config::test_coordination();
16454 config.graph.worktree_root = Some(tmp.path().join("wt"));
16455 config.graph.timeout_resource_wait = 1;
16456 let mut state = RunState::new(
16457 repo.clone(),
16458 "main".into(),
16459 "deadbeef".into(),
16460 "task".into(),
16461 config,
16462 );
16463 let root = state.worktree_root();
16464 let candidate = root.join("cand-A");
16465 let reviewer = root.join("review-1");
16466 let branch = state.branch_for('A');
16467 git::worktree_add_branch(&repo, &candidate, &branch, "main")
16468 .await
16469 .unwrap();
16470 git::worktree_add_detached(&repo, &reviewer, "main")
16471 .await
16472 .unwrap();
16473 git::acquire_worktree_config(&repo).await.unwrap();
16474 state.enabled_worktree_config = true;
16475 state.status = RunStatus::Ready;
16476 state.candidates.push(Candidate {
16477 index: 0,
16478 label: 'A',
16479 agent: "implement".into(),
16480 branch: branch.clone(),
16481 worktree: candidate.clone(),
16482 summary: String::new(),
16483 stat: String::new(),
16484 files: 0,
16485 commits: 0,
16486 empty: false,
16487 failed: None,
16488 verified_noop: None,
16489 duration_ms: 0,
16490 folded: false,
16491 });
16492 state.save_under(&home).unwrap();
16493 let config_before = std::fs::read(common.join("config")).unwrap();
16494 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
16495 .await
16496 .unwrap();
16497 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
16500 assert_eq!(
16501 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
16502 0
16503 );
16504 let mut fold = Box::pin(fold_run(&mut state, true, &home));
16505 assert!(
16506 tokio::time::timeout(Duration::from_millis(200), &mut fold)
16507 .await
16508 .is_err()
16509 );
16510 assert!(candidate.exists());
16511 assert!(reviewer.exists());
16512 assert!(git::branch_exists(&repo, &branch).await.unwrap());
16513 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
16514 assert_eq!(
16515 git::git(&repo, &["worktree", "list", "--porcelain"])
16516 .await
16517 .unwrap(),
16518 worktrees_before
16519 );
16520 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
16521 .await
16522 .unwrap()
16523 .unwrap_err();
16524 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
16525 drop(fold);
16526 assert!(candidate.exists() && reviewer.exists());
16527 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
16528 assert_eq!(
16529 state.resource_block.as_ref().unwrap().kind,
16530 "repository coordination"
16531 );
16532 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
16533 drop(external);
16534 let mut fold = Box::pin(fold_run(&mut state, true, &home));
16535 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
16536 .await
16537 .unwrap()
16538 .unwrap();
16539 drop(fold);
16540 assert!(removed.contains(&branch));
16541 assert!(!root.exists());
16542 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
16543 assert!(state.candidates[0].folded);
16544 assert!(!state.enabled_worktree_config);
16545 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
16546 let enabled = git::git_raw(
16547 &repo,
16548 &["config", "--local", "--get", "extensions.worktreeConfig"],
16549 )
16550 .await
16551 .unwrap();
16552 assert!(!enabled.ok(), "the last config reference was released");
16553 }
16554
16555 #[tokio::test]
16558 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
16559 let tmp = tempfile::tempdir().expect("tempdir");
16560 let repo = tmp.path().join("repo");
16561 std::fs::create_dir_all(&repo).unwrap();
16562 init_repo(&repo);
16563 let home = tmp.path().join("home");
16564
16565 let mut config = Config::default();
16566 config.graph.worktree_root = Some(tmp.path().join("wt"));
16567 let mut state = RunState::new(
16568 repo.clone(),
16569 "main".to_owned(),
16570 "deadbeef".to_owned(),
16571 "task".to_owned(),
16572 config,
16573 );
16574 git::git(&repo, &["branch", "magi/x/A", "main"])
16576 .await
16577 .expect("branch");
16578 state.candidates = vec![Candidate {
16579 index: 0,
16580 label: 'A',
16581 agent: "alpha".to_owned(),
16582 branch: "magi/x/A".to_owned(),
16583 worktree: state.worktree_root().join("cand-A"),
16584 summary: String::new(),
16585 stat: String::new(),
16586 files: 0,
16587 commits: 0,
16588 empty: false,
16589 failed: None,
16590 verified_noop: None,
16591 duration_ms: 0,
16592 folded: true,
16593 }];
16594 state.released_to = Some("20260901-000000-new1".to_owned());
16595 state.released_branches = vec!["magi/x/A".to_owned()];
16596
16597 fold_run(&mut state, true, &home).await.expect("fold_run");
16598 state.status = RunStatus::Merged;
16599 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
16600 state.merge = Some(crate::run::MergeOutcome {
16601 mode: MergeMode::Pr,
16602 ok: true,
16603 detail: "confirmed merged".into(),
16604 empty: false,
16605 });
16606 fold_run(&mut state, true, &home)
16607 .await
16608 .expect("merged fold");
16609
16610 assert!(
16611 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16612 "the handed-over branch survives a fold"
16613 );
16614 }
16615
16616 #[tokio::test]
16620 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
16621 let tmp = tempfile::tempdir().expect("tempdir");
16622 let repo = tmp.path().join("repo");
16623 std::fs::create_dir_all(&repo).unwrap();
16624 init_repo(&repo);
16625 let run = |args: &[&str]| {
16626 let out = std::process::Command::new("git")
16627 .quiet()
16628 .args(args)
16629 .current_dir(&repo)
16630 .output()
16631 .expect("spawn git");
16632 assert!(out.status.success(), "git {args:?}");
16633 };
16634 run(&["branch", "magi/x/A"]);
16635 run(&["checkout", "-q", "-b", "magi/x/B"]);
16636 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
16637 run(&["add", "-A"]);
16638 run(&["commit", "-q", "-m", "work"]);
16639 run(&["checkout", "-q", "main"]);
16640
16641 let probe = RunState::new(
16645 repo.clone(),
16646 "main".to_owned(),
16647 "deadbeef".to_owned(),
16648 "task".to_owned(),
16649 Config::default(),
16650 );
16651 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
16652 let bare = tmp.path().join("origin.git");
16653 let out = std::process::Command::new("git")
16654 .quiet()
16655 .args(["init", "-q", "--bare"])
16656 .arg(&bare)
16657 .output()
16658 .expect("spawn git");
16659 assert!(out.status.success(), "git init --bare");
16660 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
16661 run(&["push", "-q", "origin", "main"]);
16662
16663 let mut state = RunState::new(
16664 repo.clone(),
16665 "main".to_owned(),
16666 "deadbeef".to_owned(),
16667 "task".to_owned(),
16668 Config::default(),
16669 );
16670 state.seeds = vec![refs::Seed {
16671 token: "magi/27b2/A".to_owned(),
16672 kind: refs::SeedKind::Unresolved,
16673 sha: String::new(),
16674 branch: true,
16675 detail: "no branch or commit named magi/27b2/A".to_owned(),
16676 }];
16677
16678 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
16679 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
16680 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
16681 let detail = empty_candidate_detail(&state, "main");
16682 assert!(detail.starts_with("empty candidate"), "{detail}");
16683 assert!(!detail.contains("magi/27b2/A"), "{detail}");
16684 assert!(detail.contains("--start-from"), "{detail}");
16685 }
16686
16687 #[tokio::test]
16696 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
16697 let tmp = tempfile::tempdir().expect("tempdir");
16698 let repo = tmp.path().join("repo");
16699 std::fs::create_dir_all(&repo).unwrap();
16700 init_repo(&repo);
16701
16702 let mut config = Config::default();
16703 config.merge.mode = MergeMode::Local;
16704
16705 let mut state = RunState::new(
16706 repo.clone(),
16707 "main".to_owned(),
16708 "deadbeef".to_owned(),
16709 "task".to_owned(),
16710 config,
16711 );
16712 state.candidates = vec![Candidate {
16713 index: 0,
16714 label: 'A',
16715 agent: "alpha".to_owned(),
16716 branch: "does-not-exist".to_owned(),
16717 worktree: repo.clone(),
16718 summary: String::new(),
16719 stat: String::new(),
16720 files: 0,
16721 commits: 0,
16722 empty: false,
16723 failed: None,
16724 verified_noop: None,
16725 duration_ms: 0,
16726 folded: false,
16727 }];
16728 state.tally = Some(Tally {
16729 first_choice: BTreeMap::from([('A', 1)]),
16730 borda: BTreeMap::new(),
16731 winner: 'A',
16732 rankings: 1,
16733 unanimous_initial: true,
16734 deliberated: false,
16735 changed_votes: 0,
16736 unanimous_final: true,
16737 tie_break: None,
16738 judges: 0,
16739 present: 0,
16740 quorum: 0,
16741 met_quorum: true,
16742 uncontested: Some("only candidate A produced a change".to_owned()),
16743 });
16744 state.reviews = vec![ReviewRound {
16745 round: 1,
16746 head: "deadbeef".to_owned(),
16747 verified_head: None,
16748 verified_at: None,
16749 reviews: Vec::new(),
16750 e2e: Vec::new(),
16751 fix: None,
16752 blocking: 0,
16753 answered: 0,
16754 expected: 0,
16755 clean: true,
16756 verify_retried: false,
16757 e2e_deferred: false,
16758 e2e_defer_reason: None,
16759 progressed: false,
16760 vote_split: false,
16761 reconsideration: Vec::new(),
16762 verdict: None,
16763 }];
16764 state.gate = vec![CommandOutcome {
16765 command: "test".to_owned(),
16766 code: Some(0),
16767 output_tail: String::new(),
16768 duration_ms: 0,
16769 resource_blocked: false,
16770 }];
16771 state.gate_ran = true;
16772 state.status = RunStatus::Ready;
16777 state.merge = Some(MergeOutcome {
16778 mode: MergeMode::Local,
16779 ok: false,
16780 detail: "already concluded".to_owned(),
16781 empty: false,
16782 });
16783
16784 let mut runner = Runner {
16785 state,
16786 roles: ResolvedRoles {
16787 implementers: Vec::new(),
16788 judges: Vec::new(),
16789 reviewers: Vec::new(),
16790 fixer: None,
16791 conductor: conductor(),
16792 implementer_roster: Vec::new(),
16793 judge_roster: Vec::new(),
16794 reviewer_roster: Vec::new(),
16795 },
16796 sem: Arc::new(Semaphore::new(1)),
16797 pause: Pause::new(),
16798 interrupt: Pause::new(),
16799 };
16800
16801 runner.merge().await.expect("merge");
16802
16803 assert_eq!(
16804 runner.state.status,
16805 RunStatus::Ready,
16806 "a concluded run's status must not change on reentry"
16807 );
16808 assert_eq!(
16809 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16810 Some("already concluded"),
16811 "merge must not run again once the node already recorded an outcome"
16812 );
16813 }
16814
16815 #[tokio::test]
16824 async fn merge_refuses_a_gate_that_has_not_actually_run() {
16825 let tmp = tempfile::tempdir().expect("tempdir");
16826 let repo = tmp.path().join("repo");
16827 std::fs::create_dir_all(&repo).unwrap();
16828 init_repo(&repo);
16829
16830 let mut config = Config::default();
16831 config.merge.mode = MergeMode::Local;
16832
16833 let mut state = RunState::new(
16834 repo.clone(),
16835 "main".to_owned(),
16836 "deadbeef".to_owned(),
16837 "task".to_owned(),
16838 config,
16839 );
16840 state.candidates = vec![Candidate {
16841 index: 0,
16842 label: 'A',
16843 agent: "alpha".to_owned(),
16844 branch: "does-not-exist".to_owned(),
16845 worktree: repo.clone(),
16846 summary: String::new(),
16847 stat: String::new(),
16848 files: 0,
16849 commits: 0,
16850 empty: false,
16851 failed: None,
16852 verified_noop: None,
16853 duration_ms: 0,
16854 folded: false,
16855 }];
16856 state.tally = Some(Tally {
16857 first_choice: BTreeMap::from([('A', 1)]),
16858 borda: BTreeMap::new(),
16859 winner: 'A',
16860 rankings: 1,
16861 unanimous_initial: true,
16862 deliberated: false,
16863 changed_votes: 0,
16864 unanimous_final: true,
16865 tie_break: None,
16866 judges: 0,
16867 present: 0,
16868 quorum: 0,
16869 met_quorum: true,
16870 uncontested: Some("only candidate A produced a change".to_owned()),
16871 });
16872 state.reviews = vec![ReviewRound {
16873 round: 1,
16874 head: "deadbeef".to_owned(),
16875 verified_head: None,
16876 verified_at: None,
16877 reviews: Vec::new(),
16878 e2e: Vec::new(),
16879 fix: None,
16880 blocking: 0,
16881 answered: 0,
16882 expected: 0,
16883 clean: true,
16884 verify_retried: false,
16885 e2e_deferred: false,
16886 e2e_defer_reason: None,
16887 progressed: false,
16888 vote_split: false,
16889 reconsideration: Vec::new(),
16890 verdict: None,
16891 }];
16892 state.gate = Vec::new();
16894 state.gate_ran = false;
16895 state.status = RunStatus::Gating;
16896
16897 let mut runner = Runner {
16898 state,
16899 roles: ResolvedRoles {
16900 implementers: Vec::new(),
16901 judges: Vec::new(),
16902 reviewers: Vec::new(),
16903 fixer: None,
16904 conductor: conductor(),
16905 implementer_roster: Vec::new(),
16906 judge_roster: Vec::new(),
16907 reviewer_roster: Vec::new(),
16908 },
16909 sem: Arc::new(Semaphore::new(1)),
16910 pause: Pause::new(),
16911 interrupt: Pause::new(),
16912 };
16913
16914 runner.merge().await.expect("merge");
16915
16916 assert!(
16917 runner.state.merge.is_none(),
16918 "an empty gate must never be read as a passing one: {:?}",
16919 runner.state.merge
16920 );
16921 }
16922
16923 #[tokio::test]
16930 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16931 let tmp = tempfile::tempdir().expect("tempdir");
16932 let repo = tmp.path().join("repo");
16933 std::fs::create_dir_all(&repo).unwrap();
16934 init_repo(&repo);
16935
16936 let config = Config::default();
16938
16939 let mut state = RunState::new(
16940 repo.clone(),
16941 "main".to_owned(),
16942 "deadbeef".to_owned(),
16943 "task".to_owned(),
16944 config,
16945 );
16946 state.candidates = vec![Candidate {
16947 index: 0,
16948 label: 'A',
16949 agent: "alpha".to_owned(),
16950 branch: "does-not-exist".to_owned(),
16951 worktree: repo.clone(),
16952 summary: String::new(),
16953 stat: String::new(),
16954 files: 0,
16955 commits: 0,
16956 empty: false,
16957 failed: None,
16958 verified_noop: None,
16959 duration_ms: 0,
16960 folded: false,
16961 }];
16962 state.tally = Some(Tally {
16963 first_choice: BTreeMap::from([('A', 1)]),
16964 borda: BTreeMap::new(),
16965 winner: 'A',
16966 rankings: 1,
16967 unanimous_initial: true,
16968 deliberated: false,
16969 changed_votes: 0,
16970 unanimous_final: true,
16971 tie_break: None,
16972 judges: 0,
16973 present: 0,
16974 quorum: 0,
16975 met_quorum: true,
16976 uncontested: Some("only candidate A produced a change".to_owned()),
16977 });
16978 state.reviews = vec![ReviewRound {
16979 round: 1,
16980 head: "deadbeef".to_owned(),
16981 verified_head: None,
16982 verified_at: None,
16983 reviews: Vec::new(),
16984 e2e: Vec::new(),
16985 fix: None,
16986 blocking: 0,
16987 answered: 0,
16988 expected: 0,
16989 clean: true,
16990 verify_retried: false,
16991 e2e_deferred: false,
16992 e2e_defer_reason: None,
16993 progressed: false,
16994 vote_split: false,
16995 reconsideration: Vec::new(),
16996 verdict: None,
16997 }];
16998
16999 let mut runner = Runner {
17000 state,
17001 roles: ResolvedRoles {
17002 implementers: Vec::new(),
17003 judges: Vec::new(),
17004 reviewers: Vec::new(),
17005 fixer: None,
17006 conductor: conductor(),
17007 implementer_roster: Vec::new(),
17008 judge_roster: Vec::new(),
17009 reviewer_roster: Vec::new(),
17010 },
17011 sem: Arc::new(Semaphore::new(1)),
17012 pause: Pause::new(),
17013 interrupt: Pause::new(),
17014 };
17015
17016 runner.gate().await.expect("gate");
17017 assert!(
17018 runner.state.gate_ran,
17019 "zero configured commands is still a real attempt, not an unrun gate"
17020 );
17021 assert!(runner.state.gate.is_empty());
17022 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17023 assert_ne!(
17024 runner.state.status,
17025 RunStatus::Blocked,
17026 "a gate with nothing to check must not read as failed"
17027 );
17028
17029 runner.merge().await.expect("merge");
17030 assert_eq!(
17031 runner.state.status,
17032 RunStatus::Ready,
17033 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17034 );
17035 }
17036
17037 #[tokio::test]
17048 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17049 crate::run::pin_test_home();
17050 let home = crate::run::home();
17051
17052 let tmp = tempfile::tempdir().expect("tempdir");
17053 let repo = tmp.path().join("repo");
17054 std::fs::create_dir_all(&repo).unwrap();
17055 init_repo(&repo);
17056 let cache_dir = tmp.path().join("target");
17059
17060 let mut config = Config::default();
17061 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17062 config.graph.timeout_verify = Some(2);
17065 config.graph.timeout_resource_wait = 2;
17066
17067 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17068 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17069 .expect("no io error acquiring directly")
17070 {
17071 crate::cache::AcquireOutcome::Acquired(g) => g,
17072 crate::cache::AcquireOutcome::Busy(b) => {
17073 panic!("expected the direct acquire to win the lease first: {b:?}")
17074 }
17075 };
17076
17077 let mut state = RunState::new(
17078 repo.clone(),
17079 "main".to_owned(),
17080 "deadbeef".to_owned(),
17081 "task".to_owned(),
17082 config,
17083 );
17084 state.candidates = vec![Candidate {
17085 index: 0,
17086 label: 'A',
17087 agent: "alpha".to_owned(),
17088 branch: "does-not-exist".to_owned(),
17089 worktree: repo.clone(),
17090 summary: String::new(),
17091 stat: String::new(),
17092 files: 0,
17093 commits: 0,
17094 empty: false,
17095 failed: None,
17096 verified_noop: None,
17097 duration_ms: 0,
17098 folded: false,
17099 }];
17100 state.tally = Some(Tally {
17101 first_choice: BTreeMap::from([('A', 1)]),
17102 borda: BTreeMap::new(),
17103 winner: 'A',
17104 rankings: 1,
17105 unanimous_initial: true,
17106 deliberated: false,
17107 changed_votes: 0,
17108 unanimous_final: true,
17109 tie_break: None,
17110 judges: 0,
17111 present: 0,
17112 quorum: 0,
17113 met_quorum: true,
17114 uncontested: Some("only candidate A produced a change".to_owned()),
17115 });
17116 state.reviews = vec![ReviewRound {
17117 round: 1,
17118 head: "deadbeef".to_owned(),
17119 verified_head: None,
17120 verified_at: None,
17121 reviews: Vec::new(),
17122 e2e: Vec::new(),
17123 fix: None,
17124 blocking: 0,
17125 answered: 0,
17126 expected: 0,
17127 clean: true,
17128 verify_retried: false,
17129 e2e_deferred: false,
17130 e2e_defer_reason: None,
17131 progressed: false,
17132 vote_split: false,
17133 reconsideration: Vec::new(),
17134 verdict: None,
17135 }];
17136
17137 let mut runner = Runner {
17138 state,
17139 roles: ResolvedRoles {
17140 implementers: Vec::new(),
17141 judges: Vec::new(),
17142 reviewers: Vec::new(),
17143 fixer: None,
17144 conductor: conductor(),
17145 implementer_roster: Vec::new(),
17146 judge_roster: Vec::new(),
17147 reviewer_roster: Vec::new(),
17148 },
17149 sem: Arc::new(Semaphore::new(1)),
17150 pause: Pause::new(),
17151 interrupt: Pause::new(),
17152 };
17153
17154 let started = std::time::Instant::now();
17155 runner.gate().await.expect("gate");
17156 assert!(
17157 started.elapsed() < Duration::from_secs(1),
17158 "a gate with nothing to run must never wait on a lease it never needed"
17159 );
17160 assert!(
17161 runner.state.gate_ran,
17162 "zero commands is still a real, immediate attempt"
17163 );
17164 assert!(runner.state.gate.is_empty());
17165 assert_ne!(
17166 runner.state.status,
17167 RunStatus::Blocked,
17168 "must not read as resource-blocked on a lease it never asked for"
17169 );
17170 }
17171
17172 #[cfg(unix)]
17175 #[tokio::test]
17176 async fn verification_is_standalone_inside_a_live_worker() {
17177 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17178 if std::env::var_os(CHILD).is_none() {
17179 let home = tempfile::tempdir().unwrap();
17180 let generation = crate::worker::pin_current(home.path()).unwrap();
17181 let output = std::process::Command::new(std::env::current_exe().unwrap())
17182 .args([
17183 "--exact",
17184 "graph::tests::verification_is_standalone_inside_a_live_worker",
17185 "--nocapture",
17186 ])
17187 .env(CHILD, "1")
17188 .env("MAGI_HOME", home.path())
17189 .env("MAGI_WORKER_HOME", home.path())
17190 .env("MAGI_WORKER_ID", "abcdef")
17191 .env("MAGI_WORKER_GENERATION", &generation.id)
17192 .env("MAGI_WORKER_EXE", &generation.executable)
17193 .env("MAGI_RUN", "verification-parent")
17194 .env("MAGI_NODE", "gate")
17195 .env("MAGI_SEAT", "worker-fixture")
17196 .env("MAGI_PROCESS_OWNER", "fixture-owner")
17197 .env("MAGI_NO_AUTOUPDATE", "0")
17198 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17199 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17200 .env(
17201 "VERIFY_EXPECT_TOOLCHAIN",
17202 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17203 )
17204 .env(
17205 "VERIFY_EXPECT_CACHE",
17206 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17207 )
17208 .output()
17209 .unwrap();
17210 assert!(
17211 output.status.success()
17212 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17213 "{}\n{}",
17214 String::from_utf8_lossy(&output.stdout),
17215 String::from_utf8_lossy(&output.stderr)
17216 );
17217 return;
17218 }
17219 let home = crate::run::home();
17220 let directory = home.join("workers/jobs/abcdef");
17221 std::fs::create_dir_all(&directory).unwrap();
17222 let job = crate::worker::Job {
17223 id: "abcdef".into(),
17224 sequence: 1,
17225 kind: "task".into(),
17226 key: "task:verification".into(),
17227 payload: serde_json::json!({}),
17228 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17229 version: Some(env!("CARGO_PKG_VERSION").into()),
17230 status: crate::worker::Status::Running,
17231 result: None,
17232 pid: Some(std::process::id()),
17233 created_at: 0,
17234 finished_at: None,
17235 };
17236 std::fs::write(
17237 directory.join("job.json"),
17238 serde_json::to_vec(&job).unwrap(),
17239 )
17240 .unwrap();
17241 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17242 let assert_worker = || {
17243 assert_eq!(
17244 crate::worker::current_job(&home).unwrap().unwrap().id,
17245 job.id
17246 );
17247 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17248 assert_eq!(
17249 crate::worker::helper_executable(&home).unwrap(),
17250 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17251 );
17252 assert!(
17253 crate::updater::Checker::new(&crate::config::Update {
17254 mode: crate::config::UpdateMode::Notify,
17255 interval: None,
17256 auto_restart: false,
17257 })
17258 .is_none(),
17259 "the driver remains pinned, even while tests are standalone"
17260 );
17261 };
17262 assert_worker();
17263 let repo = home.join("repo");
17264 std::fs::create_dir_all(&repo).unwrap();
17265 init_repo(&repo);
17266 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17269 test -z "${MAGI_WORKER_GENERATION+x}" &&
17270 test -z "${MAGI_WORKER_HOME+x}" &&
17271 test -z "${MAGI_WORKER_EXE+x}" &&
17272 test "$MAGI_HOME/repo" -ef . &&
17273 test "$MAGI_RUN" = verification-parent &&
17274 test "$MAGI_NODE" = gate &&
17275 test "$MAGI_SEAT" = worker-fixture &&
17276 test -n "$MAGI_PROCESS_OWNER" &&
17277 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17278 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17279 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17280 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17281 assert!(
17283 !std::process::Command::new("sh")
17284 .args(["-c", probe])
17285 .current_dir(&repo)
17286 .status()
17287 .unwrap()
17288 .success()
17289 );
17290 let unrelated_home = tempfile::tempdir().unwrap();
17291 assert!(
17292 crate::worker::current_job(unrelated_home.path()).is_err(),
17293 "an inherited worker id cannot be resolved in a unit fixture's home"
17294 );
17295 let mut runner = runner_at(RunStatus::Gating);
17296 runner.state.repo = repo.clone();
17297 runner.state.config.disk.min_free_bytes = 0;
17298 runner.state.config.coordination = crate::config::test_coordination();
17299 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17300 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17301 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17302 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17303 let winner = Candidate {
17304 index: 0,
17305 label: 'A',
17306 agent: "fixture".into(),
17307 branch: "main".into(),
17308 worktree: repo.clone(),
17309 summary: String::new(),
17310 stat: String::new(),
17311 files: 0,
17312 commits: 0,
17313 empty: false,
17314 failed: None,
17315 verified_noop: None,
17316 duration_ms: 0,
17317 folded: false,
17318 };
17319 let (e2e, retried, pids) = run_e2e_with_retry(
17320 &mut runner.state,
17321 &["sh".into(), "-c".into()],
17322 &[format!("{probe} && printf initial")],
17323 &repo,
17324 Duration::from_secs(10),
17325 "initial",
17326 )
17327 .await;
17328 assert!(!retried && pids.is_empty());
17329 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17330 assert_eq!(e2e[0].output_tail, "initial");
17331 let gate = runner.run_gate(&winner).await.unwrap();
17332 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17333 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17336 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17337 passed: false,
17338 outcomes: Vec::new(),
17339 });
17340 runner.state.save().unwrap();
17341 let mut resumed = RunState::load(&runner.state.id).unwrap();
17342 let wait = crate::repo_lock::WaitPolicy::default();
17343 assert!(matches!(
17344 check_gate_repair_e2e(&mut resumed, &winner, &wait)
17345 .await
17346 .unwrap(),
17347 GateFix::Retry
17348 ));
17349 let saved = RunState::load(&resumed.id).unwrap();
17350 let repair = saved.gate_repair_e2e.unwrap();
17351 assert!(repair.passed);
17352 assert_eq!(repair.outcomes.len(), 1);
17353 assert!(repair.outcomes[0].ok());
17354 let landing = run_landing_gate(&mut resumed, &winner, &wait)
17355 .await
17356 .unwrap();
17357 assert_eq!(landing.len(), 1);
17358 assert!(landing[0].ok(), "{landing:?}");
17359 assert_eq!(
17360 std::fs::read_to_string(repo.join("phases")).unwrap(),
17361 "pre\ngate\ne2e\npre\ngate\n"
17362 );
17363 assert_worker();
17364 }
17365
17366 #[cfg(unix)]
17367 #[tokio::test]
17368 async fn verification_keeps_the_panic_and_the_full_output() {
17369 crate::run::pin_test_home();
17370 let tmp = tempfile::tempdir().unwrap();
17371 let mut config = Config::default();
17372 config.disk.min_free_bytes = 0;
17373 let mut state = RunState::new(
17374 tmp.path().to_owned(),
17375 "main".into(),
17376 "base".into(),
17377 "verification evidence".into(),
17378 config,
17379 );
17380 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
17381 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
17382 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
17383 let mut names = Vec::new();
17384 for _ in 0..2 {
17385 let (outcomes, _) = run_commands(
17386 &mut state,
17387 "gate",
17388 "gate",
17389 0,
17390 &["sh".into(), "-c".into()],
17391 &[script.into()],
17392 tmp.path(),
17393 Duration::from_secs(60),
17394 )
17395 .await;
17396 let tail = &outcomes[0].output_tail;
17397 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
17398 assert!(tail.contains("boom assertion"));
17399 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
17400 let line = tail.lines().next().unwrap();
17401 names.push(line.to_owned());
17402 }
17403 assert_ne!(names[0], names[1], "the second run must not overwrite");
17404 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
17405 assert!(full.len() > 500_000 && full.contains("boom assertion"));
17406 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
17407
17408 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
17410 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
17411 let (outcomes, _) = run_commands(
17412 &mut state,
17413 "gate",
17414 "gate",
17415 0,
17416 &["sh".into(), "-c".into()],
17417 &["echo failure-output; exit 7".into()],
17418 tmp.path(),
17419 Duration::from_secs(10),
17420 )
17421 .await;
17422 assert_eq!(outcomes[0].code, Some(7));
17423 assert!(state.events.iter().any(|e| {
17424 e.node == "gate" && e.message.contains("could not save the full gate output:")
17425 }));
17426 }
17427
17428 #[cfg(unix)]
17429 #[tokio::test]
17430 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
17431 crate::run::pin_test_home();
17432 let tmp = tempfile::tempdir().unwrap();
17433 let mut config = Config::default();
17434 config.disk.min_free_bytes = 0;
17435 let mut state = RunState::new(
17436 tmp.path().to_owned(),
17437 "main".into(),
17438 "base".into(),
17439 "verification evidence".into(),
17440 config,
17441 );
17442 let (outcomes, pids) = run_commands(
17443 &mut state,
17444 "gate",
17445 "gate",
17446 1,
17447 &["sh".into(), "-c".into()],
17448 &[
17449 "printf before-timeout; exec sleep 30".into(),
17450 "kill -TERM $$".into(),
17451 "exit 7".into(),
17452 "printf success".into(),
17453 ],
17454 tmp.path(),
17455 Duration::from_secs(1),
17456 )
17457 .await;
17458 wait_for_timed_out_children_to_die(&pids).await;
17459 assert_eq!(
17460 pids.len(),
17461 1,
17462 "unleased timeout retains its death-check PID"
17463 );
17464 assert_eq!(outcomes.len(), 4);
17465 assert_eq!(outcomes[0].code, None);
17466 assert!(outcomes[0].output_tail.contains("before-timeout"));
17467 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
17468 assert_eq!(outcomes[1].code, None);
17469 assert!(outcomes[1].output_tail.contains("command terminated:"));
17470 assert!(!outcomes[1].output_tail.contains("timed out"));
17471 assert_eq!(outcomes[2].code, Some(7));
17472 assert!(!outcomes[2].output_tail.contains("timed out"));
17473 assert_eq!(outcomes[3].code, Some(0));
17474 assert_eq!(outcomes[3].output_tail, "success");
17475 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
17476 }
17477
17478 #[tokio::test]
17488 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
17489 crate::run::pin_test_home();
17490
17491 let tmp = tempfile::tempdir().expect("tempdir");
17492 let repo = tmp.path().join("repo");
17493 std::fs::create_dir_all(&repo).unwrap();
17494 init_repo(&repo);
17495
17496 let mut config = Config::default();
17497 config.verify.gate = vec![
17498 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
17499 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
17500 .to_owned(),
17501 ];
17502
17503 let mut state = RunState::new(
17504 repo.clone(),
17505 "main".to_owned(),
17506 "deadbeef".to_owned(),
17507 "task".to_owned(),
17508 config,
17509 );
17510 let run_id = state.id.clone();
17511 state.candidates = vec![Candidate {
17512 index: 0,
17513 label: 'A',
17514 agent: "alpha".to_owned(),
17515 branch: "does-not-exist".to_owned(),
17516 worktree: repo.clone(),
17517 summary: String::new(),
17518 stat: String::new(),
17519 files: 0,
17520 commits: 0,
17521 empty: false,
17522 failed: None,
17523 verified_noop: None,
17524 duration_ms: 0,
17525 folded: false,
17526 }];
17527 state.tally = Some(Tally {
17528 first_choice: BTreeMap::from([('A', 1)]),
17529 borda: BTreeMap::new(),
17530 winner: 'A',
17531 rankings: 1,
17532 unanimous_initial: true,
17533 deliberated: false,
17534 changed_votes: 0,
17535 unanimous_final: true,
17536 tie_break: None,
17537 judges: 0,
17538 present: 0,
17539 quorum: 0,
17540 met_quorum: true,
17541 uncontested: Some("only candidate A produced a change".to_owned()),
17542 });
17543 state.reviews = vec![ReviewRound {
17544 round: 1,
17545 head: "deadbeef".to_owned(),
17546 verified_head: None,
17547 verified_at: None,
17548 reviews: Vec::new(),
17549 e2e: Vec::new(),
17550 fix: None,
17551 blocking: 0,
17552 answered: 0,
17553 expected: 0,
17554 clean: true,
17555 verify_retried: false,
17556 e2e_deferred: false,
17557 e2e_defer_reason: None,
17558 progressed: false,
17559 vote_split: false,
17560 reconsideration: Vec::new(),
17561 verdict: None,
17562 }];
17563
17564 let mut runner = Runner {
17565 state,
17566 roles: ResolvedRoles {
17567 implementers: Vec::new(),
17568 judges: Vec::new(),
17569 reviewers: Vec::new(),
17570 fixer: None,
17571 conductor: conductor(),
17572 implementer_roster: Vec::new(),
17573 judge_roster: Vec::new(),
17574 reviewer_roster: Vec::new(),
17575 },
17576 sem: Arc::new(Semaphore::new(1)),
17577 pause: Pause::new(),
17578 interrupt: Pause::new(),
17579 };
17580
17581 let started_marker = repo.join("started.marker");
17582 let release_marker = repo.join("release.marker");
17583 let poller = tokio::spawn(async move {
17584 for _ in 0..100 {
17589 if started_marker.exists()
17590 && let Ok(s) = crate::run::RunState::load(&run_id)
17591 && let Some(a) = s.active.get("gate")
17592 {
17593 std::fs::write(&release_marker, b"go").expect("release marker");
17594 return Some(a.clone());
17595 }
17596 tokio::time::sleep(Duration::from_millis(50)).await;
17597 }
17598 None
17599 });
17600
17601 runner.gate().await.expect("gate");
17602 let captured = poller.await.expect("poller task");
17603 let captured = captured.expect(
17604 "the poller never saw a `gate` task entry in run.json while the command was \
17605 still blocked on its own release marker",
17606 );
17607
17608 assert_eq!(captured.task.as_deref(), Some("gate"));
17609 assert_eq!(captured.node, "gate");
17610 assert_eq!(captured.index, Some(1));
17611 assert_eq!(captured.total, Some(1));
17612 assert!(
17613 captured
17614 .command
17615 .as_deref()
17616 .is_some_and(|c| c.contains("started.marker")),
17617 "{captured:?}"
17618 );
17619
17620 assert!(
17621 runner.state.active.is_empty(),
17622 "the entry must be cleared once the command actually finished: {:?}",
17623 runner.state.active
17624 );
17625 assert!(runner.state.gate_ran);
17626 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
17627 }
17628
17629 #[tokio::test]
17632 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
17633 use crate::verdict::{Finding, ReviewVote, Severity};
17634 crate::run::pin_test_home();
17635 let tmp = tempfile::tempdir().expect("tempdir");
17636 let repo = tmp.path().join("repo");
17637 std::fs::create_dir_all(&repo).unwrap();
17638 init_repo(&repo);
17639
17640 for (severity, vote, expect) in [
17641 (Severity::Major, ReviewVote::Reject, true),
17642 (Severity::Minor, ReviewVote::Reject, false),
17643 (Severity::Major, ReviewVote::Approve, false),
17644 ] {
17645 let mut round = review_round(false, 1, 1, 1, false, true);
17646 round.reviews = vec![ReviewRecord {
17647 reviewer: 1,
17648 agent: "alpha".to_owned(),
17649 summary: String::new(),
17650 findings: vec![Finding {
17651 id: "R1-1-1".to_owned(),
17652 severity,
17653 file: None,
17654 line: None,
17655 title: "t".to_owned(),
17656 detail: String::new(),
17657 }],
17658 vote: Some(vote),
17659 failed: None,
17660 duration_ms: 0,
17661 attempts: 0,
17662 }];
17663 let mut state = RunState::new(
17664 repo.clone(),
17665 "main".to_owned(),
17666 "deadbeef".to_owned(),
17667 "task".to_owned(),
17668 Config::default(),
17669 );
17670 state.reviews = vec![round];
17671 let mut runner = Runner {
17672 state,
17673 roles: ResolvedRoles {
17674 implementers: Vec::new(),
17675 judges: Vec::new(),
17676 reviewers: Vec::new(),
17677 fixer: None,
17678 conductor: conductor(),
17679 implementer_roster: Vec::new(),
17680 judge_roster: Vec::new(),
17681 reviewer_roster: Vec::new(),
17682 },
17683 sem: Arc::new(Semaphore::new(1)),
17684 pause: Pause::new(),
17685 interrupt: Pause::new(),
17686 };
17687 let shell = runner.state.config.shell();
17688 runner
17689 .stop_reviewing("round budget spent", &shell, &repo)
17690 .await
17691 .expect("stop_reviewing");
17692 assert_eq!(runner.state.status, RunStatus::Gating);
17693 assert_eq!(
17694 runner.state.contested_handoff.is_some(),
17695 expect,
17696 "{severity:?} + {vote:?}"
17697 );
17698 }
17699 }
17700
17701 #[tokio::test]
17714 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
17715 crate::run::pin_test_home();
17716 let home = crate::run::home();
17717
17718 let tmp = tempfile::tempdir().expect("tempdir");
17719 let repo = tmp.path().join("repo");
17720 std::fs::create_dir_all(&repo).unwrap();
17721 init_repo(&repo);
17722 let head = crate::git::rev_parse(&repo, "HEAD")
17723 .await
17724 .expect("rev-parse");
17725 let cache_dir = tmp.path().join("target");
17728
17729 let mut config = Config::default();
17730 config.verify.e2e = vec![format!(
17731 "CARGO_TARGET_DIR='{}' test -f README.md",
17732 cache_dir.display()
17733 )];
17734 config.graph.review_rounds = 1;
17735 config.graph.timeout_verify = Some(2);
17738 config.graph.timeout_resource_wait = 2;
17739
17740 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17741 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17742 .expect("no io error acquiring directly")
17743 {
17744 crate::cache::AcquireOutcome::Acquired(g) => g,
17745 crate::cache::AcquireOutcome::Busy(b) => {
17746 panic!("expected the direct acquire to win the lease first: {b:?}")
17747 }
17748 };
17749
17750 let mut state = RunState::new(
17751 repo.clone(),
17752 "main".to_owned(),
17753 head.clone(),
17754 "task".to_owned(),
17755 config,
17756 );
17757 state.candidates = vec![Candidate {
17758 index: 0,
17759 label: 'A',
17760 agent: "alpha".to_owned(),
17761 branch: "does-not-exist".to_owned(),
17762 worktree: repo.clone(),
17763 summary: String::new(),
17764 stat: String::new(),
17765 files: 0,
17766 commits: 0,
17767 empty: false,
17768 failed: None,
17769 verified_noop: None,
17770 duration_ms: 0,
17771 folded: false,
17772 }];
17773 state.tally = Some(Tally {
17774 first_choice: BTreeMap::from([('A', 1)]),
17775 borda: BTreeMap::new(),
17776 winner: 'A',
17777 rankings: 1,
17778 unanimous_initial: true,
17779 deliberated: false,
17780 changed_votes: 0,
17781 unanimous_final: true,
17782 tie_break: None,
17783 judges: 0,
17784 present: 0,
17785 quorum: 0,
17786 met_quorum: true,
17787 uncontested: Some("only candidate A produced a change".to_owned()),
17788 });
17789 state.reviews = vec![ReviewRound {
17793 round: 1,
17794 head: head.clone(),
17795 verified_head: None,
17796 verified_at: None,
17797 reviews: Vec::new(),
17798 e2e: Vec::new(),
17799 fix: None,
17800 blocking: 1,
17801 answered: 1,
17802 expected: 1,
17803 clean: false,
17804 verify_retried: false,
17805 e2e_deferred: true,
17806 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
17807 progressed: false,
17808 vote_split: false,
17809 reconsideration: Vec::new(),
17810 verdict: None,
17811 }];
17812
17813 let mut runner = Runner {
17814 state,
17815 roles: ResolvedRoles {
17816 implementers: Vec::new(),
17817 judges: Vec::new(),
17818 reviewers: Vec::new(),
17819 fixer: None,
17820 conductor: conductor(),
17821 implementer_roster: Vec::new(),
17822 judge_roster: Vec::new(),
17823 reviewer_roster: Vec::new(),
17824 },
17825 sem: Arc::new(Semaphore::new(1)),
17826 pause: Pause::new(),
17827 interrupt: Pause::new(),
17828 };
17829
17830 let shell = runner.state.config.shell();
17831 runner
17832 .stop_reviewing("round budget spent", &shell, &repo)
17833 .await
17834 .expect("stop_reviewing");
17835
17836 let last = runner.state.reviews.last().expect("round record");
17837 assert_eq!(
17838 last.e2e_status(),
17839 E2eStatus::ResourceBlocked,
17840 "the shared cache is still held; the attempt must read as blocked, not deferred or \
17841 failed: {last:?}"
17842 );
17843 assert_eq!(
17844 last.verified_head.as_deref(),
17845 Some(head.as_str()),
17846 "which commit this attempt targeted is known even though nothing finished checking \
17847 it"
17848 );
17849 let first_attempt_at = last
17850 .verified_at
17851 .expect("when this attempt ran is known too");
17852 assert_ne!(
17853 runner.state.status,
17854 RunStatus::Blocked,
17855 "contention is evidence about the machine, not the patch — it must not settle the \
17856 run as blocked: {:?}",
17857 runner.state.status
17858 );
17859 assert!(
17860 !runner
17861 .state
17862 .events
17863 .iter()
17864 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
17865 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
17866 runner.state.events
17867 );
17868
17869 runner
17874 .stop_reviewing("round budget spent", &shell, &repo)
17875 .await
17876 .expect("stop_reviewing retry");
17877 assert_eq!(
17878 runner.state.reviews.len(),
17879 1,
17880 "no new round was started: {:?}",
17881 runner.state.reviews
17882 );
17883 let last = runner.state.reviews.last().expect("round record");
17884 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
17885 assert!(
17886 last.verified_at.expect("still known") > first_attempt_at,
17887 "a second reentry must be a fresh attempt, not a stale copy of the first"
17888 );
17889 assert_ne!(runner.state.status, RunStatus::Blocked);
17890
17891 held.release();
17892 }
17893
17894 #[cfg(unix)]
17895 #[tokio::test]
17896 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
17897 ask_test_home();
17898 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
17899 let tmp = tempfile::tempdir().unwrap();
17900 let repo = tmp.path().join("repo");
17901 std::fs::create_dir(&repo).unwrap();
17902 init_repo(&repo);
17903 let run_git = |args: &[&str]| {
17904 let out = std::process::Command::new("git")
17905 .args(args)
17906 .current_dir(&repo)
17907 .output()
17908 .unwrap();
17909 assert!(
17910 out.status.success(),
17911 "{args:?}: {}",
17912 String::from_utf8_lossy(&out.stderr)
17913 );
17914 };
17915 let origin = tmp.path().join("origin.git");
17916 run_git(&["init", "--bare", origin.to_str().unwrap()]);
17917 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17918 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17919 run_git(&["add", "spec.rs"]);
17920 run_git(&["commit", "-m", "initial spec"]);
17921 run_git(&["push", "origin", "main"]);
17922 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17923 run_git(&["checkout", "-b", "fix/review"]);
17924 std::fs::write(
17925 repo.join("check.rs"),
17926 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17927 )
17928 .unwrap();
17929 run_git(&["add", "check.rs"]);
17930 run_git(&["commit", "-m", "use spec"]);
17931 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17932 run_git(&["checkout", "main"]);
17933 std::fs::write(
17934 repo.join("spec.rs"),
17935 "struct AgentSpec { max_turn_secs: u64 }\n",
17936 )
17937 .unwrap();
17938 run_git(&["add", "spec.rs"]);
17939 run_git(&["commit", "-m", "extend spec"]);
17940 run_git(&["push", "origin", "main"]);
17941 run_git(&["checkout", "fix/review"]);
17942
17943 let mut runner = runner_at(RunStatus::Reviewing);
17944 runner.state.repo = repo.clone();
17945 runner.state.base_commit = base;
17946 runner.state.pr_head = Some("fix/review".into());
17947 runner.state.config.graph.review_rounds = max_rounds;
17948 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17949 runner.state.config.graph.sessions = false;
17950 runner.state.config.disk.min_free_bytes = 0;
17951 runner.state.config.verify.e2e =
17952 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17953 runner.state.candidates = vec![Candidate {
17954 index: 0,
17955 label: 'A',
17956 agent: "fixture".into(),
17957 branch: "fix/review".into(),
17958 worktree: repo.clone(),
17959 summary: String::new(),
17960 stat: String::new(),
17961 files: 1,
17962 commits: 1,
17963 empty: false,
17964 failed: None,
17965 verified_noop: None,
17966 duration_ms: 0,
17967 folded: false,
17968 }];
17969 runner.state.tally = Some(Tally {
17970 first_choice: BTreeMap::new(),
17971 borda: BTreeMap::new(),
17972 winner: 'A',
17973 rankings: 0,
17974 unanimous_initial: true,
17975 deliberated: false,
17976 changed_votes: 0,
17977 unanimous_final: true,
17978 tie_break: None,
17979 judges: 0,
17980 present: 0,
17981 quorum: 0,
17982 met_quorum: true,
17983 uncontested: Some("review-only".into()),
17984 });
17985 let mut last = review_round(false, 0, 1, 1, false, false);
17986 last.head = reviewed.clone();
17987 last.e2e_deferred = true;
17988 last.e2e[0].resource_blocked = true;
17989 last.e2e[0].output_tail = "waiting for the shared build cache".into();
17990 if max_rounds == 1 {
17991 last.e2e.clear();
17992 }
17993 runner.state.reviews = vec![last];
17994 let calls = tmp.path().join("fix-calls");
17995 let mut fixer = spec("repair");
17996 fixer
17997 .env
17998 .insert("FIX_CALLS".into(), calls.display().to_string());
17999 fixer.command = vec!["sh".into(), "-c".into(), r#"
18000 echo fix >> "$FIX_CALLS"
18001 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18002 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18003 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18004 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18005 "#.into()];
18006 let mut reviewer = spec("reviewer");
18007 reviewer.command = vec![
18008 "sh".into(),
18009 "-c".into(),
18010 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18011 .into(),
18012 ];
18013 runner.roles.fixer = Some(vec![fixer]);
18014 runner.roles.reviewers = vec![reviewer];
18015
18016 runner.sync_to_base().await.unwrap();
18017 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18018 assert_ne!(rebased, reviewed);
18019 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18020 if reentry {
18021 let shell = runner.state.config.shell();
18024 runner
18025 .stop_unresolved_review("deferred verification", &shell, &repo)
18026 .await
18027 .unwrap();
18028 assert!(runner.catchup_repairable());
18029 assert!(!calls.exists());
18030 runner.state = RunState::load(&runner.state.id).unwrap();
18031 assert!(runner.catchup_repairable());
18032 }
18033 runner.review_loop().await.unwrap();
18034 let first = &runner.state.reviews[0];
18035 assert_eq!(first.head, reviewed);
18036 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18037 assert!(
18038 first.verify_retried,
18039 "a reproduced build failure gets exactly one retry: {:?}",
18040 first.e2e
18041 );
18042 assert!(first.e2e[0].output_tail.contains("E0063"));
18043 assert!(!runner.catchup_repairable());
18044 if max_rounds == 2 {
18045 assert_eq!(
18046 runner.state.status,
18047 RunStatus::Gating,
18048 "{}",
18049 crate::report::run(&runner.state)
18050 );
18051 assert_eq!(runner.state.reviews.len(), 2);
18052 assert!(first.fix.as_ref().unwrap().committed);
18053 assert!(runner.state.reviews[1].clean);
18054 assert_eq!(
18055 runner
18056 .state
18057 .reviews
18058 .iter()
18059 .filter(|r| r.fix.is_some())
18060 .count(),
18061 1
18062 );
18063 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18064 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18065 } else {
18066 assert_eq!(runner.state.status, RunStatus::Blocked);
18067 assert!(first.fix.is_none());
18068 assert!(!calls.exists());
18069 let recovery = crate::recovery::assess(
18070 "prior",
18071 Some(&runner.state),
18072 &runner.landing_base(),
18073 crate::recovery::Relation::Diverged,
18074 );
18075 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18076 let reasons = recovery.reasons.join("\n");
18077 assert!(
18078 reasons.contains(&reviewed) && reasons.contains(&rebased),
18079 "{reasons}"
18080 );
18081 assert!(
18082 reasons.contains("base sync to")
18083 && reasons.contains("additional review is required"),
18084 "{reasons}"
18085 );
18086 }
18087 }
18088 }
18089
18090 #[tokio::test]
18102 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18103 crate::run::pin_test_home();
18104 let home = crate::run::home();
18105
18106 let tmp = tempfile::tempdir().expect("tempdir");
18107 let repo = tmp.path().join("repo");
18108 std::fs::create_dir_all(&repo).unwrap();
18109 init_repo(&repo);
18110 let head = crate::git::rev_parse(&repo, "HEAD")
18111 .await
18112 .expect("rev-parse");
18113 let cache_dir = tmp.path().join("target");
18114
18115 let mut config = Config::default();
18116 config.verify.e2e = vec![format!(
18117 "CARGO_TARGET_DIR='{}' test -f README.md",
18118 cache_dir.display()
18119 )];
18120 config.graph.review_rounds = 1;
18121 config.graph.timeout_verify = Some(2);
18122 config.graph.timeout_resource_wait = 2;
18123
18124 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18125 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18126 .expect("no io error acquiring directly")
18127 {
18128 crate::cache::AcquireOutcome::Acquired(g) => g,
18129 crate::cache::AcquireOutcome::Busy(b) => {
18130 panic!("expected the direct acquire to win the lease first: {b:?}")
18131 }
18132 };
18133
18134 let mut state = RunState::new(
18135 repo.clone(),
18136 "main".to_owned(),
18137 head.clone(),
18138 "task".to_owned(),
18139 config,
18140 );
18141 state.candidates = vec![Candidate {
18142 index: 0,
18143 label: 'A',
18144 agent: "alpha".to_owned(),
18145 branch: "does-not-exist".to_owned(),
18146 worktree: repo.clone(),
18147 summary: String::new(),
18148 stat: String::new(),
18149 files: 0,
18150 commits: 0,
18151 empty: false,
18152 failed: None,
18153 verified_noop: None,
18154 duration_ms: 0,
18155 folded: false,
18156 }];
18157 state.tally = Some(Tally {
18158 first_choice: BTreeMap::from([('A', 1)]),
18159 borda: BTreeMap::new(),
18160 winner: 'A',
18161 rankings: 1,
18162 unanimous_initial: true,
18163 deliberated: false,
18164 changed_votes: 0,
18165 unanimous_final: true,
18166 tie_break: None,
18167 judges: 0,
18168 present: 0,
18169 quorum: 0,
18170 met_quorum: true,
18171 uncontested: Some("only candidate A produced a change".to_owned()),
18172 });
18173 state.reviews = vec![ReviewRound {
18177 round: 1,
18178 head: head.clone(),
18179 verified_head: Some(head.clone()),
18180 verified_at: Some(jiff::Timestamp::now()),
18181 reviews: Vec::new(),
18182 e2e: vec![CommandOutcome {
18183 command: format!(
18184 "CARGO_TARGET_DIR='{}' test -f README.md",
18185 cache_dir.display()
18186 ),
18187 code: None,
18188 output_tail: "waiting for the shared build cache".to_owned(),
18189 duration_ms: 0,
18190 resource_blocked: true,
18191 }],
18192 fix: None,
18193 blocking: 1,
18194 answered: 1,
18195 expected: 1,
18196 clean: false,
18197 verify_retried: false,
18198 e2e_deferred: false,
18199 e2e_defer_reason: None,
18200 progressed: false,
18201 vote_split: false,
18202 reconsideration: Vec::new(),
18203 verdict: None,
18204 }];
18205
18206 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18207 let mut runner = Runner {
18208 state,
18209 roles: ResolvedRoles {
18210 implementers: Vec::new(),
18211 judges: Vec::new(),
18212 reviewers: Vec::new(),
18213 fixer: None,
18214 conductor: conductor(),
18215 implementer_roster: Vec::new(),
18216 judge_roster: Vec::new(),
18217 reviewer_roster: Vec::new(),
18218 },
18219 sem: Arc::new(Semaphore::new(1)),
18220 pause: Pause::new(),
18221 interrupt: Pause::new(),
18222 };
18223
18224 runner.review_loop().await.expect("review_loop");
18229
18230 assert_eq!(
18231 runner.state.reviews.len(),
18232 1,
18233 "no new round was started on top of the unresolved one: {:?}",
18234 runner.state.reviews
18235 );
18236 let last = &runner.state.reviews[0];
18237 assert_eq!(
18238 last.e2e_status(),
18239 E2eStatus::ResourceBlocked,
18240 "still contended: {last:?}"
18241 );
18242 assert!(
18243 last.verified_at.expect("still known") > first_attempt_at,
18244 "review_loop must have actually retried the check, not left it exactly as found"
18245 );
18246 assert_ne!(
18247 runner.state.status,
18248 RunStatus::Blocked,
18249 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18250 runner.state.status
18251 );
18252
18253 held.release();
18254 }
18255
18256 #[tokio::test]
18257 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18258 crate::run::pin_test_home();
18259 for status in [RunStatus::Blocked, RunStatus::Gating] {
18260 let mut runner = runner_at(status);
18261 runner.state.config.graph.land = true;
18262 runner.state.gate_ran = true;
18263 runner.state.merge = Some(MergeOutcome {
18264 mode: MergeMode::Pr,
18265 ok: false,
18266 detail: "the fixer produced no commit".into(),
18267 empty: false,
18268 });
18269 let url = "https://example.invalid/o/r/pull/1";
18270 runner.state.pr = Some(crate::run::PrRecord {
18271 url: url.into(),
18272 number: 1,
18273 state: "open".into(),
18274 checks: "red".into(),
18275 round: 1,
18276 rounds: 1,
18277 red_at_merge: Vec::new(),
18278 });
18279 runner.state.save().unwrap();
18280 runner.execute_graph().await.unwrap();
18284 assert_eq!(runner.state.status, RunStatus::Blocked);
18285 assert!(runner.state.land_sync_pending);
18286 assert!(runner.state.events.iter().any(|event|
18287 event.node == "land" && event.message == format!("watching {url}")));
18288 assert_eq!(
18289 runner.state.merge.as_ref().unwrap().detail,
18290 "the fixer produced no commit"
18291 );
18292 }
18293 }
18294
18295 #[tokio::test]
18296 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18297 crate::run::pin_test_home();
18298 let tmp = tempfile::tempdir().expect("tempdir");
18299 let repo = tmp.path().join("repo");
18300 std::fs::create_dir_all(&repo).unwrap();
18301 init_repo(&repo);
18302
18303 let mut config = Config::default();
18304 config.merge.mode = MergeMode::Pr;
18305 config.graph.land = true;
18306 config.graph.land_approval = false;
18307
18308 let mut state = RunState::new(
18309 repo.clone(),
18310 "main".to_owned(),
18311 "deadbeef".to_owned(),
18312 "task".to_owned(),
18313 config,
18314 );
18315 state.candidates = vec![Candidate {
18316 index: 0,
18317 label: 'A',
18318 agent: "alpha".to_owned(),
18319 branch: "does-not-exist".to_owned(),
18320 worktree: repo.clone(),
18321 summary: String::new(),
18322 stat: String::new(),
18323 files: 0,
18324 commits: 0,
18325 empty: false,
18326 failed: None,
18327 verified_noop: None,
18328 duration_ms: 0,
18329 folded: false,
18330 }];
18331 state.tally = Some(Tally {
18332 first_choice: BTreeMap::from([('A', 1)]),
18333 borda: BTreeMap::new(),
18334 winner: 'A',
18335 rankings: 1,
18336 unanimous_initial: true,
18337 deliberated: false,
18338 changed_votes: 0,
18339 unanimous_final: true,
18340 tie_break: None,
18341 judges: 0,
18342 present: 0,
18343 quorum: 0,
18344 met_quorum: true,
18345 uncontested: Some("only candidate A produced a change".to_owned()),
18346 });
18347 state.reviews = vec![ReviewRound {
18348 round: 1,
18349 head: "deadbeef".to_owned(),
18350 verified_head: None,
18351 verified_at: None,
18352 reviews: Vec::new(),
18353 e2e: Vec::new(),
18354 fix: None,
18355 blocking: 0,
18356 answered: 0,
18357 expected: 0,
18358 clean: true,
18359 verify_retried: false,
18360 e2e_deferred: false,
18361 e2e_defer_reason: None,
18362 progressed: false,
18363 vote_split: false,
18364 reconsideration: Vec::new(),
18365 verdict: None,
18366 }];
18367 state.gate = vec![CommandOutcome {
18368 command: "test".to_owned(),
18369 code: Some(0),
18370 output_tail: String::new(),
18371 duration_ms: 0,
18372 resource_blocked: false,
18373 }];
18374 state.gate_ran = true;
18375 state.status = RunStatus::Landing;
18379 state.merge = Some(MergeOutcome {
18380 mode: MergeMode::Pr,
18381 ok: true,
18382 detail: "https://example.invalid/x/y/pull/1".to_owned(),
18383 empty: false,
18384 });
18385
18386 ask_test_home();
18390 state.save().expect("save landing run");
18393 let store = ask::Questions::open();
18394 let q = ask_open_question(&store, &state.id);
18395
18396 let mut runner = Runner {
18397 state,
18398 roles: ResolvedRoles {
18399 implementers: Vec::new(),
18400 judges: Vec::new(),
18401 reviewers: Vec::new(),
18402 fixer: None,
18403 conductor: conductor(),
18404 implementer_roster: Vec::new(),
18405 judge_roster: Vec::new(),
18406 reviewer_roster: Vec::new(),
18407 },
18408 sem: Arc::new(Semaphore::new(1)),
18409 pause: Pause::new(),
18410 interrupt: Pause::new(),
18411 };
18412
18413 runner.execute().await.expect("execute");
18418
18419 assert_eq!(
18420 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
18421 Some("https://example.invalid/x/y/pull/1"),
18422 "reentry must not push again or open a second pull request over the \
18423 one `land` is already watching"
18424 );
18425 assert_ne!(
18426 runner.state.status,
18427 RunStatus::Landing,
18428 "land could not actually reach the fake pull request, so it must \
18429 have given up rather than left the run silently parked forever"
18430 );
18431 assert_eq!(runner.state.status, RunStatus::Blocked);
18435 assert!(
18436 store.get(&q.id).unwrap().status.open(),
18437 "Blocked is still alive; settle_questions must have been a no-op here"
18438 );
18439 }
18440
18441 fn state_with_round(round: ReviewRound) -> RunState {
18442 let mut s = RunState::new(
18443 PathBuf::from("/repo"),
18444 "main".to_owned(),
18445 "abc1234".to_owned(),
18446 "add retries".to_owned(),
18447 Config::default(),
18448 );
18449 s.reviews = vec![round];
18450 s
18451 }
18452
18453 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
18454 crate::verdict::Finding {
18455 id: id.to_owned(),
18456 severity,
18457 file: None,
18458 line: None,
18459 title: title.to_owned(),
18460 detail: String::new(),
18461 }
18462 }
18463
18464 #[test]
18465 fn pr_body_names_open_findings_and_declined_ones() {
18466 let round = ReviewRound {
18467 round: 2,
18468 head: "deadbee".to_owned(),
18469 verified_head: None,
18470 verified_at: None,
18471 reviews: vec![ReviewRecord {
18472 attempts: 0,
18473 reviewer: 1,
18474 agent: "alpha".to_owned(),
18475 summary: String::new(),
18476 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
18477 vote: None,
18478 failed: None,
18479 duration_ms: 0,
18480 }],
18481 e2e: vec![CommandOutcome {
18482 command: "cargo test".to_owned(),
18483 code: Some(0),
18484 output_tail: String::new(),
18485 duration_ms: 0,
18486 resource_blocked: false,
18487 }],
18488 verify_retried: false,
18489 e2e_deferred: false,
18490 e2e_defer_reason: None,
18491 fix: Some(FixRecord {
18492 agent: "alpha".to_owned(),
18493 addressed: Vec::new(),
18494 rejected: vec![crate::verdict::Rejection {
18495 id: "R1-1-1".to_owned(),
18496 why: "not reachable from any caller".to_owned(),
18497 }],
18498 notes: String::new(),
18499 committed: true,
18500 failed: None,
18501 duration_ms: 0,
18502 continuation: None,
18503 }),
18504 blocking: 0,
18505 answered: 1,
18506 expected: 1,
18507 clean: false,
18508 progressed: true,
18509 vote_split: false,
18510 reconsideration: Vec::new(),
18511 verdict: None,
18512 };
18513 let state = state_with_round(round);
18514 let body = pr_message(&state, 'A').body;
18515
18516 assert!(body.contains("add retries"), "the task must still be there");
18517 assert!(body.contains("R2-1-1"), "{body}");
18518 assert!(!body.contains("unused import"), "{body}");
18519 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
18520 assert!(
18521 !body.contains("not reachable from any caller"),
18522 "the reason it was declined: {body}"
18523 );
18524 }
18525
18526 #[test]
18527 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
18528 let round = ReviewRound {
18529 round: 1,
18530 head: "deadbee".to_owned(),
18531 verified_head: None,
18532 verified_at: None,
18533 reviews: vec![ReviewRecord {
18534 attempts: 0,
18535 reviewer: 1,
18536 agent: "alpha".to_owned(),
18537 summary: String::new(),
18538 findings: Vec::new(),
18539 vote: None,
18540 failed: None,
18541 duration_ms: 0,
18542 }],
18543 e2e: Vec::new(),
18544 verify_retried: false,
18545 e2e_deferred: false,
18546 e2e_defer_reason: None,
18547 fix: None,
18548 blocking: 0,
18549 answered: 1,
18550 expected: 1,
18551 clean: true,
18552 progressed: false,
18553 vote_split: false,
18554 reconsideration: Vec::new(),
18555 verdict: None,
18556 };
18557 let state = state_with_round(round);
18558 let body = pr_message(&state, 'A').body;
18559 assert!(!body.contains("Open review findings"), "{body}");
18560 assert!(!body.contains("Declined"), "{body}");
18561 }
18562
18563 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
18564 let mut state = RunState::new(
18565 PathBuf::from("/repo"),
18566 "main".to_owned(),
18567 "abc1234".to_owned(),
18568 instruction.to_owned(),
18569 Config::default(),
18570 );
18571 state.candidates.push(Candidate {
18572 index: 0,
18573 label: 'A',
18574 agent: "alpha".to_owned(),
18575 branch: "magi/x/A".to_owned(),
18576 worktree: PathBuf::from("/wt"),
18577 summary: summary.to_owned(),
18578 stat: String::new(),
18579 files: 1,
18580 commits: 1,
18581 empty: false,
18582 failed: None,
18583 verified_noop: None,
18584 folded: false,
18585 duration_ms: 0,
18586 });
18587 state
18588 }
18589
18590 fn review_state(subjects: &[&str]) -> RunState {
18591 let mut state = state_with_summary(
18592 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
18593 "",
18594 );
18595 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18596 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
18597 state
18598 }
18599
18600 #[tokio::test]
18603 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
18604 ask_test_home();
18605 let tmp = tempfile::tempdir().unwrap();
18606 let repo = tmp.path().join("repo");
18607 std::fs::create_dir_all(&repo).unwrap();
18608 init_repo(&repo);
18609 let origin = tmp.path().join("origin.git");
18610 let g = |dir: &Path, args: &[&str]| {
18611 let out = std::process::Command::new("git")
18612 .args(args)
18613 .current_dir(dir)
18614 .quiet()
18615 .output()
18616 .expect("spawn git");
18617 assert!(
18618 out.status.success(),
18619 "git {args:?}: {}",
18620 String::from_utf8_lossy(&out.stderr)
18621 );
18622 };
18623 g(
18624 tmp.path(),
18625 &[
18626 "clone",
18627 "--bare",
18628 "-q",
18629 repo.to_str().unwrap(),
18630 origin.to_str().unwrap(),
18631 ],
18632 );
18633 g(
18634 &repo,
18635 &["remote", "add", "origin", origin.to_str().unwrap()],
18636 );
18637 let c1 = git::rev_parse(&repo, "main").await.unwrap();
18638
18639 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
18641 g(&repo, &["add", "-A"]);
18642 g(
18643 &repo,
18644 &["commit", "-q", "-m", "chore(deps): update a crate"],
18645 );
18646 g(&repo, &["push", "-q", "origin", "main"]);
18647 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
18648 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
18649 g(&repo, &["add", "-A"]);
18650 g(
18651 &repo,
18652 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
18653 );
18654 g(&repo, &["checkout", "-q", "main"]);
18655
18656 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
18657 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
18658 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
18660 assert_eq!(fallback, c1);
18661
18662 let mut state = review_state(&[
18663 "chore(deps): update a crate",
18664 "fix(daemon): apply a chosen action",
18665 ]);
18666 state.repo = repo.clone();
18667 state.base_branch = "main".to_owned();
18668 state.base_commit = c1;
18669 refresh_reviewed_commits(&mut state, "feat/own").await;
18670 assert_eq!(
18671 state.reviewed_commits,
18672 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
18673 );
18674 assert_eq!(
18675 review_title(&state).as_deref(),
18676 Some("fix(daemon): apply a chosen action")
18677 );
18678 assert_eq!(
18679 leaked_subjects(&state, "feat/own").await,
18680 Some(vec!["chore(deps): update a crate".to_owned()])
18681 );
18682 state.config.merge.remote = "nowhere".to_owned();
18685 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
18686 }
18687
18688 struct SummaryFixture {
18692 _tmp: tempfile::TempDir,
18693 repo: PathBuf,
18694 reply: PathBuf,
18695 runner: Runner,
18696 }
18697
18698 fn summary_fixture() -> SummaryFixture {
18699 ask_test_home();
18700 let tmp = tempfile::tempdir().unwrap();
18701 let repo = tmp.path().join("repo");
18702 std::fs::create_dir_all(&repo).unwrap();
18703 init_repo(&repo);
18704 let origin = tmp.path().join("origin.git");
18705 let g = |args: &[&str]| {
18706 let out = std::process::Command::new("git")
18707 .args(args)
18708 .current_dir(&repo)
18709 .quiet()
18710 .output()
18711 .expect("spawn git");
18712 assert!(
18713 out.status.success(),
18714 "git {args:?}: {}",
18715 String::from_utf8_lossy(&out.stderr)
18716 );
18717 };
18718 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
18719 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
18720 g(&["push", "-q", "origin", "main"]);
18721 g(&["checkout", "-q", "-b", "feat/own"]);
18722 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
18723 g(&["add", "-A"]);
18724 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
18725 let reply = tmp.path().join("reply.txt");
18726 std::fs::write(
18727 &reply,
18728 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
18729 )
18730 .unwrap();
18731 let mut runner = runner_at(RunStatus::Gating);
18732 let mut state = review_state(&["feat(own): add the own file"]);
18733 state.repo = repo.clone();
18734 state.base_branch = "main".to_owned();
18735 state.candidates[0].branch = "feat/own".to_owned();
18736 state.candidates[0].worktree = repo.clone();
18737 let mut author = spec("alpha");
18738 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
18739 author
18740 .env
18741 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
18742 state.config.agents = vec![author];
18743 runner.state = state;
18744 SummaryFixture {
18745 _tmp: tmp,
18746 repo,
18747 reply,
18748 runner,
18749 }
18750 }
18751
18752 async fn posted(f: &mut SummaryFixture) -> PrMessage {
18753 let winner = f.runner.state.candidates[0].clone();
18754 f.runner
18755 .guarded_pr_message(&winner, None, true)
18756 .await
18757 .unwrap()
18758 .expect("an English description never parks")
18759 }
18760
18761 fn clean_round_with_minor_findings() -> ReviewRound {
18762 let mut round = passing_round(2, None);
18763 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
18764 round.reviews = vec![ReviewRecord {
18765 attempts: 0,
18766 reviewer: 1,
18767 agent: "alpha".into(),
18768 summary: "内部レビュー".into(),
18769 findings: vec![
18770 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
18771 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
18772 ],
18773 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
18774 failed: None,
18775 duration_ms: 0,
18776 }];
18777 round
18778 }
18779
18780 #[tokio::test]
18781 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
18782 for review_only in [false, true] {
18783 let mut f = summary_fixture();
18784 if !review_only {
18785 f.runner.state.candidates[0].agent = "alpha".into();
18786 }
18787 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
18788 assert!(f.runner.state.reviews[0].clean);
18789 assert!(f.runner.state.reviews[0].fix.is_none());
18790 assert!(f.runner.state.open_findings().is_empty());
18791 assert_eq!(
18792 crate::pr_outcome::required_finding_ids(&f.runner.state),
18793 ["R2-1-1", "R2-1-2"]
18794 );
18795 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18796 let evidence = crate::pr_outcome::summary_evidence(
18797 &f.runner.state,
18798 crate::pr_outcome::FinalHead::At(&head),
18799 );
18800 assert!(
18801 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
18802 );
18803 let omitted = posted(&mut f).await;
18805 assert!(omitted.body.contains("No description"));
18806 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
18807 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
18808 assert!(f.runner.state.change_summary.is_none());
18809 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
18810 let described = posted(&mut f).await;
18811 assert!(described.body.contains("improve documentation for callers"));
18812 assert!(described.body.contains("remaining edge case"));
18813 assert!(described.body.contains("## Open review findings"));
18814 assert!(!described.body.contains("未修正"));
18815 }
18816 }
18817
18818 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
18819 let mut f = summary_fixture();
18820 if !review_only {
18821 f.runner.state.candidates[0].agent = "alpha".into();
18822 }
18823 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18824 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
18825 fixed
18826 .fix
18827 .as_mut()
18828 .unwrap()
18829 .rejected
18830 .push(crate::verdict::Rejection {
18831 id: "R1-1-1".into(),
18832 why: "呼び出されないコードのため辞退".into(),
18833 });
18834 let mut clean = clean_round_with_minor_findings();
18835 clean.verified_head = Some(head.clone());
18836 f.runner.state.reviews = vec![fixed, clean];
18837 ok_gate(&mut f.runner.state, Some(&head));
18838 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
18839 f.runner.state.config.agents.push(scripted("beta", reply));
18840 f.runner.state.config.roles.language_judge =
18841 Some(crate::config::AgentChoice::One("beta".into()));
18842 f
18843 }
18844
18845 #[tokio::test]
18846 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
18847 for review_only in [false, true] {
18848 let mut f = rewrite_fixture_with_findings(review_only,
18849 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
18850 let m = posted(&mut f).await;
18851 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18852 let expected =
18853 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
18854 assert!(m.body.contains(&expected));
18855 assert!(m.body.contains("## Review rounds"));
18856 assert!(
18857 m.body
18858 .contains("Gate (`verify.gate`): passed on the final commit")
18859 );
18860 assert!(
18861 m.body
18862 .contains("Tests (`verify.e2e`): passed on the final commit")
18863 );
18864 let c = f.runner.state.change_summary.as_ref().unwrap();
18865 assert!(
18866 m.body
18867 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
18868 );
18869 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18870 assert!(crate::pr_outcome::covers_required_findings(
18871 &f.runner.state,
18872 &m.body
18873 ));
18874 assert!(m.body.contains("documentation remains incomplete"));
18875 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
18876 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
18877 assert!(f.runner.state.github_text.is_none());
18878 }
18879 }
18880
18881 #[tokio::test]
18882 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
18883 for review_only in [false, true] {
18884 for reply in [
18885 "TITLE: fix: public description\nA short English description.",
18886 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
18887 ] {
18888 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
18889 let winner = f.runner.state.candidates[0].clone();
18890 let posted = f
18891 .runner
18892 .guarded_pr_message(&winner, None, true)
18893 .await
18894 .unwrap();
18895 assert!(posted.is_none());
18896 assert!(f.runner.state.parked);
18897 let withheld = f.runner.state.github_text.as_ref().unwrap();
18898 assert!(
18899 withheld.rewrite.is_none(),
18900 "an incomplete rewrite must not be selectable"
18901 );
18902 let q = ask::Questions::open()
18903 .get(withheld.question.as_ref().unwrap())
18904 .unwrap();
18905 assert!(
18906 !q.choices
18907 .contains(&crate::github_text::USE_REWRITE.to_owned())
18908 );
18909 assert!(
18910 q.detail.contains("R1-1-1")
18911 && q.detail.contains("R2-1-1")
18912 && q.detail.contains("R2-1-2")
18913 );
18914 }
18915 }
18916 }
18917
18918 #[tokio::test]
18919 async fn japanese_reports_stay_local_in_both_public_run_modes() {
18920 for review_only in [false, true] {
18921 let mut f = summary_fixture();
18922 if !review_only {
18923 f.runner.state.candidates[0].agent = "alpha".into();
18924 }
18925 f.runner.state.instruction = "内部タスクの説明".into();
18926 f.runner.state.candidates[0].summary = "内部の候補報告".into();
18927 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18928 f.runner.state.reviews = vec![
18929 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18930 round_with_notes(2, Some("二回目の内部修正報告です。")),
18931 passing_round(3, Some(&head)),
18932 ];
18933 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18934 ok_gate(&mut f.runner.state, Some(&head));
18935 let captured = f.repo.join("public-prompt.txt");
18936 let author = &mut f.runner.state.config.agents[0];
18937 author.command = vec![
18938 "sh".into(),
18939 "-c".into(),
18940 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18941 ];
18942 author.env.insert(
18943 "PROMPT_FILE".into(),
18944 captured.to_string_lossy().into_owned(),
18945 );
18946 let m = posted(&mut f).await;
18947 assert!(m.body.contains("Adds `own.txt`"));
18948 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18949 assert!(m.body.contains("R1-1-1"));
18950 assert!(
18951 m.body
18952 .contains("Gate (`verify.gate`): passed on the final commit")
18953 );
18954 assert!(
18955 m.body
18956 .contains("Tests (`verify.e2e`): passed on the final commit")
18957 );
18958 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18959 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18960 let prompt = std::fs::read_to_string(captured).unwrap();
18961 assert!(prompt.contains(&format!(
18962 "git diff {} {head}",
18963 f.runner.state.change_summary.as_ref().unwrap().base
18964 )));
18965 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18966 assert_eq!(
18967 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18968 "二回目の内部修正報告です。"
18969 );
18970 }
18971 }
18972
18973 #[tokio::test]
18974 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18975 for review_only in [false, true] {
18976 let mut f = summary_fixture();
18977 if !review_only {
18978 f.runner.state.candidates[0].agent = "alpha".into();
18979 }
18980 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18981 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18982 std::fs::write(&f.reply, "no usable description").unwrap();
18983 let failed = posted(&mut f).await;
18984 assert!(
18985 failed
18986 .body
18987 .contains("No description of what this change does could be generated")
18988 );
18989 assert!(failed.body.is_ascii());
18990 assert!(f.runner.state.github_text.is_none());
18991 assert!(f.runner.state.change_summary.is_none());
18992 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18993 let evidence = crate::pr_outcome::summary_evidence(
18994 &f.runner.state,
18995 crate::pr_outcome::FinalHead::At(&head),
18996 );
18997 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18998 assert!(
18999 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19000 .exists()
19001 );
19002 f.runner.state =
19003 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19004 std::fs::write(
19005 &f.reply,
19006 "TITLE: feat(own): add the own file\nAdds the committed own file.",
19007 )
19008 .unwrap();
19009 let retried = posted(&mut f).await;
19010 assert!(retried.body.contains("Adds the committed own file."));
19011 assert_eq!(
19012 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19013 "日本語の修正報告"
19014 );
19015 }
19016 }
19017
19018 #[tokio::test]
19019 async fn same_head_verification_and_findings_invalidate_public_prose() {
19020 let mut f = summary_fixture();
19021 posted(&mut f).await;
19022 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19023 let original = f
19024 .runner
19025 .state
19026 .change_summary
19027 .as_ref()
19028 .unwrap()
19029 .evidence
19030 .clone();
19031 ok_gate(&mut f.runner.state, Some(&head));
19032 std::fs::write(
19033 &f.reply,
19034 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19035 )
19036 .unwrap();
19037 let verified = posted(&mut f).await;
19038 assert!(verified.body.contains("now verified"));
19039 assert_ne!(
19040 f.runner.state.change_summary.as_ref().unwrap().evidence,
19041 original
19042 );
19043 let mut round = round_with_notes(1, Some("内部報告"));
19044 round.reviews.push(ReviewRecord {
19045 attempts: 0,
19046 reviewer: 1,
19047 agent: "alpha".into(),
19048 summary: "内部レビュー".into(),
19049 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19050 vote: None,
19051 failed: None,
19052 duration_ms: 0,
19053 });
19054 round
19055 .fix
19056 .as_mut()
19057 .unwrap()
19058 .rejected
19059 .push(crate::verdict::Rejection {
19060 id: "R1-1-1".into(),
19061 why: "呼び出し元から到達できない".into(),
19062 });
19063 f.runner.state.reviews = vec![round];
19064 let omitted = posted(&mut f).await;
19066 assert!(omitted.body.contains("No description"));
19067 assert!(omitted.body.contains("R1-1-1"));
19068 assert!(!omitted.body.contains("到達"));
19069 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19070 let described = posted(&mut f).await;
19071 assert!(described.body.contains("unreachable from callers"));
19072 assert!(described.body.contains("R1-1-1"));
19073 assert!(!described.body.contains("到達"));
19074 let evidence = f
19075 .runner
19076 .state
19077 .change_summary
19078 .as_ref()
19079 .unwrap()
19080 .evidence
19081 .clone();
19082 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19083 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19084 assert!(posted(&mut f).await.body.contains("rationale updated"));
19085 assert_ne!(
19086 f.runner.state.change_summary.as_ref().unwrap().evidence,
19087 evidence
19088 );
19089 f.runner
19091 .state
19092 .change_summary
19093 .as_mut()
19094 .unwrap()
19095 .evidence
19096 .clear();
19097 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19098 assert!(posted(&mut f).await.body.contains("current specification"));
19099 }
19100
19101 #[tokio::test]
19102 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19103 let mut f = summary_fixture();
19104 let m = posted(&mut f).await;
19105 assert_eq!(m.title, "feat(own): add the own file");
19106 assert!(
19107 m.body
19108 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19109 "{}",
19110 m.body
19111 );
19112 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19113 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19114 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19115 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19116 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19117 let kept = f.runner.state.change_summary.as_ref().unwrap();
19118 assert_eq!(
19119 (kept.head.as_str(), kept.base.as_str()),
19120 (head.as_str(), base.as_str())
19121 );
19122 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19123 }
19124
19125 #[tokio::test]
19126 async fn review_only_summary_receives_the_configured_github_language() {
19127 let mut f = summary_fixture();
19128 f.runner.state.config.graph.github_language = "ja".into();
19129 let captured = f.repo.join("summary-prompt.txt");
19130 let author = &mut f.runner.state.config.agents[0];
19131 author.command = vec![
19132 "sh".into(),
19133 "-c".into(),
19134 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19135 ];
19136 author.env.insert(
19137 "PROMPT_FILE".into(),
19138 captured.to_string_lossy().into_owned(),
19139 );
19140 std::fs::write(
19141 &f.reply,
19142 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19143 )
19144 .unwrap();
19145 let m = posted(&mut f).await;
19146 let prompt = std::fs::read_to_string(captured).unwrap();
19147 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19148 assert!(!prompt.contains("in English"));
19149 assert_eq!(m.title, "feat(own): ファイルを追加");
19150 assert!(m.body.contains("own.txt を追加します。"));
19151 }
19152
19153 #[tokio::test]
19154 async fn review_fixes_come_after_the_whole_change_summary() {
19155 let mut f = summary_fixture();
19156 f.runner.state.reviews = vec![
19157 round_with_notes(1, Some("handled the PATH case")),
19158 round_with_notes(2, None),
19159 ];
19160 let m = posted(&mut f).await;
19161 let summary = m.body.find("## Summary").expect("summary");
19162 let fixes = m.body.find("## Review rounds").expect("review rounds");
19163 assert!(summary < fixes, "{}", m.body);
19164 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19165 assert!(m.body.contains("## Final verification"), "{}", m.body);
19166 }
19167
19168 #[tokio::test]
19169 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19170 let mut f = summary_fixture();
19171 let first = posted(&mut f).await;
19172 let first_head = f.runner.state.change_summary.clone().unwrap().head;
19173 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19175 let again = posted(&mut f).await;
19176 assert_eq!((again.title, again.body), (first.title, first.body));
19177
19178 let g = |args: &[&str]| {
19179 let out = std::process::Command::new("git")
19180 .args(args)
19181 .current_dir(&f.repo)
19182 .quiet()
19183 .output()
19184 .expect("spawn git");
19185 assert!(out.status.success(), "git {args:?}");
19186 };
19187 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19188 g(&["add", "-A"]);
19189 g(&["commit", "-q", "-m", "fix(own): handle more"]);
19190 let second = posted(&mut f).await;
19191 let kept = f.runner.state.change_summary.clone().unwrap();
19192 assert_ne!(kept.head, first_head);
19193 assert!(second.body.contains("Not read yet."), "{}", second.body);
19194 assert!(!second.body.contains("so the first summary is read"));
19195 assert_eq!(second.title, "feat(own): other");
19196 }
19197
19198 #[tokio::test]
19199 async fn an_unavailable_summary_is_stated_and_never_invented() {
19200 let mut f = summary_fixture();
19201 std::fs::write(&f.reply, "no title line here").unwrap();
19202 let m = posted(&mut f).await;
19203 assert!(
19204 m.body
19205 .contains("No description of what this change does could be generated"),
19206 "{}",
19207 m.body
19208 );
19209 assert!(m.body.contains("internal reports remain in local records"));
19210 assert!(f.runner.state.change_summary.is_none());
19211 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19212 let winner = f.runner.state.candidates[0].clone();
19214 let preview = f
19215 .runner
19216 .guarded_pr_message(&winner, None, false)
19217 .await
19218 .unwrap()
19219 .unwrap();
19220 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19221 }
19222
19223 #[test]
19224 fn pr_message_review_single_commit_uses_its_subject() {
19225 let state = review_state(&["feat(nats): per-role user"]);
19226 let m = pr_message(&state, 'A');
19227 assert_eq!(m.title, "feat(nats): per-role user");
19228 assert!(!m.body.contains("Review the work already"), "{}", m.body);
19229 assert!(m.body.contains("## Commits under review"), "{}", m.body);
19230 }
19231
19232 #[test]
19233 fn pr_message_review_multi_commit_takes_the_oldest() {
19234 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19235 let m = pr_message(&state, 'A');
19236 assert_eq!(m.title, "feat: the change");
19237 for s in ["feat: the change", "fix: typo", "fix: again"] {
19238 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19239 }
19240 }
19241
19242 #[test]
19243 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19244 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19245 let mut state = review_state(&[first, "fix: later fixup"]);
19246 state.candidates[0].branch = "機能/ログイン".to_owned();
19247 let m = pr_message(&state, 'A');
19248 assert!(
19249 m.title.starts_with("chore: land candidate A of run"),
19250 "{}",
19251 m.title
19252 );
19253 }
19254 }
19255
19256 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19257 BranchFacts {
19258 commits: commits
19259 .iter()
19260 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19261 .collect(),
19262 stat: stat.to_owned(),
19263 }
19264 }
19265
19266 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19267 let mut r = review_round(true, 0, 1, 1, true, true);
19268 r.round = round;
19269 r.fix = notes.map(|n| FixRecord {
19270 agent: "fixer".to_owned(),
19271 addressed: Vec::new(),
19272 rejected: Vec::new(),
19273 notes: n.to_owned(),
19274 committed: true,
19275 failed: None,
19276 duration_ms: 0,
19277 continuation: None,
19278 });
19279 r
19280 }
19281
19282 #[test]
19283 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19284 let state = review_state(&["ignored"]);
19285 let f = facts(
19286 &[
19287 (
19288 "fix(login): resolve PATH on macOS",
19289 "Login shells skip rc files.",
19290 ),
19291 ("fix: address review", ""),
19292 ],
19293 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19294 );
19295 let m = pr_message_with(&state, 'A', Some(&f));
19296 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19297 assert!(
19298 m.body
19299 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
19300 "{}",
19301 m.body
19302 );
19303 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19304 assert!(m.body.contains("## Diff stat"), "{}", m.body);
19305 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19306 for banned in [
19307 "Review the work already",
19308 "no task statement",
19309 "Original task",
19310 ] {
19311 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19312 }
19313 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19314 }
19315
19316 #[test]
19317 fn pr_message_review_truncates_a_huge_first_commit_body() {
19318 let state = review_state(&["ignored"]);
19319 let f = facts(
19320 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19321 "s",
19322 );
19323 let m = pr_message_with(&state, 'A', Some(&f));
19324 assert!(m.body.len() < 30_000, "{}", m.body.len());
19325 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19326 assert!(
19327 m.body.contains("- ... 1 more commit(s)"),
19328 "{}",
19329 m.body.len()
19330 );
19331 assert!(m.body.ends_with("magi:candidate-a\n"));
19332 }
19333
19334 #[test]
19335 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19336 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19337 assert_eq!(m.title, "feat: x");
19338 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19339 assert!(!m.body.contains("Diff stat"), "{}", m.body);
19340 assert!(!m.body.contains("no task statement"), "{}", m.body);
19341 }
19342
19343 #[test]
19344 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19345 let mut state = review_state(&["日本語の件名"]);
19346 state.candidates[0].branch = "fix/macos-login-path".to_owned();
19347 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19348 state.candidates[0].branch = "機能/ログイン".to_owned();
19349 assert!(
19350 pr_message(&state, 'A')
19351 .title
19352 .starts_with("chore: land candidate A")
19353 );
19354 }
19355
19356 #[test]
19357 fn pr_message_review_fixes_survive_a_clean_final_round() {
19358 let mut state = review_state(&["feat: x"]);
19359 state.reviews = vec![
19360 round_with_notes(1, Some("handled the PATH case")),
19361 round_with_notes(2, None),
19362 ];
19363 let body = pr_message(&state, 'A').body;
19364 assert!(
19365 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19366 "{body}"
19367 );
19368 assert!(body.contains("### Round 1"), "{body}");
19369
19370 state.reviews = vec![
19371 round_with_notes(1, Some("first fix")),
19372 round_with_notes(2, Some("")),
19373 round_with_notes(3, Some("second fix")),
19374 round_with_notes(4, None),
19375 ];
19376 let body = pr_message(&state, 'A').body;
19377 assert!(
19378 body.contains("### Round 1") && !body.contains("first fix"),
19379 "{body}"
19380 );
19381 assert!(
19382 body.contains("### Round 3") && !body.contains("second fix"),
19383 "{body}"
19384 );
19385 assert!(body.contains("### Round 2"), "{body}");
19386 }
19387
19388 const FINAL: &str = "abcdef1234567890";
19389
19390 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
19391 let mut r = round_with_notes(round, None);
19392 r.verified_head = verified.map(str::to_owned);
19393 r
19394 }
19395
19396 fn ok_gate(state: &mut RunState, head: Option<&str>) {
19397 state.gate = vec![CommandOutcome {
19398 command: "gate".to_owned(),
19399 code: Some(0),
19400 output_tail: String::new(),
19401 duration_ms: 0,
19402 resource_blocked: false,
19403 }];
19404 state.gate_ran = true;
19405 state.gate_head = head.map(str::to_owned);
19406 }
19407
19408 fn body_at(state: &RunState) -> String {
19409 pr_message_raw(
19410 state,
19411 'A',
19412 None,
19413 crate::pr_outcome::FinalHead::At(FINAL),
19414 ChangeNote::NotAsked,
19415 )
19416 .body
19417 }
19418
19419 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
19420
19421 #[test]
19422 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
19423 let mut state = review_state(&["feat: x"]);
19424 state.reviews = vec![
19425 round_with_notes(1, Some(SANDBOXED)),
19426 passing_round(2, Some(FINAL)),
19427 ];
19428 ok_gate(&mut state, Some(FINAL));
19429 let body = body_at(&state);
19430 assert!(!body.contains("Handled the PATH case."), "{body}");
19431 assert!(!body.contains("could not commit"), "{body}");
19432 assert!(!body.contains("could not run"), "{body}");
19433 assert!(
19434 body.contains("The fix is committed on the branch"),
19435 "{body}"
19436 );
19437 assert!(!body.contains("sandbox"), "{body}");
19438 assert!(
19439 body.contains("Tests (`verify.e2e`): passed on the final commit"),
19440 "{body}"
19441 );
19442 assert!(
19443 body.contains("Gate (`verify.gate`): passed on the final commit"),
19444 "{body}"
19445 );
19446 assert!(
19448 state.reviews[0]
19449 .fix
19450 .as_ref()
19451 .unwrap()
19452 .notes
19453 .contains("could not commit")
19454 );
19455 }
19456
19457 #[test]
19458 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
19459 let mut state = review_state(&["feat: x"]);
19460 let mut r1 = round_with_notes(1, Some(SANDBOXED));
19461 r1.fix.as_mut().unwrap().committed = false;
19462 let mut r2 = passing_round(2, Some(FINAL));
19463 r2.e2e[0].code = Some(1);
19464 state.reviews = vec![r1, r2];
19465 let body = body_at(&state);
19466 assert!(!body.contains("could not commit"), "{body}");
19467 assert!(!body.contains("could not run"), "{body}");
19468 assert!(body.contains("This round produced no commit"), "{body}");
19469 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
19470 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
19471 assert!(!body.contains("passed"), "{body}");
19472 }
19473
19474 #[test]
19475 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
19476 let mut state = review_state(&["feat: x"]);
19477 state.reviews = vec![
19478 round_with_notes(1, Some(SANDBOXED)),
19479 passing_round(2, Some("0123456789abcdef")),
19480 ];
19481 ok_gate(&mut state, Some("0123456789abcdef"));
19482 let body = body_at(&state);
19483 assert!(!body.contains("HTTP tests could not run"), "{body}");
19484 assert!(!body.contains("could not commit"), "{body}");
19486 assert!(
19487 body.contains("passed on an earlier commit (`0123456`)"),
19488 "{body}"
19489 );
19490 assert!(!body.contains("passed on the final commit"), "{body}");
19491
19492 ok_gate(&mut state, None);
19494 let body = body_at(&state);
19495 assert!(
19496 body.contains("passed on a commit that was not recorded"),
19497 "{body}"
19498 );
19499 }
19500
19501 #[test]
19502 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
19503 let state = state_with_summary(
19504 "add retries to the client",
19505 "TITLE: feat: retries\n\nDid it.",
19506 );
19507 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
19508 assert_eq!(m.title, "feat: retries");
19509 assert!(
19510 m.body.contains("<summary>Original task</summary>"),
19511 "{}",
19512 m.body
19513 );
19514 assert!(!m.body.contains("Commits under review"), "{}", m.body);
19515 assert!(
19516 m.body
19517 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
19518 "{}",
19519 m.body
19520 );
19521 }
19522
19523 #[test]
19524 fn pr_message_review_bounds_a_long_english_subject() {
19525 let long = format!("feat: {}", "word ".repeat(100));
19526 let m = pr_message(&review_state(&[&long]), 'A');
19527 assert!(m.title.starts_with("feat: word"), "{}", m.title);
19528 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19529 }
19530
19531 #[test]
19532 fn pr_message_implementation_run_is_unchanged_by_review_support() {
19533 let state = state_with_summary("add retries\n\ndetails", "- did some things");
19534 let m = pr_message(&state, 'A');
19535 assert_eq!(m.title, "add retries");
19536 assert!(m.body.contains("<summary>Original task</summary>"));
19537 assert!(!m.body.contains("Commits under review"));
19538 assert_eq!(landing_subject_source(&state), state.instruction);
19539 }
19540
19541 #[test]
19542 fn review_run_squash_subject_is_the_change_not_the_prompt() {
19543 let state = review_state(&["feat: the change", "fix: typo"]);
19544 let source = landing_subject_source(&state);
19545 assert_eq!(land::merge_subject("", &source), "feat: the change");
19546 assert_eq!(
19547 land::merge_subject("magi: candidate A (uncommitted work)", &source),
19548 "feat: the change"
19549 );
19550 assert_eq!(
19552 land::merge_subject("feat: renamed by hand", &source),
19553 "feat: renamed by hand"
19554 );
19555 let blank = review_state(&["日本語"]);
19556 assert!(
19557 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
19558 );
19559 }
19560
19561 #[test]
19562 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
19563 let state = review_state(&["feat: the change"]);
19564 let source = landing_subject_source(&state);
19565 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
19566 assert_eq!(
19567 land::merge_subject(landing_title(&state, old), &source),
19568 "feat: the change"
19569 );
19570 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
19571 let task = state_with_summary("add retries", "");
19572 assert_eq!(landing_title(&task, old), old);
19573 }
19574
19575 #[test]
19576 fn pr_message_describes_the_change_not_the_task() {
19577 let state = state_with_summary(
19578 "今回やってほしいこと: results projector を直す",
19579 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
19580 );
19581 let m = pr_message(&state, 'A');
19582 assert_eq!(m.title, "fix(web): batch the runs list reads");
19583 assert!(
19584 m.body.starts_with("## Summary\n\n- reads run.json once"),
19585 "{}",
19586 m.body
19587 );
19588 assert!(!m.body.contains("TITLE:"), "{}", m.body);
19589 let task_at = m.body.find("今回やってほしいこと").unwrap();
19590 let details_at = m.body.find("<details>").unwrap();
19591 assert!(
19592 details_at < task_at,
19593 "the task lives inside <details>: {}",
19594 m.body
19595 );
19596 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
19597 assert!(m.body.contains("magi:candidate-a"));
19598 }
19599
19600 #[test]
19601 fn pr_message_falls_back_to_the_task_without_a_title_line() {
19602 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
19603 let m = pr_message(&state, 'A');
19604 assert_eq!(m.title, "add retries");
19605 assert!(
19606 m.body.contains("## Summary\n\n- did some things"),
19607 "{}",
19608 m.body
19609 );
19610
19611 let none = RunState::new(
19612 PathBuf::from("/repo"),
19613 "main".to_owned(),
19614 "abc1234".to_owned(),
19615 "add retries".to_owned(),
19616 Config::default(),
19617 );
19618 let m = pr_message(&none, 'A');
19619 assert_eq!(m.title, "add retries");
19620 assert!(!m.body.contains("## Summary"), "{}", m.body);
19621 }
19622
19623 #[test]
19624 fn pr_message_refuses_the_candidate_commit_subject() {
19625 for bad in [
19626 "TITLE: magi: candidate A (uncommitted work)",
19627 "TITLE: chore: stuff (uncommitted work)",
19628 "TITLE: ",
19629 ] {
19630 let state = state_with_summary("add retries", bad);
19631 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
19632 }
19633 }
19634
19635 #[test]
19636 fn pr_message_bounds_a_very_long_task_and_title() {
19637 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
19638 let state = state_with_summary(&long, "- nothing");
19639 let m = pr_message(&state, 'A');
19640 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19641 assert!(!m.title.contains('\n'));
19642
19643 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
19644 let m = pr_message(&state, 'A');
19645 assert!(m.title.starts_with("feat: "));
19646 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19647 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
19648 }
19649
19650 fn long_title_of(instruction: &str) -> String {
19651 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
19652 }
19653
19654 #[test]
19655 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
19656 let first = "Make the landing path keep a readable title for long tasks";
19657 let line = format!(
19658 "{first}. {}",
19659 "And then keep going with more words ".repeat(20)
19660 );
19661 let t = long_title_of(&line);
19662 assert_eq!(t, first);
19663 assert!(!t.starts_with("chore: land"));
19664 }
19665
19666 #[test]
19667 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
19668 let line = "word ".repeat(200);
19669 let t = long_title_of(&line);
19670 assert!(t.ends_with("word..."), "{t}");
19671 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
19672 }
19673
19674 #[test]
19675 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
19676 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
19677 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
19678 }
19679 }
19680
19681 #[test]
19682 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
19683 let mut state = state_with_summary("再試行を追加する", "- nothing");
19684 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
19685 state.config.graph.github_language = "ja".into();
19686 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
19687 }
19688
19689 #[test]
19690 fn pr_message_title_limit_is_exact() {
19691 let at = "a".repeat(PR_TITLE_MAX);
19692 assert_eq!(long_title_of(&at), at);
19693 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
19694 assert!(over.ends_with("..."), "{over}");
19695 assert_eq!(over.chars().count(), PR_TITLE_MAX);
19696 }
19697
19698 #[test]
19699 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
19700 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
19701 let t = long_title_of(&line);
19702 assert!(t.ends_with("..."), "{t}");
19703 assert!(t.is_ascii(), "{t}");
19704 }
19705
19706 #[test]
19707 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
19708 let line = format!(
19709 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
19710 "plus more filler words ".repeat(20)
19711 );
19712 let t = long_title_of(&line);
19713 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
19714 }
19715
19716 #[test]
19717 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
19718 let line = format!("\n\n# {}", "title words ".repeat(40));
19719 let state = state_with_summary(&line, "- nothing");
19720 let m = pr_message(&state, 'A');
19721 assert!(m.title.starts_with("title words"), "{}", m.title);
19722 assert_eq!(
19723 land::merge_subject(&m.title, &landing_subject_source(&state)),
19724 m.title
19725 );
19726 assert_eq!(
19728 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
19729 "feat: renamed by hand"
19730 );
19731 }
19732
19733 #[tokio::test]
19734 async fn github_text_rewrite_is_bounded_and_falls_back() {
19735 crate::run::pin_test_home();
19736 for (reply, accepted) in [
19737 (
19738 "TITLE: fix: retries\nAdd retries for failed requests.",
19739 true,
19740 ),
19741 (
19742 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
19743 false,
19744 ),
19745 (
19746 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
19747 true,
19748 ),
19749 (
19750 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
19751 true,
19752 ),
19753 (
19754 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
19755 true,
19756 ),
19757 (
19758 "TITLE: fix: retries\n日本語の説明です。 token=secret",
19759 false,
19760 ),
19761 ("TITLE: fix: retries\n", false),
19762 ] {
19763 let dir = tempfile::tempdir().unwrap();
19764 let mut runner = runner_at(RunStatus::Gating);
19765 runner.state = state_with_summary(
19766 "add retries",
19767 "TITLE: fix: retries\n日本語の説明を書きます。",
19768 );
19769 runner.state.repo = dir.path().to_owned();
19770 runner.state.candidates[0].worktree = dir.path().to_owned();
19771 let mut author = spec("alpha");
19772 author.command = vec![
19773 "sh".into(),
19774 "-c".into(),
19775 "printf '%s' \"$REWRITE_REPLY\"".into(),
19776 ];
19777 author.env.insert("REWRITE_REPLY".into(), reply.into());
19778 runner.state.config.agents = vec![author];
19779 seed_public_summary(&mut runner);
19780 let winner = runner.state.candidates[0].clone();
19781 let first = runner
19782 .guarded_pr_message(&winner, None, true)
19783 .await
19784 .unwrap();
19785 let message = if accepted {
19786 first.expect("a passing rewrite never asks")
19787 } else {
19788 assert!(first.is_none());
19790 assert!(runner.state.parked);
19791 let store = ask::Questions::open();
19792 let asked: Vec<_> = store
19793 .list()
19794 .into_iter()
19795 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19796 .collect();
19797 assert_eq!(asked.len(), 1);
19798 assert!(!asked[0].detail.contains("secret"));
19799 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
19800 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
19801 assert_eq!(
19802 asked[0]
19803 .choices
19804 .contains(&crate::github_text::USE_REWRITE.to_owned()),
19805 rewrite.is_some()
19806 );
19807 store
19810 .update(&asked[0].id, |q| {
19811 q.abandon("test");
19812 Ok(())
19813 })
19814 .unwrap();
19815 let again = runner
19816 .guarded_pr_message(&winner, None, true)
19817 .await
19818 .unwrap()
19819 .expect("an abandoned question falls back");
19820 let n = store
19821 .list()
19822 .into_iter()
19823 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19824 .count();
19825 assert_eq!(n, 1, "asked once per rejected text");
19826 again
19827 };
19828 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
19829 assert!(!message.body.contains("secret"));
19830 assert!(!message.body.contains("prefix"));
19831 assert!(!message.title.contains("secret"));
19832 assert_eq!(
19833 message.body.contains("Add retries for failed requests."),
19834 accepted
19835 );
19836 assert_eq!(
19837 runner.state.seats["github-text-rewrite"].turns, 1,
19838 "only one rewrite invocation"
19839 );
19840 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
19841 if accepted {
19842 assert!(runner.state.github_text.is_none());
19843 assert!(!runner.state.parked);
19844 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
19845 assert!(message.body.contains("## Final verification"));
19846 assert!(message.body.contains("Evidence: final diff"));
19847 assert!(
19848 message
19849 .body
19850 .contains(&format!("magi:run/{}", runner.state.id))
19851 );
19852 }
19853 if !accepted {
19854 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19855 assert!(
19856 message
19857 .body
19858 .contains(&format!("magi:run/{}", runner.state.id))
19859 );
19860 }
19861 }
19862 }
19863
19864 fn scripted(id: &str, reply: &str) -> AgentSpec {
19865 let mut a = spec(id);
19866 a.command = vec![
19867 "sh".into(),
19868 "-c".into(),
19869 "printf '%s' \"$REWRITE_REPLY\"".into(),
19870 ];
19871 a.env.insert("REWRITE_REPLY".into(), reply.into());
19872 a
19873 }
19874
19875 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
19876 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
19877
19878 fn seed_public_summary(runner: &mut Runner) {
19880 let repo = &runner.state.repo;
19881 let g = |args: &[&str]| {
19882 let out = std::process::Command::new("git")
19883 .args(args)
19884 .current_dir(repo)
19885 .quiet()
19886 .output()
19887 .unwrap();
19888 assert!(out.status.success(), "git {args:?}");
19889 String::from_utf8_lossy(&out.stdout).trim().to_owned()
19890 };
19891 g(&["init", "-q", "-b", "main"]);
19892 g(&[
19893 "-c",
19894 "user.name=Test",
19895 "-c",
19896 "user.email=test@example.invalid",
19897 "commit",
19898 "-q",
19899 "--allow-empty",
19900 "-m",
19901 "feat: fixture",
19902 ]);
19903 let head = g(&["rev-parse", "HEAD"]);
19904 g(&["update-ref", "refs/remotes/origin/main", &head]);
19905 let evidence = crate::pr_outcome::summary_evidence(
19906 &runner.state,
19907 crate::pr_outcome::FinalHead::At(&head),
19908 );
19909 let generated = &runner.state.candidates[0].summary;
19910 runner.state.change_summary = Some(crate::run::ChangeSummary {
19911 base: head.clone(),
19912 head,
19913 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
19914 title: summary_title(generated).unwrap(),
19915 body: summary_without_title(generated),
19916 });
19917 }
19918
19919 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19920 let mut runner = runner_at(RunStatus::Gating);
19921 runner.state = state_with_summary("add retries", JA_SUMMARY);
19922 if review_only {
19923 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19924 }
19925 runner.state.repo = dir.to_owned();
19926 runner.state.candidates[0].worktree = dir.to_owned();
19927 runner.state.config.agents = agents;
19928 seed_public_summary(&mut runner);
19929 runner
19930 }
19931
19932 fn gate_questions(runner: &Runner) -> usize {
19933 ask::Questions::open()
19934 .list()
19935 .into_iter()
19936 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19937 .count()
19938 }
19939
19940 #[tokio::test]
19941 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19942 crate::run::pin_test_home();
19943 let dir = tempfile::tempdir().unwrap();
19944 let mut runner = rewrite_runner(
19945 dir.path(),
19946 true,
19947 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19948 );
19949 runner.state.config.roles.language_judge =
19950 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19951 let winner = runner.state.candidates[0].clone();
19952 let msg = runner
19953 .guarded_pr_message(&winner, None, true)
19954 .await
19955 .unwrap()
19956 .expect("an English rewrite posts without asking");
19957 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19958 assert!(msg.body.contains("<details>kept</details>"));
19959 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19960 assert_eq!(gate_questions(&runner), 0);
19961 assert!(runner.state.github_text.is_none());
19962 assert!(!runner.state.seats.contains_key("impl-A"));
19963 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19964 }
19965
19966 #[tokio::test]
19967 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19968 crate::run::pin_test_home();
19969 let dir = tempfile::tempdir().unwrap();
19970 let mut runner = rewrite_runner(
19971 dir.path(),
19972 false,
19973 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19974 );
19975 runner.state.config.roles.language_judge =
19976 Some(crate::config::AgentChoice::One("beta".into()));
19977 runner.state.seats.insert(
19978 "impl-A".into(),
19979 SeatState::new("impl-A", "alpha", runner.state.seed),
19980 );
19981 let winner = runner.state.candidates[0].clone();
19982 let msg = runner
19983 .guarded_pr_message(&winner, None, true)
19984 .await
19985 .unwrap()
19986 .expect("rewritten");
19987 assert!(msg.body.contains("<details>kept</details>"));
19988 assert_eq!(runner.state.seats["impl-A"].turns, 0);
19989 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19990 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19991 }
19992
19993 #[tokio::test]
19994 async fn rewrite_reuses_a_resumable_implementer_session() {
19995 crate::run::pin_test_home();
19996 let dir = tempfile::tempdir().unwrap();
19997 let mut runner = rewrite_runner(
19998 dir.path(),
19999 false,
20000 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20001 );
20002 runner.state.config.roles.language_judge =
20003 Some(crate::config::AgentChoice::One("beta".into()));
20004 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20005 seat.turns = 1;
20006 runner.state.seats.insert("impl-A".into(), seat);
20007 let winner = runner.state.candidates[0].clone();
20008 runner
20009 .guarded_pr_message(&winner, None, true)
20010 .await
20011 .unwrap()
20012 .expect("rewritten");
20013 assert_eq!(runner.state.seats["impl-A"].turns, 2);
20014 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20015 }
20016
20017 #[tokio::test]
20018 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20019 crate::run::pin_test_home();
20022 let dir = tempfile::tempdir().unwrap();
20023 let mut exhausted = spec("alpha");
20024 exhausted.command = vec!["false".into()];
20025 let mut runner = rewrite_runner(
20026 dir.path(),
20027 false,
20028 vec![exhausted, scripted("beta", EN_REWRITE)],
20029 );
20030 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20032 "alpha".into(),
20033 "beta".into(),
20034 ]));
20035 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20036 seat.turns = 1;
20037 runner.state.seats.insert("impl-A".into(), seat);
20038 let winner = runner.state.candidates[0].clone();
20039 let msg = runner
20040 .guarded_pr_message(&winner, None, true)
20041 .await
20042 .unwrap()
20043 .expect("the role rewrites after the implementer fails");
20044 assert!(msg.body.contains("`src/net.rs`"));
20045 assert_eq!(
20046 runner.state.seats["impl-A"].turns, 2,
20047 "the implementer was tried once"
20048 );
20049 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20050 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20051 }
20052
20053 #[tokio::test]
20054 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20055 crate::run::pin_test_home();
20056 let dir = tempfile::tempdir().unwrap();
20057 let mut broken = spec("bad");
20058 broken.command = vec!["false".into()];
20059 let mut runner = rewrite_runner(
20060 dir.path(),
20061 true,
20062 vec![
20063 broken,
20064 scripted("noformat", "Add retries."),
20065 scripted("good", EN_REWRITE),
20066 ],
20067 );
20068 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20069 "bad".into(),
20070 "noformat".into(),
20071 "good".into(),
20072 ]));
20073 let winner = runner.state.candidates[0].clone();
20074 let msg = runner
20075 .guarded_pr_message(&winner, None, true)
20076 .await
20077 .unwrap()
20078 .expect("the third agent rewrites");
20079 assert!(msg.body.contains("`src/net.rs`"));
20080 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20081 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20082 }
20083
20084 #[tokio::test]
20085 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20086 crate::run::pin_test_home();
20087 for (agents, role, saved) in [
20089 (
20090 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20091 Some("beta"),
20092 true,
20093 ),
20094 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20095 (Vec::new(), None, false),
20096 ] {
20097 let dir = tempfile::tempdir().unwrap();
20098 let mut runner = rewrite_runner(dir.path(), true, agents);
20099 runner.state.config.roles.language_judge =
20100 role.map(|r| crate::config::AgentChoice::One(r.into()));
20101 let winner = runner.state.candidates[0].clone();
20102 let first = runner
20103 .guarded_pr_message(&winner, None, true)
20104 .await
20105 .unwrap();
20106 assert!(first.is_none(), "withheld text parks the run");
20107 assert!(runner.state.parked);
20108 assert_eq!(gate_questions(&runner), 1);
20109 let g = runner.state.github_text.as_ref().unwrap();
20110 assert_eq!(g.rewrite.is_some(), saved);
20111 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20112 }
20113 }
20114
20115 #[tokio::test]
20116 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20117 crate::run::pin_test_home();
20118 let dir = tempfile::tempdir().unwrap();
20119 let mut runner = runner_at(RunStatus::Gating);
20120 runner.state =
20121 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20122 runner.state.repo = dir.path().to_owned();
20123 runner.state.candidates[0].worktree = dir.path().to_owned();
20124 let mut author = spec("alpha");
20125 author.command = vec![
20126 "sh".into(),
20127 "-c".into(),
20128 "printf '%s' \"$REWRITE_REPLY\"".into(),
20129 ];
20130 author.env.insert(
20131 "REWRITE_REPLY".into(),
20132 format!(
20133 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20134 "English sentence. ".repeat(600)
20135 ),
20136 );
20137 runner.state.config.agents = vec![author];
20138 seed_public_summary(&mut runner);
20139 let winner = runner.state.candidates[0].clone();
20140 assert!(
20141 runner
20142 .guarded_pr_message(&winner, None, true)
20143 .await
20144 .unwrap()
20145 .is_none()
20146 );
20147 let record = runner.state.github_text.clone().unwrap();
20148 let rewrite = record.rewrite.clone().unwrap();
20149 assert_eq!(rewrite.failed, ["body-language"]);
20150 assert!(
20151 rewrite
20152 .body
20153 .ends_with(&format!("magi:run/{}", runner.state.id))
20154 );
20155 let store = ask::Questions::open();
20156 let q = store.get(record.question.as_ref().unwrap()).unwrap();
20157 assert!(
20158 q.choices
20159 .contains(&crate::github_text::USE_REWRITE.to_owned())
20160 );
20161 for label in [
20162 "ORIGINAL",
20163 "FALLBACK",
20164 "REWRITE",
20165 "Rules still failed: body-language",
20166 "truncated",
20167 ] {
20168 assert!(q.detail.contains(label), "{label}");
20169 }
20170 let artifact = std::fs::read_to_string(crate::run::artifact_path(
20171 &runner.state,
20172 &crate::github_text::artifact_name(&record.fingerprint),
20173 ))
20174 .unwrap();
20175 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20176 runner.state =
20178 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20179 store
20180 .update(&q.id, |q| {
20181 q.say("まだ日本語")?;
20182 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20183 })
20184 .unwrap();
20185 assert!(
20186 runner
20187 .guarded_pr_message(&winner, None, true)
20188 .await
20189 .unwrap()
20190 .is_none()
20191 );
20192 let retry = runner.state.github_text.as_ref().unwrap();
20193 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20194 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20195 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20196 assert!(
20197 q.posting_comparison
20198 .as_ref()
20199 .unwrap()
20200 .matches_detail(&q.detail)
20201 );
20202 store
20203 .update(&q.id, |q| {
20204 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20205 })
20206 .unwrap();
20207 let message = runner
20208 .guarded_pr_message(&winner, None, true)
20209 .await
20210 .unwrap()
20211 .unwrap();
20212 assert_eq!(message.title, rewrite.title);
20213 assert_eq!(message.body, rewrite.body);
20214 assert_eq!(
20215 runner.state.github_text.as_ref().unwrap().choice,
20216 Some(GithubTextChoice::Rewrite)
20217 );
20218 let again = runner
20219 .guarded_pr_message(&winner, None, true)
20220 .await
20221 .unwrap()
20222 .unwrap();
20223 assert_eq!(again.title, message.title);
20224 assert_eq!(again.body, message.body);
20225 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20226 assert!(
20227 !runner
20228 .state
20229 .events
20230 .iter()
20231 .any(|e| e.message.contains(&rewrite.title)
20232 || e.message.contains("English sentence."))
20233 );
20234 }
20235
20236 #[tokio::test]
20237 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20238 crate::run::pin_test_home();
20239 let dir = tempfile::tempdir().unwrap();
20240 let mut runner = runner_at(RunStatus::Gating);
20241 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20242 runner.state.repo = dir.path().to_owned();
20243 runner.state.candidates[0].worktree = dir.path().to_owned();
20244 seed_public_summary(&mut runner);
20245 let winner = runner.state.candidates[0].clone();
20246 assert!(
20247 runner
20248 .guarded_pr_message(&winner, None, true)
20249 .await
20250 .unwrap()
20251 .is_none()
20252 );
20253 let record = runner.state.github_text.as_ref().unwrap();
20254 assert!(record.rewrite.is_none());
20255 let mut old = serde_json::to_value(record).unwrap();
20257 old.as_object_mut().unwrap().remove("rewrite");
20258 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20259 assert!(old.rewrite.is_none());
20260 let id = record.question.clone().unwrap();
20261 ask::Questions::open()
20262 .update(&id, |q| {
20263 q.choices.push(crate::github_text::USE_REWRITE.into());
20264 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20265 })
20266 .unwrap();
20267 let message = runner
20268 .guarded_pr_message(&winner, None, true)
20269 .await
20270 .unwrap()
20271 .unwrap();
20272 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20273 assert_eq!(
20274 runner.state.github_text.as_ref().unwrap().choice,
20275 Some(GithubTextChoice::Fallback)
20276 );
20277 }
20278
20279 #[tokio::test]
20280 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20281 crate::run::pin_test_home();
20282 let dir = tempfile::tempdir().unwrap();
20283 let mut runner = runner_at(RunStatus::Gating);
20284 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20285 runner.state.repo = dir.path().to_owned();
20286 runner.state.candidates[0].worktree = dir.path().to_owned();
20287 seed_public_summary(&mut runner);
20288 let winner = runner.state.candidates[0].clone();
20289 assert!(
20290 runner
20291 .guarded_pr_message(&winner, None, true)
20292 .await
20293 .unwrap()
20294 .is_none()
20295 );
20296 let store = ask::Questions::open();
20297 let find = |run: &str| {
20298 store
20299 .list()
20300 .into_iter()
20301 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20302 .unwrap()
20303 };
20304 let run = runner.state.id.clone();
20305 let q = find(&run);
20306 let fp = runner
20307 .state
20308 .github_text
20309 .as_ref()
20310 .unwrap()
20311 .fingerprint
20312 .clone();
20313 let name = crate::github_text::artifact_name(&fp);
20314 let path = crate::run::artifact_path(&runner.state, &name);
20315 assert!(q.detail.contains(&path.display().to_string()));
20316 assert!(q.detail.contains("title-language: title"));
20317 let saved = std::fs::read_to_string(&path).expect("artifact written");
20318 assert!(saved.contains("日本語のタイトル"));
20319 assert!(q.cwd.is_none());
20320 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20321 let say = |id: &str, text: &str, choice: &str| {
20323 store
20324 .update(id, |q| {
20325 q.thread.push(ask::Turn {
20326 who: ask::Who::Operator,
20327 body: text.to_owned(),
20328 at: jiff::Timestamp::now(),
20329 note: None,
20330 });
20331 q.answer(ask::Answer::Choice(choice.to_owned()))
20332 })
20333 .unwrap();
20334 };
20335 say(&q.id, "まだ日本語", "use my text");
20336 assert!(
20337 runner
20338 .guarded_pr_message(&winner, None, true)
20339 .await
20340 .unwrap()
20341 .is_none()
20342 );
20343 let second = find(&run);
20344 assert_ne!(second.id, q.id);
20345 assert!(second.detail.contains("did not pass"));
20346 assert!(second.detail.contains("日本語のタイトル"));
20349 assert!(!second.detail.contains("まだ日本語"));
20350 assert!(second.detail.contains(&path.display().to_string()));
20351 say(&second.id, "fix: retry failed requests", "use my text");
20352 let message = runner
20353 .guarded_pr_message(&winner, None, true)
20354 .await
20355 .unwrap()
20356 .expect("a vetted replacement is posted");
20357 assert_eq!(message.title, "fix: retry failed requests");
20358 assert_eq!(
20359 runner
20360 .state
20361 .github_text
20362 .as_ref()
20363 .unwrap()
20364 .chosen_title
20365 .as_deref(),
20366 Some("fix: retry failed requests")
20367 );
20368 assert!(runner.state.github_text.as_ref().unwrap().resolved);
20369 let again = runner
20371 .guarded_pr_message(&winner, None, true)
20372 .await
20373 .unwrap()
20374 .unwrap();
20375 assert_eq!(again.title, message.title);
20376 assert_eq!(
20377 store
20378 .list()
20379 .into_iter()
20380 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
20381 .count(),
20382 2
20383 );
20384 }
20385
20386 #[test]
20387 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
20388 let mut state = state_with_summary(
20392 "add retries",
20393 "TITLE: fix(web): batch reads\n- reads run.json once",
20394 );
20395 state.config.graph.language = "ja".to_owned();
20396 let m = pr_message(&state, 'A');
20397 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
20398 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
20399
20400 let task = "今回やってほしいこと: results projector を直す";
20403 let mut state = state_with_summary(task, "- no title line");
20404 state.config.graph.language = "ja".to_owned();
20405 let m = pr_message(&state, 'A');
20406 assert_eq!(
20407 m.title,
20408 format!("chore: land candidate A of run {}", state.id)
20409 );
20410 assert!(
20411 m.body.contains(&format!(
20412 "<summary>Original task</summary>\n\n{task}\n\n</details>"
20413 )),
20414 "{}",
20415 m.body
20416 );
20417 }
20418
20419 #[test]
20420 fn pr_message_scrubs_home_paths_and_addresses() {
20421 let state = state_with_summary(
20422 "fix it in /Users/someone/src/x",
20423 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
20424 );
20425 let m = pr_message(&state, 'A');
20426 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
20427 assert!(!m.body.contains(leak), "{}", m.body);
20428 }
20429 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
20430 }
20431
20432 #[test]
20433 fn pr_message_survives_a_task_that_closes_details() {
20434 let state = state_with_summary("a </details> b", "TITLE: fix: x");
20435 let m = pr_message(&state, 'A');
20436 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
20437 }
20438
20439 #[test]
20440 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
20441 let cmd = manual_merge_command(
20442 MergeStyle::Squash,
20443 Path::new("/repo"),
20444 "b",
20445 "fix: \"quoted\" $(x) `y`\n\nbody",
20446 );
20447 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
20448 }
20449
20450 #[test]
20451 fn manual_merge_command_matches_the_configured_style() {
20452 let repo = Path::new("/repo");
20453 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
20454
20455 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
20456 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
20457
20458 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
20459 assert_eq!(
20460 squash,
20461 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
20462 \"Merge magi run 0832 (candidate A)\""
20463 );
20464
20465 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
20466 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
20467 }
20468
20469 #[test]
20470 fn a_nudge_gets_a_quarter_of_the_budget() {
20471 assert_eq!(retry_budget(secs(1200), true), secs(300));
20473 assert_eq!(retry_budget(secs(3600), true), secs(900));
20474 }
20475
20476 #[test]
20477 fn a_resent_prompt_keeps_the_whole_budget() {
20478 assert_eq!(retry_budget(secs(1200), false), secs(1200));
20481 assert_eq!(retry_budget(secs(60), false), secs(60));
20482 }
20483
20484 #[test]
20485 fn the_floor_never_exceeds_the_original_budget() {
20486 assert_eq!(retry_budget(secs(60), true), secs(60));
20490 assert_eq!(retry_budget(secs(480), true), secs(120));
20491 assert_eq!(retry_budget(secs(0), true), secs(0));
20492 }
20493
20494 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
20495 agent::CommandEvidence {
20496 id: "item1".to_owned(),
20497 description: "cargo test".to_owned(),
20498 exit_code,
20499 result_summary: String::new(),
20500 source: "codex".to_owned(),
20501 }
20502 }
20503
20504 #[test]
20505 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
20506 assert!(!has_unconfirmed_command(&[]));
20510 }
20511
20512 #[test]
20513 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
20514 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
20518 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
20519 assert!(!has_unconfirmed_command(&[
20520 evidence(Some(0)),
20521 evidence(Some(101))
20522 ]));
20523 }
20524
20525 #[test]
20526 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
20527 assert!(has_unconfirmed_command(&[
20528 evidence(Some(0)),
20529 evidence(None)
20530 ]));
20531 }
20532
20533 #[test]
20534 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
20535 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20536 assert_eq!(
20537 verified_noop_claim(true, &[], text).as_deref(),
20538 Some("already fixed by b32cfc4, on main.")
20539 );
20540 }
20541
20542 #[test]
20543 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
20544 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20547 assert!(verified_noop_claim(false, &[], text).is_none());
20548 }
20549
20550 #[test]
20551 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
20552 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20553 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
20554 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
20556 }
20557
20558 #[test]
20559 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
20560 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
20561 }
20562
20563 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
20566 runner.state.candidates = shape
20567 .iter()
20568 .enumerate()
20569 .map(|(i, &(empty, verified))| Candidate {
20570 index: i,
20571 label: (b'A' + i as u8) as char,
20572 agent: "sonnet".to_owned(),
20573 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
20574 worktree: PathBuf::from(format!("/wt/{i}")),
20575 summary: String::new(),
20576 stat: String::new(),
20577 files: 0,
20578 commits: 0,
20579 empty,
20580 failed: None,
20581 verified_noop: verified.map(str::to_owned),
20582 duration_ms: 0,
20583 folded: false,
20584 })
20585 .collect();
20586 }
20587
20588 #[test]
20589 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
20590 ask_test_home();
20591 let mut runner = runner_at(RunStatus::Implementing);
20592 set_candidates(
20593 &mut runner,
20594 &[
20595 (true, Some("already on main at b32cfc4")),
20596 (true, Some("same fix, see the existing test")),
20597 ],
20598 );
20599
20600 runner
20601 .after_implement()
20602 .expect("a verified no-op is not an error");
20603
20604 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
20605 }
20606
20607 #[test]
20608 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
20609 ask_test_home();
20610 let mut runner = runner_at(RunStatus::Implementing);
20611 set_candidates(
20615 &mut runner,
20616 &[(true, Some("already on main at b32cfc4")), (true, None)],
20617 );
20618
20619 let err = runner
20620 .after_implement()
20621 .expect_err("an unverified empty candidate must still fail the run");
20622
20623 assert!(
20624 err.to_string().contains("no candidate produced a change"),
20625 "{err}"
20626 );
20627 assert_eq!(runner.state.status, RunStatus::Failed);
20628 }
20629
20630 #[test]
20631 fn after_implement_still_fails_an_ordinary_all_empty_run() {
20632 ask_test_home();
20633 let mut runner = runner_at(RunStatus::Implementing);
20634 set_candidates(&mut runner, &[(true, None), (true, None)]);
20635
20636 let err = runner
20637 .after_implement()
20638 .expect_err("no candidate declared anything; this is an ordinary failure");
20639
20640 assert!(
20641 err.to_string().contains("no candidate produced a change"),
20642 "{err}"
20643 );
20644 assert_eq!(runner.state.status, RunStatus::Failed);
20645 }
20646}
20647
20648#[derive(Debug)]
20651struct GraftRestoreFailed(String);
20652
20653impl std::fmt::Display for GraftRestoreFailed {
20654 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
20655 write!(
20656 f,
20657 "could not restore the winner to its pre-graft commit: {}",
20658 self.0
20659 )
20660 }
20661}
20662
20663impl std::error::Error for GraftRestoreFailed {}
20664
20665#[cfg(test)]
20669mod resource_wait_tests {
20670 use super::*;
20671 use std::collections::BTreeMap;
20672
20673 fn sh(id: &str, script: &str) -> AgentSpec {
20674 AgentSpec {
20675 id: id.to_owned(),
20676 kind: crate::config::AgentKind::Command,
20677 model: None,
20678 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
20679 extra_args: Vec::new(),
20680 env: BTreeMap::new(),
20681 prompt_delivery: None,
20682 max_turn_secs: None,
20683 }
20684 }
20685
20686 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
20687 SeatJob {
20688 seat: SeatState::new("impl-A", &spec.id, 1),
20689 spec,
20690 cwd: dir.to_path_buf(),
20691 prompt: "do it".to_owned(),
20692 timeout,
20693 allow_write: true,
20694 sessions: false,
20695 artifacts: dir.join("art"),
20696 stem: "impl-A".to_owned(),
20697 handover: None,
20698 }
20699 }
20700
20701 fn state(resource_wait_secs: u64) -> RunState {
20702 let mut config = Config::default();
20703 config.graph.timeout_resource_wait = resource_wait_secs;
20704 RunState::new(
20705 PathBuf::from("/repo"),
20706 "main".to_owned(),
20707 "abc1234".to_owned(),
20708 "task".to_owned(),
20709 config,
20710 )
20711 }
20712
20713 fn hold(cache: &Path) -> crate::cache::Guard {
20714 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
20715 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
20716 crate::cache::AcquireOutcome::Acquired(g) => g,
20717 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
20718 }
20719 }
20720
20721 async fn run(
20722 jobs: Vec<SeatJob>,
20723 sem: Arc<Semaphore>,
20724 cache: Option<&Path>,
20725 st: &mut RunState,
20726 ) -> Vec<(usize, SeatState, AgentOutcome)> {
20727 let prompts = Prompts::default();
20728 let ctx = WaveCtx {
20729 run: "r",
20730 node: "implement",
20731 prompts: &prompts,
20732 cache,
20733 round: None,
20734 carry_seats: false,
20735 pauses: None,
20736 };
20737 wave(jobs, sem, &ctx, st, 0).await
20738 }
20739
20740 #[tokio::test]
20741 async fn verification_cache_wait_preserves_the_command_budget() {
20742 crate::run::pin_test_home();
20743 let tmp = tempfile::tempdir().unwrap();
20744 let cache = tmp.path().join("target");
20745 let guard = hold(&cache);
20746 let release = tokio::spawn(async move {
20747 tokio::time::sleep(Duration::from_millis(100)).await;
20748 guard.release();
20749 });
20750 let mut st = state(10);
20751 let budget = Duration::from_millis(50);
20752 let wait = repository_wait(&st, None);
20753 let (outcomes, _) = with_cache_lease(
20754 &mut st,
20755 Some(&cache),
20756 "gate",
20757 "gate",
20758 tmp.path(),
20759 "abc1234",
20760 budget,
20761 "test gate",
20762 &wait,
20763 |_, available| async move {
20764 assert_eq!(
20765 available, budget,
20766 "contention must not shorten verification"
20767 );
20768 (Vec::new(), false, Vec::new())
20769 },
20770 )
20771 .await
20772 .unwrap();
20773 release.await.unwrap();
20774 assert!(outcomes.is_empty(), "{outcomes:?}");
20775 assert!(
20776 st.events
20777 .iter()
20778 .any(|event| event.message.contains("waiting for"))
20779 );
20780 assert!(st.resource_wait.is_none());
20781 hold(&cache).release();
20783 }
20784
20785 #[tokio::test]
20786 async fn verification_cache_wait_expires_without_starting_commands() {
20787 crate::run::pin_test_home();
20788 let tmp = tempfile::tempdir().unwrap();
20789 let cache = tmp.path().join("target");
20790 let _held = hold(&cache);
20791 let mut st = state(0);
20792 let wait = repository_wait(&st, None);
20793 let (outcomes, _) = with_cache_lease(
20794 &mut st,
20795 Some(&cache),
20796 "gate",
20797 "gate",
20798 tmp.path(),
20799 "abc1234",
20800 Duration::from_secs(60),
20801 "test gate",
20802 &wait,
20803 |_, _| async { panic!("commands must not start without the cache lease") },
20804 )
20805 .await
20806 .unwrap();
20807 assert_eq!(outcomes.len(), 1);
20808 assert!(outcomes[0].resource_blocked);
20809 assert!(st.resource_wait.is_none());
20810 assert!(matches!(
20811 st.resource_block.unwrap().cause,
20812 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
20813 ));
20814 }
20815
20816 #[tokio::test]
20817 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
20818 crate::run::pin_test_home();
20819 let tmp = tempfile::tempdir().unwrap();
20820 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
20821 let cache = tmp.path().join("cache");
20822 let mut state = state(21600);
20823 let wait = repository_wait(&state, None);
20824 let (outcomes, _) = with_cache_lease(
20825 &mut state,
20826 Some(&cache),
20827 "gate",
20828 "gate",
20829 tmp.path(),
20830 "head",
20831 Duration::from_secs(60),
20832 "gate",
20833 &wait,
20834 |_, _| async { panic!("freshness failure must not start verification") },
20835 )
20836 .await
20837 .unwrap();
20838 assert_eq!(outcomes.len(), 1);
20839 assert!(outcomes[0].resource_blocked);
20840 let block = state.resource_block.unwrap();
20841 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
20842 assert!(!block.description().contains("timed out"));
20843 }
20844
20845 #[tokio::test]
20846 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
20847 crate::run::pin_test_home();
20848 let tmp = tempfile::tempdir().unwrap();
20849 let cache = tmp.path().join("cache");
20850 let _held = hold(&cache);
20851 let mut state = state(21600);
20852 let pause = Pause::new();
20853 let interrupt = Pause::new();
20854 let cause = ParkCause::Operator;
20855 pause.park_for(cause.clone());
20856 let prompts = Prompts::default();
20857 let ctx = WaveCtx {
20858 run: "test",
20859 node: "fix",
20860 prompts: &prompts,
20861 cache: Some(&cache),
20862 round: None,
20863 carry_seats: false,
20864 pauses: Some((&pause, &interrupt)),
20865 };
20866 let result = wave(
20867 vec![job(
20868 sh("writer", "exit 99"),
20869 tmp.path(),
20870 Duration::from_secs(1),
20871 )],
20872 Arc::new(Semaphore::new(1)),
20873 &ctx,
20874 &mut state,
20875 0,
20876 )
20877 .await;
20878 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
20879 assert_eq!(result[0].1.turns, 0);
20880 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20881 assert_eq!(
20882 state.resource_block.unwrap().cause,
20883 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
20884 );
20885 assert!(state.active.is_empty());
20886 }
20887
20888 #[tokio::test]
20889 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
20890 crate::run::pin_test_home();
20891 let tmp = tempfile::tempdir().unwrap();
20892 let cache = tmp.path().join("target-a");
20893 let guard = hold(&cache);
20894 tokio::spawn(async move {
20896 tokio::time::sleep(Duration::from_millis(1500)).await;
20897 guard.release();
20898 });
20899 let mut st = state(60);
20900 let results = run(
20901 vec![job(
20902 sh("alpha", "echo ok"),
20903 tmp.path(),
20904 Duration::from_secs(1),
20905 )],
20906 Arc::new(Semaphore::new(1)),
20907 Some(&cache),
20908 &mut st,
20909 )
20910 .await;
20911 assert!(
20912 matches!(results[0].2, AgentOutcome::Ok(_)),
20913 "the wait must not be charged to the agent"
20914 );
20915 assert!(st.resource_block.is_none());
20916 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20917 assert!(st.resource_wait.is_none() && st.active.is_empty());
20918 }
20919
20920 #[tokio::test]
20921 #[cfg(unix)]
20922 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20923 use std::os::fd::AsRawFd as _;
20924 crate::run::pin_test_home();
20925 let tmp = tempfile::tempdir().unwrap();
20926 crate::git::git(tmp.path(), &["init"]).await.unwrap();
20927 let common = tmp.path().join(".git");
20928 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20929 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20930 assert_eq!(
20931 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20932 0
20933 );
20934 let mut st = state(1);
20935 st.config.coordination = crate::config::test_coordination();
20936 let mut reader = job(
20937 sh("reader", "echo must-not-run"),
20938 tmp.path(),
20939 Duration::from_secs(30),
20940 );
20941 reader.allow_write = false;
20942 let results = run(
20943 vec![reader.clone()],
20944 Arc::new(Semaphore::new(1)),
20945 None,
20946 &mut st,
20947 )
20948 .await;
20949 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20950 assert_eq!(results[0].1.turns, 0);
20951 assert!(FailClass::of(&results[0].2).is_none());
20952 assert_eq!(
20953 st.resource_block.as_ref().unwrap().kind,
20954 "repository coordination"
20955 );
20956 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20957 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20958 drop(external);
20959 let editor = crate::repo_lock::SeatLease::for_checkout(
20960 tmp.path(),
20961 true,
20962 crate::config::test_coordination().as_ref(),
20963 )
20964 .await
20965 .unwrap();
20966 let pause = Pause::new();
20967 let interrupt = Pause::new();
20968 let prompts = Prompts::default();
20969 let ctx = WaveCtx {
20970 run: "r",
20971 node: "review",
20972 prompts: &prompts,
20973 cache: None,
20974 round: None,
20975 carry_seats: false,
20976 pauses: Some((&pause, &interrupt)),
20977 };
20978 let (results, ()) = tokio::join!(
20979 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20980 async {
20981 tokio::time::sleep(Duration::from_millis(50)).await;
20982 pause.park();
20983 }
20984 );
20985 assert!(
20986 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20987 );
20988 assert_eq!(results[0].1.turns, 0);
20989 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20990 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20991 drop(editor);
20992 }
20993
20994 #[tokio::test]
20995 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20996 crate::run::pin_test_home();
20997 let tmp = tempfile::tempdir().unwrap();
20998 let cache = tmp.path().join("target-b");
20999 let _held = hold(&cache);
21000 let mut st = state(1);
21001 let results = run(
21002 vec![job(
21003 sh("alpha", "echo ok"),
21004 tmp.path(),
21005 Duration::from_secs(1),
21006 )],
21007 Arc::new(Semaphore::new(1)),
21008 Some(&cache),
21009 &mut st,
21010 )
21011 .await;
21012 let (_, seat, out) = &results[0];
21013 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21014 assert!(wave_blocked(&results));
21015 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21016 assert_eq!(seat.turns, 0, "nothing was asked");
21017 assert!(
21018 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21019 "no agent was spawned"
21020 );
21021 assert!(FailClass::of(out).is_none(), "not a failure class");
21022 let block = st.resource_block.as_ref().expect("recorded for the resume");
21023 assert_eq!(block.kind, "shared build cache");
21024 assert!(matches!(
21025 block.cause,
21026 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21027 ));
21028 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21029 assert!(st.active.is_empty());
21030 }
21031
21032 #[tokio::test]
21033 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21034 crate::run::pin_test_home();
21035 let tmp = tempfile::tempdir().unwrap();
21036 let mut st = state(1);
21037 let results = run(
21038 vec![job(
21039 sh("alpha", "echo ok"),
21040 tmp.path(),
21041 Duration::from_secs(1),
21042 )],
21043 Arc::new(Semaphore::new(0)),
21044 None,
21045 &mut st,
21046 )
21047 .await;
21048 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21049 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21050 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21051 }
21052
21053 #[tokio::test]
21054 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21055 crate::run::pin_test_home();
21056 let tmp = tempfile::tempdir().unwrap();
21057 let mut st = state(60);
21058 let results = run(
21059 vec![job(
21060 sh("alpha", "sleep 30"),
21061 tmp.path(),
21062 Duration::from_secs(1),
21063 )],
21064 Arc::new(Semaphore::new(1)),
21065 None,
21066 &mut st,
21067 )
21068 .await;
21069 let out = &results[0].2;
21070 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21071 let class = FailClass::of(out).expect("a failure");
21072 assert_eq!(class, FailClass::Timeout);
21073 assert!(should_hand_over(None, &class), "existing handover policy");
21074 assert!(st.resource_block.is_none());
21075 }
21076}