1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615impl Runner {
616 pub async fn start(
618 repo: &Path,
619 instruction: String,
620 config: Config,
621 origin: Origin,
622 ) -> Result<Self> {
623 Self::start_naming(repo, instruction, "", config, origin).await
624 }
625
626 pub async fn start_naming(
629 repo: &Path,
630 instruction: String,
631 _also_scan: &str,
632 config: Config,
633 origin: Origin,
634 ) -> Result<Self> {
635 let repo = git::toplevel(repo).await?;
636 let missing = agent::missing_programs(&config.agents);
637 if !missing.is_empty() {
638 bail!(
639 "these agent programs are not on PATH: {}. Fix the roster in \
640 magi.toml or install them.",
641 missing.join(", ")
642 );
643 }
644 let base_branch =
645 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646 .await?;
647 let fetched_base = resolve_base(
648 &repo,
649 &base_branch,
650 &config.merge.remote,
651 config.coordination.as_ref(),
652 )
653 .await?;
654 let base_commit = fetched_base.commit.clone();
655 let roles = config.resolve_roles()?;
656 let max_parallel = config.graph.max_parallel.max(1);
657 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658 state.fetched_base = Some(fetched_base);
662 state.origin = Some(origin);
663 state.reserve()?;
664 state.event("start", format!("run {} created", state.id));
665 state.save()?;
666 Ok(Self {
667 state,
668 roles,
669 sem: Arc::new(Semaphore::new(max_parallel)),
670 pause: Pause::new(),
671 interrupt: Pause::new(),
672 })
673 }
674
675 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689 Self::review_taking_over(repo, branch, config, None, origin).await
690 }
691
692 pub async fn review_taking_over(
699 repo: &Path,
700 branch: &str,
701 config: Config,
702 takeover: Option<crate::handover::Takeover>,
703 origin: Origin,
704 ) -> Result<Self> {
705 let repo = git::toplevel(repo).await?;
706 let missing = agent::missing_programs(&config.agents);
707 if !missing.is_empty() {
708 bail!(
709 "these agent programs are not on PATH: {}. Fix the roster in \
710 magi.toml or install them.",
711 missing.join(", ")
712 );
713 }
714 let base_branch =
715 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716 .await?;
717 if base_branch == branch {
718 bail!("`{branch}` is the base branch; there is nothing to review against");
719 }
720 let fetched_base = resolve_base(
721 &repo,
722 &base_branch,
723 &config.merge.remote,
724 config.coordination.as_ref(),
725 )
726 .await?;
727 let base_commit = fetched_base.commit.clone();
728
729 let roles = config.resolve_roles()?;
730 let max_parallel = config.graph.max_parallel.max(1);
731 let mut state = RunState::new(
732 repo.clone(),
733 base_branch,
734 base_commit.clone(),
735 String::new(),
736 config,
737 );
738 state.fetched_base = Some(fetched_base);
739 state.origin = Some(origin);
740 state.reserve()?;
741
742 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749 earlier: Vec::new(),
750 home: crate::run::home(),
751 choice: None,
752 task_instruction: None,
753 });
754 let wait = repository_wait(&state, None);
755 let coordination_config = state.config.coordination.clone();
756 let coordination =
757 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758 .await?;
759 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760 if let Some(released) = &released {
761 state.event(
762 "release",
763 format!(
764 "took `{branch}` over from run {}: its worktree was released: {}",
765 crate::run::short_of(&released.old_id),
766 released.audit
767 ),
768 );
769 }
770 if let Some(choice) = takeover.choice.as_ref()
774 && let Err(e) =
775 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
776 .await
777 {
778 if let Some(released) = &released {
779 released.restore(&repo, branch).await;
780 }
781 return Err(e.context("applying the owner's answer about the diverged branch"));
782 }
783 for id in takeover.earlier.iter().rev() {
789 if let Ok(old) = RunState::load_under(id, &takeover.home)
790 && let Some(head) = old.pr_head.clone()
791 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
792 {
793 state.pr_head = Some(head);
794 break;
795 }
796 }
797 drop(coordination);
798 let opened = Self::open_review(
799 &repo,
800 branch,
801 state,
802 roles,
803 max_parallel,
804 base_commit,
805 &takeover,
806 )
807 .await;
808 if opened.is_err()
809 && let Some(released) = &released
810 {
811 let _coordination = crate::repo_lock::SeatLease::operation_with(
812 &repo,
813 coordination_config.as_ref(),
814 &wait,
815 )
816 .await?;
817 released.restore(&repo, branch).await;
818 }
819 opened
820 }
821
822 pub async fn recover_pr(
824 repo: &Path,
825 task: &str,
826 selection: crate::pr_recovery::Selection,
827 mut config: Config,
828 ) -> Result<Self> {
829 selection.configure(&mut config)?;
830 selection.check_task(task, &selection.instruction)?;
831 let base_branch =
832 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
833 .await?;
834 let fetched_base = resolve_base(
835 repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842 selection
843 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
844 .await?;
845 Box::pin(selection.verify_source(repo)).await?;
846 if let Some(proof) = &selection.source_proof {
847 anyhow::ensure!(
848 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
849 "source rebase base is outside the fetched landing base"
850 );
851 }
852 let roles = config.resolve_roles()?;
853 let max_parallel = config.graph.max_parallel.max(1);
854 let mut state = RunState::new(
855 repo.to_path_buf(),
856 base_branch,
857 base_commit.clone(),
858 selection.instruction.clone(),
859 config,
860 );
861 state.fetched_base = Some(fetched_base);
862 state.origin = Some(crate::run::Origin::queue(task));
863 state.pr_head = Some(selection.branch.clone());
864 state.pr_recovery = Some(selection.clone());
865 state.reserve()?;
866 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
867 state.discard_reserved();
869 return Err(e);
870 }
871 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
872 }
873
874 async fn open_selected(
875 repo: &Path,
876 state: RunState,
877 roles: ResolvedRoles,
878 max_parallel: usize,
879 base_commit: String,
880 selection: crate::pr_recovery::Selection,
881 ) -> Result<Self> {
882 let mut state = state;
883 let branch = state.branch_for('A');
884 let wait = repository_wait(&state, None);
885 let prepared = async {
886 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
887 git::git(repo, &["branch", &branch, selection.source()]).await?;
888 drop(coordination);
889 anyhow::Ok(())
890 }
891 .await;
892 if let Err(e) = prepared {
893 state.discard_reserved();
896 return Err(e);
897 }
898 state.event(
899 "recovery",
900 format!(
901 "selected implementation {} for {} (public lease {})",
902 selection.source(),
903 selection.url,
904 selection.head
905 ),
906 );
907 let takeover = crate::handover::Takeover {
908 earlier: Vec::new(),
909 home: crate::run::home(),
910 choice: None,
911 task_instruction: Some(selection.instruction.clone()),
912 };
913 if let Err(e) = state.save() {
914 state.discard_reserved();
915 return Err(e);
916 }
917 let id = state.id.clone();
918 let opened = Self::open_review(
919 repo,
920 &branch,
921 state,
922 roles,
923 max_parallel,
924 base_commit,
925 &takeover,
926 )
927 .await;
928 match opened {
929 Ok(runner) => Ok(runner),
930 Err(e) => {
931 if let Ok(mut partial) = RunState::load(&id) {
932 partial.status = RunStatus::Blocked;
933 partial.driver_exited = true;
934 partial.event(
935 "recovery",
936 format!("startup failed; retained source and candidate {branch}: {e:#}"),
937 );
938 partial.save()?;
939 }
940 Err(crate::pr_recovery::StartupFailed {
941 run: id,
942 cause: format!("{e:#}"),
943 }
944 .into())
945 }
946 }
947 }
948
949 async fn inherit(
954 repo: &Path,
955 branch: &str,
956 base_commit: &str,
957 takeover: &crate::handover::Takeover,
958 ) -> Option<crate::recovery::RecoveryRecord> {
959 use crate::recovery::{Relation, assess};
960 let newest = takeover.earlier.last()?;
961 let mut chosen = None;
962 for id in takeover.earlier.iter().rev() {
963 if let Ok(old) = RunState::load_under(id, &takeover.home)
964 && !old.reviews.is_empty()
965 && (old.winner().is_some_and(|w| w.branch == branch)
966 || old.pr_head.as_deref() == Some(branch))
967 {
968 chosen = Some(old);
969 break;
970 }
971 }
972 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
973 .await
974 .unwrap_or_default();
975 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
976 Some(last) if last.head == tip => Relation::Same,
977 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
978 _ => Relation::Diverged,
979 };
980 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
981 Some(assess(from, chosen.as_ref(), base_commit, relation))
982 }
983
984 async fn open_review(
987 repo: &Path,
988 branch: &str,
989 mut state: RunState,
990 roles: ResolvedRoles,
991 max_parallel: usize,
992 base_commit: String,
993 takeover: &crate::handover::Takeover,
994 ) -> Result<Self> {
995 let wait = repository_wait(&state, None);
996 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
997 if state.pr_recovery.is_none() {
998 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
999 }
1000 let start = review_base(
1003 repo,
1004 &state.config.merge.remote,
1005 &state.base_branch,
1006 &base_commit,
1007 branch,
1008 )
1009 .await;
1010 let log = git::log_oneline(repo, &start, branch)
1011 .await
1012 .unwrap_or_default();
1013 let log_text = if log.trim().is_empty() {
1014 "(no commit messages)"
1015 } else {
1016 log.trim()
1017 };
1018 let mut instruction = format!(
1019 "Review the work already on branch `{branch}`. There is no task \
1020 statement: what the change claims to do is whatever its commits \
1021 say.\n\n{log_text}"
1022 );
1023 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1027 if let Some(task) = takeover
1028 .task_instruction
1029 .as_deref()
1030 .filter(|t| !t.trim().is_empty())
1031 && rec.reviewed_head.is_some()
1032 {
1033 instruction = format!(
1034 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1035 task.trim()
1036 );
1037 rec.criteria_inherited = true;
1038 }
1039 state.event("recovery", rec.summary());
1040 state.recovery = Some(rec);
1041 }
1042 if let Some(selected) = &state.pr_recovery {
1043 instruction = selected.instruction.clone();
1044 }
1045 state.task_instruction = takeover.task_instruction.clone();
1046 state.instruction = instruction;
1047 state.reviewed_commits = Some(
1048 git::subjects(repo, &start, branch)
1049 .await
1050 .unwrap_or_default(),
1051 );
1052
1053 for id in &takeover.earlier {
1056 if let Ok(old) = RunState::load_under(id, &takeover.home)
1057 && old.repo == state.repo
1058 && (old.pr_head.as_deref() == Some(branch)
1059 || old.winner().is_some_and(|w| w.branch == branch))
1060 {
1061 for hold in old.branch_owner_holds {
1062 if !state.branch_owner_holds.contains(&hold) {
1063 state.event(
1064 "approval-policy",
1065 format!(
1066 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1067 hold.control.key(),
1068 hold.file,
1069 hold.head
1070 ),
1071 );
1072 state.branch_owner_holds.push(hold);
1073 }
1074 }
1075 }
1076 }
1077 let reviewed_head = git::rev_parse(repo, branch).await?;
1078 state.observe_branch_owner_holds(&reviewed_head).await;
1079
1080 let worktree = state.worktree_root().join("under-review");
1083 if let Some(parent) = worktree.parent() {
1084 tokio::fs::create_dir_all(parent).await.ok();
1085 }
1086 let path = worktree.to_string_lossy().to_string();
1087 git::git(repo, &["worktree", "add", &path, branch])
1088 .await
1089 .with_context(|| {
1090 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1091 })?;
1092
1093 let setup = match ensure_setup_config(&mut state, repo).await {
1094 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1095 Err(e) => Err(e),
1096 };
1097 if let Err(e) = setup {
1098 git::worktree_remove_clean(repo, &worktree).await.ok();
1099 return Err(e);
1100 }
1101 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1102 .await
1103 .unwrap_or(0);
1104 if commits == 0 {
1105 git::worktree_remove_clean(repo, &worktree).await.ok();
1106 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1107 }
1108 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1109 .await
1110 .map(|f| f.len())
1111 .unwrap_or(0);
1112 if files == 0
1113 && let (Ok(head_tree), Ok(base_tree)) = (
1114 git::tree_of(&worktree, "HEAD").await,
1115 git::tree_of(&worktree, &base_commit).await,
1116 )
1117 && head_tree == base_tree
1118 {
1119 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1120 git::worktree_remove_clean(repo, &worktree).await.ok();
1121 bail!(
1122 "`{branch}` at {} has a tree identical to base {}; this usually means \
1123 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1124 against `{}/{branch}`) rather than an empty change",
1125 short(&head),
1126 short(&base_commit),
1127 state.config.merge.remote
1128 );
1129 }
1130 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1131 .await
1132 .unwrap_or_default();
1133
1134 state.candidates.push(Candidate {
1135 index: 0,
1136 label: 'A',
1137 agent: EXISTING_BRANCH.to_owned(),
1140 branch: branch.to_owned(),
1141 worktree,
1142 summary: String::new(),
1143 stat,
1144 files,
1145 commits,
1146 empty: false,
1147 failed: None,
1148 verified_noop: None,
1149 duration_ms: 0,
1150 folded: false,
1151 });
1152 state.tally = Some(Tally {
1153 first_choice: BTreeMap::from([('A', 0)]),
1154 borda: BTreeMap::new(),
1155 winner: 'A',
1156 rankings: 0,
1157 unanimous_initial: false,
1158 deliberated: false,
1159 changed_votes: 0,
1160 unanimous_final: false,
1161 tie_break: None,
1162 judges: 0,
1166 present: 0,
1167 quorum: 0,
1168 met_quorum: true,
1169 uncontested: Some("review-only run: nothing competed".to_owned()),
1170 });
1171 state.status = RunStatus::Reviewing;
1172 state.prep_complete = Some(true);
1173 state.event(
1174 "start",
1175 format!(
1176 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1177 state.id
1178 ),
1179 );
1180 state.save()?;
1181 Ok(Self {
1182 state,
1183 roles,
1184 sem: Arc::new(Semaphore::new(max_parallel)),
1185 pause: Pause::new(),
1186 interrupt: Pause::new(),
1187 })
1188 }
1189
1190 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1193 Self::authorize_base_recovery_request(home, id, task, false).await
1194 }
1195
1196 pub(crate) async fn authorize_base_recovery_from_resumption(
1198 home: &Path,
1199 id: &str,
1200 task: &str,
1201 ) -> Result<()> {
1202 Self::authorize_base_recovery_request(home, id, task, true).await
1203 }
1204
1205 async fn authorize_base_recovery_request(
1206 home: &Path,
1207 id: &str,
1208 task: &str,
1209 resumption: bool,
1210 ) -> Result<()> {
1211 let _owner = crate::pr_head::owner_lock(home, id)?;
1212 let mut state = RunState::load_under(id, home)?;
1213 crate::pr_head::ensure_owner(&state)?;
1214 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1215 anyhow::ensure!(
1216 queued.runs.last().is_some_and(|run| run == id)
1217 && queued.status != crate::queue::TaskStatus::Done
1218 && !queued.fresh_start
1219 && queued.review_branch.is_none(),
1220 "run task ownership changed"
1221 );
1222 anyhow::ensure!(
1223 state
1224 .origin
1225 .as_ref()
1226 .and_then(|o| o.task.as_deref())
1227 .is_none_or(|owner| owner == task),
1228 "run belongs to another task"
1229 );
1230 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1231 anyhow::ensure!(
1232 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1233 "run is live or its ownership is unknown"
1234 );
1235 let _execution = crate::driver::maintenance(home, id)?;
1238 anyhow::ensure!(
1239 state.status.resumable()
1240 && state.released_to.is_none()
1241 && state.retired_to.is_none()
1242 && state.pr_taken_over_by.is_none(),
1243 "run ownership is final"
1244 );
1245 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1246 anyhow::ensure!(
1247 !state.pr_publication_pending
1248 && !crate::handover::landing_checkpoint(&state)
1249 && state.merge_decision.is_none(),
1250 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1251 );
1252 let winner = state.winner().context("run has no retained winner")?;
1253 anyhow::ensure!(
1254 git::is_clean(&winner.worktree).await?,
1255 "retained worktree has uncommitted changes"
1256 );
1257 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1258 anyhow::ensure!(
1259 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1260 "retained worktree does not match the winner branch"
1261 );
1262 state.record_base_sync_stop(&head);
1263 state.save()?;
1264 if resumption {
1265 anyhow::ensure!(
1266 crate::daemon::compatible_run_instruction(&state, &queued),
1267 "resumption instruction does not match the retained run"
1268 );
1269 let approval = queued
1270 .resumption
1271 .as_ref()
1272 .context("no operator resumption")?;
1273 anyhow::ensure!(
1274 approval.instruction == queued.instruction,
1275 "resumption approval does not match the current task instruction"
1276 );
1277 state.authorize_base_sync_resumption(task, approval)?;
1278 } else {
1279 state.authorize_base_sync(task)?;
1280 }
1281 state.event(
1282 "base-recovery",
1283 "owner authorized one additional base rebase; fresh verification required",
1284 );
1285 state.save()?;
1286 Ok(())
1287 }
1288
1289 pub fn resume(id: &str) -> Result<Self> {
1291 let state = RunState::load(id)?;
1292 crate::pr_head::ensure_owner(&state)?;
1293 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1294 bail!(
1295 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1296 );
1297 }
1298 if let Some(to) = &state.released_to {
1299 bail!(
1300 "run {} cannot be resumed: its worktree was released to run {}",
1301 state.short(),
1302 crate::run::short_of(to)
1303 );
1304 }
1305 let roles = state.config.resolve_roles()?;
1306 if let Some(id) = &state.config.graph.selected_implementer
1307 && state.candidates.iter().any(|c| c.agent != *id)
1308 {
1309 bail!(
1310 "replacement implementer pin does not match the saved candidates; task must stay held"
1311 );
1312 }
1313 let max_parallel = state.config.graph.max_parallel.max(1);
1314 Ok(Self {
1315 state,
1316 roles,
1317 sem: Arc::new(Semaphore::new(max_parallel)),
1318 pause: Pause::new(),
1319 interrupt: Pause::new(),
1320 })
1321 }
1322
1323 pub async fn execute(&mut self) -> Result<()> {
1330 if let Some(selection) = &self.state.pr_recovery
1331 && (selection.source_proof.is_some() || selection.source_run.is_some())
1332 {
1333 Box::pin(selection.verify_source(&self.state.repo)).await?;
1334 }
1335 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1338 || self.state.land_approval.is_some()
1339 || self.state.merge_decision.is_some()
1340 || !self.state.merge_repairs.is_empty()
1341 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1342 {
1343 let claim = FixClaim::acquire(&self.state.dir())?;
1344 let disk = RunState::load(&self.state.id)?;
1345 let checkpoint = |s: &RunState| {
1346 (
1347 s.merge_repairs.len(),
1348 s.merge_repairs
1349 .last()
1350 .map(|r| (r.question.clone(), r.phase)),
1351 s.land_approval
1352 .as_ref()
1353 .map(|a| (a.question.clone(), a.head.clone())),
1354 )
1355 };
1356 if checkpoint(&disk) != checkpoint(&self.state) {
1357 bail!(
1358 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1359 );
1360 }
1361 Some(claim)
1362 } else {
1363 None
1364 };
1365 let mut execution = self.claim_driver().await?;
1366 let runner = &mut *execution.runner;
1367 let wait = runner.repository_wait();
1368 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1371 let result = match result {
1372 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1373 record_repository_block(&mut runner.state, &e);
1374 runner.park_here()?;
1375 Ok(())
1376 }
1377 other => other,
1378 };
1379 execution.driver.finish(&mut runner.state).await?;
1380 let ended = if result.is_err() {
1381 Some({
1382 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1383 if let Err(error) = &result {
1384 notice.message = format!(
1385 "Run {} stopped with an error: {}",
1386 runner.state.id,
1387 error.to_string().lines().next().unwrap_or("unknown error")
1388 );
1389 }
1390 notice
1391 })
1392 } else {
1393 crate::notices::run_ended(&runner.state)
1394 };
1395 if let Some(notice) = ended {
1396 crate::notices::raise(notice);
1397 }
1398 result
1399 }
1400
1401 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1402 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1403 }
1404
1405 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1406 let wait = self.repository_wait();
1407 coordinate_operation(&mut self.state, cwd, &wait).await
1408 }
1409
1410 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1412 let owner = match crate::pr_head::owner_lock_wait(
1413 &crate::run::home(),
1414 &self.state.id,
1415 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1416 )
1417 .await
1418 {
1419 Ok(owner) => owner,
1420 Err(error) => {
1421 let detail = format!("worktree maintenance or ownership update: {error:#}");
1422 self.state.resource_block = Some(crate::run::ResourceBlock {
1426 cause: crate::run::ResourceBlockCause::Other,
1427 node: "start".into(),
1428 kind: "worktree maintenance".into(),
1429 detail: detail.clone(),
1430 at: Timestamp::now(),
1431 });
1432 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1433 }
1434 };
1435 let disk = RunState::load(&self.state.id)?;
1436 crate::pr_head::ensure_owner(&disk)?;
1437 let claimed =
1438 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1439 anyhow::ensure!(
1444 disk.liveness(claimed) != crate::run::Liveness::Live,
1445 "run already has a live driver; wait for its execution to end"
1446 );
1447 if let Some(to) = &disk.released_to {
1448 bail!(
1449 "run {} cannot continue: its worktree was released to run {}",
1450 self.state.short(),
1451 crate::run::short_of(to)
1452 );
1453 }
1454 anyhow::ensure!(
1455 serde_json::to_value(&disk.base_sync_recovery)?
1456 == serde_json::to_value(&self.state.base_sync_recovery)?,
1457 "base recovery changed since this run was loaded; reload the saved run"
1458 );
1459 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1460 let mut execution = DriverExecution {
1461 runner: self,
1462 driver,
1463 };
1464 let _owner = owner;
1467 let state = &mut execution.runner.state;
1468 state.parked = false;
1473 state.resource_wait = None;
1474 state.resource_block = None;
1475 state.clear_active();
1482 let pid = std::process::id();
1497 state.driver_pid = Some(pid);
1498 state.driver_started_at = crate::proc::process_started_at(pid);
1499 state.driver_exited = false;
1500 state.driver_claimed = Some(true);
1501 state.begin_pass();
1502 execution.driver.bind(state);
1503 state.save()?;
1504 execution.driver.record(state)?;
1505 Ok(execution)
1506 }
1507
1508 async fn execute_graph(&mut self) -> Result<()> {
1509 self.state.configure_recovery_resume()?;
1510 self.persist_candidates().await?;
1513 if self.state.base_sync_blocked() {
1516 self.sync_to_base().await?;
1517 if !self.state.base_sync_blocked()
1518 && self.state.status != RunStatus::AlreadyInBase
1519 && !self.resource_halt()?
1520 {
1521 self.finish_after_tally().await?;
1522 }
1523 return Ok(());
1524 }
1525 if self
1526 .state
1527 .merge_decision
1528 .as_ref()
1529 .is_some_and(|decision| !decision.closed)
1530 && self.state.status != RunStatus::Landing
1531 {
1532 if crate::merge_decision::resumable_hold(&self.state) {
1533 self.state.merge = Some(MergeOutcome {
1534 mode: MergeMode::Pr,
1535 ok: false,
1536 detail: self
1537 .state
1538 .merge_decision
1539 .as_ref()
1540 .unwrap()
1541 .target
1542 .pr
1543 .clone(),
1544 empty: false,
1545 });
1546 self.run_land().await?;
1547 self.settle_questions();
1548 return Ok(());
1549 }
1550 bail!(
1551 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1552 );
1553 }
1554 if self.state.config.graph.selected_implementer.is_some()
1555 && self.state.status == RunStatus::Blocked
1556 && self.state.tally.is_none()
1557 && self.state.candidate_preservation.is_empty()
1558 && self.state.interrupted_implementations.is_empty()
1559 {
1560 bail!(
1561 "selected implementation is held; explicitly release the owning task to retry it"
1562 );
1563 }
1564
1565 if self.state.merge_repairs.last().is_some_and(|r| {
1566 matches!(
1567 r.phase,
1568 crate::run::MergeRepairPhase::Requested
1569 | crate::run::MergeRepairPhase::Fixing
1570 | crate::run::MergeRepairPhase::Reviewing
1571 )
1572 }) {
1573 if self.repair_merge_review().await? {
1574 self.run_land().await?;
1575 }
1576 self.settle_questions();
1577 return Ok(());
1578 }
1579 if self.state.pr_publication_pending {
1582 self.merge().await?;
1583 if self.resource_halt()? {
1584 return Ok(());
1585 }
1586 self.state.save()?;
1587 self.settle_questions();
1588 return Ok(());
1589 }
1590 if self.state.status == RunStatus::Stalled {
1603 if self.recover_stall().await? {
1604 self.finish_after_tally().await?;
1605 } else {
1606 self.state.save()?;
1608 }
1609 return Ok(());
1610 }
1611 if self.state.status == RunStatus::Landing
1621 || crate::handover::landing_checkpoint(&self.state)
1622 {
1623 self.state.land_sync_pending = true;
1624 self.state.save()?;
1625 self.run_land().await?;
1626 self.settle_questions();
1631 return Ok(());
1632 }
1633 if self.state.github_text.is_some()
1638 && self.state.merge.is_none()
1639 && self.state.status == RunStatus::Gating
1640 {
1641 self.merge().await?;
1642 if self.resource_halt()? {
1643 return Ok(());
1644 }
1645 self.state.save()?;
1646 return Ok(());
1647 }
1648 if let Err(e) = self.prep().await {
1649 if e.is::<crate::repo_lock::ResourceBlocked>() {
1650 return Err(e);
1651 }
1652 self.state.status = RunStatus::Blocked;
1653 self.state
1654 .prep_failure
1655 .get_or_insert_with(|| format!("{e:#}"));
1656 self.state
1657 .event("prep", format!("preparation stopped: {e:#}"));
1658 self.state.save()?;
1659 return Err(e);
1660 }
1661 if self.park_here()? {
1662 return Ok(());
1663 }
1664 self.advise().await?;
1665 if self.park_here()? {
1666 return Ok(());
1667 }
1668 self.implement().await?;
1669 if self.park_here()? {
1670 return Ok(());
1671 }
1672 if self.state.status == RunStatus::VerifiedNoop {
1676 return Ok(());
1677 }
1678 self.judge().await?;
1679 if self.park_here()? {
1680 return Ok(());
1681 }
1682 self.deliberate().await?;
1683 if self.park_here()? {
1684 return Ok(());
1685 }
1686 self.vote().await?;
1687 if self.park_here()? {
1688 return Ok(());
1689 }
1690 self.tally()?;
1691 if self.state.status == RunStatus::Stalled {
1696 self.state.save()?;
1700 return Ok(());
1701 }
1702 self.finish_after_tally().await?;
1703 Ok(())
1704 }
1705
1706 fn stop_for_resource(&mut self) -> Result<()> {
1711 if self.pause.parked() || self.interrupt.parked() {
1714 self.park_here()?;
1715 return Ok(());
1716 }
1717 if self.state.resource_block.is_some() {
1718 self.state.status = RunStatus::Blocked;
1719 }
1720 self.state.save()
1721 }
1722
1723 fn resource_halt(&mut self) -> Result<bool> {
1725 if self.state.resource_block.is_none() {
1726 return Ok(false);
1727 }
1728 self.stop_for_resource()?;
1729 Ok(true)
1730 }
1731
1732 fn park_here(&mut self) -> Result<bool> {
1739 let parking = self.pause.parked() || self.interrupt.parked();
1740 if !parking && self.state.resource_block.is_some() {
1741 self.stop_for_resource()?;
1742 return Ok(true);
1743 }
1744 if !self.pause.parked() && !self.interrupt.parked() {
1749 return Ok(false);
1750 }
1751 if self.state.parked {
1752 return Ok(true);
1753 }
1754 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1755 if let Some(block) = &mut self.state.resource_block
1756 && matches!(
1757 block.cause,
1758 crate::run::ResourceBlockCause::Cancelled { .. }
1759 )
1760 {
1761 block.cause = crate::run::ResourceBlockCause::Cancelled {
1762 park: cause.clone(),
1763 };
1764 }
1765 let why = match cause.as_ref().map(ParkCause::label) {
1766 Some(reason) => format!(
1767 "parked after `{}` ({reason}) — resume to carry on from here",
1768 self.state.status.as_str()
1769 ),
1770 None => format!(
1771 "parked after `{}` — resume to carry on from here",
1772 self.state.status.as_str()
1773 ),
1774 };
1775 self.state.event("park", why);
1776 self.state
1777 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1778 self.state.parked = true;
1779 self.state.save()?;
1780 Ok(true)
1781 }
1782
1783 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1785 if self.state.prep_failure.is_none() {
1786 return Ok(());
1787 }
1788 let failed_at = self
1789 .state
1790 .events
1791 .iter()
1792 .rev()
1793 .find(|event| event.node == "prep")
1794 .map_or(self.state.updated_at, |event| event.at);
1795 anyhow::ensure!(
1796 approval_at > failed_at,
1797 "preparation retry requires a new explicit release after the failure"
1798 );
1799 self.state.prep_failure = None;
1800 self.state.prep_complete = Some(false);
1801 self.state
1802 .event("prep-retry", "operator approved retry of preparation");
1803 Ok(())
1804 }
1805
1806 pub fn on_pause(&mut self, pause: Pause) {
1808 self.pause = pause;
1809 }
1810
1811 pub fn watch_interrupt(&mut self, pause: Pause) {
1817 self.interrupt = pause;
1818 }
1819
1820 fn settle_questions(&mut self) {
1840 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1841 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1842 }
1843 }
1844
1845 async fn finish_after_tally(&mut self) -> Result<()> {
1848 self.graft().await?;
1851 if self.resource_halt()? {
1852 return Ok(());
1853 }
1854 self.fold_losers().await?;
1855 self.sync_to_base().await?;
1860 if self.hold_base_sync().await? || self.resource_halt()? {
1861 return Ok(());
1862 }
1863 if self.state.status == RunStatus::AlreadyInBase {
1864 return Ok(());
1865 }
1866 self.review_loop().await?;
1867 if self.resource_halt()? {
1868 return Ok(());
1869 }
1870 self.sync_to_base().await?;
1871 if self.hold_base_sync().await? || self.resource_halt()? {
1872 return Ok(());
1873 }
1874 if self.state.status == RunStatus::AlreadyInBase {
1875 return Ok(());
1876 }
1877 self.gate().await?;
1878 self.merge().await?;
1879 if self.resource_halt()? {
1880 return Ok(());
1881 }
1882 self.state.save()?;
1883 Ok(())
1884 }
1885
1886 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1893 let result = match ensure_setup_config(&mut self.state, repo).await {
1894 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1895 Err(e) => Err(e),
1896 };
1897 if let Err(e) = result {
1898 git::worktree_remove_clean(repo, wt).await.ok();
1899 if in_prep {
1900 self.rollback_prep(repo).await;
1901 }
1902 self.state.status = RunStatus::Blocked;
1903 self.state.event(
1904 "setup",
1905 format!("worktree setup failed in {}: {e:#}", wt.display()),
1906 );
1907 self.state.save()?;
1908 return Err(e);
1909 }
1910 Ok(())
1911 }
1912
1913 async fn rollback_prep(&mut self, repo: &Path) {
1915 let root = self.state.worktree_root();
1916 for c in std::mem::take(&mut self.state.candidates) {
1917 if matches!(
1918 git::worktree_remove_clean(repo, &c.worktree).await,
1919 Ok(true)
1920 ) {
1921 git::branch_delete(repo, &c.branch).await.ok();
1922 } else {
1923 self.state.candidates.push(c);
1924 }
1925 }
1926 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1927 for j in 1..=seats {
1928 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1929 .await
1930 .ok();
1931 }
1932 for k in 1..=self.state.config.graph.advisors {
1933 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1934 .await
1935 .ok();
1936 }
1937 }
1938
1939 async fn prep(&mut self) -> Result<()> {
1940 if let Some(reason) = &self.state.prep_failure {
1941 bail!(
1942 "candidate preparation previously failed: {reason}; correct the source before retrying"
1943 );
1944 }
1945 if self
1946 .state
1947 .prep_complete
1948 .unwrap_or(!self.state.candidates.is_empty())
1949 {
1950 return Ok(());
1951 }
1952 let repo = self.state.repo.clone();
1953 let _coordination = self.coordinate(&repo).await?;
1956 let base = self.state.base_commit.clone();
1957 let plan = self.state.sources.plan(&repo, &base).await?;
1958 self.rollback_prep(&repo).await;
1959 if !self.state.candidates.is_empty() {
1960 bail!(
1961 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1962 );
1963 }
1964 self.state.prep_complete = Some(false);
1965 self.state.status = RunStatus::Prep;
1966 self.state.save()?;
1967 let repo = self.state.repo.clone();
1968 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1969 let root = self.state.worktree_root();
1970 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1971
1972 let hooks_dir = self.state.dir().join("hooks");
1975 if self.state.config.blind.commit_msg_hook {
1976 std::fs::create_dir_all(&hooks_dir)
1977 .with_context(|| format!("create {}", hooks_dir.display()))?;
1978 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1979 let path = hooks_dir.join("commit-msg");
1980 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1981 make_executable(&path)?;
1982 git::acquire_worktree_config(&repo).await?;
1990 self.state.enabled_worktree_config = true;
1991 }
1992
1993 ensure_setup_config(&mut self.state, &repo).await?;
1996
1997 for (index, (spec, label)) in self
1998 .roles
1999 .implementers
2000 .clone()
2001 .into_iter()
2002 .zip(labels)
2003 .enumerate()
2004 {
2005 let branch = self.state.branch_for(label);
2006 let worktree = root.join(format!("cand-{label}"));
2007 self.state.event(
2011 "prep-intent",
2012 serde_json::json!({
2013 "branch": branch,
2014 "worktree": worktree,
2015 "start": start,
2016 "picks": plan.picks,
2017 })
2018 .to_string(),
2019 );
2020 self.state.save()?;
2021 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2022 if self.state.config.blind.commit_msg_hook {
2023 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2024 }
2025 git::local_exclude(&worktree, "/.magi/").await?;
2026 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2027 git::worktree_remove_clean(&repo, &worktree).await.ok();
2030 git::branch_delete(&repo, &branch).await.ok();
2031 self.rollback_prep(&repo).await;
2032 self.state.status = RunStatus::Blocked;
2033 self.state
2034 .event("prep", format!("worktree setup failed: {e:#}"));
2035 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2036 self.state.save()?;
2037 return Err(e);
2038 }
2039 self.state.candidates.push(Candidate {
2040 index,
2041 label,
2042 agent: spec.id.clone(),
2043 branch,
2044 worktree,
2045 summary: String::new(),
2046 stat: String::new(),
2047 files: 0,
2048 commits: 0,
2049 empty: false,
2050 failed: None,
2051 verified_noop: None,
2052 duration_ms: 0,
2053 folded: false,
2054 });
2055 self.state.save()?;
2056 for pick in &plan.picks {
2057 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2058 {
2059 self.state.status = RunStatus::Blocked;
2060 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2061 self.state
2062 .event("prep", format!("cannot apply referenced commit: {e}"));
2063 self.state.save()?;
2064 return Err(e);
2065 }
2066 }
2067 }
2068
2069 for j in 1..=self.roles.judges.len() {
2070 let wt = root.join(format!("judge-{j}"));
2071 if !wt.exists() {
2072 git::worktree_add_detached(&repo, &wt, &base).await?;
2073 self.setup_seat_worktree(&repo, &wt, true).await?;
2074 }
2075 }
2076
2077 if self.state.config.graph.advise {
2085 for k in 1..=self.state.config.graph.advisors {
2086 let wt = root.join(format!("advisor-{k}"));
2087 if !wt.exists() {
2088 git::worktree_add_detached(&repo, &wt, &base).await?;
2089 self.setup_seat_worktree(&repo, &wt, true).await?;
2090 }
2091 }
2092 }
2093
2094 let authors: Vec<&str> = self
2099 .roles
2100 .implementers
2101 .iter()
2102 .map(|a| a.id.as_str())
2103 .collect();
2104 let overlap: Vec<String> = self
2105 .roles
2106 .judges
2107 .iter()
2108 .enumerate()
2109 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2110 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2111 .collect();
2112 if !overlap.is_empty() {
2113 let note = format!(
2114 "{} also authored a candidate; blind, but the panel is less \
2115 independent than {} distinct agents would be",
2116 overlap.join(", "),
2117 self.roles.judges.len()
2118 );
2119 self.state.event("prep", note);
2120 }
2121
2122 self.state.event(
2123 "prep",
2124 format!(
2125 "{} candidates, {} judges, base {} ({})",
2126 self.state.candidates.len(),
2127 self.roles.judges.len(),
2128 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2129 self.state.base_branch
2130 ),
2131 );
2132 self.state.prep_complete = Some(true);
2133 self.state.status = RunStatus::Implementing;
2134 self.state.save()?;
2135 Ok(())
2136 }
2137
2138 async fn advise(&mut self) -> Result<()> {
2171 let implement_untouched = self
2172 .state
2173 .candidates
2174 .iter()
2175 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2176 if !self.state.config.graph.advise || self.state.advise_attempted {
2177 return Ok(());
2178 }
2179 if !implement_untouched {
2180 self.state.event(
2181 "advise",
2182 "skipping the design-deliberation stage: at least one \
2183 candidate already shows implementation progress, so this \
2184 run is past the point the stage exists to run before"
2185 .to_owned(),
2186 );
2187 self.state.advise_attempted = true;
2188 self.state.save()?;
2189 return Ok(());
2190 }
2191 let run_id = self.state.id.clone();
2192 let prompts = self.state.config.prompts.clone();
2193 let instruction = self.state.instruction.clone();
2194 let language = self.state.config.graph.language.clone();
2195 let root = self.state.worktree_root();
2196 let n = self.state.config.graph.advisors;
2197 let where_recorded = self.state.dir().join("run.json");
2198
2199 let seats = match self.state.config.advisors() {
2200 Ok(seats) if !seats.is_empty() => seats,
2201 Ok(_) => {
2202 self.state.event(
2203 "advise",
2204 format!(
2205 "[graph] advisors is 0; skipping the design-deliberation \
2206 stage and continuing without a synthesis brief (see {})",
2207 where_recorded.display()
2208 ),
2209 );
2210 self.state.advise_attempted = true;
2211 self.state.save()?;
2212 return Ok(());
2213 }
2214 Err(e) => {
2215 self.state.event(
2216 "advise",
2217 format!(
2218 "could not resolve advisor seats ({e:#}); continuing \
2219 without a design-deliberation brief (see {})",
2220 where_recorded.display()
2221 ),
2222 );
2223 self.state.advise_attempted = true;
2224 self.state.save()?;
2225 return Ok(());
2226 }
2227 };
2228
2229 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2230 let artifacts = agent::artifacts_dir(&self.state.dir());
2231 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2232
2233 let mut jobs = Vec::new();
2234 for (i, spec) in seats.iter().cloned().enumerate() {
2235 let seat_key = format!("advisor-{}", i + 1);
2236 let seat = self.seat(&seat_key, &spec.id);
2237 jobs.push(SeatJob {
2238 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2239 spec,
2240 seat,
2241 cwd: worktrees[i % worktrees.len()].clone(),
2242 timeout,
2243 allow_write: false,
2244 sessions: false,
2245 artifacts: artifacts.clone(),
2246 stem: seat_key,
2247 handover: None,
2248 });
2249 }
2250
2251 self.state.event(
2252 "advise",
2253 format!(
2254 "{} advisor seat(s) sketching a design in parallel",
2255 jobs.len()
2256 ),
2257 );
2258 let mut quota_losses = Vec::new();
2259 let cache = self.state.config.cache_dir();
2260 let ctx = WaveCtx {
2261 carry_seats: false,
2262 pauses: Some((&self.pause, &self.interrupt)),
2263 run: &run_id,
2264 node: "advise",
2265 prompts: &prompts,
2266 cache: cache.as_deref(),
2267 round: None,
2268 };
2269 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2270 let results = ask_json_wave::<Proposal>(
2271 jobs,
2272 Arc::clone(&self.sem),
2273 self.state.config.graph.retries,
2274 &advisor_roster,
2275 &ctx,
2276 &mut quota_losses,
2277 &mut self.state,
2278 &|p: &Proposal| p.validate(),
2279 )
2280 .await;
2281 if self.state.resource_block.is_some() {
2282 return self.stop_for_resource();
2283 }
2284 self.state.quota.extend(quota_losses);
2285
2286 let mut records = Vec::with_capacity(results.len());
2287 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2288 let agent_id = seat.agent.clone();
2289 self.state.seats.insert(seat.key.clone(), seat);
2290 match res {
2291 Ok((proposal, out)) => {
2292 self.state
2293 .event("advise", format!("advisor-{} proposed a design", i + 1));
2294 records.push(advise::AdvisorRecord::proposed(
2295 i + 1,
2296 agent_id,
2297 proposal,
2298 out.duration_ms,
2299 ));
2300 }
2301 Err(e) => {
2302 self.state.event(
2303 "advise",
2304 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2305 );
2306 records.push(advise::AdvisorRecord::failed(
2307 i + 1,
2308 agent_id,
2309 e.to_string(),
2310 ));
2311 }
2312 }
2313 }
2314
2315 let mut advice = advise::Advice {
2316 records,
2317 synthesis: None,
2318 };
2319 if advice.proposals().is_empty() {
2320 self.state.event(
2321 "advise",
2322 "no advisor produced a usable proposal; continuing without a \
2323 synthesis brief"
2324 .to_owned(),
2325 );
2326 } else {
2327 match self
2328 .synthesize_brief(
2329 &advice,
2330 &instruction,
2331 &language,
2332 &worktrees[0],
2333 &artifacts,
2334 &run_id,
2335 &prompts,
2336 cache.as_deref(),
2337 )
2338 .await
2339 {
2340 Ok(Some(text)) => {
2341 self.state.event(
2342 "advise",
2343 "synthesized a design brief for the implementer".to_owned(),
2344 );
2345 advice.synthesis = Some(text);
2346 }
2347 Ok(None) => {
2348 self.state.event(
2349 "advise",
2350 "the synthesis seat produced nothing usable; continuing \
2351 without a design brief"
2352 .to_owned(),
2353 );
2354 }
2355 Err(e) => {
2356 if e.is::<crate::repo_lock::ResourceBlocked>() {
2357 return Err(e);
2358 }
2359 self.state.event(
2360 "advise",
2361 format!("could not synthesize a design brief: {e:#}"),
2362 );
2363 }
2364 }
2365 }
2366 advise::apply_reflection(&mut advice);
2367
2368 self.state.advice = Some(advice);
2369 self.state.advise_attempted = true;
2370 self.state.save()?;
2371 Ok(())
2372 }
2373
2374 #[allow(clippy::too_many_arguments)]
2386 async fn synthesize_brief(
2387 &mut self,
2388 advice: &advise::Advice,
2389 instruction: &str,
2390 language: &str,
2391 cwd: &Path,
2392 artifacts: &Path,
2393 run_id: &str,
2394 prompts: &Prompts,
2395 cache: Option<&Path>,
2396 ) -> Result<Option<String>> {
2397 let chain = agent::pick_chain(
2398 &self.state.config.agents,
2399 self.state.config.roles.synthesizer.as_ref(),
2400 &agent::installed,
2401 "synthesizer",
2402 )?;
2403 let proposals = advice.proposals();
2404 let mut prompt = prompt::with_overlay(
2405 prompt::synthesize_brief(instruction, &proposals, language),
2406 prompts.overlay("advise"),
2407 );
2408 if cache.is_some() {
2409 prompt.push('\n');
2414 prompt.push_str(&prompt::build_cache_note("advise", false));
2415 }
2416 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2417 let mut last = None;
2422 for (n, spec) in chain.iter().enumerate() {
2423 if n > 0 {
2424 self.state
2425 .event("advise", format!("synthesis falling back to {}", spec.id));
2426 }
2427 let mut seat = self.seat("advise-synthesis", &spec.id);
2428 let wait = self.repository_wait();
2429 let outcome = wait
2430 .scope(agent::invoke(
2431 spec,
2432 &mut seat,
2433 &Invocation {
2434 coordination: self.state.config.coordination.as_ref(),
2435 fetched_base: None,
2436 cwd,
2437 prompt: &prompt,
2438 timeout,
2439 allow_write: false,
2440 sessions: false,
2441 artifacts,
2442 stem: &if n == 0 {
2443 "advise-synthesis".to_owned()
2444 } else {
2445 format!("advise-synthesis-{}", spec.id)
2446 },
2447 run: run_id,
2448 node: "advise",
2449 cache_dir: None,
2450 attachments: &[],
2451 writable: &[],
2452 },
2453 ))
2454 .await;
2455 if outcome.is_ok() {
2456 self.state.seats.insert(seat.key.clone(), seat);
2457 }
2458 let advance = agent::chain_advances(&outcome);
2459 last = Some(outcome);
2460 if !advance {
2461 break;
2462 }
2463 }
2464 let out = last.expect("a chain holds at least one agent")?;
2467 if !out.usable() {
2468 return Ok(None);
2469 }
2470 let text =
2471 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2472 Ok((!text.trim().is_empty()).then_some(text))
2473 }
2474
2475 async fn implement(&mut self) -> Result<()> {
2478 self.persist_candidates().await?;
2479 let run_id = self.state.id.clone();
2484 let prompts = self.state.config.prompts.clone();
2485 let todo: Vec<usize> = self
2486 .state
2487 .candidates
2488 .iter()
2489 .enumerate()
2490 .filter(|(_, c)| {
2491 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2492 && c.failed.is_none()
2493 && !c.empty
2494 })
2495 .map(|(i, _)| i)
2496 .collect();
2497 if todo.is_empty() {
2498 return self.after_implement();
2499 }
2500 self.state.status = RunStatus::Implementing;
2501
2502 let language = self.state.config.graph.language.clone();
2503 let github_language = self.state.config.graph.github_language.clone();
2504 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2505 let sessions = self.state.config.graph.sessions;
2506 let artifacts = agent::artifacts_dir(&self.state.dir());
2507 let brief = self
2511 .state
2512 .advice
2513 .as_ref()
2514 .and_then(|a| a.synthesis.as_deref())
2515 .map(str::to_owned);
2516 let attachments = self.state.attachments.clone();
2517
2518 let mut jobs = Vec::new();
2519 for &i in &todo {
2520 let (index, label, worktree) = {
2521 let c = &self.state.candidates[i];
2522 (c.index, c.label, c.worktree.clone())
2523 };
2524 let spec = self.roles.implementers[index].clone();
2525 let seat_key = format!("impl-{label}");
2526 let seat = self.seat(&seat_key, &spec.id);
2527 let instruction = seeded_instruction(&self.state);
2528 jobs.push(SeatJob {
2529 spec,
2530 seat,
2531 prompt: prompt::implement_in(
2532 &instruction,
2533 &worktree.to_string_lossy(),
2534 &language,
2535 &github_language,
2536 brief.as_deref(),
2537 &attachments,
2538 ),
2539 cwd: worktree,
2540 timeout,
2541 allow_write: true,
2542 sessions,
2543 artifacts: artifacts.clone(),
2544 stem: format!("impl-{label}"),
2545 handover: None,
2546 });
2547 }
2548
2549 self.state.event(
2550 "implement",
2551 format!("{} candidates in parallel", jobs.len()),
2552 );
2553 let mut sent = jobs.clone();
2559 let cache = self.state.config.cache_dir();
2560 let ctx = WaveCtx {
2561 carry_seats: false,
2562 pauses: Some((&self.pause, &self.interrupt)),
2563 run: &run_id,
2564 node: "implement",
2565 prompts: &prompts,
2566 cache: cache.as_deref(),
2567 round: None,
2568 };
2569 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2570 if !wave_blocked(&results) {
2571 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2572 .await;
2573 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2574 .await?;
2575 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2576 .await;
2577 }
2578 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2579
2580 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2581 && results
2582 .iter()
2583 .any(|(_, _, out)| FailClass::of(out).is_some());
2584 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2585 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2586 if self
2590 .state
2591 .candidate_preservation
2592 .contains_key(&self.state.candidates[i].label)
2593 {
2594 self.state.seats.insert(seat.key.clone(), seat);
2595 }
2596 continue;
2597 }
2598 let seat_key = seat.key.clone();
2599 let agent = seat.agent.clone();
2608 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2609 self.state.seats.insert(seat.key.clone(), seat);
2610 let label = self.state.candidates[i].label;
2611 self.state.interrupted_implementations.remove(&label);
2612
2613 let (summary, duration, failed, verified_claim) = match out {
2614 AgentOutcome::Ok(o) => {
2615 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2616 let failed = (!o.usable()).then(|| {
2617 if o.timed_out {
2618 "agent timed out".to_owned()
2619 } else {
2620 format!(
2621 "agent exited with {}",
2622 crate::notices::exit_status(o.exit_code)
2623 )
2624 }
2625 });
2626 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2627 (text, o.duration_ms, failed, verified_claim)
2628 }
2629 AgentOutcome::Dropped(o) => {
2635 let why = o
2636 .dropped
2637 .as_ref()
2638 .map(|d| d.why.as_str())
2639 .unwrap_or("the CLI ended the stream without delivering its answer");
2640 (
2641 String::new(),
2642 o.duration_ms,
2643 Some(format!("the CLI dropped the stream ({why})")),
2644 None,
2645 )
2646 }
2647 AgentOutcome::Quota(o) => {
2648 self.state.quota.push(QuotaLoss {
2649 seat: seat_key,
2650 node: "implement".to_owned(),
2651 at: Timestamp::now(),
2652 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2653 });
2654 (
2655 String::new(),
2656 o.duration_ms,
2657 Some("rate limited (quota); produced no change".to_owned()),
2658 None,
2659 )
2660 }
2661 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2662 (String::new(), 0, Some(e), None)
2663 }
2664 };
2665
2666 let c = &mut self.state.candidates[i];
2669 if !exhausted_the_fallback_chain {
2670 c.agent = agent;
2671 }
2672 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2673 c.duration_ms = duration;
2674 c.failed = failed;
2675 c.verified_noop = verified_claim;
2676 self.state
2677 .candidate_preservation
2678 .insert(label, "awaiting persistence".to_owned());
2679 }
2680 self.state.save()?;
2681 self.persist_candidates().await?;
2682
2683 if resource_blocked {
2684 return self.stop_for_resource();
2685 }
2686 if selected_failed {
2687 self.state.status = RunStatus::Blocked;
2688 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2689 self.state.save()?;
2690 bail!(
2691 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2692 );
2693 }
2694 self.after_implement()
2695 }
2696
2697 async fn persist_candidates(&mut self) -> Result<()> {
2699 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2700 for label in labels {
2701 let i = self
2702 .state
2703 .candidates
2704 .iter()
2705 .position(|c| c.label == label)
2706 .context("pending candidate is missing")?;
2707 let c = self.state.candidates[i].clone();
2708 let interrupted = self.state.interrupted_implementations.contains(&label);
2709 let result = async {
2710 let wait = crate::repo_lock::WaitPolicy::new(
2711 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2712 || false,
2713 );
2714 let _coordination =
2715 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2716 if let Err(e) = git::snapshot_uncommitted(
2719 &c.worktree,
2720 &self
2721 .state
2722 .dir()
2723 .join("artifacts")
2724 .join(format!("candidate-{label}-recovery")),
2725 )
2726 .await
2727 {
2728 self.state.event(
2729 "preserve",
2730 format!("candidate {label} snapshot incomplete: {e:#}"),
2731 );
2732 }
2733 let rescued = git::rescue_commit(
2734 &c.worktree,
2735 &format!("magi: candidate {label} (uncommitted work)"),
2736 )
2737 .await?;
2738 self.state.note_withheld("implement", &rescued.withheld);
2739 let base = &self.state.base_commit;
2740 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2741 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2742 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2743 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2744 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2745 if c.empty && commits > 0 && !patch.trim().is_empty()
2746 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2747 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2748 || self.state.tally.is_some())
2749 {
2750 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2754 }
2755 let c = &mut self.state.candidates[i];
2756 c.commits = commits;
2757 c.stat = stat;
2758 c.files = files;
2759 let no_changes = commits == 0 || patch.trim().is_empty();
2760 c.empty = no_changes && !interrupted;
2761 if interrupted && no_changes {
2762 c.failed = None;
2763 c.verified_noop = None;
2764 }
2765 if !c.empty {
2766 c.failed = None;
2767 c.verified_noop = None;
2768 }
2769 self.state.event(
2770 "preserve",
2771 format!("candidate {label}: {files} files, {commits} commits"),
2772 );
2773 Ok::<_, anyhow::Error>(())
2774 }
2775 .await;
2776 match result {
2777 Ok(()) => {
2778 self.state.candidate_preservation.remove(&label);
2779 if !interrupted || self.state.candidates[i].files > 0 {
2780 self.state.interrupted_implementations.remove(&label);
2781 }
2782 }
2783 Err(e) => {
2784 let reason = format!("candidate {label} preservation failed: {e:#}");
2785 self.state.event("preserve", &reason);
2786 self.state.candidate_preservation.insert(label, reason);
2787 }
2788 }
2789 if self.state.candidate_preservation.is_empty()
2790 && self.state.status == RunStatus::Blocked
2791 && self.state.candidates.iter().all(|c| c.failed.is_none())
2792 {
2793 self.state.status = RunStatus::Implementing;
2797 self.state.event(
2798 "preserve",
2799 "candidate persistence recovered; implementation can continue",
2800 );
2801 }
2802 self.state.save()?;
2803 }
2804 if !self.state.candidate_preservation.is_empty() {
2805 self.state.status = RunStatus::Blocked;
2806 self.state.save()?;
2807 bail!(
2808 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2809 );
2810 }
2811 Ok(())
2812 }
2813
2814 async fn resume_undelivered(
2842 &mut self,
2843 results: &mut [(usize, SeatState, AgentOutcome)],
2844 sent: &[SeatJob],
2845 prompts: &Prompts,
2846 run_id: &str,
2847 ) {
2848 for (wi, seat, out) in results.iter_mut() {
2849 let Some(dropped) = (match &*out {
2850 AgentOutcome::Dropped(o) => o.dropped.clone(),
2851 _ => None,
2852 }) else {
2853 continue;
2854 };
2855 let Some(job) = sent.get(*wi) else { continue };
2856 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2858 self.state.event(
2859 "implement",
2860 format!(
2861 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2862 work is in the tree",
2863 seat.key, dropped.output_tokens, dropped.why
2864 ),
2865 );
2866 continue;
2867 }
2868 if !has_context(&job.spec, seat, job.sessions) {
2876 self.state.event(
2877 "implement",
2878 format!(
2879 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2880 is no session left to resume",
2881 seat.key, dropped.output_tokens, dropped.why
2882 ),
2883 );
2884 continue;
2885 }
2886 self.state.event(
2887 "implement",
2888 format!(
2889 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2890 conversation",
2891 seat.key, dropped.output_tokens, dropped.why
2892 ),
2893 );
2894 let mut retry = job.clone();
2895 retry.seat = seat.clone();
2896 retry.prompt = prompt::resume_after_drop(&dropped.why);
2897 retry.timeout = retry_budget(job.timeout, true);
2898 retry.stem = format!("{}-resume", job.stem);
2899 let cache = self.state.config.cache_dir();
2900 let ctx = WaveCtx {
2901 carry_seats: false,
2902 pauses: Some((&self.pause, &self.interrupt)),
2903 run: run_id,
2904 node: "implement",
2905 prompts,
2906 cache: cache.as_deref(),
2907 round: None,
2908 };
2909 let (resumed_seat, resumed) =
2910 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2911 *seat = resumed_seat;
2912 *out = resumed;
2913 }
2914 }
2915
2916 async fn resume_seat_handovers(
2981 &mut self,
2982 results: &mut [(usize, SeatState, AgentOutcome)],
2983 sent: &mut [SeatJob],
2984 prompts: &Prompts,
2985 run_id: &str,
2986 ) -> Result<()> {
2987 let instruction = seeded_instruction(&self.state);
2988 let language = self.state.config.graph.language.clone();
2989 let github_language = self.state.config.graph.github_language.clone();
2990 let brief = self
2991 .state
2992 .advice
2993 .as_ref()
2994 .and_then(|a| a.synthesis.as_deref())
2995 .map(str::to_owned);
2996 let attachments = self.state.attachments.clone();
2997 for (wi, seat, out) in results.iter_mut() {
2998 let others: BTreeSet<String> = sent
3001 .iter()
3002 .enumerate()
3003 .filter(|(j, _)| j != wi)
3004 .map(|(_, j)| j.spec.id.clone())
3005 .collect();
3006 let Some(job) = sent.get_mut(*wi) else {
3007 continue;
3008 };
3009 let start = self
3014 .roles
3015 .implementer_roster
3016 .iter()
3017 .position(|s| s.id == job.spec.id)
3018 .unwrap_or(0);
3019 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3020 let mut fallback_attempt = 0usize;
3021 let mut prev: Option<FailClass> = None;
3022 while let Some(cur) = FailClass::of(&*out) {
3023 if !should_hand_over(prev.as_ref(), &cur) {
3024 break;
3025 }
3026 let Some(next) =
3027 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3028 .cloned()
3029 else {
3030 break;
3031 };
3032 tried.insert(next.id.clone());
3033 fallback_attempt += 1;
3034
3035 let coordination = self.coordinate(&job.cwd).await?;
3036 if let Ok(r) = git::rescue_commit(
3037 &job.cwd,
3038 &format!(
3039 "magi: candidate {} (uncommitted work before {} fallback)",
3040 seat.key,
3041 if cur == FailClass::Quota {
3042 "quota"
3043 } else {
3044 "handover"
3045 }
3046 ),
3047 )
3048 .await
3049 {
3050 self.state.note_withheld("implement", &r.withheld);
3051 }
3052 drop(coordination);
3053
3054 record_handover(
3055 &mut self.state,
3056 "implement",
3057 &seat.key,
3058 &seat.agent,
3059 &next.id,
3060 &cur,
3061 &fail_reason(&*out),
3062 );
3063 prev = Some(cur.clone());
3064
3065 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3066 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3067 job.spec = next.clone();
3075 let mut retry = job.clone();
3076 retry.seat = new_seat;
3077 retry.prompt = prompt::implement_in(
3078 &instruction,
3079 &job.cwd.to_string_lossy(),
3080 &language,
3081 &github_language,
3082 brief.as_deref(),
3083 &attachments,
3084 );
3085 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3086 let cache = self.state.config.cache_dir();
3087 let ctx = WaveCtx {
3088 carry_seats: false,
3089 pauses: Some((&self.pause, &self.interrupt)),
3090 run: run_id,
3091 node: "implement",
3092 prompts,
3093 cache: cache.as_deref(),
3094 round: None,
3095 };
3096 let (fallback_seat, fallback_out) = run_one(
3097 retry,
3098 Arc::clone(&self.sem),
3099 &ctx,
3100 &mut self.state,
3101 fallback_attempt,
3102 )
3103 .await;
3104 *seat = fallback_seat;
3105 *out = fallback_out;
3106 }
3107 }
3108 Ok(())
3109 }
3110
3111 async fn resume_unconfirmed_commands(
3135 &mut self,
3136 results: &mut [(usize, SeatState, AgentOutcome)],
3137 sent: &[SeatJob],
3138 prompts: &Prompts,
3139 run_id: &str,
3140 ) {
3141 for (wi, seat, out) in results.iter_mut() {
3142 let AgentOutcome::Ok(o) = &*out else {
3143 continue;
3144 };
3145 if !has_unconfirmed_command(&o.commands) {
3146 continue;
3147 }
3148 let Some(job) = sent.get(*wi) else { continue };
3149 if !has_context(&job.spec, seat, job.sessions) {
3150 self.state.event(
3151 "implement",
3152 format!(
3153 "{}: the reply named a command whose own CLI never confirmed the exit \
3154 status of, but there is no session left to resume",
3155 seat.key
3156 ),
3157 );
3158 continue;
3159 }
3160 self.state.event(
3161 "implement",
3162 format!(
3163 "{}: the reply named a command whose own CLI never confirmed the exit \
3164 status of; resuming the conversation",
3165 seat.key
3166 ),
3167 );
3168 let mut retry = job.clone();
3169 retry.seat = seat.clone();
3170 retry.prompt = prompt::resume_incomplete(
3171 "a command in your last reply had no confirmed exit status",
3172 );
3173 retry.timeout = retry_budget(job.timeout, true);
3174 retry.stem = format!("{}-confirm", job.stem);
3175 let cache = self.state.config.cache_dir();
3176 let ctx = WaveCtx {
3177 carry_seats: false,
3178 pauses: Some((&self.pause, &self.interrupt)),
3179 run: run_id,
3180 node: "implement",
3181 prompts,
3182 cache: cache.as_deref(),
3183 round: None,
3184 };
3185 let (resumed_seat, resumed) =
3186 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3187 *seat = resumed_seat;
3188 *out = resumed;
3189 }
3190 }
3191
3192 async fn continue_fix_report(
3213 &mut self,
3214 mut seat: SeatState,
3215 parse_err: String,
3216 job: &SeatJob,
3217 prompts: &Prompts,
3218 run_id: &str,
3219 round: usize,
3220 ) -> (
3221 SeatState,
3222 Option<FixReport>,
3223 Option<String>,
3224 ContinuationRecord,
3225 ) {
3226 let mut last_err = parse_err;
3227 let mut cumulative_wait_ms = 0u64;
3228 let mut attempts = 0usize;
3229 loop {
3230 if !has_context(&job.spec, &seat, job.sessions) {
3231 self.state.event(
3232 "fix",
3233 format!(
3234 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3235 session left to resume into"
3236 ),
3237 );
3238 let outcome = if attempts == 0 {
3239 ContinuationOutcome::NoSession
3240 } else {
3241 ContinuationOutcome::Exhausted
3242 };
3243 return (
3244 seat,
3245 None,
3246 Some(format!("unparsable fix report: {last_err}")),
3247 ContinuationRecord {
3248 attempts,
3249 cumulative_wait_ms,
3250 outcome,
3251 },
3252 );
3253 }
3254 if attempts >= MAX_FIX_CONTINUATIONS {
3255 self.state.event(
3256 "fix",
3257 format!(
3258 "round {round}: fixer's reply still had no adoption report after \
3259 {attempts} continuation(s) ({last_err}); giving up"
3260 ),
3261 );
3262 return (
3263 seat,
3264 None,
3265 Some(format!(
3266 "unparsable fix report after {attempts} continuation(s): {last_err}"
3267 )),
3268 ContinuationRecord {
3269 attempts,
3270 cumulative_wait_ms,
3271 outcome: ContinuationOutcome::Exhausted,
3272 },
3273 );
3274 }
3275 attempts += 1;
3276 self.state.event(
3277 "fix",
3278 format!(
3279 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3280 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3281 ),
3282 );
3283 let mut retry = job.clone();
3284 retry.seat = seat.clone();
3285 retry.prompt = prompt::resume_incomplete(&last_err);
3286 retry.timeout = retry_budget(job.timeout, true);
3287 retry.stem = format!("{}-continue{attempts}", job.stem);
3288 let cache = self.state.config.cache_dir();
3289 let ctx = WaveCtx {
3290 carry_seats: false,
3291 pauses: Some((&self.pause, &self.interrupt)),
3292 run: run_id,
3293 node: "fix",
3294 prompts,
3295 cache: cache.as_deref(),
3296 round: Some(round),
3297 };
3298 let (resumed_seat, resumed_out) = run_one(
3299 retry,
3300 Arc::clone(&self.sem),
3301 &ctx,
3302 &mut self.state,
3303 attempts,
3304 )
3305 .await;
3306 seat = resumed_seat;
3307 match resumed_out {
3308 AgentOutcome::Ok(o) => {
3309 cumulative_wait_ms += o.duration_ms;
3310 match verdict::extract_json::<FixReport>(&o.text) {
3311 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3312 self.state.event(
3313 "fix",
3314 format!(
3315 "round {round}: fixer's adoption report recovered after \
3316 {attempts} continuation(s)"
3317 ),
3318 );
3319 return (
3320 seat,
3321 Some(report),
3322 None,
3323 ContinuationRecord {
3324 attempts,
3325 cumulative_wait_ms,
3326 outcome: ContinuationOutcome::Resumed,
3327 },
3328 );
3329 }
3330 Ok(_) => {
3338 last_err = "the reply parsed, but it reported a command whose own CLI \
3339 never confirmed an exit status"
3340 .to_owned();
3341 }
3342 Err(e) => last_err = e.to_string(),
3343 }
3344 }
3345 AgentOutcome::Quota(o) => {
3346 cumulative_wait_ms += o.duration_ms;
3347 self.state.quota.push(QuotaLoss {
3348 seat: seat.key.clone(),
3349 node: "fix".to_owned(),
3350 at: Timestamp::now(),
3351 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3352 });
3353 self.state.event(
3354 "fix",
3355 format!(
3356 "round {round}: continuation rate limited (quota); not retrying now"
3357 ),
3358 );
3359 return (
3360 seat,
3361 None,
3362 Some("rate limited (quota) while recovering the fix report".to_owned()),
3363 ContinuationRecord {
3364 attempts,
3365 cumulative_wait_ms,
3366 outcome: ContinuationOutcome::QuotaLost,
3367 },
3368 );
3369 }
3370 AgentOutcome::Dropped(o) => {
3371 cumulative_wait_ms += o.duration_ms;
3372 let why = o
3373 .dropped
3374 .as_ref()
3375 .map(|d| d.why.as_str())
3376 .unwrap_or("the CLI ended the stream without delivering its answer");
3377 last_err = format!("the CLI dropped the stream ({why})");
3378 }
3379 AgentOutcome::Failed(e) => last_err = e,
3380 AgentOutcome::ResourceBlocked(e) => {
3383 return (
3384 seat,
3385 None,
3386 Some(format!("the continuation was not started: {e}")),
3387 ContinuationRecord {
3388 attempts: attempts.saturating_sub(1),
3389 cumulative_wait_ms,
3390 outcome: ContinuationOutcome::NoSession,
3391 },
3392 );
3393 }
3394 }
3395 }
3396 }
3397
3398 fn after_implement(&mut self) -> Result<()> {
3399 if !self.state.candidate_preservation.is_empty() {
3400 bail!("candidate preservation incomplete; judging is unsafe");
3401 }
3402 if self.state.leaks.is_empty() {
3404 let cfg = self.state.config.blind.clone();
3405 let mut leaks = Vec::new();
3406 for c in &self.state.candidates {
3407 let Some(patch) =
3408 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3409 else {
3410 continue;
3411 };
3412 leaks.extend(blind::scan(
3413 &format!("candidate {} patch", c.label),
3414 &patch,
3415 &cfg.vendor_tokens,
3416 ));
3417 }
3418 if !leaks.is_empty() {
3419 let summary = leaks
3420 .iter()
3421 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3422 .collect::<Vec<_>>()
3423 .join(", ");
3424 match cfg.on_leak {
3425 LeakPolicy::Fail => {
3426 self.state.status = RunStatus::Failed;
3427 self.state
3428 .event("blind", format!("vendor text in a patch: {summary}"));
3429 self.state.leaks = leaks;
3430 self.state.save()?;
3431 self.settle_questions();
3432 bail!(
3433 "blind.on_leak = \"fail\" and vendor text reached a \
3434 judged patch: {summary}"
3435 );
3436 }
3437 LeakPolicy::Redact => self.state.event(
3438 "blind",
3439 format!("redacting vendor text for judging: {summary}"),
3440 ),
3441 LeakPolicy::Warn => self.state.event(
3442 "blind",
3443 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3444 ),
3445 }
3446 self.state.leaks = leaks;
3447 }
3448 }
3449
3450 if self.state.viable().is_empty() {
3451 if self.state.all_candidates_verified_noop() {
3452 self.state.status = RunStatus::VerifiedNoop;
3463 self.state.save()?;
3464 self.settle_questions();
3465 return Ok(());
3466 }
3467 self.state.status = RunStatus::Failed;
3468 self.state.save()?;
3469 self.settle_questions();
3470 bail!("no candidate produced a change; nothing to judge");
3471 }
3472 self.state.status = RunStatus::Judging;
3473 self.state.save()?;
3474 Ok(())
3475 }
3476
3477 async fn judge(&mut self) -> Result<()> {
3480 let run_id = self.state.id.clone();
3485 let prompts = self.state.config.prompts.clone();
3486 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3487 return Ok(());
3488 }
3489 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3490 if viable.len() == 1 {
3491 self.state.judge_skipped = true;
3498 self.state.event(
3499 "judge",
3500 format!(
3501 "only candidate {} produced a change; judging skipped",
3502 viable[0].label
3503 ),
3504 );
3505 self.state.save()?;
3506 return Ok(());
3507 }
3508 self.state.status = RunStatus::Judging;
3509
3510 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3511 let language = self.state.config.graph.language.clone();
3512 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3513 let sessions = self.state.config.graph.sessions;
3514 let artifacts = agent::artifacts_dir(&self.state.dir());
3515 let root = self.state.worktree_root();
3516 let base_short = short(&self.state.base_commit);
3517
3518 let mut jobs = Vec::new();
3519 let mut orders = Vec::new();
3520 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3521 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3522 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3523 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3524 let seat_key = format!("judge-{}", j + 1);
3525 let seat = self.seat(&seat_key, &spec.id);
3526 jobs.push(SeatJob {
3527 prompt: prompt::judge(
3528 &self.state.instruction,
3529 &views,
3530 self.roles.judges.len(),
3531 &base_short,
3532 &language,
3533 ),
3534 spec,
3535 seat,
3536 cwd: root.join(format!("judge-{}", j + 1)),
3537 timeout,
3538 allow_write: false,
3539 sessions,
3540 artifacts: artifacts.clone(),
3541 stem: format!("judge-{}", j + 1),
3542 handover: None,
3543 });
3544 }
3545
3546 self.state.event(
3547 "judge",
3548 format!(
3549 "{} judges ranking {} candidates blind",
3550 jobs.len(),
3551 viable.len()
3552 ),
3553 );
3554 let labels_for_check = labels.clone();
3555 let mut quota_losses = Vec::new();
3556 let cache = self.state.config.cache_dir();
3557 let ctx = WaveCtx {
3558 carry_seats: false,
3559 pauses: Some((&self.pause, &self.interrupt)),
3560 run: &run_id,
3561 node: "judge",
3562 prompts: &prompts,
3563 cache: cache.as_deref(),
3564 round: None,
3565 };
3566 let results = ask_json_wave::<Ranking>(
3567 jobs,
3568 Arc::clone(&self.sem),
3569 self.state.config.graph.retries,
3570 &self.roles.judge_roster,
3571 &ctx,
3572 &mut quota_losses,
3573 &mut self.state,
3574 &move |r: &Ranking| r.validate(&labels_for_check),
3575 )
3576 .await;
3577 if self.state.resource_block.is_some() {
3578 return self.stop_for_resource();
3579 }
3580 self.state.quota.extend(quota_losses);
3581
3582 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3583 let agent_id = seat.agent.clone();
3584 self.state.seats.insert(seat.key.clone(), seat);
3585 let mut record = Judgement {
3586 judge: j + 1,
3587 seat: format!("judge-{}", j + 1),
3588 agent: agent_id,
3589 ranking: Vec::new(),
3590 reasons: BTreeMap::new(),
3591 confidence: None,
3592 strengths: BTreeMap::new(),
3593 order: orders[j].clone(),
3594 failed: None,
3595 duration_ms: 0,
3596 };
3597 match res {
3598 Ok((ranking, out)) => {
3599 record.ranking = ranking.normalized();
3600 record.reasons = ranking.reasons;
3601 record.strengths = ranking.strengths;
3602 record.confidence = ranking.confidence;
3603 record.duration_ms = out.duration_ms;
3604 self.state.event(
3605 "judge",
3606 format!(
3607 "judge {} ranked {}",
3608 j + 1,
3609 record.ranking.iter().collect::<String>()
3610 ),
3611 );
3612 }
3613 Err(e) => {
3614 record.failed = Some(e.to_string());
3615 self.state
3616 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3617 }
3618 }
3619 self.state.judgements.push(record);
3620 self.state.save()?;
3621 }
3622 Ok(())
3623 }
3624
3625 async fn deliberate(&mut self) -> Result<()> {
3628 if self.hold_base_sync().await? {
3629 return Ok(());
3630 }
3631 let run_id = self.state.id.clone();
3636 let prompts = self.state.config.prompts.clone();
3637 if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3638 return Ok(());
3639 }
3640 let tops: Vec<char> = self
3641 .state
3642 .judgements
3643 .iter()
3644 .filter_map(|j| j.ranking.first().copied())
3645 .collect();
3646 let rounds = self.state.config.graph.deliberate_rounds;
3647 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3648 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3649 self.state.event(
3650 "deliberate",
3651 format!("judges agreed on {} outright; no deliberation", tops[0]),
3652 );
3653 }
3654 self.state.status = RunStatus::Voting;
3655 self.state.save()?;
3656 return Ok(());
3657 }
3658
3659 self.state.status = RunStatus::Deliberating;
3660 self.state.event(
3661 "deliberate",
3662 format!(
3663 "split: first choices were {} — opening {rounds} round(s)",
3664 tops.iter().collect::<String>()
3665 ),
3666 );
3667
3668 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3669 let language = self.state.config.graph.language.clone();
3670 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3671 let sessions = self.state.config.graph.sessions;
3672 let artifacts = agent::artifacts_dir(&self.state.dir());
3673 let root = self.state.worktree_root();
3674 let base_short = short(&self.state.base_commit);
3675
3676 for round in 1..=rounds {
3680 let mut turns: Vec<DeliberationTurn> = Vec::new();
3681 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3682 if self.state.judgements[j].failed.is_some() {
3683 continue;
3684 }
3685 let seat_key = format!("judge-{}", j + 1);
3686 let spec = self.occupant(&seat_key, spec);
3687 let mut seat = self.seat(&seat_key, &spec.id);
3688 let transcript = self.transcript(&turns, j);
3689 let build = |context: Option<&str>| {
3690 prompt::deliberate(
3691 &self.state.instruction,
3692 context,
3693 &transcript,
3694 round,
3695 rounds,
3696 &language,
3697 )
3698 };
3699 let block = self.candidate_block(&viable, &base_short);
3700 let full = build(Some(&block));
3701 let text = if has_context(&spec, &seat, sessions) {
3702 build(None)
3703 } else {
3704 full.clone()
3705 };
3706 let job = SeatJob {
3707 spec,
3708 seat: seat.clone(),
3709 prompt: text,
3710 cwd: root.join(format!("judge-{}", j + 1)),
3711 timeout,
3712 allow_write: false,
3713 sessions,
3714 artifacts: artifacts.clone(),
3715 stem: format!("delib-{round}-judge-{}", j + 1),
3716 handover: Some(full),
3717 };
3718 let cache = self.state.config.cache_dir();
3719 let ctx = WaveCtx {
3720 carry_seats: false,
3721 pauses: Some((&self.pause, &self.interrupt)),
3722 run: &run_id,
3723 node: "deliberate",
3724 prompts: &prompts,
3725 cache: cache.as_deref(),
3726 round: None,
3727 };
3728 let mut losses = Vec::new();
3732 let mut results = ask_wave_with::<String>(
3733 vec![job],
3734 Arc::clone(&self.sem),
3735 0,
3736 &self.roles.judge_roster,
3737 &ctx,
3738 &mut losses,
3739 &mut self.state,
3740 &|text: &str| {
3741 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3742 },
3743 )
3744 .await;
3745 self.state.quota.extend(losses);
3746 let (updated, res, _) = results.pop().expect("one job in, one result out");
3747 seat = updated;
3748 let agent_id = seat.agent.clone();
3749 self.state.seats.insert(seat.key.clone(), seat);
3750 let body = match res {
3751 Ok((body, _)) => body,
3752 Err(e) => {
3755 self.state
3756 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3757 continue;
3758 }
3759 };
3760 let tentative = verdict::extract_json::<Position>(&body)
3761 .ok()
3762 .and_then(|p| p.tentative)
3763 .and_then(|s| s.trim().chars().next())
3764 .map(|c| c.to_ascii_uppercase());
3765 self.state.event(
3766 "deliberate",
3767 format!(
3768 "round {round}: judge {} now favours {}",
3769 j + 1,
3770 tentative.map_or("—".to_owned(), |c| c.to_string())
3771 ),
3772 );
3773 turns.push(DeliberationTurn {
3774 judge: j + 1,
3775 agent: agent_id,
3776 body: blind::sanitize_prose(&body, &self.state.config.blind),
3777 tentative,
3778 });
3779 }
3780 self.state
3781 .deliberation
3782 .push(DeliberationRound { round, turns });
3783 self.state.save()?;
3784 }
3785
3786 self.state.status = RunStatus::Voting;
3787 self.state.save()?;
3788 Ok(())
3789 }
3790
3791 async fn vote(&mut self) -> Result<()> {
3794 if self.hold_base_sync().await? {
3795 return Ok(());
3796 }
3797 let run_id = self.state.id.clone();
3802 let prompts = self.state.config.prompts.clone();
3803 if !self.state.votes.is_empty() || self.state.tally.is_some() {
3804 return Ok(());
3805 }
3806 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3807 if viable.len() == 1 {
3808 return Ok(());
3809 }
3810 self.state.status = RunStatus::Voting;
3811
3812 let language = self.state.config.graph.language.clone();
3813 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3814 let sessions = self.state.config.graph.sessions;
3815 let artifacts = agent::artifacts_dir(&self.state.dir());
3816 let root = self.state.worktree_root();
3817 let base_short = short(&self.state.base_commit);
3818 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3819
3820 let mut jobs = Vec::new();
3821 let mut seats_at = Vec::new();
3822 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3823 if self
3824 .state
3825 .judgements
3826 .get(j)
3827 .is_some_and(|r| r.failed.is_some())
3828 {
3829 continue;
3830 }
3831 let seat_key = format!("judge-{}", j + 1);
3832 let spec = self.occupant(&seat_key, spec);
3833 let seat = self.seat(&seat_key, &spec.id);
3834 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3835 let text = if has_context(&spec, &seat, sessions) {
3836 prompt::final_vote(&viable, &language)
3837 } else {
3838 full.clone()
3839 };
3840 jobs.push(SeatJob {
3841 spec,
3842 seat,
3843 prompt: text,
3844 cwd: root.join(format!("judge-{}", j + 1)),
3845 timeout,
3846 allow_write: false,
3847 sessions,
3848 artifacts: artifacts.clone(),
3849 stem: format!("vote-judge-{}", j + 1),
3850 handover: Some(full),
3851 });
3852 seats_at.push(j);
3853 }
3854
3855 self.state.event(
3856 "vote",
3857 format!(
3858 "collecting {} final votes one by one, privately",
3859 jobs.len()
3860 ),
3861 );
3862 let allowed = viable.clone();
3863 let mut quota_losses = Vec::new();
3864 let cache = self.state.config.cache_dir();
3865 let ctx = WaveCtx {
3866 carry_seats: false,
3867 pauses: Some((&self.pause, &self.interrupt)),
3868 run: &run_id,
3869 node: "vote",
3870 prompts: &prompts,
3871 cache: cache.as_deref(),
3872 round: None,
3873 };
3874 let results = ask_json_wave::<FinalVote>(
3875 jobs,
3876 Arc::clone(&self.sem),
3877 self.state.config.graph.retries,
3878 &self.roles.judge_roster,
3879 &ctx,
3880 &mut quota_losses,
3881 &mut self.state,
3882 &move |v: &FinalVote| match v.label() {
3883 Some(c) if allowed.contains(&c) => Ok(()),
3884 other => bail!("vote {other:?} is not one of {allowed:?}"),
3885 },
3886 )
3887 .await;
3888 if self.state.resource_block.is_some() {
3889 return self.stop_for_resource();
3890 }
3891 self.state.quota.extend(quota_losses);
3892
3893 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3894 let agent_id = seat.agent.clone();
3895 self.state.seats.insert(seat.key.clone(), seat);
3896 let initial = self
3897 .state
3898 .judgements
3899 .get(j)
3900 .and_then(|r| r.ranking.first().copied());
3901 let mut record = VoteRecord {
3902 judge: j + 1,
3903 agent: agent_id,
3904 vote: None,
3905 reason: String::new(),
3906 changed: false,
3907 };
3908 match res {
3909 Ok((v, _)) => {
3910 record.vote = v.label();
3911 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3912 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3913 self.state.event(
3914 "vote",
3915 format!(
3916 "judge {} voted {}{}",
3917 j + 1,
3918 record.vote.unwrap_or('?'),
3919 if record.changed { " (changed)" } else { "" }
3920 ),
3921 );
3922 }
3923 Err(e) => {
3924 self.state
3925 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3926 }
3927 }
3928 self.state.votes.push(record);
3929 self.state.save()?;
3930 }
3931 Ok(())
3932 }
3933
3934 fn tally(&mut self) -> Result<()> {
3937 if self.state.tally.is_some() {
3938 return Ok(());
3939 }
3940 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3941 let tops: Vec<char> = self
3942 .state
3943 .judgements
3944 .iter()
3945 .filter_map(|j| j.ranking.first().copied())
3946 .collect();
3947 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3948
3949 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3952 let mut cast: Vec<char> = Vec::new();
3953 for (i, j) in self.state.judgements.iter().enumerate() {
3954 let vote = self
3955 .state
3956 .votes
3957 .iter()
3958 .find(|v| v.judge == i + 1)
3959 .and_then(|v| v.vote)
3960 .or_else(|| j.ranking.first().copied());
3961 if let Some(v) = vote {
3962 *first_choice.entry(v).or_insert(0) += 1;
3963 cast.push(v);
3964 }
3965 }
3966
3967 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3968 for j in &self.state.judgements {
3969 let n = j.ranking.len();
3970 for (pos, label) in j.ranking.iter().enumerate() {
3971 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3972 }
3973 }
3974
3975 let best = first_choice.values().copied().max().unwrap_or(0);
3976 let mut leaders: Vec<char> = first_choice
3977 .iter()
3978 .filter(|(_, v)| **v == best)
3979 .map(|(k, _)| *k)
3980 .collect();
3981 let mut tie_break = None;
3982 if leaders.len() > 1 {
3983 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3984 let borda_leaders: Vec<char> = leaders
3985 .iter()
3986 .copied()
3987 .filter(|l| borda[l] == top_borda)
3988 .collect();
3989 tie_break = Some(if borda_leaders.len() == 1 {
3990 format!(
3991 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3992 leaders.len()
3993 )
3994 } else {
3995 format!(
3996 "{} way tie on both first-choice votes and Borda points, broken by label order",
3997 leaders.len()
3998 )
3999 });
4000 leaders = borda_leaders;
4001 leaders.sort_unstable();
4002 }
4003 let winner = *leaders
4004 .first()
4005 .or(viable.first())
4006 .context("no candidate to declare a winner from")?;
4007
4008 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4009 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4010 let deliberated = !self.state.deliberation.is_empty();
4011
4012 let quota_seats: std::collections::BTreeSet<&str> =
4016 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4017 let mut present = 0usize;
4018 for (i, j) in self.state.judgements.iter().enumerate() {
4019 if quota_seats.contains(j.seat.as_str()) {
4020 continue;
4021 }
4022 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4023 let voted = self
4024 .state
4025 .votes
4026 .iter()
4027 .any(|v| v.judge == i + 1 && v.vote.is_some());
4028 if ranked || voted {
4029 present += 1;
4030 }
4031 }
4032 let needs_quorum = viable.len() > 1;
4038 let judges_total = if needs_quorum {
4039 self.roles.judges.len()
4040 } else {
4041 0
4042 };
4043 let quorum = if needs_quorum {
4044 judges_total / 2 + 1
4045 } else {
4046 0
4047 };
4048 let met_quorum = !needs_quorum || present >= quorum;
4049 let uncontested = (!needs_quorum).then(|| {
4050 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4051 });
4052
4053 self.state.event(
4054 "tally",
4055 match &uncontested {
4056 Some(reason) => format!("winner {winner} — {reason}"),
4057 None => format!(
4058 "winner {winner} — votes {} | initial {} | {} changed | \
4059 {present}/{judges_total} judges{}",
4060 first_choice
4061 .iter()
4062 .map(|(k, v)| format!("{k}:{v}"))
4063 .collect::<Vec<_>>()
4064 .join(" "),
4065 if unanimous_initial {
4066 "unanimous"
4067 } else {
4068 "split"
4069 },
4070 changed_votes,
4071 if met_quorum {
4072 String::new()
4073 } else {
4074 format!(" — below quorum ({quorum} required)")
4075 },
4076 ),
4077 },
4078 );
4079 if !met_quorum {
4080 self.state.event(
4081 "stall",
4082 format!(
4083 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4084 the run stops here, resumable"
4085 ),
4086 );
4087 }
4088 self.state.tally = Some(Tally {
4089 first_choice,
4090 borda,
4091 winner,
4092 rankings: tops.len(),
4093 unanimous_initial,
4094 deliberated,
4095 changed_votes,
4096 unanimous_final,
4097 tie_break,
4098 judges: judges_total,
4099 present,
4100 quorum,
4101 met_quorum,
4102 uncontested,
4103 });
4104 self.state.status = if met_quorum {
4105 RunStatus::Reviewing
4106 } else {
4107 RunStatus::Stalled
4108 };
4109 self.state.save()?;
4110 Ok(())
4111 }
4112
4113 #[allow(clippy::too_many_lines)]
4134 async fn recover_stall(&mut self) -> Result<bool> {
4135 let run_id = self.state.id.clone();
4140 let prompts = self.state.config.prompts.clone();
4141 let quota_seats: BTreeSet<&str> =
4146 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4147 let absent: Vec<String> = self
4148 .state
4149 .judgements
4150 .iter()
4151 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4152 .map(|j| j.seat.clone())
4153 .collect();
4154 if absent.is_empty() {
4155 return Ok(false);
4156 }
4157 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4158 if viable.len() <= 1 {
4159 return Ok(false);
4160 }
4161 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4162 let language = self.state.config.graph.language.clone();
4163 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4164 let sessions = self.state.config.graph.sessions;
4165 let artifacts = agent::artifacts_dir(&self.state.dir());
4166 let root = self.state.worktree_root();
4167 let base_short = short(&self.state.base_commit);
4168 let candidates: Vec<Candidate> = viable.clone();
4169
4170 let mut positions: Vec<usize> = absent
4172 .iter()
4173 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4174 .collect();
4175 if positions.is_empty() {
4176 return Ok(false);
4177 }
4178 positions.sort_unstable();
4179 positions.dedup();
4180
4181 let mut judge_jobs = Vec::new();
4183 for &j in &positions {
4184 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4185 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4186 let seat_key = format!("judge-{}", j + 1);
4187 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4188 let seat = self.seat(&seat_key, &spec.id);
4189 judge_jobs.push(SeatJob {
4190 spec,
4191 seat,
4192 prompt: prompt::judge(
4193 &self.state.instruction,
4194 &views,
4195 self.roles.judges.len(),
4196 &base_short,
4197 &language,
4198 ),
4199 cwd: root.join(seat_key),
4200 timeout,
4201 allow_write: false,
4202 sessions,
4203 artifacts: artifacts.clone(),
4204 stem: format!("judge-{}-recover", j + 1),
4205 handover: None,
4206 });
4207 }
4208
4209 let labels_for_check = labels.clone();
4210 let mut judge_losses = Vec::new();
4211 let retries = self.state.config.graph.retries;
4212 let cache = self.state.config.cache_dir();
4213 let ctx = WaveCtx {
4214 carry_seats: false,
4215 pauses: Some((&self.pause, &self.interrupt)),
4216 run: &run_id,
4217 node: "judge",
4218 prompts: &prompts,
4219 cache: cache.as_deref(),
4220 round: None,
4221 };
4222 let results = ask_json_wave::<Ranking>(
4223 judge_jobs,
4224 Arc::clone(&self.sem),
4225 retries,
4226 &self.roles.judge_roster,
4227 &ctx,
4228 &mut judge_losses,
4229 &mut self.state,
4230 &move |r: &Ranking| r.validate(&labels_for_check),
4231 )
4232 .await;
4233 if self.state.resource_block.is_some() {
4234 self.stop_for_resource()?;
4235 return Ok(false);
4236 }
4237
4238 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4240 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4241 let agent_id = seat.agent.clone();
4242 self.state.seats.insert(seat.key.clone(), seat);
4243 let record = &mut self.state.judgements[j];
4244 match res {
4245 Ok((ranking, out)) => {
4246 record.agent = agent_id;
4247 record.ranking = ranking.normalized();
4248 record.reasons = ranking.reasons;
4249 record.strengths = ranking.strengths;
4250 record.confidence = ranking.confidence;
4251 record.failed = None;
4252 record.duration_ms = out.duration_ms;
4253 recovered.insert(j);
4254 self.state.event(
4255 "recover",
4256 format!("judge {} ranked again after the limit", j + 1),
4257 );
4258 }
4259 Err(e) => {
4260 self.state
4261 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4262 }
4263 }
4264 }
4265
4266 let mut vote_jobs = Vec::new();
4268 let mut vote_pos: Vec<usize> = Vec::new();
4269 for &j in &recovered {
4270 let seat_key = format!("judge-{}", j + 1);
4271 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4272 let seat = self.seat(&seat_key, &spec.id);
4273 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4274 let text = if has_context(&spec, &seat, sessions) {
4275 prompt::final_vote(&labels, &language)
4276 } else {
4277 full.clone()
4278 };
4279 vote_jobs.push(SeatJob {
4280 spec,
4281 seat,
4282 prompt: text,
4283 cwd: root.join(seat_key),
4284 timeout,
4285 allow_write: false,
4286 sessions,
4287 artifacts: artifacts.clone(),
4288 stem: format!("vote-judge-{}-recover", j + 1),
4289 handover: Some(full),
4290 });
4291 vote_pos.push(j);
4292 }
4293 let allowed = labels.clone();
4294 let mut vote_losses = Vec::new();
4295 let vote_retries = self.state.config.graph.retries;
4296 let vote_cache = self.state.config.cache_dir();
4297 let ctx = WaveCtx {
4298 carry_seats: false,
4299 pauses: Some((&self.pause, &self.interrupt)),
4300 run: &run_id,
4301 node: "vote",
4302 prompts: &prompts,
4303 cache: vote_cache.as_deref(),
4304 round: None,
4305 };
4306 let votes = ask_json_wave::<FinalVote>(
4307 vote_jobs,
4308 Arc::clone(&self.sem),
4309 vote_retries,
4310 &self.roles.judge_roster,
4311 &ctx,
4312 &mut vote_losses,
4313 &mut self.state,
4314 &move |v: &FinalVote| match v.label() {
4315 Some(c) if allowed.contains(&c) => Ok(()),
4316 other => bail!("vote {other:?} is not one of {allowed:?}"),
4317 },
4318 )
4319 .await;
4320 if self.state.resource_block.is_some() {
4321 self.stop_for_resource()?;
4322 return Ok(false);
4323 }
4324 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4325 let agent_id = seat.agent.clone();
4326 self.state.seats.insert(seat.key.clone(), seat);
4327 match res {
4328 Ok((v, _)) => {
4329 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4330 rec.vote = v.label();
4331 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4332 } else {
4333 self.state.votes.push(VoteRecord {
4334 judge: j + 1,
4335 agent: agent_id,
4336 vote: v.label(),
4337 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4338 changed: false,
4339 });
4340 }
4341 self.state.event(
4342 "recover",
4343 format!("judge {} voted again after the limit", j + 1),
4344 );
4345 }
4346 Err(e) => {
4347 self.state
4348 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4349 }
4350 }
4351 }
4352
4353 let recovered_keys: BTreeSet<String> = recovered
4357 .iter()
4358 .map(|&j| format!("judge-{}", j + 1))
4359 .collect();
4360 self.state
4361 .quota
4362 .retain(|q| !recovered_keys.contains(&q.seat));
4363 for loss in judge_losses.into_iter().chain(vote_losses) {
4367 if recovered_keys.contains(&loss.seat) {
4368 continue;
4369 }
4370 self.state.quota.retain(|q| q.seat != loss.seat);
4371 self.state.quota.push(loss);
4372 }
4373
4374 self.state.tally = None;
4376 self.tally()?;
4377 Ok(self
4378 .state
4379 .tally
4380 .as_ref()
4381 .map(|t| t.met_quorum)
4382 .unwrap_or(false))
4383 }
4384
4385 async fn graft(&mut self) -> Result<()> {
4401 let Err(e) = self.graft_inner().await else {
4402 return Ok(());
4403 };
4404 if e.is::<crate::repo_lock::ResourceBlocked>() {
4405 return Err(e);
4406 }
4407 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4408 self.state.status = RunStatus::Blocked;
4409 self.state.event("graft", format!("blocked: {e:#}"));
4410 self.state.save()?;
4411 return Err(e);
4412 }
4413 if let Some(rec) = self.state.graft.clone()
4418 && !rec.outcome.settled()
4419 {
4420 if let Some(winner) = self.state.winner().cloned() {
4421 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4422 if re.is::<crate::repo_lock::ResourceBlocked>() {
4423 return Err(re);
4424 }
4425 self.state.status = RunStatus::Blocked;
4426 self.state.event("graft", format!("blocked: {re:#}"));
4427 self.state.save()?;
4428 return Err(re);
4429 }
4430 }
4431 if let Some(g) = self.state.graft.as_mut() {
4432 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4433 }
4434 }
4435 self.state.event("graft", format!("skipped: {e:#}"));
4436 if let Err(e) = self.state.save() {
4437 tracing::warn!("save after graft: {e:#}");
4438 }
4439 Ok(())
4440 }
4441
4442 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4445 let _coordination = self.coordinate(wt).await?;
4446 crate::recovery::restore_repair(wt, pre)
4447 .await
4448 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4449 }
4450
4451 fn graft_skipped(&mut self, why: &str) {
4453 let message = format!("skipped: {why}");
4454 if !self
4455 .state
4456 .events
4457 .iter()
4458 .any(|e| e.node == "graft" && e.message == message)
4459 {
4460 self.state.event("graft", message);
4461 }
4462 }
4463
4464 async fn graft_inner(&mut self) -> Result<()> {
4465 if let Some(rec) = self.state.graft.clone()
4468 && !rec.outcome.settled()
4469 {
4470 if self.state.reviews.is_empty()
4474 && let Some(winner) = self.state.winner().cloned()
4475 {
4476 self.graft_restore(&winner.worktree, &rec.pre).await?;
4477 }
4478 if rec.outcome == GraftOutcome::WaitingForRestore {
4479 self.state.graft = None;
4480 self.state.save()?;
4481 } else {
4482 if let Some(g) = self.state.graft.as_mut() {
4483 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4484 }
4485 self.state.event(
4486 "graft",
4487 "rolled back: the previous round was interrupted; the winner is untouched",
4488 );
4489 self.state.save()?;
4490 return Ok(());
4491 }
4492 }
4493 let plan = match graft::plan(&self.state) {
4494 Ok(plan) => plan,
4495 Err(skip) => {
4496 if skip != graft::Skip::AlreadyDone {
4497 self.graft_skipped(skip.reason());
4498 self.state.save()?;
4499 }
4500 return Ok(());
4501 }
4502 };
4503 let Some(winner) = self.state.winner().cloned() else {
4504 return Ok(());
4505 };
4506 let wt = winner.worktree.clone();
4507 if !git::is_clean(&wt).await? {
4508 self.graft_skipped("the winner's worktree has uncommitted changes");
4509 self.state.save()?;
4510 return Ok(());
4511 }
4512 let pre = git::rev_parse(&wt, "HEAD").await?;
4513 self.state.graft = Some(GraftRecord {
4516 pre: pre.clone(),
4517 post: None,
4518 lines: None,
4519 limit: None,
4520 agent: None,
4521 outcome: GraftOutcome::Running,
4522 requested: graft::requested(&plan),
4523 adoption: None,
4524 adoption_error: None,
4525 });
4526 self.state.event(
4527 "graft",
4528 format!(
4529 "porting strengths of candidate(s) {} onto {}",
4530 plan.strengths
4531 .iter()
4532 .map(|(l, _, _)| l.to_string())
4533 .collect::<Vec<_>>()
4534 .join(", "),
4535 plan.winner
4536 ),
4537 );
4538 self.state.save()?;
4539
4540 let fix_prompt = prompt::graft(
4541 &self.state.instruction,
4542 plan.winner,
4543 &plan.strengths,
4544 &self.state.config.graph.language,
4545 );
4546 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4547 let sessions = self.state.config.graph.sessions;
4548 let artifacts = agent::artifacts_dir(&self.state.dir());
4549 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4550 let (_, seat, out) = self
4551 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4552 SeatJob {
4553 prompt: fix_prompt.clone(),
4554 spec,
4555 seat,
4556 cwd: wt.clone(),
4557 timeout,
4558 allow_write: true,
4559 sessions,
4560 artifacts: artifacts.clone(),
4561 stem: "graft".to_owned(),
4562 handover: None,
4563 }
4564 })
4565 .await;
4566 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4567 self.state.graft.as_mut().expect("graft record").outcome =
4570 GraftOutcome::WaitingForRestore;
4571 return self.stop_for_resource();
4572 }
4573 let mut why: Option<String> = None;
4574 match &out {
4575 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4576 why = Some("the fixer produced nothing usable".to_owned());
4577 }
4578 AgentOutcome::Ok(_) => {}
4579 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4580 AgentOutcome::Quota(o) => {
4581 self.state.quota.push(QuotaLoss {
4582 seat: seat.key.clone(),
4583 node: "graft".to_owned(),
4584 at: Timestamp::now(),
4585 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4586 });
4587 why = Some("rate limited (quota)".to_owned());
4588 }
4589 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4590 why = Some(format!(
4591 "the fixer failed: {}",
4592 e.lines()
4593 .next()
4594 .unwrap_or("")
4595 .chars()
4596 .take(160)
4597 .collect::<String>()
4598 ));
4599 }
4600 }
4601 let agent_id = seat.agent.clone();
4602 self.state.seats.insert(seat.key.clone(), seat);
4603
4604 let mut post = None;
4605 let mut lines = None;
4606 let _coordination = self.coordinate(&wt).await?;
4607 if why.is_none() {
4608 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4609 if let Ok(r) = &rescued {
4610 self.state.note_withheld("graft", &r.withheld);
4611 }
4612 let head = git::rev_parse(&wt, "HEAD").await?;
4613 let dirty = match &rescued {
4617 Ok(r) => git::status_porcelain(&wt)
4618 .await
4619 .map(|st| {
4620 st.lines().any(|l| {
4621 !r.withheld
4622 .iter()
4623 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4624 })
4625 })
4626 .unwrap_or(true),
4627 Err(_) => true,
4628 };
4629 if dirty {
4630 why = Some("the fixer left changes that could not be committed".to_owned());
4631 } else if head == pre {
4632 why = Some("the fixer changed nothing".to_owned());
4633 } else {
4634 lines = git::changed_lines(&wt, &pre, &head).await;
4635 post = Some(head);
4636 }
4637 }
4638 let (adoption, adoption_error) = match &out {
4639 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4640 _ => (None, None),
4641 };
4642 let requested = graft::requested(&plan);
4643 let outcome = match why {
4644 None => {
4645 self.state.event(
4646 "graft",
4647 format!(
4648 "applied: {}",
4649 lines.map_or_else(
4650 || "changed lines unavailable".to_owned(),
4651 |n| format!("{n} changed lines")
4652 )
4653 ),
4654 );
4655 GraftOutcome::Applied
4656 }
4657 Some(why) => {
4658 crate::recovery::restore_repair(&wt, &pre)
4661 .await
4662 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4663 self.state.event(
4664 "graft",
4665 format!("rolled back: {why}; the winner is untouched"),
4666 );
4667 GraftOutcome::RolledBack(why)
4668 }
4669 };
4670 self.state.graft = Some(GraftRecord {
4671 pre,
4672 post,
4673 lines,
4674 limit: None,
4675 agent: Some(agent_id),
4676 outcome,
4677 requested,
4678 adoption,
4679 adoption_error,
4680 });
4681 self.state.save()?;
4682 Ok(())
4683 }
4684
4685 async fn fold_losers(&mut self) -> Result<()> {
4688 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4689 return Ok(());
4690 };
4691 if !self.state.candidate_preservation.is_empty() {
4692 bail!("candidate preservation incomplete; refusing cleanup");
4693 }
4694 let repo = self.state.repo.clone();
4695 let _coordination = self.coordinate(&repo).await?;
4696 let mut folded = Vec::new();
4697 for i in 0..self.state.candidates.len() {
4698 let c = &self.state.candidates[i];
4699 if c.label == winner || c.folded {
4700 continue;
4701 }
4702 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4703 remove_preserved_worktree(&repo, &wt).await?;
4704 if git::branch_exists(&repo, &branch).await?
4705 && !git::branch_delete(&repo, &branch).await?
4706 {
4707 bail!("candidate branch removal refused");
4708 }
4709 self.state.candidates[i].folded = true;
4710 self.state.save()?;
4711 folded.push(label.to_string());
4712 }
4713 let root = self.state.worktree_root();
4715 for j in 1..=self.roles.judges.len() {
4716 let wt = root.join(format!("judge-{j}"));
4717 if wt.exists() {
4718 remove_preserved_worktree(&repo, &wt).await?;
4719 }
4720 }
4721 if self.state.config.graph.advise {
4724 for k in 1..=self.state.config.graph.advisors {
4725 let wt = root.join(format!("advisor-{k}"));
4726 if wt.exists() {
4727 remove_preserved_worktree(&repo, &wt).await?;
4728 }
4729 }
4730 }
4731 if !folded.is_empty() {
4732 self.state
4733 .event("fold", format!("folded candidates {}", folded.join(", ")));
4734 self.state.save()?;
4735 }
4736 Ok(())
4737 }
4738
4739 async fn hold_base_sync(&mut self) -> Result<bool> {
4742 self.hold_base_sync_as(None).await
4743 }
4744
4745 async fn hold_base_sync_kind(
4746 &mut self,
4747 kind: crate::base_sync_recovery::StopKind,
4748 ) -> Result<()> {
4749 self.hold_base_sync_as(Some(kind)).await?;
4750 Ok(())
4751 }
4752
4753 async fn hold_base_sync_as(
4754 &mut self,
4755 kind: Option<crate::base_sync_recovery::StopKind>,
4756 ) -> Result<bool> {
4757 if !self.state.base_sync_blocked() {
4758 self.state.base_sync_recovery.stop = None;
4759 return Ok(false);
4760 }
4761 let head = match self.state.winner() {
4762 Some(w) => git::rev_parse(&w.worktree, "HEAD")
4763 .await
4764 .unwrap_or_default(),
4765 None => String::new(),
4766 };
4767 self.state.record_base_sync_stop(&head);
4768 if let Some(kind) = kind {
4769 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4770 }
4771 self.state.save()?;
4772 Ok(true)
4773 }
4774
4775 async fn sync_to_base(&mut self) -> Result<()> {
4813 if self.state.status == RunStatus::AlreadyInBase {
4814 return Ok(());
4815 }
4816 self.hold_base_sync().await?;
4817 let conflicted = self
4818 .state
4819 .base_sync
4820 .as_ref()
4821 .is_some_and(|s| s.conflict.is_some());
4822 let Some(winner) = self.state.winner().cloned() else {
4823 return Ok(());
4824 };
4825
4826 let repo = self.state.repo.clone();
4827 let remote = self.state.config.merge.remote.clone();
4828 let base_branch = self.state.base_branch.clone();
4829 let tracking = format!("{remote}/{base_branch}");
4830
4831 let fetched_base =
4832 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4833 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4838 return Ok(());
4839 };
4840
4841 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4842 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4843 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4844
4845 if (behind > 0 || conflicted || head == tip)
4854 && self
4855 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4856 .await?
4857 {
4858 return Ok(());
4859 }
4860 if conflicted && self.state.base_recovery_pending() {
4861 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4862 let owner =
4863 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4864 if !fetched_base
4865 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4866 || !git::is_clean(&winner.worktree).await?
4867 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4868 {
4869 return Ok(());
4870 }
4871 }
4872 let recovery = conflicted && self.state.consume_base_recovery(&head);
4873 if conflicted {
4874 self.state.save()?;
4876 if !recovery {
4877 return Ok(());
4878 }
4879 }
4880
4881 if behind == 0 {
4882 if let Some(from) = self
4891 .state
4892 .rebase_fixes
4893 .iter()
4894 .rev()
4895 .find_map(|r| r.from.clone())
4896 && from != head
4897 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4898 .await
4899 .is_ok_and(|o| o.ok())
4900 {
4901 git::sync_to_head(&winner.worktree).await?;
4902 }
4903 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4909 if let Some(why) = &conflict {
4910 self.state.status = RunStatus::Blocked;
4911 self.state.event("land", why.clone());
4912 }
4913 self.state.base_sync = Some(BaseSync {
4914 tip,
4915 behind: 0,
4916 attempts: attempts + usize::from(recovery),
4917 conflict,
4918 already_in: None,
4919 });
4920 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4921 .await?;
4922 self.state.save()?;
4923 return Ok(());
4924 }
4925
4926 if attempts >= BASE_SYNC_ROUNDS && !recovery {
4927 let why = format!(
4928 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4929 rebase(s); rebasing again would only race it",
4930 winner.branch
4931 );
4932 self.state.status = RunStatus::Blocked;
4933 self.state.base_sync = Some(BaseSync {
4934 tip,
4935 behind,
4936 attempts,
4937 conflict: Some(why.clone()),
4938 already_in: None,
4939 });
4940 self.state.event("land", why);
4941 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4942 .await?;
4943 self.state.save()?;
4944 return Ok(());
4945 }
4946
4947 self.state.event(
4948 "land",
4949 format!(
4950 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4951 winner.branch
4952 ),
4953 );
4954 self.state.save()?;
4955
4956 let branch_tracking = format!("{remote}/{}", winner.branch);
4962 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4963 let remote_tip = if self.state.task_pr_branch().is_none()
4964 && matches!(&fetched_branch, Ok(o) if o.ok())
4965 {
4966 git::rev_parse(&repo, &branch_tracking).await.ok()
4967 } else {
4968 None
4969 };
4970 if let Some(theirs) = &remote_tip
4974 && !git::is_ancestor(&repo, theirs, &head).await
4975 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4976 .await
4977 .is_ok_and(|missing| missing.is_empty())
4978 {
4979 let why = format!(
4980 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4981 them",
4982 short(theirs),
4983 winner.branch
4984 );
4985 self.state.status = RunStatus::Blocked;
4986 self.state.base_sync = Some(BaseSync {
4987 tip,
4988 behind,
4989 attempts: attempts + usize::from(recovery),
4990 conflict: Some(why.clone()),
4991 already_in: None,
4992 });
4993 self.state.event("land", why);
4994 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4995 .await?;
4996 self.state.save()?;
4997 return Ok(());
4998 }
4999
5000 let scratch = self.state.dir().join("base-sync");
5001 let rebased =
5002 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5003 .await
5004 {
5005 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5006 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5007 Err(e) => Err(e),
5008 };
5009 let attempts = attempts + 1;
5010 let stop_kind = match &rebased {
5011 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5012 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5013 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5014 };
5015 match rebased {
5016 Ok(None) => {
5017 git::sync_to_head(&winner.worktree).await?;
5021 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5022 let mut conflict = None;
5023 if let Some(pinned) = &remote_tip {
5024 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5025 match pushed {
5026 Ok(o) if o.ok() => self.state.event(
5027 "land",
5028 format!("pushed rebased {} to {remote}", winner.branch),
5029 ),
5030 Ok(o) => {
5031 conflict = Some(format!(
5032 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5033 winner.branch,
5034 short(pinned),
5035 o.stderr.chars().take(600).collect::<String>()
5036 ));
5037 }
5038 Err(e) => {
5039 conflict = Some(format!(
5040 "rebased {} locally but could not push it: {e:#}",
5041 winner.branch
5042 ));
5043 }
5044 }
5045 }
5046 if let Some(why) = &conflict {
5047 self.state.status = RunStatus::Blocked;
5048 self.state.event("land", why.clone());
5049 }
5050 self.state.base_sync = Some(BaseSync {
5051 tip: tip.clone(),
5052 behind: 0,
5053 attempts,
5054 conflict,
5055 already_in: None,
5056 });
5057 self.state
5058 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5059 }
5060 Ok(Some(conflict)) => {
5061 let why = format!(
5062 "{} conflicts with {tracking} and did not rebase: {}",
5063 winner.branch,
5064 conflict.chars().take(600).collect::<String>()
5065 );
5066 self.state.status = RunStatus::Blocked;
5067 self.state.base_sync = Some(BaseSync {
5068 tip,
5069 behind,
5070 attempts,
5071 conflict: Some(why.clone()),
5072 already_in: None,
5073 });
5074 self.state.event("land", why);
5075 }
5076 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5080 self.state.save()?;
5081 return Ok(());
5082 }
5083 Err(e) => {
5084 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5085 self.state.status = RunStatus::Blocked;
5086 self.state.base_sync = Some(BaseSync {
5087 tip,
5088 behind,
5089 attempts,
5090 conflict: Some(why.clone()),
5091 already_in: None,
5092 });
5093 self.state.event("land", why);
5094 }
5095 }
5096 if !self.state.base_sync_blocked() {
5097 self.state.base_sync_recovery.stop = None;
5098 }
5099 self.hold_base_sync_kind(stop_kind).await?;
5100 self.state.save()?;
5101 Ok(())
5102 }
5103
5104 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5110 if self.state.task_pr_branch().is_some() {
5113 return None;
5114 }
5115 let repo = self.state.repo.clone();
5116 let remote = self.state.config.merge.remote.clone();
5117 let fetched = git::fetch(&repo, &remote, branch).await;
5118 if !matches!(&fetched, Ok(o) if o.ok()) {
5119 return None;
5120 }
5121 let branch_tracking = format!("{remote}/{branch}");
5122 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5123 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5124 return None;
5125 }
5126 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5127 .await
5128 .is_ok_and(|missing| missing.is_empty())
5129 {
5130 return Some(format!(
5131 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5132 them",
5133 short(&theirs)
5134 ));
5135 }
5136 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5137 Ok(o) if o.ok() => {
5138 self.state
5139 .event("land", format!("pushed rebased {branch} to {remote}"));
5140 None
5141 }
5142 Ok(o) => Some(format!(
5143 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5144 someone may have pushed): {}",
5145 short(&theirs),
5146 o.stderr.chars().take(600).collect::<String>()
5147 )),
5148 Err(e) => Some(format!(
5149 "{branch} is rebased locally but could not be pushed: {e:#}"
5150 )),
5151 }
5152 }
5153
5154 async fn settle_already_in(
5171 &mut self,
5172 branch: &str,
5173 tip: &str,
5174 head: &str,
5175 attempts: usize,
5176 behind: usize,
5177 ) -> Result<bool> {
5178 let repo = self.state.repo.clone();
5179 let remote = self.state.config.merge.remote.clone();
5180 let start = self.state.base_commit.clone();
5181 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5182 Ok(Some(e)) => e,
5183 Ok(None) => return Ok(false),
5184 Err(e) => {
5185 tracing::warn!("already-in-base check for {branch}: {e:#}");
5186 return Ok(false);
5187 }
5188 };
5189 let mut verified = vec![head.to_owned()];
5190 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5191 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5192 if matches!(&fetched, Ok(o) if o.ok())
5193 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5194 && theirs != head
5195 {
5196 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5197 Ok(Some(_)) => verified.push(theirs),
5198 _ => return Ok(false),
5199 }
5200 }
5201 let base_branch = self.state.base_branch.clone();
5202 let message = format!(
5203 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5204 land",
5205 evidence.names(),
5206 evidence.proof.as_str()
5207 );
5208 let closed =
5209 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5210 .await;
5211 match closed {
5212 Ok(Ok(url)) => self
5213 .state
5214 .event("land", format!("closed {url}: superseded on {base_branch}")),
5215 Ok(Err(why)) => self
5216 .state
5217 .event("land", format!("did not close a pull request: {why}")),
5218 Err(e) => {
5219 let why = format!(
5220 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5221 not be closed ({e:#}); resume to retry"
5222 );
5223 self.state.status = RunStatus::Blocked;
5224 self.state.base_sync = Some(BaseSync {
5225 tip: tip.to_owned(),
5226 behind,
5227 attempts,
5228 conflict: Some(why.clone()),
5229 already_in: None,
5230 });
5231 self.state.event("land", why);
5232 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5233 .await?;
5234 self.state.save()?;
5235 return Ok(true);
5236 }
5237 }
5238 self.state.status = RunStatus::AlreadyInBase;
5239 self.state.base_sync_recovery.stop = None;
5240 self.state.base_sync = Some(BaseSync {
5241 tip: tip.to_owned(),
5242 behind,
5243 attempts,
5244 conflict: None,
5245 already_in: Some(evidence),
5246 });
5247 self.state.event("land", message);
5248 self.state.save()?;
5249 self.settle_questions();
5250 Ok(true)
5251 }
5252
5253 fn landing_base(&self) -> String {
5263 self.state
5264 .base_sync
5265 .as_ref()
5266 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5267 }
5268
5269 pub async fn fix_selected(
5302 &mut self,
5303 ids: &[String],
5304 reason: &str,
5305 allow_stale: bool,
5306 ) -> Result<()> {
5307 let reason = reason.trim();
5308 if reason.is_empty() {
5309 bail!("a fix request needs a reason — that is the operator's own record of why");
5310 }
5311 if ids.is_empty() {
5312 bail!("no finding id given");
5313 }
5314 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5315 bail!(
5316 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5317 has already concluded — can be given a targeted fix. A run still \
5318 in progress should simply be resumed; a `merged` run's branch has \
5319 already landed, so its answer is a fresh `magi review <branch>`, \
5320 not reopening this run's own record",
5321 self.state.id,
5322 self.state.status.as_str()
5323 );
5324 }
5325 let Some(winner) = self.state.winner().cloned() else {
5326 bail!("run {} has no winning candidate to fix", self.state.id);
5327 };
5328 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5329 bail!(
5330 "branch `{}` no longer exists; this run cannot be extended",
5331 winner.branch
5332 );
5333 }
5334 let home = crate::run::home();
5335 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5336 bail!(
5337 "run {} is currently being worked on by another magi process",
5338 self.state.id
5339 );
5340 }
5341 let _claim = FixClaim::acquire(&self.state.dir())?;
5347
5348 let mut seen = BTreeSet::new();
5352 let mut findings = Vec::new();
5353 let mut missing = Vec::new();
5354 for id in ids {
5355 if !seen.insert(id.clone()) {
5356 continue;
5357 }
5358 match self.state.finding(id) {
5359 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5360 id: f.id.clone(),
5361 severity: f.severity,
5362 reviewer_vote: rec.vote,
5363 round: round.round,
5364 round_head: round.head.clone(),
5365 reviewer: rec.reviewer,
5366 agent: rec.agent.clone(),
5367 file: f.file.clone(),
5368 line: f.line,
5369 title: f.title.clone(),
5370 detail: f.detail.clone(),
5371 outcome: OperatorFixOutcome::Pending,
5372 }),
5373 None => missing.push(id.clone()),
5374 }
5375 }
5376 if !missing.is_empty() {
5377 bail!(
5378 "unknown finding id(s): {}; nothing was changed",
5379 missing.join(", ")
5380 );
5381 }
5382
5383 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5384 let stale_details: Vec<(String, String)> = findings
5385 .iter()
5386 .filter(|f| f.round_head != head_at_request)
5387 .map(|f| (f.id.clone(), f.round_head.clone()))
5388 .collect();
5389 let stale = !stale_details.is_empty();
5390 if stale && !allow_stale {
5391 bail!(
5392 "the branch has moved since some finding(s) were raised — {} — now \
5393 at {}; pass --allow-stale to fix anyway, or re-run review first",
5394 stale_details
5395 .iter()
5396 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5397 .collect::<Vec<_>>()
5398 .join(", "),
5399 short(&head_at_request)
5400 );
5401 }
5402
5403 let request = OperatorFixRequest {
5404 requested_at: Timestamp::now(),
5405 reason: reason.to_owned(),
5406 findings,
5407 head_at_request: head_at_request.clone(),
5408 allow_stale,
5409 stale,
5410 fix: None,
5411 result_head: None,
5412 follow_up_review_run: None,
5413 };
5414 self.state.event(
5415 "fix",
5416 format!(
5417 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5418 request.findings.len(),
5419 request
5420 .findings
5421 .iter()
5422 .map(|f| f.id.as_str())
5423 .collect::<Vec<_>>()
5424 .join(", "),
5425 ),
5426 );
5427 self.state.operator_fixes.push(request);
5434 self.state.save()?;
5435 let request_index = self.state.operator_fixes.len() - 1;
5436
5437 let repo = self.state.repo.clone();
5446 let coordination = self.coordinate(&repo).await?;
5447 if winner.worktree.exists() {
5448 let dirty = git::git(
5451 &winner.worktree,
5452 &["status", "--porcelain", "--untracked-files=all"],
5453 )
5454 .await?;
5455 let only_withheld = dirty.lines().all(|l| {
5456 l.strip_prefix("?? ")
5457 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5458 });
5459 if !only_withheld {
5460 bail!(
5461 "`{}` has uncommitted changes; refusing to touch it — commit or \
5462 discard them first",
5463 winner.worktree.display()
5464 );
5465 }
5466 git::worktree_remove(&self.state.repo, &winner.worktree)
5467 .await
5468 .ok();
5469 }
5470 let fix_worktree = self.state.worktree_root().join("operator-fix");
5471 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5472 git::git(
5473 &self.state.repo,
5474 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5475 )
5476 .await
5477 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5478 if !git::is_clean(&fix_worktree).await? {
5479 git::worktree_remove(&self.state.repo, &fix_worktree)
5480 .await
5481 .ok();
5482 bail!(
5483 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5484 winner.branch
5485 );
5486 }
5487 let repo_for_setup = self.state.repo.clone();
5488 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5489 Ok(()) => {
5490 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5491 }
5492 Err(e) => Err(e),
5493 };
5494 if let Err(e) = setup {
5495 git::worktree_remove(&self.state.repo, &fix_worktree)
5496 .await
5497 .ok();
5498 return Err(e);
5499 }
5500
5501 let run_id = self.state.id.clone();
5502 let prompts = self.state.config.prompts.clone();
5503 let language = self.state.config.graph.language.clone();
5504 let sessions = self.state.config.graph.sessions;
5505 let artifacts = agent::artifacts_dir(&self.state.dir());
5506 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5507 .findings
5508 .iter()
5509 .map(|f| Finding {
5510 id: f.id.clone(),
5511 severity: f.severity,
5512 file: f.file.clone(),
5513 line: f.line,
5514 title: f.title.clone(),
5515 detail: f.detail.clone(),
5516 })
5517 .collect();
5518 let fix_prompt = prompt::operator_fix_in(
5519 &self.state.instruction,
5520 &finding_list,
5521 reason,
5522 &stale_details,
5523 &head_at_request,
5524 &language,
5525 &self.state.config.graph.github_language,
5526 );
5527 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5528 drop(coordination);
5529 let (job, seat, out) = self
5530 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5531 prompt: fix_prompt.clone(),
5532 spec,
5533 seat,
5534 cwd: fix_worktree.clone(),
5535 timeout,
5536 allow_write: true,
5537 sessions,
5538 artifacts: artifacts.clone(),
5539 stem: "operator-fix".to_owned(),
5540 handover: None,
5541 })
5542 .await;
5543 let agent_id = seat.agent.clone();
5544
5545 let mut fix = FixRecord {
5546 agent: agent_id,
5547 addressed: Vec::new(),
5548 rejected: Vec::new(),
5549 notes: String::new(),
5550 committed: false,
5551 failed: None,
5552 duration_ms: 0,
5553 continuation: None,
5554 };
5555 let mut final_seat = seat.clone();
5556 match out {
5557 AgentOutcome::Ok(o) => {
5558 fix.duration_ms = o.duration_ms;
5559 let parsed = verdict::extract_json::<FixReport>(&o.text);
5560 let incomplete_reason = match &parsed {
5561 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5562 "the reply parsed, but it reported a command whose own CLI \
5563 never confirmed an exit status"
5564 .to_owned(),
5565 ),
5566 Ok(_) => None,
5567 Err(e) => Some(e.to_string()),
5568 };
5569 match incomplete_reason {
5570 None => {
5571 let report = parsed.expect("checked Ok above");
5572 fix.addressed = report.addressed;
5573 fix.rejected = report.rejected;
5574 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5575 }
5576 Some(reason) => {
5577 let (resumed_seat, resolved, failure, cont) = self
5578 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5579 .await;
5580 fix.duration_ms += cont.cumulative_wait_ms;
5581 fix.continuation = Some(cont);
5582 final_seat = resumed_seat;
5583 match resolved {
5584 Some(report) => {
5585 fix.addressed = report.addressed;
5586 fix.rejected = report.rejected;
5587 fix.notes =
5588 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5589 }
5590 None => fix.failed = failure,
5591 }
5592 }
5593 }
5594 }
5595 AgentOutcome::Dropped(o) => {
5596 fix.duration_ms = o.duration_ms;
5597 let why = o
5598 .dropped
5599 .as_ref()
5600 .map(|d| d.why.as_str())
5601 .unwrap_or("the CLI ended the stream without delivering its answer");
5602 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5603 }
5604 AgentOutcome::Quota(o) => {
5605 self.state.quota.push(QuotaLoss {
5606 seat: final_seat.key.clone(),
5607 node: "fix".to_owned(),
5608 at: Timestamp::now(),
5609 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5610 });
5611 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5612 }
5613 AgentOutcome::Failed(e) => fix.failed = Some(e),
5614 AgentOutcome::ResourceBlocked(e) => {
5615 fix.failed = Some(format!("the fixer was not started: {e}"));
5616 }
5617 }
5618 if fix.continuation.is_none() {
5619 fix.continuation = Some(ContinuationRecord::not_needed());
5620 }
5621 self.state.seats.insert(final_seat.key.clone(), final_seat);
5622
5623 let rescue_message = format!(
5624 "magi: operator-selected fix ({}) (uncommitted work)",
5625 self.state.operator_fixes[request_index]
5626 .findings
5627 .iter()
5628 .map(|f| f.id.as_str())
5629 .collect::<Vec<_>>()
5630 .join(", ")
5631 );
5632 let repo = self.state.repo.clone();
5633 let coordination = self.coordinate(&repo).await?;
5634 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5635 self.state.note_withheld("fix", &r.withheld);
5636 }
5637 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5638 fix.committed = after != head_at_request;
5639 git::worktree_remove(&self.state.repo, &fix_worktree)
5640 .await
5641 .ok();
5642 drop(coordination);
5643
5644 self.state.event(
5645 "fix",
5646 match &fix.failed {
5647 Some(reason) => format!(
5648 "operator fix: adoption report was lost ({reason}); {}",
5649 if fix.committed {
5650 "committed"
5651 } else {
5652 "NO new commit"
5653 }
5654 ),
5655 None => format!(
5656 "operator fix: {} addressed, {} rejected, {}",
5657 fix.addressed.len(),
5658 fix.rejected.len(),
5659 if fix.committed {
5660 "committed"
5661 } else {
5662 "NO new commit"
5663 }
5664 ),
5665 },
5666 );
5667
5668 for f in &mut self.state.operator_fixes[request_index].findings {
5675 f.outcome = if fix.failed.is_some() {
5676 OperatorFixOutcome::Unreported
5677 } else if fix.addressed.contains(&f.id) {
5678 OperatorFixOutcome::Addressed
5679 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5680 OperatorFixOutcome::Rejected { why: r.why.clone() }
5681 } else {
5682 OperatorFixOutcome::Unreported
5683 };
5684 }
5685
5686 let committed = fix.committed;
5687 if committed {
5688 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5689 }
5690 self.state.operator_fixes[request_index].fix = Some(fix);
5691 self.state.save()?;
5694
5695 if committed {
5696 self.state.event(
5697 "fix",
5698 format!(
5699 "operator fix committed {}; opening a follow-up review-only run",
5700 short(&after)
5701 ),
5702 );
5703 let origin =
5706 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5707 match Self::review(
5708 &self.state.repo,
5709 &winner.branch,
5710 self.state.config.clone(),
5711 origin,
5712 )
5713 .await
5714 {
5715 Ok(mut follow_up) => {
5716 follow_up.state.event(
5717 "start",
5718 format!(
5719 "requested by an operator fix on run {} for finding(s) {}",
5720 self.state.id,
5721 self.state.operator_fixes[request_index]
5722 .findings
5723 .iter()
5724 .map(|f| f.id.as_str())
5725 .collect::<Vec<_>>()
5726 .join(", "),
5727 ),
5728 );
5729 follow_up.state.save()?;
5730 let follow_up_id = follow_up.state.id.clone();
5731 let adopted = match crate::direct::adopt(
5734 &follow_up.state,
5735 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5736 ) {
5737 Ok(a) => a,
5738 Err(e) => {
5739 self.state.event(
5742 "fix",
5743 format!(
5744 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5745 ),
5746 );
5747 self.state.save()?;
5748 return Ok(());
5749 }
5750 };
5751 let executed = follow_up.execute().await;
5752 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5753 if let Some(a) = adopted {
5754 a.finish(&follow_up.state, executed);
5755 }
5756 if let Some(e) = failure {
5757 self.state.event(
5758 "fix",
5759 format!(
5760 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5761 ),
5762 );
5763 }
5764 self.state.operator_fixes[request_index].follow_up_review_run =
5765 Some(follow_up_id);
5766 }
5767 Err(e) => {
5768 self.state.event(
5769 "fix",
5770 format!("committed the fix but could not open a follow-up review: {e:#}"),
5771 );
5772 }
5773 }
5774 self.state.save()?;
5775 }
5776
5777 Ok(())
5778 }
5779
5780 async fn ask_fixer(
5800 &mut self,
5801 winner: &Candidate,
5802 node: &'static str,
5803 round: Option<usize>,
5804 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5805 ) -> (SeatJob, SeatState, AgentOutcome) {
5806 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5807 self.ask_fixer_among(attempts, node, round, None, build)
5808 .await
5809 }
5810
5811 async fn ask_fixer_among(
5816 &mut self,
5817 attempts: Vec<(AgentSpec, String)>,
5818 node: &'static str,
5819 round: Option<usize>,
5820 restore: Option<(&Path, &str)>,
5821 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5822 ) -> (SeatJob, SeatState, AgentOutcome) {
5823 let run_id = self.state.id.clone();
5824 let prompts = self.state.config.prompts.clone();
5825 let cache = self.state.config.cache_dir();
5826 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5827 let mut last = None;
5828 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5829 let seat = self.seat(&key, &spec.id);
5830 let mut job = build(spec.clone(), seat);
5831 if i > 0 {
5832 job.stem = format!("{}-{}", job.stem, spec.id);
5833 }
5834 let ctx = WaveCtx {
5835 carry_seats: false,
5836 pauses: Some((&self.pause, &self.interrupt)),
5837 run: &run_id,
5838 node,
5839 prompts: &prompts,
5840 cache: cache.as_deref(),
5841 round,
5842 };
5843 let (seat, out) =
5844 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5845 let advances = match &out {
5848 AgentOutcome::Ok(o) => agent::output_advances(o),
5849 AgentOutcome::ResourceBlocked(_) => false,
5850 _ => true,
5851 };
5852 if let Some(next) = ids.get(i + 1)
5853 && advances
5854 {
5855 self.state.seats.insert(seat.key.clone(), seat.clone());
5856 let class =
5857 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5858 let reason = match &out {
5859 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5860 other => fail_reason(other),
5861 };
5862 if let Some((wt, rev)) = restore
5866 && let Err(e) = self.graft_restore(wt, rev).await
5867 {
5868 let why =
5869 format!("could not restore the worktree before the next fixer: {e:#}");
5870 tracing::warn!("{why}");
5871 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5872 AgentOutcome::ResourceBlocked(why)
5873 } else {
5874 AgentOutcome::Failed(why)
5875 };
5876 last = Some((job, seat, out));
5877 break;
5878 }
5879 record_handover(
5880 &mut self.state,
5881 node,
5882 &seat.key,
5883 &spec.id,
5884 next,
5885 &class,
5886 &reason,
5887 );
5888 continue;
5889 }
5890 last = Some((job, seat, out));
5891 break;
5892 }
5893 last.expect("the fixer chain always has an entry")
5894 }
5895
5896 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5907 use crate::recovery::{Mode, Repair};
5908 if !self.state.reviews.is_empty() {
5909 return Ok(());
5910 }
5911 let Some(mut rec) = self.state.recovery.clone() else {
5912 return Ok(());
5913 };
5914 if rec.mode != Mode::VerifyRepair {
5915 return Ok(());
5916 }
5917 let wt = winner.worktree.clone();
5918 let mut head = git::rev_parse(&wt, "HEAD").await?;
5919 match rec.repair.clone() {
5920 Repair::Done { .. } => return Ok(()),
5921 Repair::WaitingForRestore { before } => {
5922 self.graft_restore(&wt, &before).await?;
5923 head = before;
5924 rec.repair = Repair::NotStarted;
5925 self.state.recovery = Some(rec.clone());
5926 self.state.save()?;
5927 }
5928 Repair::Started { before } => {
5929 if let Err(e) = self.graft_restore(&wt, &before).await {
5933 if e.is::<crate::repo_lock::ResourceBlocked>() {
5934 return Err(e);
5935 }
5936 return self
5937 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5938 .await;
5939 }
5940 rec.repair = Repair::Done {
5941 before: before.clone(),
5942 after: before,
5943 committed: false,
5944 failed: Some("interrupted; rolled back".to_owned()),
5945 };
5946 self.state
5947 .event("recovery", "the repair was interrupted and rolled back");
5948 self.state.recovery = Some(rec);
5949 self.state.save()?;
5950 return Ok(());
5951 }
5952 Repair::NotStarted => {}
5953 }
5954 if crate::recovery::recheck(&mut rec, &head) {
5955 self.state.event("recovery", rec.summary());
5956 self.state.recovery = Some(rec);
5957 self.state.save()?;
5958 return Ok(());
5959 }
5960 rec.repair = Repair::Started {
5961 before: head.clone(),
5962 };
5963 self.state.recovery = Some(rec.clone());
5964 self.state.save()?;
5965
5966 let language = self.state.config.graph.language.clone();
5967 let sessions = self.state.config.graph.sessions;
5968 let artifacts = agent::artifacts_dir(&self.state.dir());
5969 let fix_prompt = prompt::recovery_fix(
5970 &self.state.instruction,
5971 &rec,
5972 self.state.config.graph.review_rounds,
5973 &language,
5974 );
5975 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5976 self.state.event(
5977 "recovery",
5978 format!(
5979 "repair: the fixer starts from the failed verification on {}",
5980 short(&head)
5981 ),
5982 );
5983 let (_job, seat, out) = self
5984 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5985 prompt: fix_prompt.clone(),
5986 spec,
5987 seat,
5988 cwd: wt.clone(),
5989 timeout,
5990 allow_write: true,
5991 sessions,
5992 artifacts: artifacts.clone(),
5993 stem: "fix-recovery".to_owned(),
5994 handover: None,
5995 })
5996 .await;
5997 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5998 rec.repair = Repair::WaitingForRestore { before: head };
5999 self.state.recovery = Some(rec);
6000 return self.stop_for_resource();
6001 }
6002 let failed = match &out {
6003 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6004 AgentOutcome::Quota(o) => {
6005 self.state.quota.push(QuotaLoss {
6006 seat: seat.key.clone(),
6007 node: "fix".to_owned(),
6008 at: Timestamp::now(),
6009 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6010 });
6011 Some("rate limited (quota); the fixer could not run".to_owned())
6012 }
6013 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6014 AgentOutcome::Failed(e) => Some(e.clone()),
6015 };
6016 self.state.seats.insert(seat.key.clone(), seat);
6017 let coordination = self.coordinate(&wt).await?;
6018 let failed = match failed {
6019 Some(why) => {
6021 if let Err(e) = git::restore_to(&wt, &head).await {
6022 return self
6023 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6024 .await;
6025 }
6026 Some(why)
6027 }
6028 None => {
6029 if let Ok(r) = git::rescue_commit(
6030 &wt,
6031 "magi: recovery repair of the failed verification (uncommitted work)",
6032 )
6033 .await
6034 {
6035 self.state.note_withheld("fix", &r.withheld);
6036 }
6037 None
6038 }
6039 };
6040 let after = git::rev_parse(&wt, "HEAD").await?;
6041 drop(coordination);
6042 self.state.event(
6043 "recovery",
6044 match &failed {
6045 Some(why) => format!("repair: nothing kept ({why})"),
6046 None if after != head => {
6047 format!("repair: committed {}", short(&after))
6048 }
6049 None => "repair: the fixer made no commit".to_owned(),
6050 },
6051 );
6052 rec.repair = Repair::Done {
6053 before: head.clone(),
6054 committed: after != head,
6055 after,
6056 failed,
6057 };
6058 self.state.recovery = Some(rec);
6059 self.state.save()?;
6060 Ok(())
6061 }
6062
6063 async fn stop_recovery(
6066 &mut self,
6067 mut rec: crate::recovery::RecoveryRecord,
6068 head: &str,
6069 why: String,
6070 ) -> Result<()> {
6071 rec.repair = crate::recovery::Repair::Started {
6073 before: head.to_owned(),
6074 };
6075 self.state.recovery = Some(rec);
6076 self.state
6077 .event("recovery", format!("repair could not be undone: {why}"));
6078 self.state.status = RunStatus::Blocked;
6079 self.state.save()?;
6080 bail!("recovery repair could not be rolled back: {why}")
6081 }
6082
6083 async fn review_loop(&mut self) -> Result<()> {
6084 if self.hold_base_sync().await? {
6085 return Ok(());
6086 }
6087 loop {
6088 if self
6089 .state
6090 .base_sync
6091 .as_ref()
6092 .is_some_and(|s| s.conflict.is_some())
6093 {
6094 return Ok(());
6095 }
6096 if self.catchup_repairable() {
6097 let record = self.state.reviews.last().expect("repairable round").clone();
6098 let winner = self
6099 .state
6100 .winner()
6101 .cloned()
6102 .context("catch-up repair has no winner")?;
6103 if !self.fix_review_round(&winner, record).await? {
6104 return Ok(());
6105 }
6106 }
6107 self.review_loop_pass().await?;
6108 if !self.catchup_repairable() {
6109 return Ok(());
6110 }
6111 }
6112 }
6113
6114 fn catchup_repairable(&self) -> bool {
6117 self.state.reviews.len() < self.state.config.graph.review_rounds
6118 && self.state.reviews.last().is_some_and(|r| {
6119 !r.clean
6120 && r.blocking == 0
6121 && r.fix.is_none()
6122 && r.e2e_status() == E2eStatus::Failed
6123 })
6124 }
6125
6126 async fn review_loop_pass(&mut self) -> Result<()> {
6127 if self
6132 .state
6133 .base_sync
6134 .as_ref()
6135 .is_some_and(|s| s.conflict.is_some())
6136 {
6137 return Ok(());
6138 }
6139 let run_id = self.state.id.clone();
6144 let prompts = self.state.config.prompts.clone();
6145 let Some(winner) = self.state.winner().cloned() else {
6146 return Ok(());
6147 };
6148 let max_rounds = self.state.config.graph.review_rounds;
6149 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6159 if status == RunStatus::Gating {
6162 self.record_contested_handoff();
6163 }
6164 self.state.status = status;
6165 self.state.save()?;
6166 return Ok(());
6167 }
6168 self.state.status = RunStatus::Reviewing;
6169 if self.state.reviews.last().is_some_and(|r| {
6179 r.e2e_status() == E2eStatus::ResourceBlocked
6180 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6181 }) {
6182 let shell = self.state.config.shell();
6183 return self
6184 .stop_unresolved_review(
6185 "the last round's own verification never resolved",
6186 &shell,
6187 &winner.worktree,
6188 )
6189 .await;
6190 }
6191
6192 self.recovery_repair(&winner).await?;
6193 if self.resource_halt()? {
6194 return Ok(());
6195 }
6196
6197 let repo = self.state.repo.clone();
6198 let root = self.state.worktree_root();
6199 let language = self.state.config.graph.language.clone();
6200 let sessions = self.state.config.graph.sessions;
6201 let artifacts = self.merge_repair_artifacts();
6202 let base = self.landing_base();
6203 let base_short = short(&base);
6204 let reviewers = self.roles.reviewers.clone();
6205 let shell = self.state.config.shell();
6206
6207 for round in (self.state.reviews.len() + 1)..=max_rounds {
6208 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6209 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6210 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6211 let prev_verification = self
6220 .state
6221 .reviews
6222 .last()
6223 .and_then(|r| r.verification_summary(&head));
6224 let prior_section = match (&self.state.recovery, round) {
6227 (Some(rec), 1) if self.state.reviews.is_empty() => {
6228 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6229 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6230 Some(crate::recovery::head_of(
6231 &git::diff(&winner.worktree, reviewed, "HEAD")
6232 .await
6233 .unwrap_or_default(),
6234 crate::recovery::SINCE_DIFF_MAX,
6235 ))
6236 }
6237 _ => None,
6238 };
6239 Some(prompt::recovery_review(rec, since.as_deref()))
6240 }
6241 _ => None,
6242 };
6243
6244 let mut jobs = Vec::new();
6248 let coordination = self.coordinate(&repo).await?;
6249 for (r, spec) in reviewers.iter().cloned().enumerate() {
6250 let wt = root.join(format!("review-{}", r + 1));
6251 if wt.exists() {
6254 git::reset_detached(&wt, &head).await?;
6255 } else {
6256 git::worktree_add_detached(&repo, &wt, &head).await?;
6257 }
6258 self.setup_seat_worktree(&repo, &wt, false).await?;
6259 let seat_key = format!("review-{}", r + 1);
6260 let spec = pick_start_spec(
6263 &self.roles.reviewer_roster,
6264 spec,
6265 self.state.seat_history.get(&seat_key),
6266 );
6267 let seat = self.seat(&seat_key, &spec.id);
6268 jobs.push(SeatJob {
6269 prompt: prompt::review(&prompt::ReviewCtx {
6270 instruction: &self.state.instruction,
6271 branch: &winner.branch,
6272 base_short: &base_short,
6273 stat: &stat,
6274 patch: &patch,
6275 verification: prev_verification.as_ref(),
6276 prior: prior_section.as_deref(),
6277 reviewers: reviewers.len(),
6278 round,
6279 rounds: max_rounds,
6280 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6283 lens: Lens::for_seat(r),
6284 language: &language,
6285 github_language: &self.state.config.graph.github_language,
6286 }),
6287 spec,
6288 seat,
6289 cwd: wt,
6290 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6291 allow_write: false,
6292 sessions,
6293 artifacts: artifacts.clone(),
6294 stem: format!("review-{round}-{}", r + 1),
6295 handover: None,
6296 });
6297 }
6298
6299 drop(coordination);
6300 self.state.event(
6301 "review",
6302 format!(
6303 "round {round}: {} reviewers on {}",
6304 jobs.len(),
6305 short(&head)
6306 ),
6307 );
6308 let mut quota_losses = Vec::new();
6309 let review_retries = self.state.config.graph.retries;
6310 let review_cache = self.state.config.cache_dir();
6311 let ctx = WaveCtx {
6312 carry_seats: true,
6313 pauses: Some((&self.pause, &self.interrupt)),
6314 run: &run_id,
6315 node: "review",
6316 prompts: &prompts,
6317 cache: review_cache.as_deref(),
6318 round: Some(round),
6319 };
6320 let results = ask_json_wave::<Review>(
6321 jobs,
6322 Arc::clone(&self.sem),
6323 review_retries,
6324 &self.roles.reviewer_roster,
6325 &ctx,
6326 &mut quota_losses,
6327 &mut self.state,
6328 &|_: &Review| Ok(()),
6329 )
6330 .await;
6331 if self.state.resource_block.is_some() {
6332 return self.stop_for_resource();
6333 }
6334 let round_quota_missing = quota_losses.len();
6338 self.state.quota.extend(quota_losses);
6339
6340 let mut records = Vec::new();
6341 let mut all_findings = Vec::new();
6342 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6343 let agent_id = seat.agent.clone();
6344 self.state.seats.insert(seat.key.clone(), seat);
6345 let mut record = ReviewRecord {
6346 reviewer: r + 1,
6347 agent: agent_id,
6348 summary: String::new(),
6349 findings: Vec::new(),
6350 vote: None,
6351 failed: None,
6352 duration_ms: 0,
6353 attempts,
6359 };
6360 match res {
6361 Ok((review, out)) => {
6362 record.summary =
6370 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6371 record.vote = Some(review.vote);
6372 record.duration_ms = out.duration_ms;
6373 for (n, mut f) in review.findings.into_iter().enumerate() {
6374 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6377 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6378 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6379 f.file = f
6385 .file
6386 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6387 all_findings.push(f.clone());
6388 record.findings.push(f);
6389 }
6390 self.state.event(
6391 "review",
6392 format!(
6393 "round {round}: reviewer {} voted {} with {} finding(s)",
6394 r + 1,
6395 review.vote.label(),
6396 record.findings.len()
6397 ),
6398 );
6399 }
6400 Err(e) => {
6401 record.failed = Some(e.to_string());
6402 self.state.event(
6403 "review",
6404 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6405 );
6406 }
6407 }
6408 records.push(record);
6409 }
6410
6411 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6418 let vote_split =
6419 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6420 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6421 if vote_split {
6422 self.state.event(
6423 "review",
6424 format!(
6425 "round {round}: votes split ({}) — one round of reconsideration",
6426 initial_votes
6427 .iter()
6428 .map(|v| v.label())
6429 .collect::<Vec<_>>()
6430 .join(", ")
6431 ),
6432 );
6433 let panel: Vec<ReviewSeatReport<'_>> = records
6436 .iter()
6437 .filter_map(|r| {
6438 r.vote.map(|vote| ReviewSeatReport {
6439 reviewer: r.reviewer,
6440 vote,
6441 summary: &r.summary,
6442 findings: &r.findings,
6443 })
6444 })
6445 .collect();
6446
6447 let mut jobs = Vec::new();
6448 let mut seats_at = Vec::new();
6449 for (r, spec) in reviewers.iter().cloned().enumerate() {
6450 if records[r].vote.is_none() {
6454 continue;
6455 }
6456 let wt = root.join(format!("review-{}", r + 1));
6457 let seat_key = format!("review-{}", r + 1);
6458 let spec = self.occupant(&seat_key, spec);
6459 let seat = self.seat(&seat_key, &spec.id);
6460 let build = |with_patch: bool| {
6468 prompt::review_reconsider(&ReviewReconsiderCtx {
6469 instruction: &self.state.instruction,
6470 reviewer: r + 1,
6471 lens: Lens::for_seat(r),
6472 panel: &panel,
6473 patch: with_patch.then_some(ReviewPatch {
6474 branch: &winner.branch,
6475 base_short: &base_short,
6476 stat: &stat,
6477 patch: &patch,
6478 }),
6479 round,
6480 rounds: max_rounds,
6481 language: &language,
6482 })
6483 };
6484 let full = build(true);
6485 let prompt = if has_context(&spec, &seat, sessions) {
6486 build(false)
6487 } else {
6488 full.clone()
6489 };
6490 jobs.push(SeatJob {
6491 prompt,
6492 spec,
6493 seat,
6494 cwd: wt,
6495 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6496 allow_write: false,
6497 sessions,
6498 artifacts: artifacts.clone(),
6499 stem: format!("review-{round}-reconsider-{}", r + 1),
6500 handover: Some(full),
6501 });
6502 seats_at.push(r);
6503 }
6504
6505 let mut recon_quota_losses = Vec::new();
6506 let recon_cache = self.state.config.cache_dir();
6507 let recon_ctx = WaveCtx {
6508 carry_seats: true,
6509 pauses: Some((&self.pause, &self.interrupt)),
6510 run: &run_id,
6511 node: "review",
6512 prompts: &prompts,
6513 cache: recon_cache.as_deref(),
6514 round: Some(round),
6515 };
6516 let recon_results = ask_json_wave::<ReviewRevote>(
6517 jobs,
6518 Arc::clone(&self.sem),
6519 review_retries,
6520 &self.roles.reviewer_roster,
6521 &recon_ctx,
6522 &mut recon_quota_losses,
6523 &mut self.state,
6524 &|_: &ReviewRevote| Ok(()),
6525 )
6526 .await;
6527 if self.state.resource_block.is_some() {
6528 return self.stop_for_resource();
6529 }
6530 self.state.quota.extend(recon_quota_losses);
6531
6532 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6533 let agent_id = seat.agent.clone();
6534 self.state.seats.insert(seat.key.clone(), seat);
6535 let mut rec = ReviewRevoteRecord {
6536 reviewer: r + 1,
6537 agent: agent_id,
6538 vote: None,
6539 reason: String::new(),
6540 failed: None,
6541 };
6542 match res {
6543 Ok((rv, _)) => {
6544 rec.vote = Some(rv.vote);
6545 rec.reason =
6546 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6547 self.state.event(
6548 "review",
6549 format!(
6550 "round {round}: reviewer {} revoted {}",
6551 r + 1,
6552 rv.vote.label()
6553 ),
6554 );
6555 }
6556 Err(e) => {
6557 rec.failed = Some(e.to_string());
6558 self.state.event(
6559 "review",
6560 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6561 );
6562 }
6563 }
6564 reconsideration.push(rec);
6565 }
6566 } else if initial_votes.len() > 1 {
6567 self.state.event(
6568 "review",
6569 format!(
6570 "round {round}: votes agreed ({}) — no reconsideration",
6571 initial_votes[0].label()
6572 ),
6573 );
6574 }
6575
6576 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6577 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6578 let defer_e2e =
6589 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6590 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6591 let reason =
6592 format!("{blocking} blocking finding(s) already required a fix this round");
6593 self.state.event(
6594 "verify",
6595 format!(
6596 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6597 {}); it will run once a round has none left",
6598 short(&head)
6599 ),
6600 );
6601 (Vec::new(), false, true, Some(reason))
6602 } else {
6603 let e2e_commands = self.state.config.verify.e2e.clone();
6604 let cache_dir = self.state.config.cache_dir();
6605 let context = format!("round {round}");
6606 let wait = self.repository_wait();
6607 let (e2e, verify_retried) = with_cache_lease(
6608 &mut self.state,
6609 cache_dir.as_deref(),
6610 "e2e",
6611 "e2e",
6612 &winner.worktree,
6613 &head,
6614 verify_timeout,
6615 &context,
6616 &wait,
6617 |state, budget| {
6618 let shell = shell.clone();
6619 let e2e_commands = e2e_commands.clone();
6620 let worktree = winner.worktree.clone();
6621 let context = context.clone();
6622 async move {
6623 run_e2e_with_retry(
6624 state,
6625 &shell,
6626 &e2e_commands,
6627 &worktree,
6628 budget,
6629 &context,
6630 )
6631 .await
6632 }
6633 },
6634 )
6635 .await?;
6636 (e2e, verify_retried, false, None)
6637 };
6638
6639 let expected = records.len();
6640 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6641 let incomplete = answered < expected;
6642 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6643 let policy = self.state.config.graph.incomplete_review;
6644 let clean = round_is_clean(
6645 blocking,
6646 e2e_ok,
6647 answered,
6648 expected,
6649 round_quota_missing,
6650 policy,
6651 );
6652
6653 let mut round_record = ReviewRound {
6654 round,
6655 head: head.clone(),
6656 verified_head: None,
6657 verified_at: None,
6658 reviews: records,
6659 e2e,
6660 verify_retried,
6661 e2e_deferred,
6662 e2e_defer_reason,
6663 fix: None,
6664 blocking,
6665 answered,
6666 expected,
6667 clean,
6668 progressed: false,
6669 vote_split,
6670 reconsideration,
6671 verdict: None,
6672 };
6673 round_record.verdict = ReviewVote::worst(
6677 round_record
6678 .final_votes()
6679 .into_iter()
6680 .map(|(_, _, vote)| vote),
6681 );
6682 if !matches!(
6693 round_record.e2e_status(),
6694 E2eStatus::Deferred | E2eStatus::NotConfigured
6695 ) {
6696 round_record.verified_head = Some(head.clone());
6697 round_record.verified_at = Some(Timestamp::now());
6698 }
6699
6700 if incomplete {
6701 let missing: Vec<String> = round_record
6702 .reviews
6703 .iter()
6704 .filter(|r| r.failed.is_some())
6705 .map(|r| format!("review-{}", r.reviewer))
6706 .collect();
6707 self.state.event(
6708 "review",
6709 format!(
6710 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6711 missing.join(", ")
6712 ),
6713 );
6714 }
6715
6716 if clean {
6717 self.state.event(
6718 "review",
6719 if incomplete && policy == IncompleteReviewPolicy::Warn {
6720 format!(
6721 "round {round}: clean (warn policy, incomplete panel) — no \
6722 blocking findings from the seats that answered, verification green"
6723 )
6724 } else if incomplete {
6725 format!(
6726 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6727 quorum) — no blocking findings from the seats that answered, \
6728 verification green",
6729 expected - answered
6730 )
6731 } else {
6732 format!("round {round}: clean — no blocking findings, verification green")
6733 },
6734 );
6735 self.state.reviews.push(round_record);
6736 self.state.status = RunStatus::Gating;
6737 self.state.save()?;
6738 return Ok(());
6739 }
6740
6741 if incomplete && blocking == 0 && e2e_ok {
6749 self.state.reviews.push(round_record);
6750 self.state.save()?;
6751 if round == max_rounds {
6752 self.state.status = RunStatus::Blocked;
6753 self.state.event(
6754 "review",
6755 format!(
6756 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6757 refusing to call it clean",
6758 expected - answered
6759 ),
6760 );
6761 return Ok(());
6762 }
6763 continue;
6764 }
6765
6766 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6778 self.state.reviews.push(round_record);
6779 return self
6780 .stop_unresolved_review(
6781 "the round's own verification could not run",
6782 &shell,
6783 &winner.worktree,
6784 )
6785 .await;
6786 }
6787
6788 if round == max_rounds {
6789 self.state.reviews.push(round_record);
6790 return self
6791 .stop_reviewing(
6792 &format!(
6793 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6794 ),
6795 &shell,
6796 &winner.worktree,
6797 )
6798 .await;
6799 }
6800
6801 if !self.fix_review_round(&winner, round_record).await? {
6802 return Ok(());
6803 }
6804 }
6805 Ok(())
6806 }
6807
6808 async fn fix_review_round(
6811 &mut self,
6812 winner: &Candidate,
6813 mut round_record: ReviewRound,
6814 ) -> Result<bool> {
6815 let round = round_record.round;
6816 let shell = self.state.config.shell();
6817 let step = self.run_fix_round(winner, &mut round_record).await?;
6818 if matches!(step, FixStep::Resource) {
6819 self.stop_for_resource()?;
6820 return Ok(false);
6821 }
6822 if self.state.reviews.len() < round {
6823 self.state.reviews.push(round_record);
6824 } else {
6825 self.state.reviews[round - 1] = round_record;
6826 }
6827 self.state.save()?;
6828
6829 if matches!(step, FixStep::Recorded { report_lost: true }) {
6842 self.stop_reviewing(
6843 "the fixer's adoption report never came back, even after resuming its \
6844 own seat; refusing to start another round against the same worktree \
6845 while that is unresolved",
6846 &shell,
6847 &winner.worktree,
6848 )
6849 .await?;
6850 return Ok(false);
6851 }
6852
6853 let streak = self
6854 .state
6855 .reviews
6856 .iter()
6857 .rev()
6858 .take_while(|r| !r.progressed)
6859 .count();
6860 if streak >= STAGNANT_LIMIT {
6861 self.stop_reviewing(
6862 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6863 &shell,
6864 &winner.worktree,
6865 )
6866 .await?;
6867 return Ok(false);
6868 }
6869 Ok(true)
6870 }
6871
6872 async fn run_fix_round(
6874 &mut self,
6875 winner: &Candidate,
6876 round_record: &mut ReviewRound,
6877 ) -> Result<FixStep> {
6878 let round = round_record.round;
6879 let max_rounds = self.state.config.graph.review_rounds;
6880 let run_id = self.state.id.clone();
6881 let prompts = self.state.config.prompts.clone();
6882 let language = self.state.config.graph.language.clone();
6883 let sessions = self.state.config.graph.sessions;
6884 let artifacts = self.merge_repair_artifacts();
6885 let base = self.landing_base();
6886 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6887 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6888 let this_round_verification = round_record.verification_summary(&head);
6889 let all_findings: Vec<_> = round_record
6890 .reviews
6891 .iter()
6892 .flat_map(|r| r.findings.iter().cloned())
6893 .collect();
6894 let blocking_findings: Vec<_> = all_findings
6897 .iter()
6898 .filter(|f| f.severity.blocks())
6899 .cloned()
6900 .collect();
6901 let fix_prompt = prompt::fix_in(
6902 &self.state.instruction,
6903 &blocking_findings,
6904 this_round_verification.as_ref(),
6905 round,
6906 max_rounds,
6907 &language,
6908 &self.state.config.graph.github_language,
6909 );
6910 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6911 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6912 let (job, seat, out) = self
6913 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6914 prompt: fix_prompt.clone(),
6915 spec,
6916 seat,
6917 cwd: winner.worktree.clone(),
6918 timeout,
6919 allow_write: true,
6920 sessions,
6921 artifacts: artifacts.clone(),
6922 stem: format!("fix-{round}"),
6923 handover: None,
6924 })
6925 .await;
6926 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6929 return Ok(FixStep::Resource);
6930 }
6931 let agent_id = seat.agent.clone();
6932
6933 let mut fix = FixRecord {
6934 agent: agent_id,
6935 addressed: Vec::new(),
6936 rejected: Vec::new(),
6937 notes: String::new(),
6938 committed: false,
6939 failed: None,
6940 duration_ms: 0,
6941 continuation: None,
6942 };
6943 let mut continuation = ContinuationRecord::not_needed();
6944 let mut final_seat = seat.clone();
6945 match out {
6946 AgentOutcome::Ok(o) => {
6947 fix.duration_ms = o.duration_ms;
6948 let parsed = verdict::extract_json::<FixReport>(&o.text);
6949 let incomplete_reason = match &parsed {
6956 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6957 "the reply parsed, but it reported a command whose own CLI never \
6958 confirmed an exit status"
6959 .to_owned(),
6960 ),
6961 Ok(_) => None,
6962 Err(e) => Some(e.to_string()),
6963 };
6964 match incomplete_reason {
6965 None => {
6966 let report = parsed.expect("checked Ok above");
6967 fix.addressed = report.addressed;
6968 fix.rejected = report.rejected;
6969 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6970 }
6971 Some(reason) => {
6972 let (resumed_seat, resolved, failure, cont) = self
6973 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6974 .await;
6975 fix.duration_ms += cont.cumulative_wait_ms;
6976 continuation = cont;
6977 final_seat = resumed_seat;
6978 match resolved {
6979 Some(report) => {
6980 fix.addressed = report.addressed;
6981 fix.rejected = report.rejected;
6982 fix.notes =
6983 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6984 }
6985 None => fix.failed = failure,
6986 }
6987 }
6988 }
6989 }
6990 AgentOutcome::Dropped(o) => {
6992 fix.duration_ms = o.duration_ms;
6993 let why = o
6994 .dropped
6995 .as_ref()
6996 .map(|d| d.why.as_str())
6997 .unwrap_or("the CLI ended the stream without delivering its answer");
6998 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6999 }
7000 AgentOutcome::Quota(o) => {
7001 self.state.quota.push(QuotaLoss {
7002 seat: final_seat.key.clone(),
7003 node: "fix".to_owned(),
7004 at: Timestamp::now(),
7005 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7006 });
7007 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7008 }
7009 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7010 }
7011 fix.continuation = Some(continuation);
7012 self.state.seats.insert(final_seat.key.clone(), final_seat);
7013 let coordination = self.coordinate(&winner.worktree).await?;
7014 if let Ok(r) = git::rescue_commit(
7015 &winner.worktree,
7016 &format!("magi: review round {round} fixes (uncommitted work)"),
7017 )
7018 .await
7019 {
7020 self.state.note_withheld("fix", &r.withheld);
7021 }
7022 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7023 fix.committed = after != before;
7024 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7032 drop(coordination);
7033 let progressed = diff_after != patch;
7034 let commit_note = if fix.committed {
7035 "committed"
7036 } else {
7037 "NO new commit"
7038 };
7039 let tree_note = if progressed {
7040 "changed vs base"
7041 } else {
7042 "unchanged vs base"
7043 };
7044 self.state.event(
7045 "fix",
7046 match &fix.failed {
7047 Some(reason) => {
7053 format!(
7054 "round {round}: fixer's adoption report was lost ({reason}); \
7055 {commit_note}, tree {tree_note}"
7056 )
7057 }
7058 None => format!(
7059 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7060 {tree_note}{}",
7061 fix.addressed.len(),
7062 fix.rejected.len(),
7063 if continuation.outcome == ContinuationOutcome::Resumed {
7064 format!(
7065 " (adoption report recovered after {} continuation(s))",
7066 continuation.attempts
7067 )
7068 } else {
7069 String::new()
7070 },
7071 ),
7072 },
7073 );
7074 round_record.fix = Some(fix);
7075 round_record.progressed = progressed;
7076 Ok(FixStep::Recorded {
7077 report_lost: matches!(
7078 continuation.outcome,
7079 ContinuationOutcome::Exhausted
7080 | ContinuationOutcome::QuotaLost
7081 | ContinuationOutcome::NoSession
7082 ),
7083 })
7084 }
7085
7086 fn record_contested_handoff(&mut self) {
7120 if self.state.contested_handoff.is_some() {
7121 return;
7122 }
7123 let Some(contested) = self
7124 .state
7125 .reviews
7126 .last()
7127 .and_then(ReviewRound::contested_handoff)
7128 else {
7129 return;
7130 };
7131 self.state.event(
7132 "review",
7133 format!(
7134 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7135 wait for the owner's approval",
7136 contested.findings.len(),
7137 contested.rejecters.len()
7138 ),
7139 );
7140 self.state.contested_handoff = Some(contested);
7141 }
7142
7143 async fn stop_unresolved_review(
7144 &mut self,
7145 why: &str,
7146 shell: &[String],
7147 worktree: &Path,
7148 ) -> Result<()> {
7149 self.stop_reviewing_inner(why, shell, worktree, true).await
7150 }
7151
7152 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7153 self.stop_reviewing_inner(why, shell, worktree, false).await
7154 }
7155
7156 async fn stop_reviewing_inner(
7157 &mut self,
7158 why: &str,
7159 shell: &[String],
7160 worktree: &Path,
7161 repairable: bool,
7162 ) -> Result<()> {
7163 let round_idx = self.state.reviews.len() - 1;
7164 let needs_catchup_run = matches!(
7172 self.state.reviews[round_idx].e2e_status(),
7173 E2eStatus::Deferred | E2eStatus::ResourceBlocked
7174 );
7175 if needs_catchup_run {
7176 let round = self.state.reviews[round_idx].round;
7177 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7178 let commands = self.state.config.verify.e2e.clone();
7179 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7180 let cache_dir = self.state.config.cache_dir();
7181 let context = format!(
7182 "round {round}: verification unresolved, catching up before the final decision"
7183 );
7184 let wait = self.repository_wait();
7185 let (outcomes, verify_retried) = with_cache_lease(
7186 &mut self.state,
7187 cache_dir.as_deref(),
7188 "e2e",
7189 "e2e",
7190 worktree,
7191 &attempted_head,
7192 timeout,
7193 &context,
7194 &wait,
7195 |state, budget| {
7196 let shell = shell.to_vec();
7197 let commands = commands.clone();
7198 let context = context.clone();
7199 async move {
7200 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7201 .await
7202 }
7203 },
7204 )
7205 .await?;
7206 let last = &mut self.state.reviews[round_idx];
7207 last.e2e = outcomes;
7208 last.verify_retried = verify_retried;
7209 last.verified_head = Some(attempted_head);
7216 last.verified_at = Some(Timestamp::now());
7217 if verify_inconclusive(&last.e2e) {
7218 self.state.save()?;
7225 return Ok(());
7226 }
7227 last.e2e_deferred = false;
7228 }
7229 let last = &self.state.reviews[round_idx];
7230 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7231
7232 match last.e2e_status() {
7233 E2eStatus::Failed => {
7234 let red: Vec<String> = last
7235 .e2e
7236 .iter()
7237 .filter(|o| !o.ok())
7238 .map(|o| {
7239 format!(
7240 "`{}` -> {:?}\n{}",
7241 o.command,
7242 o.code,
7243 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7244 )
7245 })
7246 .collect();
7247 let can_repair = last.blocking == 0 && last.fix.is_none();
7248 self.state.reviews[round_idx].clean = false;
7249 self.state
7250 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7251 self.state.status = RunStatus::Blocked;
7252 if repairable
7253 && can_repair
7254 && self.state.reviews.len() < self.state.config.graph.review_rounds
7255 {
7256 self.state.status = RunStatus::Reviewing;
7257 }
7258 }
7259 E2eStatus::ResourceBlocked => {
7264 self.state.event(
7265 "review",
7266 format!(
7267 "{why}; e2e could not run (resource unavailable); not \
7268 deciding yet"
7269 ),
7270 );
7271 }
7272 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7273 self.state.event(
7274 "review",
7275 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7276 );
7277 self.record_contested_handoff();
7278 self.state.status = RunStatus::Gating;
7279 }
7280 }
7281 self.state.save()?;
7282 Ok(())
7283 }
7284
7285 async fn gate(&mut self) -> Result<()> {
7288 if self.hold_base_sync().await? {
7289 return Ok(());
7290 }
7291 if self.state.status == RunStatus::Failed
7303 || self
7304 .state
7305 .base_sync
7306 .as_ref()
7307 .is_some_and(|s| s.conflict.is_some())
7308 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7309 != Some(RunStatus::Gating)
7310 {
7311 return Ok(());
7312 }
7313 let Some(winner) = self.state.winner().cloned() else {
7314 return Ok(());
7315 };
7316 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7317 self.state.reconcile_verification_head(&head);
7318 self.state.save()?;
7319 let wait = self.repository_wait();
7320 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7321 return Ok(());
7322 }
7323 if self.state.gate_ran {
7324 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7335 self.state.status = RunStatus::Blocked;
7336 self.state.save()?;
7337 }
7338 return Ok(());
7339 }
7340 self.state.status = RunStatus::Gating;
7341 let mut outcomes = self.run_gate(&winner).await?;
7342 let mut attempted_head = self.state.gate_head.clone();
7343 loop {
7344 if verify_inconclusive(&outcomes) {
7355 self.state.save()?;
7356 return Ok(());
7357 }
7358 if outcomes.iter().all(CommandOutcome::ok) {
7359 break;
7360 }
7361 let fixes_before = self.state.gate_fixes.len();
7362 match self.gate_fix_round(&winner, &outcomes).await? {
7363 GateFix::Retry => {
7364 outcomes = self.run_gate(&winner).await?;
7365 attempted_head = self.state.gate_head.clone();
7366 }
7367 GateFix::Stop => {
7368 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7369 if self.state.gate_fixes.len() > fixes_before
7370 && attempted_head.as_deref() != Some(current.as_str())
7371 && verify_current_head(&mut self.state, &winner, &wait).await?
7372 {
7373 outcomes = self.run_gate(&winner).await?;
7376 attempted_head = self.state.gate_head.clone();
7377 } else {
7378 break;
7379 }
7380 }
7381 GateFix::Defer => {
7382 self.state.save()?;
7383 return Ok(());
7384 }
7385 }
7386 }
7387 let passed = outcomes.iter().all(CommandOutcome::ok);
7388 self.state.gate = outcomes;
7389 self.state.gate_ran = true;
7390 self.state.gate_head = attempted_head;
7391 self.state.save()?;
7392 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7393 return Ok(());
7394 }
7395 if !passed {
7396 self.state.status = RunStatus::Blocked;
7397 let spent = self.state.gate_fixes.len();
7398 self.state.event(
7399 "gate",
7400 if spent == 0 {
7401 "gate failed; not merging".to_owned()
7402 } else {
7403 format!("gate failed after {spent} gate-fix round(s); not merging")
7404 },
7405 );
7406 }
7407 self.state.save()?;
7408 Ok(())
7409 }
7410
7411 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7421 let commands = state.config.verify.pre_gate.clone();
7422 if commands.is_empty() {
7423 return;
7424 }
7425 let shell = state.config.shell();
7426 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7427 let (outcomes, _) = run_commands(
7428 state,
7429 "pre_gate",
7430 "pre_gate",
7431 0,
7432 &shell,
7433 &commands,
7434 &winner.worktree,
7435 timeout,
7436 )
7437 .await;
7438 for o in &outcomes {
7439 if !o.ok() {
7440 tracing::warn!(
7441 "pre_gate `{}` failed ({:?}); the gate decides",
7442 o.command,
7443 o.code
7444 );
7445 }
7446 state.event(
7447 "pre_gate",
7448 format!(
7449 "`{}` -> {}",
7450 o.command,
7451 if o.ok() {
7452 "pass".to_owned()
7453 } else {
7454 format!(
7455 "FAIL ({:?})\n{}",
7456 o.code,
7457 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7458 )
7459 }
7460 ),
7461 );
7462 }
7463 state.pre_gate = outcomes;
7464 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7465 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7466 Ok(head) => {
7467 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7468 state.pre_gate_commit = Some(head);
7469 }
7470 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7471 },
7472 Ok(false) => {}
7473 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7474 }
7475 if let Err(e) = state.save() {
7476 tracing::warn!("could not persist the pre_gate record: {e:#}");
7477 }
7478 }
7479
7480 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7483 let wait = self.repository_wait();
7484 run_landing_gate(&mut self.state, winner, &wait).await
7485 }
7486
7487 async fn gate_fix_round(
7500 &mut self,
7501 winner: &Candidate,
7502 outcomes: &[CommandOutcome],
7503 ) -> Result<GateFix> {
7504 let cap = self.state.config.graph.gate_fix_rounds;
7505 let spent = self.state.gate_fixes.len();
7506 if spent >= cap {
7507 if cap > 0 {
7508 self.state.event(
7509 "gate",
7510 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7511 );
7512 }
7513 return Ok(GateFix::Stop);
7514 }
7515 if !gate_fixable(outcomes) {
7516 self.state.event(
7517 "gate",
7518 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7519 command or similar); not spending a fix round on it",
7520 );
7521 return Ok(GateFix::Stop);
7522 }
7523 let min_free = self.state.config.disk.min_free_bytes;
7524 if min_free > 0 {
7525 match crate::disk::free_bytes(&winner.worktree) {
7526 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7527 Ok(free) => {
7528 self.state.event(
7529 "gate",
7530 format!(
7531 "only {free} bytes free ({min_free} required by `[disk] \
7532 min_free_bytes`); not spending a fix round on a failure the disk \
7533 may explain"
7534 ),
7535 );
7536 return Ok(GateFix::Stop);
7537 }
7538 Err(e) => {
7539 self.state.event(
7540 "gate",
7541 format!("free disk space could not be measured ({e:#}); no fix round"),
7542 );
7543 return Ok(GateFix::Stop);
7544 }
7545 }
7546 }
7547
7548 let attempt = spent + 1;
7549 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7550 let base = self.landing_base();
7551 let fix_prompt = prompt::gate_fix_in(
7552 &self.state.instruction,
7553 &failed,
7554 attempt,
7555 cap,
7556 &self.state.config.graph.language,
7557 &self.state.config.graph.github_language,
7558 );
7559 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7560 let sessions = self.state.config.graph.sessions;
7561 let artifacts = self.merge_repair_artifacts();
7562 self.state.event(
7563 "gate",
7564 format!("gate failed; gate-fix round {attempt} of {cap}"),
7565 );
7566 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7567 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7568 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7571 head: before.clone(),
7572 passed: false,
7573 outcomes: Vec::new(),
7574 });
7575 self.state.save()?;
7576 let (_, seat, out) = self
7577 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7578 prompt: fix_prompt.clone(),
7579 spec,
7580 seat,
7581 cwd: winner.worktree.clone(),
7582 timeout,
7583 allow_write: true,
7584 sessions,
7585 artifacts: artifacts.clone(),
7586 stem: format!("gate-fix-{attempt}"),
7587 handover: None,
7588 })
7589 .await;
7590 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7593 return Ok(GateFix::Defer);
7594 }
7595 let mut record = GateFixRecord {
7596 agent: seat.agent.clone(),
7597 failed,
7598 notes: String::new(),
7599 committed: false,
7600 error: None,
7601 };
7602 match out {
7603 AgentOutcome::Ok(o) => {
7604 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7607 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7608 }
7609 }
7610 AgentOutcome::Dropped(_) => {
7611 record.error = Some("the CLI dropped the stream".to_owned());
7612 }
7613 AgentOutcome::Quota(o) => {
7614 self.state.quota.push(QuotaLoss {
7615 seat: seat.key.clone(),
7616 node: "gate-fix".to_owned(),
7617 at: Timestamp::now(),
7618 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7619 });
7620 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7621 }
7622 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7623 }
7624 self.state.seats.insert(seat.key.clone(), seat);
7625 let coordination = self.coordinate(&winner.worktree).await?;
7626 if let Ok(r) = git::rescue_commit(
7627 &winner.worktree,
7628 &format!("magi: gate fix {attempt} (uncommitted work)"),
7629 )
7630 .await
7631 {
7632 self.state.note_withheld("gate-fix", &r.withheld);
7633 }
7634 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7635 record.committed = after != before;
7636 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7637 let note = record.error.clone();
7638 self.state.gate_fixes.push(record);
7639 self.state.save()?;
7640 if !changed {
7641 self.state.event(
7642 "gate",
7643 match note {
7644 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7645 None => format!("gate-fix round {attempt}: the tree did not change"),
7646 },
7647 );
7648 return Ok(GateFix::Stop);
7649 }
7650 self.state.event(
7651 "gate",
7652 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7653 );
7654
7655 drop(coordination);
7656 let wait = self.repository_wait();
7657 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7658 }
7659
7660 async fn guarded_pr_message(
7667 &mut self,
7668 winner: &Candidate,
7669 facts: Option<&BranchFacts>,
7670 posting: bool,
7671 ) -> Result<Option<PrMessage>> {
7672 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7673 let head = match final_head.as_deref() {
7674 Some(h) => crate::pr_outcome::FinalHead::At(h),
7675 None => crate::pr_outcome::FinalHead::Unknown,
7676 };
7677 let summarized = if posting {
7679 Some(self.summarize_change(winner, final_head.as_deref()).await)
7680 } else {
7681 None
7682 };
7683 if self.state.resource_block.is_some() {
7688 self.state.event(
7689 "pr-summary",
7690 "a shared resource was busy; nothing was published, resume to summarize again",
7691 );
7692 return Ok(None);
7693 }
7694 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7695 anyhow::bail!(
7696 "the final commit moved during public description generation; retry before publishing"
7697 );
7698 }
7699 let note = match (&summarized, &self.state.change_summary) {
7700 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7701 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7702 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7703 (None, _) => ChangeNote::NotAsked,
7704 };
7705 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7706 if !posting {
7707 let identity = crate::scrub::Identity::current();
7708 return Ok(Some(PrMessage {
7709 title: crate::scrub::scrub(&pr.title, &identity),
7710 body: crate::scrub::scrub(&pr.body, &identity),
7711 }));
7712 }
7713 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7716 if let Some(known) = self
7717 .state
7718 .github_text
7719 .clone()
7720 .filter(|g| g.fingerprint == raw_fp)
7721 {
7722 return self.settle_withheld(winner, &pr, known).await;
7723 }
7724 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7725 let mut violations =
7726 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7727 let identity = crate::scrub::Identity::current();
7728 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7729 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7730 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7731 {
7732 violations.push(crate::github_text::Violation::SensitiveData);
7733 }
7734 let has_language = violations
7738 .iter()
7739 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7740 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7741 self.state.event(
7742 "github-text",
7743 if has_language {
7744 format!("description rejected: {violations:?}; requesting one rewrite")
7745 } else {
7746 format!("description rejected: {violations:?}; asking the owner")
7747 },
7748 );
7749 let mut rewritten = false;
7750 let mut rejected_rewrite = None;
7751 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7752 {
7753 let output = text;
7754 if let Some(title) = summary_title(&output) {
7755 let raw_body = summary_without_title(&output);
7758 let clean_title = crate::scrub::scrub(&title, &identity);
7759 let mut body = crate::scrub::scrub(&raw_body, &identity);
7760 let redacted = clean_title != title || body != raw_body;
7761 let title = clean_title;
7762 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7763 if !complete {
7764 self.state.event(
7765 "github-text",
7766 "description rewrite omitted required findings",
7767 );
7768 }
7769 if !body.trim().is_empty() {
7770 body.push_str("\n\n");
7771 body.push_str(&public_pr_evidence(&self.state, head));
7772 }
7773 let marker = format!("magi:run/{}", self.state.id);
7774 if !body.trim().is_empty() && !body.contains(&marker) {
7775 body.push_str(&format!("\n\n{marker}"));
7776 }
7777 let rewrite = PrMessage {
7778 title: title.clone(),
7779 body: body.clone(),
7780 };
7781 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7782 let mut failed = crate::github_text::check_for(
7783 &self.state.config.graph,
7784 &title,
7785 &body,
7786 redecision,
7787 );
7788 if (!crate::github_text::shareable(&title)
7789 || !crate::github_text::shareable(&body))
7790 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7791 {
7792 failed.push(crate::github_text::Violation::SensitiveData);
7793 }
7794 if complete
7795 && !title.trim().is_empty()
7796 && !body.trim().is_empty()
7797 && !failed.is_empty()
7798 {
7799 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7800 title: title.clone(),
7801 body: body.clone(),
7802 failed: failed
7803 .iter()
7804 .map(|v| crate::github_text::category(*v).to_owned())
7805 .collect(),
7806 });
7807 }
7808 if complete
7809 && !title.trim().is_empty()
7810 && !raw_body.trim().is_empty()
7811 && failed.is_empty()
7812 {
7813 decision = redecision;
7814 pr = PrMessage { title, body };
7815 rewritten = true;
7816 self.state
7817 .event("github-text", "description rewrite passed");
7818 if redacted {
7819 self.state.event(
7820 "github-text",
7821 "sensitive data removed from description rewrite",
7822 );
7823 }
7824 }
7825 }
7826 }
7827 if !rewritten {
7828 self.state.event(
7829 "github-text",
7830 "description rewrite unavailable or rejected; asking the owner",
7831 );
7832 if let Some(w) =
7833 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7834 {
7835 self.retire_github_text_question("the withheld text changed");
7838 let record = crate::run::GithubTextAsk {
7839 rewrite: rejected_rewrite,
7840 fingerprint: raw_fp,
7841 title: w.title,
7842 body: w.body,
7843 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7844 question: None,
7845 asks: 0,
7846 resolved: false,
7847 chosen_title: None,
7848 choice: None,
7849 };
7850 return self.settle_withheld(winner, &pr, record).await;
7851 }
7852 }
7853 }
7854 let (title, body) =
7855 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7856 Ok(Some(PrMessage { title, body }))
7857 }
7858
7859 async fn summarize_change(
7870 &mut self,
7871 winner: &Candidate,
7872 head: Option<&str>,
7873 ) -> std::result::Result<(), String> {
7874 let Some(head) = head else {
7875 return Err("the final commit could not be read".to_owned());
7876 };
7877 let remote = self.state.config.merge.remote.clone();
7878 let tracking = format!("{remote}/{}", self.state.base_branch);
7879 if !git::rev_exists(&winner.worktree, &tracking).await {
7880 return Err(format!("the remote base `{tracking}` could not be read"));
7881 }
7882 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7883 Ok(b) if !b.is_empty() => b,
7884 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7885 };
7886 let evidence = crate::pr_outcome::summary_evidence(
7887 &self.state,
7888 crate::pr_outcome::FinalHead::At(head),
7889 );
7890 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7891 if self
7892 .state
7893 .change_summary
7894 .as_ref()
7895 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7896 {
7897 return Ok(());
7898 }
7899 if self.state.change_summary.take().is_some() {
7900 self.state.event(
7901 "pr-summary",
7902 "the final diff or evidence changed; the earlier change summary was discarded",
7903 );
7904 }
7905 if let Err(e) = write_artifact(
7907 &self.state,
7908 &format!("public-description-{fingerprint}.md"),
7909 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7910 ) {
7911 tracing::warn!("could not save public description evidence: {e:#}");
7912 }
7913 let chain = match agent::pick_chain(
7914 &self.state.config.agents,
7915 self.state.config.roles.synthesizer.as_ref(),
7916 &agent::installed,
7917 "synthesizer",
7918 ) {
7919 Ok(chain) => chain,
7920 Err(e) => {
7921 tracing::warn!("no agent for the change summary: {e:#}");
7922 self.state.event("pr-summary", "no summary agent available");
7923 return Err("no agent was available to summarize it".to_owned());
7924 }
7925 };
7926 let subjects = git::subjects(&winner.worktree, &base, head)
7927 .await
7928 .unwrap_or_default();
7929 let stat = git::diff_stat(&winner.worktree, &base, head)
7930 .await
7931 .unwrap_or_default();
7932 let stat: String = stat.chars().take(6_000).collect();
7933 let mut prompt = crate::prompt::change_summary_in(
7934 &base,
7935 head,
7936 &subjects,
7937 stat.trim(),
7938 &self.state.config.graph.github_language,
7939 );
7940 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7941 let sessions = self.state.config.graph.sessions;
7942 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7943 let prompts = self.state.config.prompts.clone();
7944 let cache = self.state.config.cache_dir();
7945 let run = self.state.id.clone();
7946 for (i, spec) in chain.into_iter().enumerate() {
7947 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7948 let github_spec = match agent::for_github_text(
7949 &spec,
7950 &self.state.config.graph.github_language,
7951 &winner.worktree,
7952 ) {
7953 Ok(spec) => spec,
7954 Err(e) => {
7955 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7956 continue;
7957 }
7958 };
7959 let job = SeatJob {
7960 spec: github_spec,
7961 seat,
7962 cwd: winner.worktree.clone(),
7963 prompt: prompt.clone(),
7964 timeout,
7965 allow_write: false,
7966 sessions,
7967 artifacts: agent::artifacts_dir(&self.state.dir()),
7968 stem: if i == 0 {
7969 "change-summary".to_owned()
7970 } else {
7971 format!("change-summary-{}", spec.id)
7972 },
7973 handover: None,
7974 };
7975 let ctx = WaveCtx {
7976 run: &run,
7977 node: "pr-summary",
7978 prompts: &prompts,
7979 cache: cache.as_deref(),
7980 round: None,
7981 carry_seats: false,
7982 pauses: Some((&self.pause, &self.interrupt)),
7983 };
7984 let (seat, outcome) =
7985 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7986 self.state.seats.insert(seat.key.clone(), seat);
7987 let text = match outcome {
7988 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7989 AgentOutcome::ResourceBlocked(why) => {
7990 if self.state.resource_block.is_none() {
7993 self.state.resource_block = Some(crate::run::ResourceBlock {
7994 cause: crate::run::ResourceBlockCause::Other,
7995 node: "pr-summary".to_owned(),
7996 kind: "shared resource".to_owned(),
7997 detail: why,
7998 at: Timestamp::now(),
7999 });
8000 }
8001 return Err("a shared resource was busy".to_owned());
8002 }
8003 _ => continue,
8004 };
8005 let blind = &self.state.config.blind;
8006 let Some(title) = summary_title(&text) else {
8007 continue;
8008 };
8009 let body: String = summary_without_title(&text)
8010 .lines()
8011 .filter(|l| !l.contains("magi:run/"))
8012 .collect::<Vec<_>>()
8013 .join("\n");
8014 let body = blind::sanitize_prose(body.trim(), blind);
8015 let title = blind::sanitize_prose(&title, blind);
8016 if body.trim().is_empty() || title.trim().is_empty() {
8017 continue;
8018 }
8019 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8020 continue;
8021 }
8022 if git::rev_parse(&winner.worktree, "HEAD")
8024 .await
8025 .ok()
8026 .as_deref()
8027 != Some(head)
8028 {
8029 return Err("the branch moved while the summary was written".to_owned());
8030 }
8031 if git::merge_base(&winner.worktree, &tracking, head)
8032 .await
8033 .ok()
8034 .as_deref()
8035 != Some(base.as_str())
8036 || crate::pr_outcome::summary_evidence(
8037 &self.state,
8038 crate::pr_outcome::FinalHead::At(head),
8039 ) != evidence
8040 {
8041 return Err("the summary evidence changed while it was written".to_owned());
8042 }
8043 self.state.change_summary = Some(crate::run::ChangeSummary {
8044 head: head.to_owned(),
8045 base,
8046 evidence: fingerprint,
8047 title,
8048 body,
8049 });
8050 self.state.event(
8051 "pr-summary",
8052 format!("change summary written by {} for the final diff", spec.id),
8053 );
8054 if let Err(e) = self.state.save() {
8055 tracing::warn!("could not save the change summary: {e:#}");
8056 }
8057 return Ok(());
8058 }
8059 self.state
8060 .event("pr-summary", "no agent produced a change summary");
8061 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8062 }
8063
8064 async fn rewrite_pr_text(
8074 &mut self,
8075 winner: &Candidate,
8076 pr: &PrMessage,
8077 violations: &[crate::github_text::Violation],
8078 ) -> Option<String> {
8079 let prompt = format!(
8080 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8081 pr.title, pr.body
8082 );
8083 let sessions = self.state.config.graph.sessions;
8084 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8085 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8088 let key = format!("impl-{}", winner.label);
8089 if let Some(spec) = self
8090 .state
8091 .config
8092 .agents
8093 .iter()
8094 .find(|a| a.id == winner.agent)
8095 .cloned()
8096 && let Some(existing) = self.state.seats.get(&key)
8097 && existing.agent == spec.id
8098 && agent::has_session(spec.kind, existing, sessions)
8099 {
8100 attempts.push((spec, Some(existing.clone())));
8101 }
8102 match agent::pick_chain(
8103 &self.state.config.agents,
8104 self.state.config.roles.language_judge.as_ref(),
8105 &agent::installed,
8106 "language_judge",
8107 ) {
8108 Ok(chain) => {
8109 for spec in chain {
8110 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8111 continue;
8112 }
8113 attempts.push((spec, None));
8114 }
8115 }
8116 Err(e) if attempts.is_empty() => {
8117 tracing::warn!("no agent for the description rewrite: {e:#}");
8118 self.state
8119 .event("github-text", "no rewrite agent available");
8120 return None;
8121 }
8122 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8123 }
8124 let prompts = self.state.config.prompts.clone();
8125 let cache = self.state.config.cache_dir();
8126 let run = self.state.id.clone();
8127 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8128 let resumed = resumed_seat.is_some();
8129 let seat = resumed_seat.unwrap_or_else(|| {
8130 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8131 });
8132 let github_spec = match agent::for_github_text(
8133 &spec,
8134 &self.state.config.graph.github_language,
8135 &winner.worktree,
8136 ) {
8137 Ok(spec) => spec,
8138 Err(e) => {
8139 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8140 continue;
8141 }
8142 };
8143 let job = SeatJob {
8144 spec: github_spec,
8145 seat,
8146 cwd: winner.worktree.clone(),
8147 prompt: prompt.clone(),
8148 timeout: retry_budget(timeout_full, resumed),
8149 allow_write: false,
8150 sessions,
8151 artifacts: agent::artifacts_dir(&self.state.dir()),
8152 stem: if i == 0 {
8153 "github-text-rewrite".to_owned()
8154 } else {
8155 format!("github-text-rewrite-{}", spec.id)
8156 },
8157 handover: None,
8158 };
8159 let ctx = WaveCtx {
8160 run: &run,
8161 node: "github-text",
8162 prompts: &prompts,
8163 cache: cache.as_deref(),
8164 round: None,
8165 carry_seats: false,
8166 pauses: Some((&self.pause, &self.interrupt)),
8167 };
8168 let (seat, outcome) =
8169 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8170 self.state.seats.insert(seat.key.clone(), seat);
8171 match outcome {
8172 AgentOutcome::Ok(o)
8173 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8174 {
8175 return Some(o.text);
8176 }
8177 AgentOutcome::ResourceBlocked(_) => return None,
8178 _ => {}
8179 }
8180 }
8181 None
8182 }
8183
8184 fn retire_github_text_question(&mut self, why: &str) {
8186 let Some(id) = self
8187 .state
8188 .github_text
8189 .as_ref()
8190 .and_then(|g| g.question.clone())
8191 else {
8192 return;
8193 };
8194 let _ = ask::Questions::open().update(&id, |q| {
8195 q.abandon(why);
8196 Ok(())
8197 });
8198 }
8199
8200 async fn settle_withheld(
8204 &mut self,
8205 winner: &Candidate,
8206 pr: &PrMessage,
8207 mut g: crate::run::GithubTextAsk,
8208 ) -> Result<Option<PrMessage>> {
8209 use crate::github_text as gt;
8210 let store = ask::Questions::open();
8211 let timeout = self.state.config.graph.answer_timeout;
8212 if !g.resolved {
8213 let mut standing = g
8214 .question
8215 .as_ref()
8216 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8217 if let Some(q) = standing.as_ref()
8218 && q.status.open()
8219 && gt::expired(q, timeout)
8220 {
8221 let why = format!("no answer within {}s of asking", timeout.max(1));
8222 standing = store
8223 .update(&q.id, |q| {
8224 q.abandon(&why);
8225 Ok(())
8226 })
8227 .ok()
8228 .map(|(q, ())| q);
8229 }
8230 let mut ask_again: Option<Option<Vec<String>>> = None;
8231 match standing {
8232 None if g.asks <= 1 => {
8235 g.asks = 0;
8236 ask_again = Some(None);
8237 }
8238 None => {
8239 g.resolved = true;
8240 g.chosen_title = None;
8241 g.choice = Some(GithubTextChoice::Fallback);
8242 }
8243 Some(q) if q.status.open() => {
8244 self.park_on_github_text(&g)?;
8245 return Ok(None);
8246 }
8247 Some(q) => match gt::read_reply(&q) {
8248 gt::Reply::Fallback => {
8249 g.resolved = true;
8250 g.chosen_title = None;
8251 g.choice = Some(GithubTextChoice::Fallback);
8252 }
8253 gt::Reply::Original => {
8254 g.resolved = true;
8255 g.chosen_title = None;
8256 g.choice = Some(GithubTextChoice::Original);
8257 }
8258 gt::Reply::Rewrite => {
8259 g.resolved = true;
8260 g.chosen_title = None;
8261 g.choice = Some(if g.rewrite.is_some() {
8262 GithubTextChoice::Rewrite
8263 } else {
8264 GithubTextChoice::Fallback
8265 });
8266 }
8267 gt::Reply::Title(text) => {
8268 let first = text
8269 .lines()
8270 .map(str::trim)
8271 .find(|l| !l.is_empty())
8272 .unwrap_or("");
8273 let decision = if first.is_empty() {
8274 None
8275 } else {
8276 let candidate = PrMessage {
8277 title: first.to_owned(),
8278 body: String::new(),
8279 };
8280 self.judge_pr_language(&winner.worktree, &candidate).await
8281 };
8282 match gt::vet_title(
8283 &text,
8284 decision,
8285 self.state.config.graph.github_english(),
8286 ) {
8287 Ok(title) => {
8288 g.resolved = true;
8289 g.chosen_title = Some(title);
8290 g.choice = Some(GithubTextChoice::MyTitle);
8291 }
8292 Err(cats) if g.asks < 2 => {
8293 ask_again = Some(Some(
8294 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8295 ))
8296 }
8297 Err(_) => {
8298 g.resolved = true;
8299 g.chosen_title = None;
8300 g.choice = Some(GithubTextChoice::Fallback);
8301 }
8302 }
8303 }
8304 },
8305 }
8306 if let Some(rejected) = ask_again {
8307 let w = gt::Withheld {
8310 title: g.title,
8311 body: g.body,
8312 categories: g.categories.clone(),
8313 };
8314 let artifact = match crate::run::write_artifact(
8317 &self.state,
8318 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8319 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8320 ) {
8321 Ok(p) => Some(p),
8322 Err(e) => {
8323 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8324 None
8325 }
8326 };
8327 let (detail, comparison) = gt::question_content(
8328 &w,
8329 &pr.title,
8330 &pr.body,
8331 &self.state.id,
8332 rejected.as_deref(),
8333 artifact.as_deref(),
8334 g.rewrite.as_ref(),
8335 );
8336 let mut q = ask::Question::new(
8337 self.state.id.clone(),
8338 gt::ASK_NODE.to_owned(),
8339 gt::ASK_SEAT.to_owned(),
8340 gt::question_summary(&self.state.config.graph.language, &w),
8341 detail,
8342 gt::question_choices(&w, g.rewrite.as_ref()),
8343 );
8344 q.posting_comparison = Some(comparison);
8345 q.answer_timeout = timeout;
8346 g.question = Some(q.id.clone());
8349 g.asks += 1;
8350 self.state.github_text = Some(g.clone());
8351 self.state.event(
8352 "github-text",
8353 format!("asking the owner about the withheld text ({})", q.short()),
8354 );
8355 self.state.save()?;
8356 store
8357 .put_executable(&mut q)
8358 .context("file the posting-gate question")?;
8359 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8360 tracing::warn!(
8361 "could not notify about posting-gate question {}: {e:#}",
8362 q.short()
8363 );
8364 }
8365 self.park_on_github_text(&g)?;
8366 return Ok(None);
8367 }
8368 }
8369 let resolved = g.chosen_title.clone();
8370 let original = g.choice == Some(GithubTextChoice::Original);
8371 self.state.event(
8373 "github-text",
8374 if original {
8375 format!(
8376 "posting the original text for: {}",
8377 match (g.title, g.body) {
8378 (true, true) => "title, description",
8379 (true, false) => "title",
8380 _ => "description",
8381 }
8382 )
8383 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8384 "posting the saved rewrite title and description".to_owned()
8385 } else if resolved.is_some() {
8386 "posting the owner's replacement title".to_owned()
8387 } else {
8388 "posting the neutral text for the withheld fields".to_owned()
8389 },
8390 );
8391 self.state.github_text = Some(g.clone());
8392 self.state.save()?;
8393 if g.choice == Some(GithubTextChoice::Rewrite)
8394 && let Some(r) = g.rewrite.as_ref()
8395 {
8396 return Ok(Some(PrMessage {
8397 title: r.title.clone(),
8398 body: r.body.clone(),
8399 }));
8400 }
8401 let w = gt::Withheld {
8402 title: g.title,
8403 body: g.body,
8404 categories: g.categories.clone(),
8405 };
8406 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8407 if let Some(t) = resolved {
8408 title = t;
8409 }
8410 let english = gt::LanguageDecision {
8412 title_english: true,
8413 body_english: true,
8414 body_complete: true,
8415 };
8416 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8417 if original {
8418 if g.title {
8421 title = pr.title.clone();
8422 }
8423 if g.body {
8424 body = pr.body.clone();
8425 }
8426 }
8427 Ok(Some(PrMessage { title, body }))
8428 }
8429
8430 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8431 self.state.github_text = Some(g.clone());
8432 self.state.end_pass_parked(ParkCause::Question);
8433 self.state.parked = true;
8434 self.state.event(
8435 "github-text",
8436 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8437 );
8438 self.state.save()
8439 }
8440
8441 async fn judge_pr_language(
8444 &mut self,
8445 cwd: &Path,
8446 pr: &PrMessage,
8447 ) -> Option<crate::github_text::LanguageDecision> {
8448 self.state.config.roles.language_judge.as_ref()?;
8449 if !self.state.config.graph.language_gate() {
8450 return None;
8451 }
8452 let decision =
8453 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8454 self.state.event(
8455 "github-text",
8456 if decision.is_some() {
8457 "language decided by the language judge"
8458 } else {
8459 "language judge unavailable; using built-in heuristics"
8460 },
8461 );
8462 decision
8463 }
8464
8465 async fn merge(&mut self) -> Result<()> {
8466 if self.resource_halt()? || self.hold_base_sync().await? {
8467 return Ok(());
8468 }
8469 if self
8484 .state
8485 .base_sync
8486 .as_ref()
8487 .is_some_and(|s| s.conflict.is_some())
8488 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8489 != Some(RunStatus::Gating)
8490 || !self.state.gate_allows_progress()
8498 {
8499 return Ok(());
8500 }
8501 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8510 return Ok(());
8511 }
8512 let Some(winner) = self.state.winner().cloned() else {
8513 return Ok(());
8514 };
8515 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8516 if self.state.gate_head.as_deref() != Some(head.as_str()) {
8517 self.state.status = RunStatus::Blocked;
8518 self.state.event(
8519 "gate",
8520 "winner changed after gate; fresh verification required",
8521 );
8522 self.state.save()?;
8523 return Ok(());
8524 }
8525 let wait = self.repository_wait();
8526 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8527 return Ok(());
8528 }
8529 let repo = self.state.repo.clone();
8530 let base = self.state.base_branch.clone();
8531 let mode = self.state.config.merge.mode;
8532 let style = self.state.config.merge.style;
8533 let facts = if is_review_run(&self.state) {
8534 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8535 let start = review_base(
8536 &repo,
8537 &self.state.config.merge.remote,
8538 &base,
8539 &self.state.base_commit,
8540 &winner.branch,
8541 )
8542 .await;
8543 branch_facts(&repo, &start, &winner.branch).await
8544 } else {
8545 None
8546 };
8547 let leaked = if is_review_run(&self.state) {
8550 leaked_subjects(&self.state, &winner.branch).await
8551 } else {
8552 Some(Vec::new())
8553 };
8554 let Some(pr) = self
8558 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8559 .await?
8560 else {
8561 return Ok(());
8562 };
8563 let message = pr.commit_message();
8564
8565 let outcome = match mode {
8566 MergeMode::None => MergeOutcome {
8567 mode,
8568 ok: true,
8569 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8570 empty: false,
8571 },
8572 MergeMode::Pr | MergeMode::Local
8573 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8574 {
8575 MergeOutcome {
8576 mode,
8577 ok: false,
8578 detail: empty_candidate_detail(&self.state, &base),
8579 empty: true,
8580 }
8581 }
8582 MergeMode::Local => {
8583 let on = git::current_branch(&repo).await?;
8584 if on.as_deref() != Some(base.as_str()) {
8585 MergeOutcome {
8586 mode,
8587 ok: false,
8588 detail: format!(
8589 "{} has {} checked out, not the base branch {base}",
8590 repo.display(),
8591 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8592 ),
8593 empty: false,
8594 }
8595 } else if !git::is_clean(&repo).await? {
8596 MergeOutcome {
8597 mode,
8598 ok: false,
8599 detail: format!("{} is dirty; refusing to merge", repo.display()),
8600 empty: false,
8601 }
8602 } else {
8603 let out = match style {
8604 MergeStyle::Merge => {
8605 git::merge_no_ff(&repo, &winner.branch, &message).await?
8606 }
8607 MergeStyle::Squash => {
8608 git::merge_squash(&repo, &winner.branch, &message).await?
8609 }
8610 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8611 };
8612 MergeOutcome {
8613 mode,
8614 ok: out.ok(),
8615 detail: if out.ok() { out.stdout } else { out.stderr },
8616 empty: false,
8617 }
8618 }
8619 }
8620 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8621 self.state.pr_publication_pending = true;
8622 self.state.save()?;
8623 let out =
8624 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8625 self.state.pr_publication_pending = out.is_err();
8626 MergeOutcome {
8627 mode,
8628 ok: out.is_ok(),
8629 detail: match out {
8630 Ok(url) => url,
8631 Err(e) => format!("{e:#}"),
8632 },
8633 empty: false,
8634 }
8635 }
8636 MergeMode::Pr => {
8637 let remote = self.state.config.merge.remote.clone();
8638 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8639 if !pushed.ok() {
8640 MergeOutcome {
8641 mode,
8642 ok: false,
8643 detail: pushed.stderr,
8644 empty: false,
8645 }
8646 } else {
8647 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8653 let out = match pr_merge_plan(found) {
8654 PrPlan::Create => {
8655 gh_pr_create(
8656 &winner.worktree,
8657 &base,
8658 &winner.branch,
8659 &pr.title,
8660 &pr.body,
8661 )
8662 .await
8663 }
8664 PrPlan::Adopt { url, title } => {
8665 self.state
8666 .event("merge", format!("Pr: adopted open pull request {url}"));
8667 if self.state.github_text.as_ref().is_some_and(|g| {
8668 g.resolved
8669 && g.choice == Some(GithubTextChoice::Rewrite)
8670 && g.rewrite.is_some()
8671 }) {
8672 land::set_pr_text_verbatim(
8675 &winner.worktree,
8676 &url,
8677 &pr.title,
8678 &pr.body,
8679 )
8680 .await
8681 .map(|()| url)
8682 } else {
8683 if title != pr.title
8684 && (!is_review_run(&self.state)
8685 || leaked
8686 .as_deref()
8687 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8688 && let Err(e) =
8689 if self.state.github_text.as_ref().is_some_and(|g| {
8690 g.resolved
8691 && g.title
8692 && g.choice == Some(GithubTextChoice::Original)
8693 }) {
8694 land::set_pr_title_verbatim(
8696 &winner.worktree,
8697 &url,
8698 &pr.title,
8699 )
8700 .await
8701 } else {
8702 land::set_pr_title(
8703 &mut self.state,
8704 &winner.worktree,
8705 &url,
8706 &pr.title,
8707 )
8708 .await
8709 }
8710 {
8711 tracing::warn!("could not refresh title of {url}: {e:#}");
8712 self.state
8713 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8714 }
8715 Ok(url)
8716 }
8717 }
8718 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8719 };
8720 match out {
8721 Ok(url) => MergeOutcome {
8722 mode,
8723 ok: true,
8724 detail: url,
8725 empty: false,
8726 },
8727 Err(e) => MergeOutcome {
8728 mode,
8729 ok: false,
8730 detail: e.to_string(),
8731 empty: false,
8732 },
8733 }
8734 }
8735 }
8736 };
8737
8738 self.state.status = match (mode, outcome.ok) {
8739 (MergeMode::None, _) => RunStatus::Ready,
8740 (_, true) if outcome.empty => RunStatus::Ready,
8741 (MergeMode::Pr, true) => RunStatus::Ready,
8742 (_, true) => RunStatus::Merged,
8743 (_, false) => RunStatus::Blocked,
8744 };
8745 self.state.event(
8746 "merge",
8747 format!(
8748 "{:?}: {}",
8749 mode,
8750 outcome.detail.lines().next().unwrap_or("")
8751 ),
8752 );
8753 self.state.merge = Some(outcome);
8754 self.state.save()?;
8755
8756 if self.state.config.graph.land
8762 && mode == MergeMode::Pr
8763 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8764 {
8765 self.run_land().await?;
8766 }
8767 self.settle_questions();
8772 Ok(())
8773 }
8774
8775 async fn run_land(&mut self) -> Result<()> {
8786 crate::pr_head::ensure_owner(&self.state)?;
8787 let url = self
8788 .state
8789 .pr
8790 .as_ref()
8791 .map(|pr| pr.url.clone())
8792 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8793 .unwrap_or_default();
8794 let url = url.lines().next().unwrap_or("").trim().to_owned();
8795 if !url.starts_with("http") {
8796 return Ok(());
8797 }
8798 loop {
8801 match land::land(&mut self.state, &url).await {
8802 Ok(_)
8803 if self
8804 .state
8805 .merge_repairs
8806 .last()
8807 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8808 {
8809 if self.repair_merge_review().await? {
8810 continue;
8811 }
8812 }
8813 Ok(pr) if self.state.parked => {
8814 let _ = pr;
8818 }
8819 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8820 self.state.status = RunStatus::Superseded;
8821 self.state.save()?;
8822 }
8823 Ok(pr) => {
8824 self.state.status = match pr.state {
8825 land::PrLifecycle::Merged => RunStatus::Merged,
8826 _ => RunStatus::Blocked,
8827 };
8828 if bump::should_release_bump(self.state.status)
8835 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8836 {
8837 self.state
8843 .event("bump", format!("release bump skipped: {e:#}"));
8844 }
8845 if self.state.status == RunStatus::Merged {
8848 crate::followup::after_merge(&mut self.state, &pr.url).await;
8849 }
8850 self.state.save()?;
8851 }
8852 Err(e) => {
8853 if e.is::<crate::repo_lock::ResourceBlocked>() {
8854 return Err(e);
8855 }
8856 self.state.status = RunStatus::Blocked;
8857 self.state.event("land", format!("gave up: {e}"));
8858 self.state.save()?;
8859 }
8860 }
8861 break;
8862 }
8863 Ok(())
8864 }
8865
8866 async fn repair_merge_review(&mut self) -> Result<bool> {
8869 use crate::run::MergeRepairPhase as Phase;
8870 crate::pr_head::ensure_owner(&self.state)?;
8871 let i = self.state.merge_repairs.len() - 1;
8872 let request = self.state.merge_repairs[i].clone();
8873 if request.phase == Phase::Fixing {
8874 let restored = match self.state.winner().cloned() {
8875 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8876 None => Err(anyhow::anyhow!("repair has no winning branch")),
8877 };
8878 let why = match restored {
8879 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8880 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8881 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8882 };
8883 self.hold_merge_repair(i, why)?;
8884 return Ok(false);
8885 }
8886 if request.phase == Phase::Reviewing && self.state.gate_ran {
8891 let winner = self
8892 .state
8893 .winner()
8894 .context("repair has no winning branch")?;
8895 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8896 if self.state.gate_head.as_deref() != Some(head.as_str())
8897 || !git::is_clean(&winner.worktree).await?
8898 {
8899 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8900 return Ok(false);
8901 }
8902 }
8903 let failure = if i >= self.state.config.graph.land_rounds
8904 || self.state.config.graph.review_rounds == 0
8905 {
8906 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8907 } else {
8908 None
8909 };
8910 if let Some(why) = failure {
8911 self.hold_merge_repair(i, why)?;
8912 return Ok(false);
8913 }
8914 if request.phase == Phase::Requested {
8915 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8916 Ok(pr) => pr,
8917 Err(e) => {
8918 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8919 return Ok(false);
8920 }
8921 };
8922 self.state.merge_repairs[i].phase = Phase::Fixing;
8923 self.state.land_approval = None;
8924 self.state.land_armed_head = None;
8925 self.state.save()?;
8926 let evidence = serde_json::to_string_pretty(&request)?;
8929 let reason = format!(
8930 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8931 );
8932 let budget = self.state.config.graph.land_rounds;
8933 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8934 match result {
8935 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8936 Ok(land::Fixed::Failed(why)) => {
8937 self.hold_merge_repair(
8938 i,
8939 format!(
8940 "repair failed: {why}; inspect the branch and run an explicit review"
8941 ),
8942 )?;
8943 return Ok(false);
8944 }
8945 Err(e) => {
8946 self.hold_merge_repair(
8947 i,
8948 format!(
8949 "repair failed: {e:#}; inspect the branch and run an explicit review"
8950 ),
8951 )?;
8952 return Ok(false);
8953 }
8954 }
8955 self.begin_merge_repair_review(i)?;
8959 }
8960 self.review_loop().await?;
8961 self.gate().await?;
8962 if self.resource_halt()? || self.state.parked {
8963 return Ok(false);
8964 }
8965 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8966 != Some(RunStatus::Gating)
8967 || !self.state.gate_allows_progress()
8968 {
8969 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8970 return Ok(false);
8971 }
8972 let winner = self
8975 .state
8976 .winner()
8977 .cloned()
8978 .context("repair has no winning branch")?;
8979 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8980 if self.state.gate_head.as_deref() != Some(after.as_str())
8981 || !git::is_clean(&winner.worktree).await?
8982 {
8983 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8984 return Ok(false);
8985 }
8986 let expected = self
8987 .state
8988 .pr_pushed_head
8989 .as_deref()
8990 .unwrap_or(&request.head)
8991 .to_owned();
8992 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8993 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8996 let refspec = format!("{after}:refs/heads/{public}");
8997 let push = git::git_raw(
8998 &winner.worktree,
8999 &["push", &lease, &self.state.config.merge.remote, &refspec],
9000 )
9001 .await;
9002 match push {
9003 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9004 Ok(out) => {
9005 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9006 return Ok(false);
9007 }
9008 Err(e) => {
9009 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9010 return Ok(false);
9011 }
9012 }
9013 self.state.merge_repairs[i].phase = Phase::Complete;
9014 self.state.status = RunStatus::Landing;
9015 self.state.event(
9016 "repair",
9017 "new review and gate completed; fresh owner merge decision required",
9018 );
9019 self.state.save()?;
9020 Ok(true)
9021 }
9022
9023 fn merge_repair_artifacts(&self) -> PathBuf {
9024 let artifacts = agent::artifacts_dir(&self.state.dir());
9025 match self
9026 .state
9027 .merge_repairs
9028 .last()
9029 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9030 {
9031 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9032 None => artifacts,
9033 }
9034 }
9035
9036 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9037 if let Some(e2e) = self.state.head_e2e.take() {
9038 self.state
9039 .verification_history
9040 .push(crate::run::VerificationEvidence {
9041 superseded_by: None,
9042 archived_at: Some(Timestamp::now()),
9043 fixes_spent: Some(self.state.gate_fixes.len()),
9044 gate_head: None,
9045 gate_ran: false,
9046 gate: Vec::new(),
9047 e2e: Some(e2e),
9048 });
9049 }
9050 self.state.reviews.clear();
9051 self.state.gate.clear();
9052 self.state.gate_ran = false;
9053 self.state.gate_head = None;
9054 self.state.gate_fixes.clear();
9055 self.state.gate_repair_e2e = None;
9056 self.state.pre_gate.clear();
9057 self.state.pre_gate_commit = None;
9058 self.state.contested_handoff = None;
9059 self.state.land_reruns = None;
9060 self.state.land_approval = None;
9061 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9062 self.state.status = RunStatus::Reviewing;
9063 self.state.save()
9064 }
9065
9066 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9067 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9068 self.state.merge_repairs[index].failure = Some(why.clone());
9069 self.state.status = RunStatus::Blocked;
9070 self.state.event("repair", why);
9071 self.state.save()
9072 }
9073
9074 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9078 if let Some(existing) = self.state.seats.get(key)
9079 && existing.agent == agent
9080 {
9081 return existing.clone();
9082 }
9083 let fresh = if self.state.seats.contains_key(key) {
9086 handover_seat(key, agent, self.state.next_seat_seed())
9087 } else {
9088 SeatState::new(key, agent, self.state.seed)
9089 };
9090 self.state.seats.insert(key.to_owned(), fresh.clone());
9091 fresh
9092 }
9093
9094 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9100 match self.state.seats.get(key) {
9101 Some(s) if s.agent != spec.id => {
9102 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9103 }
9104 _ => spec,
9105 }
9106 }
9107
9108 fn view(&self, c: &Candidate) -> CandidateView {
9110 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9111 .unwrap_or_default();
9112 let (patch, _) = blind::sanitize_patch(
9113 &format!("candidate {} patch", c.label),
9114 &raw,
9115 &self.state.config.blind,
9116 );
9117 CandidateView {
9118 label: c.label,
9119 branch: c.branch.clone(),
9120 summary: c.summary.clone(),
9121 stat: c.stat.clone(),
9122 patch,
9123 }
9124 }
9125
9126 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9128 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9129 prompt::judge(
9130 "(see above)",
9131 &views,
9132 self.roles.judges.len(),
9133 base_short,
9134 "en",
9135 )
9136 }
9137
9138 fn vote_prompt_full(
9144 &self,
9145 j: usize,
9146 labels: &[char],
9147 language: &str,
9148 candidates: &[Candidate],
9149 base_short: &str,
9150 ) -> String {
9151 let mut text = format!(
9152 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9153 prompt::final_vote(labels, language),
9154 self.state.instruction,
9155 self.candidate_block(candidates, base_short)
9156 );
9157 if let Some(own) = self
9158 .state
9159 .judgements
9160 .get(j)
9161 .filter(|r| !r.ranking.is_empty())
9162 {
9163 let reasons = own
9164 .reasons
9165 .iter()
9166 .map(|(k, v)| format!("- {k}: {v}"))
9167 .collect::<Vec<_>>()
9168 .join("\n");
9169 text.push_str(&format!(
9170 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9171 own.ranking.iter().collect::<String>(),
9172 if reasons.is_empty() {
9173 ""
9174 } else {
9175 ", because:\n"
9176 }
9177 ));
9178 }
9179 if !self.state.deliberation.is_empty() {
9180 text.push_str("\n# What was argued before this vote\n");
9181 for t in self.transcript(&[], j) {
9182 text.push_str(&format!(
9183 "\n## {}{}\n\n{}\n",
9184 t.who,
9185 if t.is_self { " (you)" } else { "" },
9186 t.body.trim()
9187 ));
9188 }
9189 }
9190 text
9191 }
9192
9193 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9200 let mut turns = Vec::new();
9201 for j in &self.state.judgements {
9202 if j.ranking.is_empty() {
9203 continue;
9204 }
9205 let reasons = j
9206 .reasons
9207 .iter()
9208 .map(|(k, v)| format!("- {k}: {v}"))
9209 .collect::<Vec<_>>()
9210 .join("\n");
9211 turns.push(Turn {
9212 who: format!("Judge {} (opening ranking)", j.judge),
9213 is_self: j.judge == self_idx + 1,
9214 body: format!(
9215 "Ranked {}{}{reasons}",
9216 j.ranking.iter().collect::<String>(),
9217 if reasons.is_empty() {
9218 ""
9219 } else {
9220 ", because:\n"
9221 }
9222 ),
9223 });
9224 }
9225 for t in self
9226 .state
9227 .deliberation
9228 .iter()
9229 .flat_map(|r| r.turns.iter())
9230 .chain(current)
9231 {
9232 turns.push(Turn {
9233 who: format!("Judge {}", t.judge),
9234 is_self: t.judge == self_idx + 1,
9235 body: t.body.clone(),
9236 });
9237 }
9238 turns
9239 }
9240}
9241
9242fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9244 agent::has_session(spec.kind, seat, sessions)
9245}
9246
9247fn next_untried_in_roster<'a>(
9268 roster: &'a [AgentSpec],
9269 start: usize,
9270 tried: &BTreeSet<String>,
9271) -> Option<&'a AgentSpec> {
9272 roster
9273 .get(start + 1..)?
9274 .iter()
9275 .find(|s| !tried.contains(&s.id))
9276}
9277
9278fn pick_successor<'a>(
9299 roster: &'a [AgentSpec],
9300 start: usize,
9301 tried: &BTreeSet<String>,
9302 carried: Option<&BTreeSet<String>>,
9303 others: &BTreeSet<String>,
9304) -> Option<&'a AgentSpec> {
9305 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9306 roster
9307 .get(start + 1..)
9308 .and_then(|tail| {
9309 tail.iter()
9310 .filter(free)
9311 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9312 })
9313 .or_else(|| {
9314 carried
9315 .is_some()
9316 .then(|| roster.iter().find(free))
9317 .flatten()
9318 })
9319 .or_else(|| match carried {
9320 Some(c) => next_for_seat(roster, start, tried, c),
9321 None => next_untried_in_roster(roster, start, tried),
9322 })
9323}
9324
9325fn next_for_seat<'a>(
9335 roster: &'a [AgentSpec],
9336 start: usize,
9337 round_tried: &BTreeSet<String>,
9338 carried_failed: &BTreeSet<String>,
9339) -> Option<&'a AgentSpec> {
9340 roster
9341 .get(start + 1..)?
9342 .iter()
9343 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9344 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9345}
9346
9347fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9353 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9354 return spec;
9355 };
9356 let ok = |id: &str| !h.failed.contains(id);
9357 if let Some(last) = h.last_ok.as_deref()
9358 && ok(last)
9359 && let Some(s) = roster.iter().find(|s| s.id == last)
9360 {
9361 return s.clone();
9362 }
9363 if ok(&spec.id) {
9364 return spec;
9365 }
9366 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9367 roster
9368 .get(from + 1..)
9369 .into_iter()
9370 .flatten()
9371 .chain(roster.iter())
9372 .find(|s| ok(&s.id))
9373 .cloned()
9374 .unwrap_or(spec)
9375}
9376
9377pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9381 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9382}
9383
9384const TIMED_OUT: &str = "timed out";
9387
9388impl FailClass {
9389 fn of(out: &AgentOutcome) -> Option<Self> {
9391 match out {
9392 AgentOutcome::Ok(_) => None,
9393 AgentOutcome::Quota(_) => Some(Self::Quota),
9394 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9395 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9396 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9397 AgentOutcome::ResourceBlocked(_) => None,
9400 }
9401 }
9402
9403 fn stem_word(&self) -> &'static str {
9405 match self {
9406 Self::Quota => "quota",
9407 _ => "handover",
9408 }
9409 }
9410}
9411
9412fn failure_signature(msg: &str) -> String {
9416 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9417 if let Some(code) = line
9420 .strip_prefix("process exited with no usable output (exit code ")
9421 .and_then(|s| s.strip_suffix(')'))
9422 .and_then(|s| s.parse::<i32>().ok())
9423 {
9424 line = format!("exited with some({code}) and no usable output");
9425 } else if matches!(
9426 line.as_str(),
9427 "process exited with no usable output (no exit code reported)"
9428 | "process exited with no usable output (exit code unavailable)"
9429 ) {
9430 line = "exited with none and no usable output".to_owned();
9431 }
9432 let mut out = Vec::new();
9433 for word in line.split_whitespace() {
9434 if word.contains('/') || word.contains('\\') {
9435 out.push("<path>".to_owned());
9436 continue;
9437 }
9438 let mut w = String::new();
9439 let mut in_digits = false;
9440 for c in word.chars() {
9441 if c.is_ascii_digit() {
9442 if !in_digits {
9443 w.push('#');
9444 }
9445 in_digits = true;
9446 } else {
9447 in_digits = false;
9448 w.push(c);
9449 }
9450 }
9451 out.push(w);
9452 }
9453 out.join(" ").chars().take(120).collect()
9454}
9455
9456fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9463 match cur {
9464 FailClass::Quota | FailClass::Timeout => true,
9465 FailClass::Other(_) => prev != Some(cur),
9466 }
9467}
9468
9469fn fail_reason(out: &AgentOutcome) -> String {
9471 match out {
9472 AgentOutcome::Ok(_) => String::new(),
9473 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9474 AgentOutcome::Dropped(o) => format!(
9475 "the CLI dropped the stream ({})",
9476 o.dropped
9477 .as_ref()
9478 .map(|d| d.why.as_str())
9479 .unwrap_or("it ended without delivering its answer")
9480 ),
9481 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9482 e.lines().next().unwrap_or("").chars().take(160).collect()
9483 }
9484 }
9485}
9486
9487pub(crate) fn record_handover(
9490 state: &mut RunState,
9491 node: &str,
9492 seat: &str,
9493 from: &str,
9494 to: &str,
9495 class: &FailClass,
9496 reason: &str,
9497) {
9498 let message = if *class == FailClass::Quota {
9499 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9500 } else {
9501 format!("{seat}: handed over {from} -> {to} ({reason})")
9502 };
9503 state.event(node, message);
9504 state.handovers.push(Handover {
9505 at: Timestamp::now(),
9506 node: node.to_owned(),
9507 seat: seat.to_owned(),
9508 from: from.to_owned(),
9509 to: to.to_owned(),
9510 reason: reason.to_owned(),
9511 });
9512}
9513
9514fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9528 commands.iter().any(|c| c.exit_code.is_none())
9529}
9530
9531fn verified_noop_claim(
9544 usable: bool,
9545 commands: &[agent::CommandEvidence],
9546 text: &str,
9547) -> Option<String> {
9548 (usable && !has_unconfirmed_command(commands))
9549 .then(|| verdict::verified_noop(text))
9550 .flatten()
9551}
9552
9553fn short(commit: &str) -> String {
9554 commit.chars().take(7).collect()
9555}
9556
9557fn make_executable(path: &Path) -> Result<()> {
9558 #[cfg(unix)]
9559 {
9560 use std::os::unix::fs::PermissionsExt as _;
9561 let mut perms = std::fs::metadata(path)?.permissions();
9562 perms.set_mode(0o755);
9563 std::fs::set_permissions(path, perms)?;
9564 }
9565 #[cfg(not(unix))]
9566 {
9567 let _ = path;
9568 }
9569 Ok(())
9570}
9571
9572struct WaveCtx<'a> {
9579 run: &'a str,
9582 node: &'a str,
9584 prompts: &'a Prompts,
9585 cache: Option<&'a Path>,
9587 round: Option<usize>,
9590 carry_seats: bool,
9594 pauses: Option<(&'a Pause, &'a Pause)>,
9597}
9598
9599fn repository_wait(
9601 state: &RunState,
9602 pauses: Option<(&Pause, &Pause)>,
9603) -> crate::repo_lock::WaitPolicy {
9604 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9605 let inherited = crate::repo_lock::WaitPolicy::current();
9606 crate::repo_lock::WaitPolicy::new(
9607 Duration::from_secs(state.config.graph.timeout_resource_wait),
9608 move || {
9609 inherited.cancelled()
9610 || pauses
9611 .as_ref()
9612 .is_some_and(|(a, b)| a.parked() || b.parked())
9613 },
9614 )
9615}
9616
9617fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9618 if error.is::<crate::repo_lock::ResourceBlocked>() {
9619 state.resource_block = Some(crate::run::ResourceBlock {
9620 cause: crate::run::ResourceBlockCause::Other,
9621 node: "repository".into(),
9622 kind: if error.to_string().contains("disk pressure") {
9623 "disk pressure"
9624 } else {
9625 "repository coordination"
9626 }
9627 .into(),
9628 detail: error.to_string(),
9629 at: Timestamp::now(),
9630 });
9631 state.event("resource", error.to_string());
9632 }
9633}
9634
9635async fn coordinate_operation(
9636 state: &mut RunState,
9637 cwd: &Path,
9638 wait: &crate::repo_lock::WaitPolicy,
9639) -> Result<Option<crate::repo_lock::SeatLease>> {
9640 let result =
9641 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9642 .await;
9643 if let Err(e) = &result {
9644 record_repository_block(state, e);
9645 }
9646 result
9647}
9648
9649async fn run_one(
9651 job: SeatJob,
9652 sem: Arc<Semaphore>,
9653 ctx: &WaveCtx<'_>,
9654 state: &mut RunState,
9655 attempt: usize,
9656) -> (SeatState, AgentOutcome) {
9657 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9658 .await
9659 .pop()
9660 .expect("one job in, one result out");
9661 (seat, out)
9662}
9663
9664async fn wave(
9670 jobs: Vec<SeatJob>,
9671 sem: Arc<Semaphore>,
9672 ctx: &WaveCtx<'_>,
9673 state: &mut RunState,
9674 attempt: usize,
9675) -> Vec<(usize, SeatState, AgentOutcome)> {
9676 let WaveCtx {
9677 run,
9678 node,
9679 prompts,
9680 cache,
9681 round,
9682 carry_seats: _,
9683 pauses,
9684 } = *ctx;
9685 for job in &jobs {
9686 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9687 }
9688 if let Err(e) = state.save() {
9689 tracing::warn!("could not persist in-progress seats: {e:#}");
9694 }
9695 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9711 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9715 let cache_guard = if let Some(cache_dir) = cache {
9716 if jobs_had_a_writer {
9717 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9718 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9719 match acquire_cache_lease(
9720 state,
9721 cache_dir,
9722 &owner,
9723 resource_limit,
9724 node,
9725 node,
9726 &cancel,
9727 )
9728 .await
9729 {
9730 Ok(Some(g)) => Some(g),
9731 Ok(None) => {
9734 if let Some(block) = &mut state.resource_block {
9735 block.cause = crate::run::ResourceBlockCause::Cancelled {
9736 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9737 };
9738 }
9739 return block_wave(
9740 jobs,
9741 state,
9742 node,
9743 attempt,
9744 "parked while waiting for the shared build cache",
9745 );
9746 }
9747 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9750 Err(e) => {
9751 let why = format!("{e:#}");
9755 state.event(
9756 "resource",
9757 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9758 );
9759 return block_wave(jobs, state, node, attempt, &why);
9760 }
9761 }
9762 } else {
9763 None
9764 }
9765 } else {
9766 None
9767 };
9768 state.seats_spawning(node, attempt);
9771 if let Err(e) = state.save() {
9772 tracing::warn!("could not persist the start of a wave: {e:#}");
9773 }
9774 let slot_limit = resource_limit
9778 + jobs
9779 .iter()
9780 .map(|j| j.timeout)
9781 .max()
9782 .unwrap_or_default()
9783 .saturating_mul(jobs.len() as u32);
9784 let mut set = tokio::task::JoinSet::new();
9785 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9788 let overlay = prompts.overlay(node);
9789 let repository_wait = repository_wait(state, pauses);
9790 for (i, mut job) in jobs.into_iter().enumerate() {
9791 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9792 if cache.is_some() {
9793 job.prompt.push('\n');
9794 job.prompt
9795 .push_str(&prompt::build_cache_note(node, job.allow_write));
9796 }
9797 let sem = Arc::clone(&sem);
9798 let run = run.to_owned();
9799 let node = node.to_owned();
9800 let attachments = if node == "implement" {
9803 state.attachments.clone()
9804 } else {
9805 Vec::new()
9806 };
9807 let cache = cache
9818 .filter(|_| job.allow_write && cache_guard.is_some())
9819 .map(Path::to_path_buf);
9820 let spawned_tx = spawned_tx.clone();
9821 let repository_wait = repository_wait.clone();
9822 let coordination = state.config.coordination.clone();
9823 let fetched_base = state.fetched_base.clone();
9824 set.spawn(async move {
9825 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9829 Ok(p) => p,
9830 Err(_) => {
9831 return (
9832 i,
9833 job.seat,
9834 AgentOutcome::ResourceBlocked(format!(
9835 "no parallel slot became free within {}s",
9836 slot_limit.as_secs()
9837 )),
9838 false,
9839 );
9840 }
9841 };
9842 spawned_tx.send(job.seat.key.clone()).ok();
9843 let mut seat = job.seat;
9844 let out = repository_wait
9845 .scope(agent::invoke(
9846 &job.spec,
9847 &mut seat,
9848 &Invocation {
9849 coordination: coordination.as_ref(),
9850 fetched_base: fetched_base.as_ref(),
9851 cwd: &job.cwd,
9852 prompt: &job.prompt,
9853 timeout: job.timeout,
9854 allow_write: job.allow_write,
9855 sessions: job.sessions,
9856 artifacts: &job.artifacts,
9857 stem: &job.stem,
9858 run: &run,
9859 node: &node,
9860 cache_dir: cache.as_deref(),
9861 attachments: &attachments,
9862 writable: &[],
9863 },
9864 ))
9865 .await;
9866 let interrupted = out
9867 .as_ref()
9868 .err()
9869 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9870 let out = match out {
9871 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9872 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9873 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9881 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9882 Ok(o) => AgentOutcome::Failed(format!(
9883 "process exited with no usable output ({})",
9884 crate::notices::exit_status(o.exit_code)
9885 )),
9886 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9887 AgentOutcome::ResourceBlocked(e.to_string())
9888 }
9889 Err(e) => AgentOutcome::Failed(e.to_string()),
9890 };
9891 (i, seat, out, interrupted)
9892 });
9893 }
9894 drop(spawned_tx);
9895 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9896 loop {
9897 let joined = tokio::select! {
9898 Some(key) = spawned_rx.recv() => {
9899 state.seat_spawned(&key);
9901 if let Err(e) = state.save() {
9902 tracing::warn!("could not persist a seat's start: {e:#}");
9903 }
9904 continue;
9905 }
9906 joined = set.join_next() => joined,
9907 };
9908 let Some(joined) = joined else {
9909 break;
9910 };
9911 let (i, seat, out, interrupted) = match joined {
9912 Ok(v) => v,
9913 Err(e) => {
9917 tracing::error!("agent task panicked: {e}");
9918 continue;
9919 }
9920 };
9921 state.seat_finished(&seat.key);
9922 if interrupted
9923 && node == "implement"
9924 && let Some(label) = state
9925 .candidates
9926 .iter()
9927 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9928 .map(|candidate| candidate.label)
9929 {
9930 state.candidate_preservation.insert(
9931 label,
9932 "interrupted implementation awaiting persistence".into(),
9933 );
9934 state.interrupted_implementations.insert(label);
9935 }
9936 if let AgentOutcome::ResourceBlocked(why) = &out
9937 && state.resource_block.is_none()
9938 {
9939 state.resource_block = Some(crate::run::ResourceBlock {
9940 cause: crate::run::ResourceBlockCause::Other,
9941 node: node.to_owned(),
9942 kind: if why.contains("disk pressure") {
9943 "disk pressure"
9944 } else if why.contains("repository coordination") {
9945 "repository coordination"
9946 } else {
9947 "parallel slot"
9948 }
9949 .to_owned(),
9950 detail: why.clone(),
9951 at: Timestamp::now(),
9952 });
9953 let activity = if interrupted {
9954 "agent execution interrupted"
9955 } else {
9956 "no agent was started"
9957 };
9958 state.event("resource", format!("{node}: {activity} - {why}"));
9959 }
9960 record_jobs(state, node, round, &seat.key, &out);
9961 if let Err(e) = state.save() {
9962 tracing::warn!("could not persist a seat's completion: {e:#}");
9963 }
9964 if collected.len() <= i {
9965 collected.resize_with(i + 1, || None);
9966 }
9967 collected[i] = Some((i, seat, out));
9968 }
9969 if state
9975 .active
9976 .values()
9977 .any(|a| a.node == node && a.attempt == attempt)
9978 {
9979 state
9980 .active
9981 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9982 if let Err(e) = state.save() {
9983 tracing::warn!("could not persist the end of a wave: {e:#}");
9984 }
9985 }
9986 if let Some(cache_dir) = cache
9993 && jobs_had_a_writer
9994 {
9995 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9996 }
9997 if let Some(guard) = cache_guard {
9998 guard.release();
9999 }
10000 collected.into_iter().flatten().collect()
10001}
10002
10003fn record_jobs(
10014 state: &mut RunState,
10015 node: &str,
10016 round: Option<usize>,
10017 seat: &str,
10018 out: &AgentOutcome,
10019) {
10020 let commands: &[agent::CommandEvidence] = match out {
10021 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10022 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10023 };
10024 let checked_at = Timestamp::now();
10025 for c in commands {
10026 state.jobs.push(JobRecord {
10027 node: node.to_owned(),
10028 round,
10029 seat: seat.to_owned(),
10030 id: c.id.clone(),
10031 description: c.description.clone(),
10032 checked_at,
10033 status: match c.exit_code {
10034 Some(0) => JobStatus::Completed,
10035 Some(_) => JobStatus::Failed,
10036 None => JobStatus::Unknown,
10037 },
10038 exit_code: c.exit_code,
10039 result_summary: c.result_summary.clone(),
10040 source: c.source.clone(),
10041 });
10042 }
10043}
10044
10045fn round_is_clean(
10066 blocking: usize,
10067 e2e_ok: bool,
10068 answered: usize,
10069 expected: usize,
10070 quota_missing: usize,
10071 policy: IncompleteReviewPolicy,
10072) -> bool {
10073 if blocking != 0 || !e2e_ok {
10074 return false;
10075 }
10076 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10077 return true;
10078 }
10079 answered > 0 && expected - answered <= quota_missing
10080}
10081
10082pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10106 if max_rounds == 0
10107 || reviews.last().is_some_and(|r| {
10108 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10109 })
10110 {
10111 return Some(RunStatus::Gating);
10112 }
10113 let last = reviews.last()?;
10114 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10115 if reviews.len() < max_rounds && !stagnant {
10116 return None;
10117 }
10118 if last.incomplete() && last.blocking == 0 {
10119 return Some(RunStatus::Blocked);
10120 }
10121 if last.e2e_status() == E2eStatus::ResourceBlocked
10122 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10123 {
10124 return None;
10125 }
10126 Some(if last.e2e.iter().all(CommandOutcome::ok) {
10127 RunStatus::Gating
10128 } else {
10129 RunStatus::Blocked
10130 })
10131}
10132
10133pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10148 if nudged {
10149 (full / 4).max(Duration::from_secs(120)).min(full)
10150 } else {
10151 full
10152 }
10153}
10154
10155#[allow(clippy::too_many_arguments)]
10184async fn ask_json_wave<T>(
10185 jobs: Vec<SeatJob>,
10186 sem: Arc<Semaphore>,
10187 retries: usize,
10188 roster: &[AgentSpec],
10189 ctx: &WaveCtx<'_>,
10190 losses: &mut Vec<QuotaLoss>,
10191 state: &mut RunState,
10192 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10193) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10194where
10195 T: serde::de::DeserializeOwned + Send + 'static,
10196{
10197 ask_wave_with(
10198 jobs,
10199 sem,
10200 retries,
10201 roster,
10202 ctx,
10203 losses,
10204 state,
10205 &|text: &str| {
10206 let v = verdict::extract_json::<T>(text)?;
10207 validate(&v)?;
10208 Ok(v)
10209 },
10210 )
10211 .await
10212}
10213
10214#[allow(clippy::too_many_arguments)]
10223async fn ask_wave_with<T>(
10224 jobs: Vec<SeatJob>,
10225 sem: Arc<Semaphore>,
10226 retries: usize,
10227 roster: &[AgentSpec],
10228 ctx: &WaveCtx<'_>,
10229 losses: &mut Vec<QuotaLoss>,
10230 state: &mut RunState,
10231 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10232) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10233where
10234 T: Send + 'static,
10235{
10236 let n = jobs.len();
10237 let originals: Vec<SeatJob> = jobs;
10238 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10239 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10240 let mut nudges: Vec<usize> = vec![0; n];
10247 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10251 let mut tried: Vec<BTreeSet<String>> = specs
10252 .iter()
10253 .map(|s| BTreeSet::from([s.id.clone()]))
10254 .collect();
10255 let starts: Vec<usize> = specs
10256 .iter()
10257 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10258 .collect();
10259 let carry = ctx.carry_seats && !roster.is_empty();
10260 let carried: Vec<BTreeSet<String>> = originals
10263 .iter()
10264 .map(|j| {
10265 state
10266 .seat_history
10267 .get(&j.seat.key)
10268 .filter(|_| carry)
10269 .map(|h| h.failed.clone())
10270 .unwrap_or_default()
10271 })
10272 .collect();
10273 let mut prev: Vec<Option<FailClass>> = originals
10274 .iter()
10275 .map(|j| {
10276 state
10277 .seat_history
10278 .get(&j.seat.key)
10279 .filter(|_| carry)
10280 .and_then(|h| h.last_fail.clone())
10281 })
10282 .collect();
10283 let next_agent =
10284 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10285 let others: BTreeSet<String> = specs
10286 .iter()
10287 .enumerate()
10288 .filter(|(j, _)| *j != i)
10289 .map(|(_, s)| s.id.clone())
10290 .collect();
10291 pick_successor(
10292 roster,
10293 starts[i],
10294 tried,
10295 carry.then(|| &carried[i]),
10296 &others,
10297 )
10298 .cloned()
10299 };
10300 let mut fresh: Vec<Option<String>> = vec![None; n];
10301 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10302 let mut pending: Vec<usize> = (0..n).collect();
10303
10304 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10309 for round in 0..max_rounds {
10310 if pending.is_empty() {
10311 break;
10312 }
10313 let mut batch = Vec::with_capacity(pending.len());
10314 let mut renudged: Vec<&str> = Vec::new();
10315 for &i in &pending {
10316 let src = &originals[i];
10317 let full: &str = match &src.handover {
10322 Some(h) if specs[i].id != src.spec.id => h,
10323 _ => &src.prompt,
10324 };
10325 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10328 (
10329 full.to_owned(),
10330 src.timeout,
10331 format!("{}-{word}-{}", src.stem, specs[i].id),
10332 )
10333 } else if nudges[i] == 0 {
10334 (full.to_owned(), src.timeout, src.stem.clone())
10335 } else {
10336 renudged.push(src.seat.key.as_str());
10337 let why = done[i]
10338 .as_ref()
10339 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10340 .unwrap_or_else(|| "no parsable answer".to_owned());
10341 let nudge = prompt::nudge(&why);
10342 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10343 let prompt = if nudged {
10344 nudge
10345 } else {
10346 format!("{full}\n\n---\n\n{nudge}")
10347 };
10348 (
10349 prompt,
10350 retry_budget(src.timeout, nudged),
10351 format!("{}-retry{}", src.stem, nudges[i]),
10352 )
10353 };
10354 batch.push(SeatJob {
10355 spec: specs[i].clone(),
10356 seat: seats[i].clone(),
10357 cwd: src.cwd.clone(),
10358 prompt,
10359 timeout,
10360 allow_write: src.allow_write,
10361 sessions: src.sessions,
10362 artifacts: src.artifacts.clone(),
10363 stem,
10364 handover: None,
10365 });
10366 }
10367
10368 if !renudged.is_empty() {
10369 state.event(
10370 ctx.node,
10371 format!("retry {round}: re-asking {}", renudged.join(", ")),
10372 );
10373 }
10374 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10375 let mut still = Vec::new();
10376 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10377 seats[i] = seat;
10378 if let AgentOutcome::ResourceBlocked(why) = &out {
10381 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10382 continue;
10383 }
10384 let class = FailClass::of(&out);
10385 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10388 && nudges[i] < retries
10389 && !roster.is_empty();
10390 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10391 let next = should_hand_over(prev[i].as_ref(), &cur)
10392 .then(|| next_agent(i, &tried[i], &specs))
10393 .flatten();
10394 if carry {
10395 let h = state
10396 .seat_history
10397 .entry(originals[i].seat.key.clone())
10398 .or_default();
10399 h.failed.insert(specs[i].id.clone());
10400 h.last_fail = Some(cur.clone());
10401 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10402 h.last_ok = None;
10403 }
10404 if let Err(e) = state.save() {
10407 tracing::warn!("could not persist a seat's failure history: {e:#}");
10408 }
10409 }
10410 if let Some(next) = next {
10411 record_handover(
10412 state,
10413 ctx.node,
10414 &originals[i].seat.key,
10415 &specs[i].id,
10416 &next.id,
10417 &cur,
10418 &fail_reason(&out),
10419 );
10420 tried[i].insert(next.id.clone());
10421 prev[i] = Some(cur.clone());
10422 seats[i] =
10423 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10424 specs[i] = next;
10425 fresh[i] = Some(cur.stem_word().to_owned());
10426 nudges[i] = 0;
10427 done[i] = Some(Err(anyhow::anyhow!(
10428 "handed over after: {}",
10429 fail_reason(&out)
10430 )));
10431 still.push(i);
10432 continue;
10433 }
10434 }
10435 if carry
10436 && !nudge_first
10437 && let Some(cur) = class.clone()
10438 {
10439 let h = state
10441 .seat_history
10442 .entry(originals[i].seat.key.clone())
10443 .or_default();
10444 h.failed.insert(specs[i].id.clone());
10445 h.last_fail = Some(cur);
10446 }
10447 let parsed = match out {
10448 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10449 AgentOutcome::Quota(o) => {
10450 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10451 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10452 }
10453 AgentOutcome::Dropped(o) => {
10457 let why = o
10458 .dropped
10459 .as_ref()
10460 .map(|d| d.why.as_str())
10461 .unwrap_or("the CLI ended the stream without delivering its answer");
10462 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10463 }
10464 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10465 Err(anyhow::anyhow!(e))
10466 }
10467 };
10468 let quota = class == Some(FailClass::Quota);
10469 let failed = parsed.is_err();
10470 done[i] = Some(parsed);
10471 if carry && !failed {
10472 let h = state
10473 .seat_history
10474 .entry(originals[i].seat.key.clone())
10475 .or_default();
10476 h.failed.remove(&specs[i].id);
10477 h.last_ok = Some(specs[i].id.clone());
10478 h.last_fail = None;
10479 }
10480 let agent_failure = class.is_some()
10488 && !nudge_first
10489 && !roster.is_empty()
10490 && next_agent(i, &tried[i], &specs).is_some();
10491 if failed && !quota && !agent_failure && nudges[i] < retries {
10492 nudges[i] += 1;
10493 still.push(i);
10494 }
10495 }
10496 pending = still;
10497 }
10498
10499 for (i, q) in last_quota.into_iter().enumerate() {
10502 if let Some(reset) = q {
10503 losses.push(QuotaLoss {
10504 seat: originals[i].seat.key.clone(),
10505 node: ctx.node.to_owned(),
10506 at: Timestamp::now(),
10507 reset,
10508 });
10509 }
10510 }
10511
10512 seats
10513 .into_iter()
10514 .zip(done)
10515 .zip(nudges)
10516 .map(|((seat, res), attempts)| {
10517 (
10518 seat,
10519 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10520 attempts,
10521 )
10522 })
10523 .collect()
10524}
10525
10526fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10528 results
10529 .iter()
10530 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10531}
10532
10533fn block_wave(
10537 jobs: Vec<SeatJob>,
10538 state: &mut RunState,
10539 node: &str,
10540 attempt: usize,
10541 why: &str,
10542) -> Vec<(usize, SeatState, AgentOutcome)> {
10543 state.resource_wait = None;
10544 state
10545 .active
10546 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10547 if let Err(e) = state.save() {
10548 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10549 }
10550 jobs.into_iter()
10551 .enumerate()
10552 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10553 .collect()
10554}
10555
10556async fn acquire_cache_lease(
10569 state: &mut RunState,
10570 cache_dir: &Path,
10571 owner: &crate::cache::Owner,
10572 budget: Duration,
10573 node: &str,
10574 context: &str,
10575 cancel: &(dyn Fn() -> bool + Sync),
10576) -> Result<Option<crate::cache::Guard>> {
10577 let home = crate::run::home();
10578 let started = Instant::now();
10579 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10580 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10581 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10582 Err(e) => {
10583 state.event(
10584 "verify",
10585 format!("{context}: could not check the shared build cache: {e:#}"),
10586 );
10587 if let Err(e2) = state.save() {
10588 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10589 }
10590 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10591 }
10592 };
10593 state.event(
10594 "verify",
10595 format!(
10596 "{context}: waiting for the shared build cache at {} ({})",
10597 cache_dir.display(),
10598 busy.describe()
10599 ),
10600 );
10601 state.resource_wait = Some(crate::run::ResourceWait {
10604 node: context.to_owned(),
10605 kind: "shared build cache".to_owned(),
10606 holder: busy.describe(),
10607 since: Timestamp::now(),
10608 limit_secs: budget.as_secs(),
10609 });
10610 if let Err(e) = state.save() {
10611 tracing::warn!("could not persist a cache wait: {e:#}");
10612 }
10613 let remaining = budget.saturating_sub(started.elapsed());
10614 let waited = crate::cache::wait_for_unless(
10615 &home,
10616 cache_dir,
10617 owner,
10618 remaining,
10619 Duration::from_secs(5),
10620 cancel,
10621 )
10622 .await;
10623 state.resource_wait = None;
10624 let cause = match &waited {
10625 Ok(Some(_)) => None,
10626 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10627 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10628 Some(crate::run::ResourceBlockCause::Timeout {
10629 waited_secs: started.elapsed().as_secs(),
10630 limit_secs: budget.as_secs(),
10631 })
10632 }
10633 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10634 };
10635 if let Some(cause) = cause {
10636 let detail = match &waited {
10637 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10638 Err(error) => format!("{error:#}"),
10639 };
10640 state.resource_block = Some(crate::run::ResourceBlock {
10641 cause,
10642 node: node.to_owned(),
10643 kind: "shared build cache".into(),
10644 detail,
10645 at: Timestamp::now(),
10646 });
10647 state.event(
10648 "resource",
10649 state.resource_block.as_ref().unwrap().description(),
10650 );
10651 }
10652 if let Err(error) = state.save() {
10653 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10654 }
10655 waited
10656}
10657
10658#[derive(Debug)]
10663struct CacheCheckError(String);
10664
10665impl std::fmt::Display for CacheCheckError {
10666 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10667 f.write_str(&self.0)
10668 }
10669}
10670
10671impl std::error::Error for CacheCheckError {}
10672
10673fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10675 state
10676 .resource_block
10677 .get_or_insert_with(|| crate::run::ResourceBlock {
10678 cause: crate::run::ResourceBlockCause::Other,
10679 node: node.to_owned(),
10680 kind: "shared build cache".into(),
10681 detail: format!("{error:#}"),
10682 at: Timestamp::now(),
10683 });
10684 if let Err(error) = state.save() {
10685 tracing::warn!("could not persist cache resource block: {error:#}");
10686 }
10687}
10688
10689#[allow(clippy::too_many_arguments)]
10710async fn with_cache_lease<'s, F, Fut>(
10711 state: &'s mut RunState,
10712 cache_dir: Option<&Path>,
10713 node: &str,
10714 seat: &str,
10715 worktree: &Path,
10716 head: &str,
10717 budget: Duration,
10718 context: &str,
10719 wait: &crate::repo_lock::WaitPolicy,
10720 body: F,
10721) -> Result<(Vec<CommandOutcome>, bool)>
10722where
10723 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10724 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10725{
10726 let started = Instant::now();
10729 let home = crate::run::home();
10730 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10731 let guard = if let Some(cache_dir) = cache_dir {
10732 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10733 match acquire_cache_lease(
10734 state,
10735 cache_dir,
10736 &owner,
10737 resource_limit,
10738 node,
10739 context,
10740 &|| wait.cancelled(),
10741 )
10742 .await
10743 .and_then(|g| {
10744 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10745 }) {
10746 Ok(g) => Some(g),
10747 Err(e) => {
10748 record_cache_error(state, node, &e);
10749 return Ok((
10750 vec![CommandOutcome {
10751 command: "(waiting for the shared build cache)".into(),
10752 code: None,
10753 output_tail: e.to_string(),
10754 duration_ms: started.elapsed().as_millis() as u64,
10755 resource_blocked: true,
10756 }],
10757 false,
10758 ));
10759 }
10760 }
10761 } else {
10762 None
10763 };
10764 let _coordination = coordinate_operation(state, worktree, wait).await?;
10765 if let Some(cache_dir) = cache_dir {
10766 let identity = crate::cache::Identity::new(worktree, head);
10767 if let Err(e) = crate::cache::ensure_fresh(
10768 &home,
10769 cache_dir,
10770 &identity,
10771 &state.config.verify.cargo_manifests(),
10772 ) {
10773 record_cache_error(state, node, &e);
10774 state.event(
10775 "verify",
10776 format!(
10777 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10778 worktree.display(),
10779 short(head)
10780 ),
10781 );
10782 if let Some(guard) = guard {
10783 guard.release();
10784 }
10785 return Ok((
10786 vec![CommandOutcome {
10787 command: "(confirming the shared build cache is fresh)".into(),
10788 code: None,
10789 output_tail: e.to_string(),
10790 duration_ms: started.elapsed().as_millis() as u64,
10791 resource_blocked: true,
10792 }],
10793 false,
10794 ));
10795 }
10796 }
10797 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10798 if !timed_out_pids.is_empty() {
10799 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10800 }
10801 drop(_coordination);
10804 if let Some(guard) = guard {
10805 guard.release();
10806 }
10807 Ok((outcomes, retried))
10808}
10809
10810async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10822 wait_for_pids_with(
10823 pids,
10824 crate::proc::pid_alive,
10825 LEASE_RELEASE_POLL,
10826 LEASE_RELEASE_MAX_WAIT,
10827 )
10828 .await;
10829}
10830
10831async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10837 pids: &[u32],
10838 alive: F,
10839 poll: Duration,
10840 max_wait: Duration,
10841) {
10842 let deadline = Instant::now() + max_wait;
10843 loop {
10844 if pids.iter().all(|&pid| !alive(pid)) {
10845 return;
10846 }
10847 if Instant::now() >= deadline {
10848 return;
10849 }
10850 tokio::time::sleep(poll).await;
10851 }
10852}
10853
10854fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10861 outcomes.iter().any(|o| o.resource_blocked)
10862}
10863
10864pub(crate) enum GateFix {
10866 Retry,
10868 Stop,
10871 Defer,
10874}
10875
10876fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10884 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10885 red.peek().is_some()
10886 && red.all(|o| {
10887 !o.resource_blocked
10888 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10889 && !o.output_tail.trim().is_empty()
10890 })
10891}
10892
10893fn e2e_outcome_label(o: &CommandOutcome) -> String {
10897 if o.ok() {
10898 return "pass".to_owned();
10899 }
10900 let reason = if o.build_failed() {
10901 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10902 } else {
10903 format!("FAIL ({:?})", o.code)
10904 };
10905 format!(
10906 "{reason}\n{}",
10907 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
10908 )
10909}
10910
10911async fn run_e2e_with_retry(
10919 state: &mut RunState,
10920 shell: &[String],
10921 commands: &[String],
10922 worktree: &Path,
10923 timeout: Duration,
10924 context: &str,
10925) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10926 let (mut e2e, mut timed_out_pids) = run_commands(
10927 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10928 )
10929 .await;
10930 for o in &e2e {
10931 state.event(
10932 "verify",
10933 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10934 );
10935 }
10936 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10940 if verify_retried {
10941 state.event(
10942 "verify",
10943 format!(
10944 "{context}: verify could not build/link, not a test result — retrying once \
10945 before concluding"
10946 ),
10947 );
10948 let retried = run_commands(
10949 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10950 )
10951 .await;
10952 e2e = retried.0;
10953 timed_out_pids.extend(retried.1);
10956 for o in &e2e {
10957 state.event(
10958 "verify",
10959 format!(
10960 "{context}: retry `{}` -> {}",
10961 o.command,
10962 e2e_outcome_label(o)
10963 ),
10964 );
10965 }
10966 }
10967 (e2e, verify_retried, timed_out_pids)
10968}
10969
10970#[allow(clippy::too_many_arguments)]
10986async fn run_commands(
10987 state: &mut RunState,
10988 node: &str,
10989 task: &str,
10990 attempt: usize,
10991 shell: &[String],
10992 commands: &[String],
10993 cwd: &Path,
10994 timeout: Duration,
10995) -> (Vec<CommandOutcome>, Vec<u32>) {
10996 if commands.is_empty() {
10997 return (Vec::new(), Vec::new());
11002 }
11003 let mut out = Vec::new();
11004 let mut timed_out_pids = Vec::new();
11005 let total = commands.len();
11006 for (idx, command) in commands.iter().enumerate() {
11007 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11008 if let Err(e) = state.save() {
11009 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11010 }
11011 let started = Instant::now();
11012 let mut cmd = tokio::process::Command::new(&shell[0]);
11013 cmd.quiet();
11014 for key in crate::worker::INVOCATION_ENV {
11018 cmd.env_remove(key);
11019 }
11020 cmd.args(&shell[1..])
11021 .arg(command)
11022 .current_dir(cwd)
11023 .stdin(std::process::Stdio::null())
11024 .stdout(std::process::Stdio::piped())
11025 .stderr(std::process::Stdio::piped())
11026 .kill_on_drop(true);
11027 let mut outputs = Vec::new();
11028 let protected = (|| -> Result<()> {
11029 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11030 outputs.push(guard);
11031 }
11032 if let Some(cache) = state.config.cache_dir() {
11033 outputs.push(crate::retention::OutputLease::cache(
11034 &crate::run::home(),
11035 &cache,
11036 false,
11037 )?);
11038 }
11039 Ok(())
11040 })();
11041 if let Err(e) = protected {
11042 out.push(CommandOutcome {
11043 command: command.clone(),
11044 code: None,
11045 output_tail: format!("output ownership unavailable: {e:#}"),
11046 duration_ms: 0,
11047 resource_blocked: true,
11048 });
11049 continue;
11050 }
11051 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11052 let (code, body) = match spawned {
11053 Ok(mut running) => {
11054 let pid = running.child().id();
11055 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11056 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11057 let mut paths = vec![cwd.to_owned()];
11058 if let Some(cache) = state.config.cache_dir() {
11059 paths.push(cache);
11060 }
11061 let mut pressure = None;
11062 let status = tokio::select! {
11063 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11064 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11065 pressure = Some(reason); None
11066 }
11067 };
11068 let (code, failure) = match status {
11069 Some(Ok(Ok(status))) => (
11070 status.code(),
11071 status
11072 .code()
11073 .is_none()
11074 .then(|| format!("command terminated: {status}")),
11075 ),
11076 Some(Ok(Err(error))) => {
11077 (None, Some(format!("failed waiting for command: {error}")))
11078 }
11079 Some(Err(_)) => {
11080 if !running.coordinated()
11084 && let Some(pid) = pid
11085 {
11086 timed_out_pids.push(pid);
11087 }
11088 (
11089 None,
11090 Some(format!("timed out after {}s", timeout.as_secs())),
11091 )
11092 }
11093 None => (None, None),
11094 };
11095 running.finish().await;
11096 if let Some(reason) = pressure {
11097 state.resource_block = Some(crate::run::ResourceBlock {
11098 cause: crate::run::ResourceBlockCause::Other,
11099 node: node.to_owned(),
11100 kind: "disk pressure".to_owned(),
11101 detail: reason.clone(),
11102 at: Timestamp::now(),
11103 });
11104 out.push(CommandOutcome {
11105 command: command.clone(),
11106 code: None,
11107 output_tail: reason,
11108 duration_ms: started.elapsed().as_millis() as u64,
11109 resource_blocked: true,
11110 });
11111 break;
11112 }
11113 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11114 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11115 if let Some(failure) = failure {
11116 body.push('\n');
11117 body.push_str(&failure);
11118 }
11119 (code, body)
11120 }
11121 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11122 };
11123 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11124 if !body.is_empty() {
11125 match crate::run::write_verification_artifact(state, task, &body) {
11129 Ok(name) if summary != body => {
11130 summary = format!("[full output: artifacts/{name}]\n{summary}");
11131 }
11132 Ok(_) => {}
11133 Err(e) => {
11134 let warning = format!("could not save the full {task} output: {e:#}");
11135 tracing::warn!("{warning}");
11136 state.event(node, warning);
11137 if summary != body {
11138 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11139 }
11140 }
11141 }
11142 }
11143 out.push(CommandOutcome {
11144 command: command.clone(),
11145 code,
11146 output_tail: summary,
11147 duration_ms: started.elapsed().as_millis() as u64,
11148 resource_blocked: false,
11149 });
11150 }
11151 state.task_finished(task);
11152 if let Err(e) = state.save() {
11153 tracing::warn!("could not persist the end of {task}: {e:#}");
11154 }
11155 (out, timed_out_pids)
11156}
11157
11158fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11170 let repo = repo.display();
11171 match style {
11172 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11173 MergeStyle::Squash => {
11174 let subject = message
11177 .lines()
11178 .next()
11179 .unwrap_or(branch)
11180 .replace(['\\', '"', '$', '`'], "");
11181 format!(
11182 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11183 )
11184 }
11185 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11186 }
11187}
11188
11189const PR_TITLE_MAX: usize = 240;
11200
11201fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11212 let line = queue::first_line(text)?;
11213 let chars: Vec<char> = line.chars().collect();
11214 let (kept, mark) = if chars.len() <= max {
11215 (line.to_owned(), "")
11216 } else if let Some(end) = sentence_end(&chars, max) {
11217 (chars[..end].iter().collect::<String>(), "")
11218 } else {
11219 let room = max.saturating_sub(3);
11220 let cut = if chars[room].is_whitespace() {
11223 room
11224 } else {
11225 chars[..room]
11226 .iter()
11227 .rposition(|c| c.is_whitespace())
11228 .unwrap_or(room)
11229 };
11230 let head: String = chars[..cut].iter().collect();
11231 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11232 (head.to_owned(), "...")
11233 };
11234 if kept.is_empty() {
11235 return None;
11236 }
11237 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11238 return None;
11239 }
11240 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11241 return None;
11242 }
11243 Some(format!("{kept}{mark}"))
11244}
11245
11246fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11250 const MIN: usize = 20;
11251 for i in MIN..max.min(chars.len()) {
11252 if !matches!(chars[i], '.' | '!' | '?') {
11253 continue;
11254 }
11255 let Some(&next) = chars.get(i + 1) else {
11256 continue;
11257 };
11258 if !next.is_whitespace() {
11259 continue;
11260 }
11261 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11262 if after.is_some_and(|c| c.is_ascii_lowercase()) {
11263 continue;
11264 }
11265 let word: String = chars[..i]
11266 .iter()
11267 .rev()
11268 .take_while(|c| !c.is_whitespace())
11269 .collect::<Vec<_>>()
11270 .into_iter()
11271 .rev()
11272 .collect();
11273 let word = word.to_ascii_lowercase();
11274 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11275 continue;
11276 }
11277 let end = chars[..i]
11278 .iter()
11279 .rposition(|c| !c.is_whitespace())
11280 .map_or(i, |p| p + 1);
11281 return Some(end);
11282 }
11283 None
11284}
11285
11286struct PrMessage {
11291 title: String,
11292 body: String,
11293}
11294
11295impl PrMessage {
11296 fn commit_message(&self) -> String {
11300 format!("{}\n\n{}", self.title, self.body)
11301 }
11302}
11303
11304fn title_marker(line: &str) -> Option<&str> {
11306 let line = line.trim();
11307 let head = line.get(..6)?;
11308 head.eq_ignore_ascii_case("title:")
11309 .then(|| line[6..].trim())
11310}
11311
11312fn summary_title(summary: &str) -> Option<String> {
11317 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11318 let raw = title_marker(first)?;
11319 if raw.is_empty() {
11320 return None;
11321 }
11322 let title = queue::title_from(raw, PR_TITLE_MAX);
11323 let lower = title.to_ascii_lowercase();
11324 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11325 return None;
11326 }
11327 Some(title)
11328}
11329
11330const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11332
11333pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11335
11336pub(crate) fn is_review_run(state: &RunState) -> bool {
11340 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11341}
11342
11343fn review_title(state: &RunState) -> Option<String> {
11348 english_subject(
11349 state.reviewed_commits.as_ref()?.first()?,
11350 state.config.graph.github_english(),
11351 )
11352}
11353
11354fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11357 let raw = raw.trim();
11358 if raw.is_empty() {
11359 return None;
11360 }
11361 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11362 return None;
11363 }
11364 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11365 return None;
11366 }
11367 let title = queue::title_from(raw, PR_TITLE_MAX);
11368 let lower = title.to_ascii_lowercase();
11369 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11370 return None;
11371 }
11372 Some(title)
11373}
11374
11375#[derive(Debug, Clone, PartialEq, Eq)]
11378struct BranchFacts {
11379 commits: Vec<(String, String)>,
11381 stat: String,
11383}
11384
11385const STAT_FILE_LINES: usize = 25;
11387const COMMITS_MAX_CHARS: usize = 20_000;
11389
11390async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11393 let commits = git::commit_log(repo, base, branch).await.ok()?;
11394 if commits.is_empty() {
11395 return None;
11396 }
11397 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11398 let lines: Vec<&str> = stat.lines().collect();
11399 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11400 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11401 let more = format!(" ... {omitted} more file(s)");
11402 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11403 kept.push(&more);
11404 kept.push(lines[lines.len() - 1]);
11405 kept.join("\n")
11406 } else {
11407 lines.join("\n")
11408 };
11409 Some(BranchFacts { commits, stat })
11410}
11411
11412fn markdown_safe(text: &str) -> String {
11415 text.replace("</details>", "</details>")
11416 .replace("\x60\x60\x60", "~~~")
11417}
11418
11419fn neutral_title(state: &RunState, winner: char) -> String {
11420 format!(
11421 "chore: land candidate {} of run {}",
11422 winner.to_ascii_uppercase(),
11423 state.id
11424 )
11425}
11426
11427pub(crate) async fn repair_landing_gate(
11430 state: &mut RunState,
11431 winner: &Candidate,
11432 outcomes: &[CommandOutcome],
11433) -> Result<GateFix> {
11434 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11435 return Ok(GateFix::Stop);
11436 }
11437 let mut runner = Runner {
11438 roles: state.config.resolve_roles()?,
11439 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11440 state: state.clone(),
11441 pause: Pause::new(),
11442 interrupt: Pause::new(),
11443 };
11444 let result = runner.gate_fix_round(winner, outcomes).await;
11445 *state = runner.state;
11446 result
11447}
11448
11449pub(crate) async fn verify_current_head(
11452 state: &mut RunState,
11453 winner: &Candidate,
11454 wait: &crate::repo_lock::WaitPolicy,
11455) -> Result<bool> {
11456 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11457 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11459 state
11460 .verification_history
11461 .push(crate::run::VerificationEvidence {
11462 superseded_by: Some(head.clone()),
11463 archived_at: Some(Timestamp::now()),
11464 fixes_spent: Some(state.gate_fixes.len()),
11465 gate_head: None,
11466 gate_ran: false,
11467 gate: Vec::new(),
11468 e2e: state.head_e2e.take(),
11469 });
11470 }
11471 if let Some(e) = &state.head_e2e
11472 && e.head == head
11473 {
11474 if e.passed {
11475 return Ok(true);
11476 }
11477 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11478 state.status = RunStatus::Blocked;
11479 state.save()?;
11480 return Ok(false);
11481 }
11482 } else if state.reviews.last().is_some_and(|r| {
11483 r.verified_head.as_deref() == Some(head.as_str())
11484 && (r.e2e_status() == E2eStatus::Passed
11485 || (r.e2e_status() == E2eStatus::NotConfigured
11486 && state.config.verify.e2e.is_empty()))
11487 }) || state
11488 .gate_repair_e2e
11489 .as_ref()
11490 .is_some_and(|e| e.head == head && e.passed)
11491 {
11492 return Ok(true);
11493 }
11494 if let Some(e) = &state.gate_repair_e2e
11495 && e.head == head
11496 && !e.passed
11497 && !e.outcomes.is_empty()
11498 && !verify_inconclusive(&e.outcomes)
11499 {
11500 state.status = RunStatus::Blocked;
11501 state.save()?;
11502 return Ok(false);
11503 }
11504 state.head_e2e = Some(crate::run::GateRepairE2e {
11505 head: head.clone(),
11506 passed: false,
11507 outcomes: Vec::new(),
11508 });
11509 state.save()?;
11510 let commands = state.config.verify.e2e.clone();
11511 let shell = state.config.shell();
11512 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11513 let cache_dir = state.config.cache_dir();
11514 let outcomes = if commands.is_empty() {
11515 Vec::new()
11516 } else {
11517 let (outcomes, _) = with_cache_lease(
11518 state,
11519 cache_dir.as_deref(),
11520 "e2e",
11521 "e2e",
11522 &winner.worktree,
11523 &head,
11524 timeout,
11525 "current head e2e",
11526 wait,
11527 |state, budget| async move {
11528 run_e2e_with_retry(
11529 state,
11530 &shell,
11531 &commands,
11532 &winner.worktree,
11533 budget,
11534 "current head e2e",
11535 )
11536 .await
11537 },
11538 )
11539 .await?;
11540 outcomes
11541 };
11542 let unresolved = verify_inconclusive(&outcomes);
11543 let passed = !unresolved
11544 && outcomes.iter().all(CommandOutcome::ok)
11545 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11546 state.head_e2e = Some(crate::run::GateRepairE2e {
11547 head: head.clone(),
11548 passed,
11549 outcomes,
11550 });
11551 if !passed && !unresolved {
11552 state.status = RunStatus::Blocked;
11553 state.event(
11554 "e2e",
11555 format!("verify.e2e failed on head {head}; nothing published"),
11556 );
11557 }
11558 state.save()?;
11559 Ok(passed)
11560}
11561
11562pub(crate) async fn check_gate_repair_e2e(
11565 state: &mut RunState,
11566 winner: &Candidate,
11567 wait: &crate::repo_lock::WaitPolicy,
11568) -> Result<GateFix> {
11569 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11570 return Ok(GateFix::Retry);
11571 }
11572 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11573 if state
11574 .gate_repair_e2e
11575 .as_ref()
11576 .is_some_and(|r| r.head == head && r.passed)
11577 {
11578 return Ok(GateFix::Retry);
11579 }
11580 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11581 head: head.clone(),
11582 passed: false,
11583 outcomes: Vec::new(),
11584 });
11585 state.save()?;
11586 let commands = state.config.verify.e2e.clone();
11587 let outcomes = if commands.is_empty() {
11588 Vec::new()
11589 } else {
11590 let shell = state.config.shell();
11591 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11592 let cache_dir = state.config.cache_dir();
11593 let (outcomes, _) = with_cache_lease(
11594 state,
11595 cache_dir.as_deref(),
11596 "e2e",
11597 "e2e",
11598 &winner.worktree,
11599 &head,
11600 timeout,
11601 "gate repair e2e",
11602 wait,
11603 |state, budget| async move {
11604 run_e2e_with_retry(
11605 state,
11606 &shell,
11607 &commands,
11608 &winner.worktree,
11609 budget,
11610 "gate repair e2e",
11611 )
11612 .await
11613 },
11614 )
11615 .await?;
11616 outcomes
11617 };
11618 let deferred = verify_inconclusive(&outcomes);
11619 let passed = outcomes.iter().all(CommandOutcome::ok)
11620 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11621 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11622 head,
11623 passed,
11624 outcomes,
11625 });
11626 state.save()?;
11627 if deferred {
11628 Ok(GateFix::Defer)
11629 } else if !passed {
11630 state.event(
11631 "gate",
11632 "verify.e2e failed after gate repair; nothing published",
11633 );
11634 Ok(GateFix::Stop)
11635 } else {
11636 Ok(GateFix::Retry)
11637 }
11638}
11639
11640pub(crate) async fn run_landing_gate(
11642 state: &mut RunState,
11643 winner: &Candidate,
11644 wait: &crate::repo_lock::WaitPolicy,
11645) -> Result<Vec<CommandOutcome>> {
11646 let shell = state.config.shell();
11647 let gate_commands = state.config.verify.gate.clone();
11648 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11649 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11650 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
11651 Vec::new()
11652 } else {
11653 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11654 let cache_dir = state.config.cache_dir();
11655 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11656 let (outcomes, _) = with_cache_lease(
11657 state,
11658 cache_dir.as_deref(),
11659 "gate",
11660 "gate",
11661 &winner.worktree,
11662 &head,
11663 timeout,
11664 "final gate",
11665 wait,
11666 |state, budget| {
11667 let shell = shell.clone();
11668 let gate_commands = gate_commands.clone();
11669 let winner = winner.clone();
11670 async move {
11671 Runner::run_pre_gate(state, &winner).await;
11672 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
11674 let (outcomes, timed_out_pids) = run_commands(
11675 state,
11676 "gate",
11677 "gate",
11678 0,
11679 &shell,
11680 &gate_commands,
11681 &winner.worktree,
11682 budget,
11683 )
11684 .await;
11685 (outcomes, false, timed_out_pids)
11686 }
11687 },
11688 )
11689 .await?;
11690 outcomes
11691 };
11692 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
11693 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
11694 let mut outcomes = outcomes;
11695 outcomes.push(CommandOutcome {
11696 command: "gate head stability".into(), code: None,
11697 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
11698 duration_ms: 0, resource_blocked: false,
11699 });
11700 return Ok(outcomes);
11701 }
11702 if outcomes.is_empty() {
11703 state.event("gate", "no gate commands configured; not tested");
11708 }
11709 for o in &outcomes {
11710 state.event(
11711 "gate",
11712 format!(
11713 "`{}` -> {}",
11714 o.command,
11715 if o.ok() {
11716 "pass".to_owned()
11717 } else {
11718 format!(
11719 "FAIL ({:?})\n{}",
11720 o.code,
11721 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11722 )
11723 }
11724 ),
11725 );
11726 }
11727 Ok(outcomes)
11728}
11729
11730pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11736 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11737 ""
11738 } else {
11739 pr_title
11740 }
11741}
11742
11743pub fn landing_subject_source(state: &RunState) -> String {
11747 if is_review_run(state) {
11748 let winner = state.candidates.first().map_or('A', |c| c.label);
11749 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11750 }
11751 state.instruction.clone()
11752}
11753
11754fn summary_without_title(summary: &str) -> String {
11757 let mut lines = summary.trim().lines().peekable();
11758 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11759 lines.next();
11760 }
11761 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11762}
11763
11764#[cfg(test)]
11776fn pr_message(state: &RunState, winner: char) -> PrMessage {
11777 pr_message_with(state, winner, None)
11778}
11779
11780#[cfg(test)]
11783fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11784 let raw = pr_message_raw(
11785 state,
11786 winner,
11787 facts,
11788 crate::pr_outcome::FinalHead::Unreported,
11789 ChangeNote::NotAsked,
11790 );
11791 let id = crate::scrub::Identity::current();
11792 PrMessage {
11793 title: crate::scrub::scrub(&raw.title, &id),
11794 body: crate::scrub::scrub(&raw.body, &id),
11795 }
11796}
11797
11798#[derive(Debug, Clone, Copy)]
11800enum ChangeNote<'a> {
11801 NotAsked,
11803 Summary(&'a crate::run::ChangeSummary),
11805 Unavailable(&'a str),
11807}
11808
11809fn pr_message_raw(
11810 state: &RunState,
11811 winner: char,
11812 facts: Option<&BranchFacts>,
11813 head: crate::pr_outcome::FinalHead<'_>,
11814 change: ChangeNote<'_>,
11815) -> PrMessage {
11816 if !matches!(change, ChangeNote::NotAsked) {
11817 let (title, mut body) = match change {
11818 ChangeNote::Summary(c) => (
11819 c.title.clone(),
11820 format!("## Summary\n\n{}\n\n", c.body.trim()),
11821 ),
11822 ChangeNote::Unavailable(why) => (
11823 neutral_title(state, winner),
11824 format!(
11825 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11826 ),
11827 ),
11828 ChangeNote::NotAsked => unreachable!(),
11829 };
11830 body.push_str(&public_pr_evidence(state, head));
11831 body.push_str(&format!(
11832 "\n---\nmagi:run/{} magi:candidate-{}\n",
11833 state.id,
11834 winner.to_ascii_lowercase()
11835 ));
11836 return PrMessage { title, body };
11837 }
11838 pr_preview(state, winner, facts, head)
11839}
11840
11841fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11844 let mut body = crate::pr_outcome::review_rounds(state);
11845 body.push_str(&crate::pr_outcome::final_verification(state, head));
11846 let open = state.last_round_findings();
11847 if !open.is_empty() {
11848 body.push_str("## Open review findings\n\n");
11849 for f in open {
11850 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11851 }
11852 body.push('\n');
11853 }
11854 if let Some(c) = state.change_summary.as_ref()
11855 && head == crate::pr_outcome::FinalHead::At(&c.head)
11856 {
11857 body.push_str(&format!(
11858 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11859 c.base, c.head
11860 ));
11861 }
11862 body
11863}
11864
11865fn pr_preview(
11867 state: &RunState,
11868 winner: char,
11869 facts: Option<&BranchFacts>,
11870 head: crate::pr_outcome::FinalHead<'_>,
11871) -> PrMessage {
11872 let summary = state
11873 .candidates
11874 .iter()
11875 .find(|c| c.label == winner)
11876 .map(|c| c.summary.as_str())
11877 .unwrap_or_default();
11878 let review = is_review_run(state);
11881 let title = if review {
11882 facts
11883 .and_then(|f| {
11884 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11885 })
11886 .or_else(|| review_title(state))
11887 .or_else(|| {
11888 state
11889 .candidates
11890 .iter()
11891 .find(|c| c.label == winner)
11892 .filter(|c| !c.branch.starts_with("magi/"))
11893 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11894 })
11895 .unwrap_or_else(|| neutral_title(state, winner))
11896 } else {
11897 summary_title(summary).unwrap_or_else(|| {
11898 english_title(
11899 &state.instruction,
11900 PR_TITLE_MAX,
11901 state.config.graph.github_english(),
11902 )
11903 .unwrap_or_else(|| neutral_title(state, winner))
11904 })
11905 };
11906
11907 let mut body = String::new();
11908 let what = summary_without_title(summary);
11909 if !what.is_empty() {
11910 body.push_str("## Summary\n\n");
11911 body.push_str(&what);
11912 body.push_str("\n\n");
11913 }
11914
11915 body.push_str(&crate::pr_outcome::review_rounds(state));
11917 body.push_str(&crate::pr_outcome::final_verification(state, head));
11918 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11919
11920 let open = state.open_findings();
11921 if !open.is_empty() {
11922 body.push_str("## Open review findings\n\n");
11923 for f in &open {
11924 body.push_str(&format!(
11925 "- `{}` [{:?}] {}\n",
11926 f.id, f.severity, "Context retained in local review records."
11927 ));
11928 }
11929 body.push('\n');
11930 }
11931
11932 if let Some(fix) = fix
11933 && !fix.rejected.is_empty()
11934 {
11935 body.push_str("## Declined by the fixer\n\n");
11936 for r in &fix.rejected {
11937 body.push_str(&format!(
11938 "- `{}`: {}\n",
11939 r.id, "Reason retained in local review records."
11940 ));
11941 }
11942 body.push('\n');
11943 }
11944
11945 if review {
11946 body.push_str("## Commits under review\n\n");
11948 if let Some(facts) = facts {
11949 let mut left = COMMITS_MAX_CHARS;
11950 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11951 let mut entry = format!("- {}\n", markdown_safe(subject));
11952 for l in markdown_safe(text).lines() {
11953 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11954 }
11955 if left == 0 {
11956 body.push_str(&format!(
11957 "- ... {} more commit(s)\n",
11958 facts.commits.len() - i
11959 ));
11960 break;
11961 }
11962 if entry.len() > left {
11963 let mut end = left;
11966 while !entry.is_char_boundary(end) {
11967 end -= 1;
11968 }
11969 entry.truncate(end);
11970 entry.push_str("\n ... (truncated)\n");
11971 left = 0;
11972 } else {
11973 left -= entry.len();
11974 }
11975 body.push_str(&entry);
11976 }
11977 if !facts.stat.trim().is_empty() {
11978 body.push_str(&format!(
11979 "\n## Diff stat\n\n```\n{}\n```\n",
11980 markdown_safe(facts.stat.trim())
11981 ));
11982 }
11983 } else {
11984 match &state.reviewed_commits {
11985 Some(subjects) => {
11986 for s in subjects {
11987 body.push_str(&format!("- {}\n", s.trim()));
11988 }
11989 }
11990 None => {
11991 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11994 body.push_str(rest.trim());
11995 body.push('\n');
11996 }
11997 }
11998 }
11999 } else {
12000 let task = state.instruction.trim();
12001 let task = if task.is_empty() {
12002 "(empty task)"
12003 } else {
12004 task
12005 };
12006 body.push_str(&format!(
12007 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12008 task.replace("</details>", "</details>")
12009 ));
12010 }
12011
12012 body.push_str(&format!(
12013 "\n---\nmagi:run/{} magi:candidate-{}\n",
12014 state.id,
12015 winner.to_ascii_lowercase()
12016 ));
12017
12018 PrMessage { title, body }
12019}
12020
12021fn seeded_instruction(state: &RunState) -> String {
12024 let mut inputs = Vec::new();
12025 if let Some(start) = &state.sources.start_from {
12026 inputs.push(format!(
12027 "Start from `{}` at `{}` (including ancestry).",
12028 start.token, start.sha
12029 ));
12030 }
12031 for pick in &state.sources.pick_commits {
12032 inputs.push(format!(
12033 "Pick `{}` at `{}` (single commit).",
12034 pick.token, pick.sha
12035 ));
12036 }
12037 if inputs.is_empty() {
12038 state.instruction.clone()
12039 } else {
12040 format!(
12041 "{}\n\n# Explicit source inputs already prepared\n\n{}",
12042 state.instruction,
12043 inputs.join("\n")
12044 )
12045 }
12046}
12047
12048async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12052 let base = &state.base_branch;
12053 let mut against = base.clone();
12054 if mode == MergeMode::Pr {
12055 let remote = &state.config.merge.remote;
12059 let tracking = format!("{remote}/{base}");
12060 let fetched = git::fetch(repo, remote, base).await;
12061 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12062 against = tracking;
12063 } else {
12064 return false;
12065 }
12066 }
12067 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12068}
12069
12070fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12073 let mut detail = format!(
12074 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12075 nothing to open a pull request for"
12076 );
12077 detail.push_str(
12078 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12079 );
12080 detail
12081}
12082
12083#[derive(Debug, PartialEq, Eq)]
12086enum PrPlan {
12087 Create,
12088 Adopt { url: String, title: String },
12089 Stop(String),
12090}
12091
12092fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12095 match found {
12096 Ok(land::OpenPr::None) => PrPlan::Create,
12097 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12098 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12099 "several open pull requests exist for this branch, not picking one: {}",
12100 urls.join(" ")
12101 )),
12102 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12103 }
12104}
12105
12106pub(crate) async fn gh_pr_create(
12108 cwd: &Path,
12109 base: &str,
12110 head: &str,
12111 title: &str,
12112 body: &str,
12113) -> Result<String> {
12114 let out = tokio::process::Command::new("gh")
12115 .args([
12116 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12117 ])
12118 .current_dir(cwd)
12119 .quiet()
12120 .stdin(std::process::Stdio::null())
12121 .output()
12122 .await
12123 .context("spawn gh")?;
12124 if out.status.success() {
12125 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12126 } else {
12127 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12128 }
12129}
12130
12131async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12133 match std::fs::symlink_metadata(path) {
12134 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12135 git::worktree_remove_clean(repo, path).await?;
12138 return Ok(());
12139 }
12140 Err(e) => return Err(e.into()),
12141 Ok(_) => {}
12142 }
12143 if !git::is_clean(path).await? {
12144 bail!(
12145 "worktree has uncommitted work; refusing removal: {}",
12146 path.display()
12147 );
12148 }
12149 if !git::worktree_remove_clean(repo, path).await? {
12150 bail!("worktree removal refused: {}", path.display());
12151 }
12152 Ok(())
12153}
12154
12155pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12164 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12165 bail!("refusing fold: {reason}");
12166 }
12167 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12168 bail!("candidate preservation incomplete; refusing cleanup");
12169 }
12170 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12174 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12175 let _driver = crate::driver::maintenance(home, &state.id)?;
12176 let mut output_guards = Vec::new();
12177 for entry in std::fs::read_dir(state.worktree_root())
12178 .into_iter()
12179 .flatten()
12180 .flatten()
12181 {
12182 if entry.path().join(".git").is_file() {
12183 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12184 }
12185 }
12186 let repo = state.repo.clone();
12187 let wait = repository_wait(state, None);
12190 let _coordination = match coordinate_operation(state, &repo, &wait).await {
12191 Ok(lease) => lease,
12192 Err(e) => {
12193 state.save_under(home)?;
12194 return Err(e);
12195 }
12196 };
12197 let _source_outputs = if state.status == RunStatus::Merged
12200 && state.merge.as_ref().is_some_and(|merge| merge.ok)
12201 {
12202 verify_merged_fold_sources(state, drop_winner, home).await?
12203 } else {
12204 Vec::new()
12205 };
12206 let root = state.worktree_root();
12207 let winner = state.tally.as_ref().map(|t| t.winner);
12208 let mut removed = Vec::new();
12209
12210 for i in 0..state.candidates.len() {
12211 let c = state.candidates[i].clone();
12212 let is_winner = Some(c.label) == winner;
12213 if is_winner && !drop_winner {
12214 continue;
12215 }
12216 let existed = c.worktree.exists();
12217 remove_preserved_worktree(&repo, &c.worktree).await?;
12220 if existed {
12221 removed.push(c.worktree.to_string_lossy().into_owned());
12222 }
12223 let handed_over = state.released_branches.contains(&c.branch);
12226 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12227 if !git::branch_delete(&repo, &c.branch).await? {
12228 bail!("candidate branch removal refused");
12229 }
12230 removed.push(c.branch.clone());
12231 }
12232 state.candidates[i].folded = true;
12233 }
12234
12235 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12236 let path = name.path();
12237 let keep = !drop_winner
12238 && winner.is_some_and(|w| {
12239 path.file_name()
12240 .is_some_and(|n| n == format!("cand-{w}").as_str())
12241 });
12242 if keep {
12243 continue;
12244 }
12245 remove_preserved_worktree(&repo, &path).await?;
12246 removed.push(path.to_string_lossy().into_owned());
12247 }
12248
12249 remove_if_empty(&root);
12258
12259 if state.enabled_worktree_config && drop_winner {
12260 git::release_worktree_config(&repo).await.ok();
12264 state.enabled_worktree_config = false;
12265 }
12266 state.save_under(home)?;
12267 Ok(removed)
12268}
12269
12270async fn verify_merged_fold_sources(
12271 state: &RunState,
12272 drop_winner: bool,
12273 home: &Path,
12274) -> Result<Vec<std::fs::File>> {
12275 let winner = state.tally.as_ref().map(|t| t.winner);
12276 let mut heads = BTreeSet::new();
12277 for candidate in &state.candidates {
12278 if Some(candidate.label) == winner && !drop_winner {
12279 continue;
12280 }
12281 anyhow::ensure!(
12282 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12283 "candidate recovery source is outside the inventoried run root: {}",
12284 candidate.worktree.display()
12285 );
12286 match std::fs::symlink_metadata(&candidate.worktree) {
12287 Ok(_) => {}
12288 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12289 Err(error) => return Err(error.into()),
12290 }
12291 if !state.released_branches.contains(&candidate.branch)
12292 && git::branch_exists(&state.repo, &candidate.branch).await?
12293 {
12294 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12295 }
12296 }
12297 let mut cargo_guards = Vec::new();
12298 let entries = match std::fs::read_dir(state.worktree_root()) {
12299 Ok(entries) => Some(entries),
12300 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12301 Err(error) => return Err(error.into()),
12302 };
12303 for entry in entries.into_iter().flatten() {
12304 let path = entry?.path();
12305 let metadata = std::fs::symlink_metadata(&path)?;
12306 anyhow::ensure!(
12307 metadata.is_dir() && !metadata.file_type().is_symlink(),
12308 "non-directory or symlink recovery source retained: {}",
12309 path.display()
12310 );
12311 if !drop_winner
12312 && winner.is_some_and(|w| {
12313 path.file_name()
12314 .is_some_and(|name| name == format!("cand-{w}").as_str())
12315 })
12316 {
12317 continue;
12318 }
12319 anyhow::ensure!(
12320 path.join(".git").is_file(),
12321 "unregistered recovery source retained: {}",
12322 path.display()
12323 );
12324 anyhow::ensure!(
12325 git::git(
12326 &path,
12327 &[
12328 "status",
12329 "--porcelain",
12330 "--untracked-files=all",
12331 "--ignored",
12332 "--",
12333 ".",
12334 ":(exclude)target"
12335 ]
12336 )
12337 .await?
12338 .is_empty(),
12339 "source changes or ignored recovery files retained: {}",
12340 path.display()
12341 );
12342 match std::fs::symlink_metadata(path.join("target")) {
12343 Ok(_) => {
12344 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12345 }
12346 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12347 Err(error) => return Err(error.into()),
12348 }
12349 for checkpoint in [
12350 "rebase-merge",
12351 "rebase-apply",
12352 "sequencer",
12353 "CHERRY_PICK_HEAD",
12354 "MERGE_HEAD",
12355 "REBASE_HEAD",
12356 "REVERT_HEAD",
12357 "BISECT_LOG",
12358 "AUTO_MERGE",
12359 "MERGE_MSG",
12360 ] {
12361 let location = git::git(
12362 &path,
12363 &[
12364 "rev-parse",
12365 "--path-format=absolute",
12366 "--git-path",
12367 checkpoint,
12368 ],
12369 )
12370 .await?;
12371 anyhow::ensure!(
12372 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12373 if e.kind() == std::io::ErrorKind::NotFound),
12374 "Git recovery checkpoint retained: {checkpoint}"
12375 );
12376 }
12377 heads.insert(git::rev_parse(&path, "HEAD").await?);
12378 }
12379 crate::fold_evidence::preserve(state, home, &heads).await?;
12380
12381 Ok(cargo_guards)
12382}
12383
12384fn remove_if_empty(dir: &Path) {
12395 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12396 std::fs::remove_dir(dir).ok();
12397 }
12398}
12399
12400pub fn worst_open(state: &RunState) -> Option<Severity> {
12402 state
12403 .reviews
12404 .last()?
12405 .reviews
12406 .iter()
12407 .flat_map(|r| r.findings.iter())
12408 .map(|f| f.severity)
12409 .max()
12410}
12411
12412async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12416 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12417 git::acquire_worktree_config(repo).await?;
12418 state.enabled_worktree_config = true;
12419 }
12420 Ok(())
12421}
12422
12423#[cfg(test)]
12424mod tests {
12425 #[tokio::test]
12426 async fn launcher_base_fetch_requires_remote_success() {
12427 let tmp = tempfile::tempdir().unwrap();
12428 let repo = tmp.path().join("repo");
12429 let remote = tmp.path().join("remote");
12430 std::fs::create_dir(&repo).unwrap();
12431 std::fs::create_dir(&remote).unwrap();
12432 git::git(&remote, &["init", "--bare", "-b", "main"])
12433 .await
12434 .unwrap();
12435 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12436 git::git(
12437 &repo,
12438 &[
12439 "-c",
12440 "user.name=Fixture",
12441 "-c",
12442 "user.email=fixture@example.invalid",
12443 "commit",
12444 "--allow-empty",
12445 "-m",
12446 "fixture",
12447 ],
12448 )
12449 .await
12450 .unwrap();
12451 git::git(
12452 &repo,
12453 &["remote", "add", "origin", remote.to_str().unwrap()],
12454 )
12455 .await
12456 .unwrap();
12457 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12458 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12459 assert_eq!(
12460 resolve_base(&repo, "main", "origin", None)
12461 .await
12462 .unwrap()
12463 .commit,
12464 head
12465 );
12466 #[cfg(unix)]
12467 {
12468 let policy: crate::config::Coordination = toml::from_str(
12469 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12470 ).unwrap();
12471 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12472 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12473 let held = crate::repo_lock::SeatLease::operation_with(
12474 &repo,
12475 Some(&policy),
12476 &crate::repo_lock::WaitPolicy::default(),
12477 )
12478 .await
12479 .unwrap();
12480 assert!(held.is_some());
12481 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12482 tokio::pin!(fetch);
12483 assert!(
12484 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12485 .await
12486 .is_err()
12487 );
12488 assert!(
12489 !repo.join(".git/FETCH_HEAD").exists(),
12490 "no fetch before admission"
12491 );
12492 drop(held);
12493 let fetched = fetch.await.unwrap();
12494 assert_eq!(fetched.commit, head);
12495 assert_eq!(fetched.remote, "origin");
12496 assert_eq!(fetched.branch, "main");
12497 }
12498 git::git(
12500 &repo,
12501 &[
12502 "remote",
12503 "set-url",
12504 "origin",
12505 tmp.path().join("missing").to_str().unwrap(),
12506 ],
12507 )
12508 .await
12509 .unwrap();
12510 let error = resolve_base(&repo, "main", "origin", None)
12511 .await
12512 .unwrap_err()
12513 .to_string();
12514 assert!(error.contains("cannot read origin/main"));
12515 assert!(error.contains("refusing to branch off"));
12516 }
12517
12518 #[test]
12519 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12520 let leaked = vec!["chore(deps): update a crate".to_owned()];
12521 let own = "fix(daemon): apply a chosen action";
12522 assert!(should_retitle("", own, &leaked));
12523 assert!(should_retitle(
12524 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12525 own,
12526 &leaked
12527 ));
12528 assert!(should_retitle(
12529 "chore: land candidate A of run 1",
12530 own,
12531 &leaked
12532 ));
12533 assert!(should_retitle(
12534 "magi: candidate A (uncommitted work)",
12535 own,
12536 &leaked
12537 ));
12538 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12539 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12540 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12541 }
12542
12543 #[test]
12544 fn pr_merge_plan_creates_adopts_or_stops() {
12545 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12546 assert_eq!(
12547 pr_merge_plan(Ok(land::OpenPr::One {
12548 url: "u".into(),
12549 title: "t".into()
12550 })),
12551 PrPlan::Adopt {
12552 url: "u".into(),
12553 title: "t".into()
12554 }
12555 );
12556 let PrPlan::Stop(many) =
12557 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12558 else {
12559 panic!("many must stop");
12560 };
12561 assert!(many.contains('a') && many.contains('b'));
12562 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12563 panic!("a failed lookup must stop");
12564 };
12565 assert!(err.contains("bad token"));
12566 }
12567
12568 use super::*;
12569 use crate::run::GateStatus;
12570 use std::collections::BTreeMap;
12571 use std::time::Duration;
12572
12573 fn conductor() -> AgentSpec {
12574 AgentSpec {
12575 id: "conductor".to_owned(),
12576 kind: crate::config::AgentKind::Command,
12577 model: None,
12578 command: vec!["true".to_owned()],
12579 extra_args: Vec::new(),
12580 env: BTreeMap::new(),
12581 prompt_delivery: None,
12582 max_turn_secs: None,
12583 }
12584 }
12585
12586 fn spec(id: &str) -> AgentSpec {
12587 AgentSpec {
12588 id: id.to_owned(),
12589 kind: crate::config::AgentKind::Command,
12590 model: None,
12591 command: vec!["true".to_owned()],
12592 extra_args: Vec::new(),
12593 env: BTreeMap::new(),
12594 prompt_delivery: None,
12595 max_turn_secs: None,
12596 }
12597 }
12598
12599 fn ids(xs: &[&str]) -> BTreeSet<String> {
12600 xs.iter().map(|s| (*s).to_owned()).collect()
12601 }
12602
12603 #[test]
12604 fn pick_successor_skips_an_agent_another_seat_holds() {
12605 let roster = [spec("a"), spec("b"), spec("c")];
12607 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12608 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12609 }
12610
12611 #[test]
12612 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12613 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12615 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12616 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12617 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12618 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12619 }
12620
12621 #[test]
12622 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12623 let roster = [spec("a"), spec("b")];
12624 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12625 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12626 let carried = ids(&["b"]);
12627 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12628 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12629 }
12630
12631 #[test]
12632 fn pick_successor_returns_none_without_any_untried_successor() {
12633 let roster = [spec("a"), spec("b")];
12634 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12635 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12636 let carried = ids(&["a"]);
12637 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12638 }
12639
12640 #[test]
12641 fn pick_successor_rescue_avoids_another_seats_occupant() {
12642 let roster = [spec("a"), spec("b"), spec("c")];
12644 let carried = ids(&["b", "c"]);
12645 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12646 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12647 }
12648
12649 #[test]
12650 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12651 let roster = [spec("a"), spec("b"), spec("c")];
12652 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12653 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12654 }
12655
12656 #[test]
12657 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12658 let roster = [spec("a"), spec("b"), spec("c")];
12659 let failed = ids(&["a", "b", "c"]);
12660 let mut tried = ids(&["b"]);
12662 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12663 assert_eq!(first.id, "a");
12664 tried.insert(first.id.clone());
12665 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12666 assert_eq!(second.id, "c");
12667 tried.insert(second.id.clone());
12668 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12669 }
12670
12671 #[test]
12672 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12673 let roster = [spec("a"), spec("b")];
12674 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12675 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12676 }
12677
12678 #[test]
12679 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12680 let roster = [spec("a"), spec("b"), spec("c")];
12681 let h = SeatHistory {
12682 failed: ids(&["a"]),
12683 last_ok: Some("b".to_owned()),
12684 last_fail: None,
12685 };
12686 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12687 let gone = SeatHistory {
12689 last_ok: Some("zzz".to_owned()),
12690 ..h.clone()
12691 };
12692 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12693 let failed = SeatHistory {
12694 failed: ids(&["a", "b"]),
12695 last_ok: Some("b".to_owned()),
12696 last_fail: None,
12697 };
12698 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12699 }
12700
12701 #[test]
12702 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12703 let first = SeatState::new("review-1", "alpha", 7);
12704 let next = handover_seat("review-1", "gamma", 7);
12705 assert_ne!(first.claude_session, next.claude_session);
12706 }
12707
12708 #[test]
12709 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12710 let mut state = state_with_summary("x", "y");
12711 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12712 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12713 assert_ne!(a.claude_session, b.claude_session);
12714 }
12715
12716 #[test]
12717 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12718 let roster = [spec("a"), spec("b")];
12719 let h = SeatHistory {
12720 failed: ids(&["a", "b"]),
12721 ..SeatHistory::default()
12722 };
12723 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12724 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12725 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12726 }
12727
12728 #[test]
12735 fn failure_signature_ignores_numbers_and_paths() {
12736 assert_eq!(
12737 failure_signature("exited with Some(2) and no usable output"),
12738 failure_signature("exited with Some(137) and no usable output")
12739 );
12740 assert_eq!(
12741 failure_signature("process exited with no usable output (exit code 2)"),
12742 failure_signature("process exited with no usable output (exit code 137)")
12743 );
12744 for (old, new) in [
12745 (
12746 "exited with Some(2) and no usable output",
12747 "process exited with no usable output (exit code 2)",
12748 ),
12749 (
12750 "exited with None and no usable output",
12751 "process exited with no usable output (exit code unavailable)",
12752 ),
12753 ] {
12754 let previous = FailClass::Other(failure_signature(old));
12755 let current = FailClass::Other(failure_signature(new));
12756 assert_eq!(previous, current, "resumption retains the failure class");
12757 assert!(!should_hand_over(Some(&previous), ¤t));
12758 }
12759 assert_eq!(
12760 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12761 failure_signature("cannot open /var/x.txt: denied")
12762 );
12763 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12764 }
12765
12766 #[test]
12767 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12768 let other = FailClass::Other("x".into());
12769 assert!(should_hand_over(None, &FailClass::Quota));
12770 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12771 assert!(should_hand_over(
12772 Some(&FailClass::Timeout),
12773 &FailClass::Timeout
12774 ));
12775 assert!(should_hand_over(None, &other));
12776 assert!(!should_hand_over(Some(&other), &other));
12777 assert!(should_hand_over(
12778 Some(&other),
12779 &FailClass::Other("y".into())
12780 ));
12781 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12783 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12784 }
12785
12786 #[test]
12787 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12788 let a = SeatState::new("judge-1", "alpha", 7);
12789 let b = handover_seat("judge-1", "beta", 7);
12790 assert_ne!(a.claude_session, b.claude_session);
12791 assert_eq!(b.turns, 0);
12792 }
12793
12794 #[test]
12795 fn a_timeout_is_classified_apart_from_other_failures() {
12796 assert_eq!(
12797 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12798 Some(FailClass::Timeout)
12799 );
12800 assert!(matches!(
12801 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12802 Some(FailClass::Other(_))
12803 ));
12804 }
12805
12806 #[test]
12807 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12808 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12809 let tried = BTreeSet::from(["beta".to_owned()]);
12810 let next = next_untried_in_roster(&roster, 1, &tried);
12813 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12814 }
12815
12816 #[test]
12817 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12818 let roster = vec![spec("alpha"), spec("beta")];
12819 let tried = BTreeSet::from(["beta".to_owned()]);
12820 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12824 }
12825
12826 #[test]
12827 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12828 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12829 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12830 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12834 }
12835
12836 #[test]
12837 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12838 let roster = vec![spec("a"), spec("a"), spec("b")];
12839 let tried = BTreeSet::from(["a".to_owned()]);
12840 let next = next_untried_in_roster(&roster, 0, &tried);
12841 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12842 }
12843
12844 #[test]
12845 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12846 let roster = vec![spec("a"), spec("b")];
12847 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12848 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12849 }
12850
12851 #[test]
12852 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12853 let dir = tempfile::tempdir().unwrap();
12854 let bay = dir.path().join("ffff");
12855
12856 remove_if_empty(&bay);
12858 assert!(!bay.exists());
12859
12860 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12863 remove_if_empty(&bay);
12864 assert!(bay.exists(), "non-empty directory must survive");
12865
12866 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12868 remove_if_empty(&bay);
12869 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12870 }
12871
12872 #[test]
12881 fn a_full_panel_that_found_nothing_is_clean() {
12882 assert!(round_is_clean(
12883 0,
12884 true,
12885 2,
12886 2,
12887 0,
12888 IncompleteReviewPolicy::Block
12889 ));
12890 }
12891
12892 #[test]
12893 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12894 assert!(!round_is_clean(
12895 0,
12896 true,
12897 1,
12898 2,
12899 0,
12900 IncompleteReviewPolicy::Block
12901 ));
12902 }
12903
12904 #[test]
12905 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12906 assert!(!round_is_clean(
12907 1,
12908 true,
12909 1,
12910 2,
12911 0,
12912 IncompleteReviewPolicy::Warn
12913 ));
12914 }
12915
12916 #[test]
12917 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12918 assert!(round_is_clean(
12919 0,
12920 true,
12921 1,
12922 2,
12923 0,
12924 IncompleteReviewPolicy::Warn
12925 ));
12926 }
12927
12928 #[test]
12929 fn a_full_panel_with_an_open_finding_is_not_clean() {
12930 assert!(!round_is_clean(
12931 1,
12932 true,
12933 2,
12934 2,
12935 0,
12936 IncompleteReviewPolicy::Block
12937 ));
12938 }
12939
12940 #[test]
12941 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12942 assert!(!round_is_clean(
12943 0,
12944 false,
12945 2,
12946 2,
12947 0,
12948 IncompleteReviewPolicy::Block
12949 ));
12950 }
12951
12952 #[test]
12959 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12960 assert!(round_is_clean(
12963 0,
12964 true,
12965 1,
12966 2,
12967 1,
12968 IncompleteReviewPolicy::Block
12969 ));
12970 }
12971
12972 #[test]
12973 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12974 assert!(!round_is_clean(
12977 0,
12978 true,
12979 1,
12980 2,
12981 0,
12982 IncompleteReviewPolicy::Block
12983 ));
12984 }
12985
12986 #[test]
12987 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12988 assert!(!round_is_clean(
12989 1,
12990 true,
12991 1,
12992 2,
12993 1,
12994 IncompleteReviewPolicy::Block
12995 ));
12996 assert!(!round_is_clean(
12997 0,
12998 false,
12999 1,
13000 2,
13001 1,
13002 IncompleteReviewPolicy::Block
13003 ));
13004 }
13005
13006 #[test]
13007 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13008 assert!(!round_is_clean(
13012 0,
13013 true,
13014 0,
13015 2,
13016 2,
13017 IncompleteReviewPolicy::Block
13018 ));
13019 }
13020
13021 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13022 CommandOutcome {
13023 command: "test".to_owned(),
13024 code,
13025 output_tail: String::new(),
13026 duration_ms: 0,
13027 resource_blocked,
13028 }
13029 }
13030
13031 #[test]
13032 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13033 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13034 assert!(
13035 !verify_inconclusive(&[outcome(Some(1), false)]),
13036 "an ordinary failure is still evidence about the patch"
13037 );
13038 assert!(verify_inconclusive(&[outcome(None, true)]));
13039 assert!(
13040 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13041 "one inconclusive outcome taints the whole batch"
13042 );
13043 assert!(!verify_inconclusive(&[]));
13044 }
13045
13046 #[tokio::test]
13047 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13048 let calls = std::sync::atomic::AtomicUsize::new(0);
13052 let started = Instant::now();
13053 wait_for_pids_with(
13054 &[123],
13055 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13056 Duration::from_millis(5),
13057 Duration::from_secs(5),
13058 )
13059 .await;
13060 assert!(
13061 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13062 "must keep checking rather than deciding on the first answer"
13063 );
13064 assert!(
13065 started.elapsed() < Duration::from_secs(1),
13066 "must return the moment it is confirmed dead, not wait out the ceiling"
13067 );
13068 }
13069
13070 #[tokio::test]
13071 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13072 let started = Instant::now();
13073 wait_for_pids_with(
13074 &[123],
13075 |_| true, Duration::from_millis(5),
13077 Duration::from_millis(30),
13078 )
13079 .await;
13080 let elapsed = started.elapsed();
13081 assert!(
13082 elapsed >= Duration::from_millis(30),
13083 "must not give up before its own ceiling: {elapsed:?}"
13084 );
13085 assert!(
13086 elapsed < Duration::from_secs(1),
13087 "must not wait past its own ceiling either: {elapsed:?}"
13088 );
13089 }
13090
13091 #[tokio::test]
13092 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13093 let started = Instant::now();
13094 wait_for_pids_with(
13095 &[],
13096 |_| true,
13097 Duration::from_secs(5),
13098 Duration::from_secs(5),
13099 )
13100 .await;
13101 assert!(
13102 started.elapsed() < Duration::from_millis(200),
13103 "an empty pid list has nothing to confirm"
13104 );
13105 }
13106
13107 fn review_round(
13113 clean: bool,
13114 blocking: usize,
13115 answered: usize,
13116 expected: usize,
13117 progressed: bool,
13118 e2e_ok: bool,
13119 ) -> ReviewRound {
13120 ReviewRound {
13121 round: 1,
13122 head: "h".to_owned(),
13123 verified_head: None,
13124 verified_at: None,
13125 reviews: Vec::new(),
13126 e2e: vec![CommandOutcome {
13127 command: "test".to_owned(),
13128 code: Some(if e2e_ok { 0 } else { 1 }),
13129 output_tail: String::new(),
13130 duration_ms: 0,
13131 resource_blocked: false,
13132 }],
13133 verify_retried: false,
13134 e2e_deferred: false,
13135 e2e_defer_reason: None,
13136 fix: None,
13137 blocking,
13138 answered,
13139 expected,
13140 clean,
13141 progressed,
13142 vote_split: false,
13143 reconsideration: Vec::new(),
13144 verdict: None,
13145 }
13146 }
13147
13148 #[test]
13149 fn review_conclusion_is_none_when_nothing_has_run() {
13150 assert_eq!(review_conclusion(&[], 3), None);
13151 }
13152
13153 #[test]
13154 fn review_conclusion_is_none_while_rounds_remain() {
13155 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13156 assert_eq!(review_conclusion(&rounds, 3), None);
13157 }
13158
13159 #[test]
13160 fn review_conclusion_is_gating_once_a_round_is_clean() {
13161 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13162 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13163 }
13164
13165 #[test]
13166 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13167 let rounds = vec![
13168 review_round(false, 1, 2, 2, true, true),
13169 review_round(false, 1, 2, 2, true, true),
13170 ];
13171 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13172 }
13173
13174 #[test]
13175 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13176 let rounds = vec![
13177 review_round(false, 1, 2, 2, true, true),
13178 review_round(false, 1, 2, 2, true, false),
13179 ];
13180 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13181 }
13182
13183 #[test]
13184 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13185 let mut blocked = review_round(false, 1, 2, 2, true, false);
13192 blocked.e2e[0].resource_blocked = true;
13193 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13194 assert_eq!(review_conclusion(&rounds, 2), None);
13195 }
13196
13197 #[test]
13198 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13199 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13201 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13202 }
13203
13204 #[test]
13205 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13206 let rounds = vec![
13207 review_round(false, 1, 2, 2, false, true),
13208 review_round(false, 1, 2, 2, false, true),
13209 ];
13210 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13211 }
13212
13213 fn secs(n: u64) -> Duration {
13214 Duration::from_secs(n)
13215 }
13216
13217 #[tokio::test]
13220 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13221 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13222 if std::env::var_os(CHILD).is_none() {
13223 let output = std::process::Command::new(std::env::current_exe().unwrap())
13227 .args([
13228 "--exact",
13229 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13230 "--nocapture",
13231 ])
13232 .env(CHILD, "1")
13233 .env_remove("MAGI_WORKER_HOME")
13234 .env_remove("MAGI_WORKER_ID")
13235 .output()
13236 .unwrap();
13237 assert!(
13238 output.status.success()
13239 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13240 "{}\n{}",
13241 String::from_utf8_lossy(&output.stdout),
13242 String::from_utf8_lossy(&output.stderr)
13243 );
13244 return;
13245 }
13246 ask_test_home();
13247 let tmp = tempfile::tempdir().unwrap();
13248 let repo = tmp.path().join("repo");
13249 std::fs::create_dir_all(&repo).unwrap();
13250 init_repo(&repo);
13251 let g = |args: &[&str]| {
13252 let out = std::process::Command::new("git")
13253 .args(args)
13254 .current_dir(&repo)
13255 .quiet()
13256 .output()
13257 .unwrap();
13258 assert!(
13259 out.status.success(),
13260 "{}",
13261 String::from_utf8_lossy(&out.stderr)
13262 );
13263 };
13264 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13265 g(&["checkout", "-b", "historical"]);
13266 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13267 g(&["add", "."]);
13268 g(&["commit", "-m", "old alternative"]);
13269 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13270 g(&["checkout", "-b", "magi/task-selected", &base]);
13271 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13272 g(&["add", "."]);
13273 g(&["commit", "-m", "measurement infrastructure"]);
13274 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13275 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13276 g(&["add", "."]);
13277 g(&["commit", "-m", "dependent implementation"]);
13278 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13279 g(&["checkout", "main"]);
13280 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13281 g(&["add", "."]);
13282 g(&["commit", "-m", "unrelated base change"]);
13283 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13284 let origin = tmp.path().join("origin.git");
13285 g(&[
13286 "clone",
13287 "--bare",
13288 repo.to_str().unwrap(),
13289 origin.to_str().unwrap(),
13290 ]);
13291 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13292 g(&["fetch", "origin"]);
13293 let selection = crate::pr_recovery::Selection {
13294 url: "https://example.test/project/pull/1".into(),
13295 branch: "magi/task-selected".into(),
13296 head: head.clone(),
13297 source_head: None,
13298 source_run: None,
13299 source_proof: None,
13300 disposition: Some(crate::pr_recovery::Disposition::Review),
13301 instruction: "preserve measured behavior".into(),
13302 approved_at: jiff::Timestamp::now(),
13303 };
13304 let mut template = runner_at(RunStatus::Reviewing);
13305 template.state.repo = repo.clone();
13306 template.state.base_commit = current.clone();
13307 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13308 template.state.origin = Some(crate::run::Origin::queue("selected"));
13309 template.state.pr_head = Some(selection.branch.clone());
13310 template.state.pr_recovery = Some(selection.clone());
13311 let mut runner =
13312 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13313 .await
13314 .unwrap();
13315 let wt = runner.state.winner().unwrap().worktree.clone();
13316 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13317 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13318 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13319 assert!(runner.state.seeds.is_empty());
13320 assert_eq!(runner.state.instruction, "preserve measured behavior");
13321 runner.sync_to_base().await.unwrap();
13322 assert_eq!(
13323 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13324 "prerequisite"
13325 );
13326 assert_eq!(
13327 std::fs::read_to_string(wt.join("implementation")).unwrap(),
13328 "measured implementation"
13329 );
13330 assert_eq!(
13331 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13332 "current main change"
13333 );
13334 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13335 assert_eq!(
13336 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13337 head
13338 );
13339
13340 let mut partial = runner_at(RunStatus::Reviewing);
13342 partial.state.repo = repo.clone();
13343 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13344 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13345 run: Some("exit 1".into()),
13346 ..Default::default()
13347 }];
13348 let selection = runner.state.pr_recovery.clone().unwrap();
13349 partial.state.pr_recovery = Some(selection.clone());
13350 partial.state.pr_head = Some(selection.branch.clone());
13351 partial.state.origin = Some(crate::run::Origin::queue("selected"));
13352 let id = partial.state.id.clone();
13353 let error = match Runner::open_selected(
13354 &repo,
13355 partial.state,
13356 partial.roles,
13357 1,
13358 runner.state.base_commit.clone(),
13359 selection,
13360 )
13361 .await
13362 {
13363 Ok(_) => panic!("setup must fail"),
13364 Err(e) => e,
13365 };
13366 assert_eq!(
13367 error
13368 .downcast_ref::<crate::pr_recovery::StartupFailed>()
13369 .unwrap()
13370 .run,
13371 id
13372 );
13373 let failed = RunState::load(&id).unwrap();
13374 assert_eq!(failed.status, RunStatus::Blocked);
13375 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13376 assert!(Runner::resume(&id).is_err());
13377 runner.state.status = RunStatus::Ready;
13378 runner.state.driver_exited = true;
13379 runner.state.save().unwrap();
13380 let mut retry = runner_at(RunStatus::Reviewing);
13381 retry.state.repo = repo.clone();
13382 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13383 let mut reselected = runner.state.pr_recovery.clone().unwrap();
13384 reselected.approved_at = jiff::Timestamp::now();
13385 retry.state.origin = Some(crate::run::Origin::queue("selected"));
13386 retry.state.pr_head = Some(reselected.branch.clone());
13387 retry.state.pr_recovery = Some(reselected.clone());
13388 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13389 .await
13390 .unwrap();
13391 let restarted = Runner::open_selected(
13392 &repo,
13393 retry.state,
13394 retry.roles,
13395 1,
13396 runner.state.base_commit.clone(),
13397 reselected,
13398 )
13399 .await
13400 .unwrap();
13401 assert_eq!(
13402 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13403 .await
13404 .unwrap(),
13405 head
13406 );
13407 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13408 assert_eq!(
13409 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13410 head
13411 );
13412 }
13413
13414 #[tokio::test]
13415 #[cfg(unix)]
13416 async fn fallback_and_interrupted_restoration_require_coordination() {
13417 use std::os::fd::AsRawFd as _;
13418 ask_test_home();
13419 let tmp = tempfile::tempdir().unwrap();
13420 let mut runner = registered_runner(tmp.path()).await;
13421 runner.state.config.graph.timeout_resource_wait = 1;
13422 let winner = runner.state.candidates[0].clone();
13423 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13424 let leftover = winner.worktree.join("leftover");
13425 std::fs::write(&leftover, "untrusted edits").unwrap();
13426 let next = tmp.path().join("next-fixer");
13427 let mut failed = spec("first");
13428 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13429 let mut successor = spec("second");
13430 successor.command = vec![
13431 "sh".into(),
13432 "-c".into(),
13433 format!("touch '{}'; echo done", next.display()),
13434 ];
13435 let attempts = vec![
13436 (failed.clone(), "first".into()),
13437 (successor.clone(), "second".into()),
13438 ];
13439 let lock_path = runner.state.repo.join(".git/updater.lock");
13440 let external = std::fs::File::create(&lock_path).unwrap();
13441 assert_eq!(
13442 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13443 0
13444 );
13445 let artifacts = tmp.path().join("artifacts");
13446 let build = |spec, seat| SeatJob {
13447 spec,
13448 seat,
13449 cwd: winner.worktree.clone(),
13450 prompt: "fixture".into(),
13451 timeout: Duration::from_secs(10),
13452 allow_write: true,
13453 sessions: false,
13454 artifacts: artifacts.clone(),
13455 stem: "fix".into(),
13456 handover: None,
13457 };
13458 let (_, _, out) = runner
13459 .ask_fixer_among(
13460 attempts.clone(),
13461 "graft",
13462 None,
13463 Some((&winner.worktree, &head)),
13464 build,
13465 )
13466 .await;
13467 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13468 assert!(leftover.exists(), "contention must prevent clean");
13469 assert!(!next.exists());
13470 assert!(runner.state.handovers.is_empty());
13471 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13473 from_run: "earlier".into(),
13474 generation: 1,
13475 mode: crate::recovery::Mode::VerifyRepair,
13476 reasons: Vec::new(),
13477 reviewed_head: None,
13478 rounds: Vec::new(),
13479 open_findings: Vec::new(),
13480 evidence: None,
13481 criteria_inherited: true,
13482 repair: crate::recovery::Repair::Started {
13483 before: head.clone(),
13484 },
13485 });
13486 assert!(
13487 runner
13488 .recovery_repair(&winner)
13489 .await
13490 .unwrap_err()
13491 .is::<crate::repo_lock::ResourceBlocked>()
13492 );
13493 assert!(matches!(
13494 runner.state.recovery.as_ref().unwrap().repair,
13495 crate::recovery::Repair::Started { .. }
13496 ));
13497 assert!(leftover.exists());
13498 drop(external);
13499 runner.recovery_repair(&winner).await.unwrap();
13500 assert!(!leftover.exists());
13501 assert!(matches!(
13502 runner.state.recovery.as_ref().unwrap().repair,
13503 crate::recovery::Repair::Done { .. }
13504 ));
13505 std::fs::remove_file(&lock_path).unwrap();
13507 std::fs::create_dir(&lock_path).unwrap();
13508 std::fs::write(&leftover, "untrusted edits").unwrap();
13509 let (_, _, out) = runner
13510 .ask_fixer_among(
13511 attempts,
13512 "graft",
13513 None,
13514 Some((&winner.worktree, &head)),
13515 build,
13516 )
13517 .await;
13518 assert!(matches!(out, AgentOutcome::Failed(_)));
13519 assert!(leftover.exists());
13520 assert!(!next.exists());
13521 assert_eq!(
13522 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13523 head
13524 );
13525 }
13526
13527 #[cfg(unix)]
13528 async fn registered_runner(root: &Path) -> Runner {
13529 ask_test_home();
13530 let repo = root.join("repo");
13531 std::fs::create_dir_all(&repo).unwrap();
13532 init_repo(&repo);
13533 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13534 let mut runner = runner_at(RunStatus::Prep);
13535 runner.state.repo = repo.clone();
13536 runner.state.config.coordination = crate::config::test_coordination();
13537 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13538 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13539 runner.state.config.graph.advise = false;
13540 runner.roles.implementers = vec![spec("implement")];
13541 runner.roles.judges = vec![spec("judge")];
13542 runner.prep().await.unwrap();
13543 runner
13544 }
13545
13546 #[tokio::test]
13547 #[cfg(unix)]
13548 async fn parked_implementer_persists_completed_work() {
13549 for registered in [false, true] {
13550 let tmp = tempfile::tempdir().unwrap();
13551 let mut runner = registered_runner(tmp.path()).await;
13552 if !registered {
13553 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13554 }
13555 let marker = tmp.path().join("started");
13556 let release = tmp.path().join("release");
13557 let mut implementer = spec("implement");
13558 implementer.command = vec![
13559 "sh".into(),
13560 "-c".into(),
13561 format!(
13562 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13563 marker.display(),
13564 release.display()
13565 ),
13566 ];
13567 runner.roles.implementers = vec![implementer.clone()];
13568 runner.state.config.agents = vec![implementer];
13569 let pause = runner.pause.clone();
13570 let (result, ()) = tokio::join!(runner.implement(), async {
13571 tokio::time::timeout(Duration::from_secs(10), async {
13572 while !marker.exists() {
13573 tokio::time::sleep(Duration::from_millis(10)).await;
13574 }
13575 })
13576 .await
13577 .unwrap();
13578 pause.park();
13579 std::fs::write(&release, "go").unwrap();
13580 });
13581 result.unwrap();
13582 let candidate = &runner.state.candidates[0];
13583 assert!(candidate.commits > 0, "{candidate:?}");
13584 assert!(candidate.summary.contains("saved-result"));
13585 assert_eq!(candidate.files, 1);
13586 let before = candidate.commits;
13587 runner.implement().await.unwrap();
13588 assert_eq!(runner.state.candidates[0].commits, before);
13589 }
13590 }
13591
13592 #[tokio::test]
13593 #[cfg(unix)]
13594 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13595 let tmp = tempfile::tempdir().unwrap();
13596 let mut runner = registered_runner(tmp.path()).await;
13597 let winner = runner.state.candidates[0].clone();
13598 let cache = tmp.path().join("cache");
13599 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13600 runner.state.config.graph.timeout_resource_wait = 0;
13601 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13602 let crate::cache::AcquireOutcome::Acquired(held) =
13603 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13604 else {
13605 panic!("cache must be free")
13606 };
13607 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13608 from_run: "earlier".into(),
13609 generation: 1,
13610 mode: crate::recovery::Mode::VerifyRepair,
13611 reasons: Vec::new(),
13612 reviewed_head: None,
13613 rounds: Vec::new(),
13614 open_findings: Vec::new(),
13615 evidence: None,
13616 criteria_inherited: true,
13617 repair: crate::recovery::Repair::NotStarted,
13618 });
13619 runner.recovery_repair(&winner).await.unwrap();
13620 assert!(matches!(
13621 runner.state.recovery.as_ref().unwrap().repair,
13622 crate::recovery::Repair::WaitingForRestore { .. }
13623 ));
13624 drop(held);
13625 runner.state.resource_block = None;
13626 runner.state.config.graph.timeout_resource_wait = 1;
13627 runner.recovery_repair(&winner).await.unwrap();
13628 assert!(matches!(
13629 runner.state.recovery.as_ref().unwrap().repair,
13630 crate::recovery::Repair::Done { .. }
13631 ));
13632 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13633 }
13634
13635 #[tokio::test]
13636 #[cfg(unix)]
13637 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13638 use std::os::fd::AsRawFd as _;
13639 for graft in [false, true] {
13640 let tmp = tempfile::tempdir().unwrap();
13641 let mut runner = registered_runner(tmp.path()).await;
13642 runner.state.config.graph.timeout_resource_wait = 1;
13643 let winner = runner.state.candidates[0].clone();
13644 let marker = tmp.path().join("fixer-started");
13645 let mut fixer = spec("fixer");
13646 fixer.command = vec![
13647 "sh".into(),
13648 "-c".into(),
13649 format!("echo started >> '{}'; echo done", marker.display()),
13650 ];
13651 runner.roles.fixer = Some(vec![fixer]);
13652 if graft {
13653 let mut loser = winner.clone();
13654 loser.label = 'B';
13655 loser.index = 1;
13656 runner.state.candidates.push(loser);
13657 runner.state.config.graph.graft_runner_up = true;
13658 runner.state.tally = Some(Tally {
13659 first_choice: BTreeMap::new(),
13660 borda: BTreeMap::new(),
13661 winner: winner.label,
13662 rankings: 1,
13663 unanimous_initial: true,
13664 deliberated: false,
13665 changed_votes: 0,
13666 unanimous_final: true,
13667 tie_break: None,
13668 judges: 1,
13669 present: 1,
13670 quorum: 1,
13671 met_quorum: true,
13672 uncontested: None,
13673 });
13674 runner.state.judgements.push(crate::run::Judgement {
13675 judge: 1,
13676 seat: "judge-1".into(),
13677 agent: "judge".into(),
13678 ranking: vec![winner.label, 'B'],
13679 reasons: BTreeMap::new(),
13680 confidence: None,
13681 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13682 order: vec![0, 1],
13683 failed: None,
13684 duration_ms: 0,
13685 });
13686 } else {
13687 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13688 from_run: "earlier".into(),
13689 generation: 1,
13690 mode: crate::recovery::Mode::VerifyRepair,
13691 reasons: Vec::new(),
13692 reviewed_head: None,
13693 rounds: Vec::new(),
13694 open_findings: Vec::new(),
13695 evidence: None,
13696 criteria_inherited: true,
13697 repair: crate::recovery::Repair::NotStarted,
13698 });
13699 }
13700 let reader = crate::repo_lock::SeatLease::for_checkout(
13701 &winner.worktree,
13702 false,
13703 crate::config::test_coordination().as_ref(),
13704 )
13705 .await
13706 .unwrap();
13707 let pause = runner.pause.clone();
13708 let (result, ()) = tokio::join!(
13709 async {
13710 if graft {
13711 runner.graft().await
13712 } else {
13713 runner.recovery_repair(&winner).await
13714 }
13715 },
13716 async {
13717 tokio::time::sleep(Duration::from_millis(100)).await;
13718 pause.park();
13719 }
13720 );
13721 result.unwrap();
13722 assert!(!marker.exists(), "a blocked seat must not spawn");
13723 runner.state = RunState::load(&runner.state.id).unwrap();
13725 if graft {
13726 assert_eq!(
13727 runner.state.graft.as_ref().unwrap().outcome,
13728 GraftOutcome::WaitingForRestore
13729 );
13730 } else {
13731 assert!(matches!(
13732 runner.state.recovery.as_ref().unwrap().repair,
13733 crate::recovery::Repair::WaitingForRestore { .. }
13734 ));
13735 }
13736 drop(reader);
13737 runner.pause = Pause::new();
13738 runner.state.resource_block = None;
13739 let external =
13741 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13742 assert_eq!(
13743 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13744 0
13745 );
13746 let leftover = winner.worktree.join("leftover");
13747 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13748 let result = if graft {
13749 runner.graft().await
13750 } else {
13751 runner.recovery_repair(&winner).await
13752 };
13753 assert!(
13754 result
13755 .unwrap_err()
13756 .is::<crate::repo_lock::ResourceBlocked>()
13757 );
13758 assert!(leftover.exists());
13759 assert!(!marker.exists());
13760 drop(external);
13761 runner.state.resource_block = None;
13762 let result = if graft {
13763 runner.graft().await
13764 } else {
13765 runner.recovery_repair(&winner).await
13766 };
13767 result.unwrap();
13768 assert!(!leftover.exists(), "restore must precede retry");
13769 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13770 if graft {
13771 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13772 } else {
13773 assert!(matches!(
13774 runner.state.recovery.as_ref().unwrap().repair,
13775 crate::recovery::Repair::Done { .. }
13776 ));
13777 }
13778 }
13779 }
13780
13781 #[tokio::test]
13782 #[cfg(unix)]
13783 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13784 use std::os::fd::AsRawFd as _;
13785 let tmp = tempfile::tempdir().unwrap();
13786 let mut runner = registered_runner(tmp.path()).await;
13787 runner.state.config.graph.timeout_resource_wait = 1;
13788 let winner = runner.state.candidates[0].clone();
13789 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13790 runner.state.tally = Some(Tally {
13791 first_choice: BTreeMap::new(),
13792 borda: BTreeMap::new(),
13793 winner: winner.label,
13794 rankings: 0,
13795 unanimous_initial: false,
13796 deliberated: false,
13797 changed_votes: 0,
13798 unanimous_final: false,
13799 tie_break: None,
13800 judges: 0,
13801 present: 0,
13802 quorum: 0,
13803 met_quorum: true,
13804 uncontested: Some("fixture".into()),
13805 });
13806 runner.state.graft = Some(GraftRecord {
13807 pre: pre.clone(),
13808 post: None,
13809 lines: None,
13810 limit: None,
13811 agent: None,
13812 outcome: GraftOutcome::Running,
13813 requested: Vec::new(),
13814 adoption: None,
13815 adoption_error: None,
13816 });
13817 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13818 git::commit_all(&winner.worktree, "unfinished graft")
13819 .await
13820 .unwrap();
13821 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13822 let leftover = winner.worktree.join("leftover");
13823 std::fs::write(&leftover, "untrusted edits").unwrap();
13824 let path = runner.state.repo.join(".git/updater.lock");
13825 let external = std::fs::File::create(&path).unwrap();
13826 assert_eq!(
13827 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13828 0
13829 );
13830 assert!(
13831 runner
13832 .graft()
13833 .await
13834 .unwrap_err()
13835 .is::<crate::repo_lock::ResourceBlocked>()
13836 );
13837 assert!(matches!(
13838 runner.state.graft.as_ref().unwrap().outcome,
13839 GraftOutcome::Running
13840 ));
13841 assert_eq!(
13842 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13843 unfinished
13844 );
13845 assert!(leftover.exists());
13846 drop(external);
13847 std::fs::remove_file(&path).unwrap();
13848 std::fs::create_dir(&path).unwrap();
13849 assert!(runner.graft().await.is_err());
13850 assert!(matches!(
13851 runner.state.graft.as_ref().unwrap().outcome,
13852 GraftOutcome::Running
13853 ));
13854 assert_eq!(
13855 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13856 unfinished
13857 );
13858 assert!(leftover.exists());
13859 std::fs::remove_dir(&path).unwrap();
13860 runner.graft().await.unwrap();
13861 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13862 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13863 assert!(!leftover.exists());
13864 }
13865
13866 #[tokio::test]
13867 #[cfg(unix)]
13868 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13869 use std::os::fd::AsRawFd as _;
13870 let tmp = tempfile::tempdir().unwrap();
13871 let mut runner = registered_runner(tmp.path()).await;
13872 runner.state.prep_complete = Some(false);
13873 runner.state.config.graph.timeout_resource_wait = 1;
13874 runner.state.save().unwrap();
13875 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13876 assert_eq!(
13877 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13878 0
13879 );
13880 runner.execute().await.unwrap();
13881 let disk = RunState::load(&runner.state.id).unwrap();
13882 assert!(disk.driver_exited);
13883 assert_eq!(disk.driver_pid, None);
13884 assert_eq!(runner.state.status, RunStatus::Blocked);
13885 assert_eq!(
13886 runner.state.resource_block.as_ref().unwrap().kind,
13887 "repository coordination"
13888 );
13889 assert!(runner.state.prep_failure.is_none());
13890 assert!(runner.state.seats.is_empty());
13891 runner.pause.park();
13892 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13895 .await
13896 .unwrap()
13897 .unwrap();
13898 assert!(runner.state.parked);
13899 assert!(runner.state.prep_failure.is_none());
13900 let disk = RunState::load(&runner.state.id).unwrap();
13901 assert!(disk.driver_exited);
13902 assert_eq!(disk.driver_pid, None);
13903 }
13904
13905 #[tokio::test]
13906 #[cfg(unix)]
13907 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13908 use std::os::fd::AsRawFd as _;
13909 let tmp = tempfile::tempdir().unwrap();
13910 let mut runner = registered_runner(tmp.path()).await;
13911 runner.state.prep_complete = Some(false);
13912 runner.state.config.graph.timeout_resource_wait = 30;
13913 runner.state.save().unwrap();
13914 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13915 assert_eq!(
13916 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13917 0
13918 );
13919 assert!(
13920 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13921 .await
13922 .is_err()
13923 );
13924 let disk = RunState::load(&runner.state.id).unwrap();
13925 assert!(disk.driver_exited);
13926 assert_eq!(disk.driver_pid, None);
13927 assert!(disk.status.resumable());
13928 assert!(runner.state.driver_exited);
13929 assert_eq!(runner.state.driver_pid, None);
13930 runner.state.save().unwrap();
13931 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13932 }
13933
13934 #[tokio::test]
13935 #[cfg(unix)]
13936 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13937 let tmp = tempfile::tempdir().unwrap();
13938 let mut runner = registered_runner(tmp.path()).await;
13939 let winner = runner.state.candidates[0].clone();
13940 let cache = tmp.path().join("cache");
13941 let started = tmp.path().join("pre-gate-started");
13942 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13943 runner.state.config.verify.gate =
13944 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13945 let home = crate::run::home();
13946 let owner =
13947 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13948 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13949 crate::cache::AcquireOutcome::Acquired(g) => g,
13950 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13951 };
13952 let mut gate = Box::pin(runner.run_gate(&winner));
13953 assert!(
13954 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13955 .await
13956 .is_err()
13957 );
13958 assert!(!started.exists(), "pre-gate must wait for the cache too");
13959 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13963 let editor = crate::repo_lock::SeatLease::checkout_with(
13964 &winner.worktree,
13965 true,
13966 crate::config::test_coordination().as_ref(),
13967 &wait,
13968 )
13969 .await
13970 .unwrap();
13971 drop(editor);
13972 guard.release();
13973 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13974 .await
13975 .unwrap()
13976 .unwrap();
13977 drop(gate);
13978 assert!(outcomes.iter().all(CommandOutcome::ok));
13979 assert!(started.exists());
13980 }
13981
13982 #[tokio::test]
13983 #[cfg(unix)]
13984 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13985 use std::os::fd::AsRawFd as _;
13986
13987 ask_test_home();
13988 let tmp = tempfile::tempdir().unwrap();
13989 let repo = tmp.path().join("repo");
13990 std::fs::create_dir_all(&repo).unwrap();
13991 init_repo(&repo);
13992 let common = repo.join(".git");
13993 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13994 let mut runner = runner_at(RunStatus::Prep);
13995 runner.state.repo = repo.clone();
13996 runner.state.config.coordination = crate::config::test_coordination();
13997 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13998 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13999 runner.state.config.graph.advise = false;
14000 runner.roles.implementers = vec![spec("implement")];
14001 runner.roles.judges = vec![spec("judge")];
14002 runner.prep().await.unwrap();
14003 let winner = runner.state.candidates[0].clone();
14004 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14007 let completed = tmp.path().join("seat-completed");
14008 runner.roles.implementers[0].command = vec![
14009 "sh".into(),
14010 "-c".into(),
14011 format!("touch '{}'; echo completed", completed.display()),
14012 ];
14013 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14014 assert_eq!(
14015 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14016 0
14017 );
14018 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14019 let mut implement = Box::pin(runner.implement());
14020 tokio::select! {
14021 result = &mut implement => panic!("rescue must wait: {result:?}"),
14022 result = tokio::time::timeout(Duration::from_secs(10), async {
14023 while !completed.exists() {
14024 tokio::time::sleep(Duration::from_millis(20)).await;
14025 }
14026 }) => result.unwrap(),
14027 }
14028 assert!(
14029 tokio::time::timeout(Duration::from_millis(200), &mut implement)
14030 .await
14031 .is_err()
14032 );
14033 assert_eq!(
14034 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14035 before
14036 );
14037 assert_eq!(
14038 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14039 0
14040 );
14041 tokio::time::timeout(Duration::from_secs(10), &mut implement)
14042 .await
14043 .unwrap()
14044 .unwrap();
14045 drop(implement);
14046 assert_ne!(
14047 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14048 before
14049 );
14050 assert!(git::is_clean(&winner.worktree).await.unwrap());
14051
14052 let started = tmp.path().join("pre-gate-started");
14053 let proceed = tmp.path().join("pre-gate-proceed");
14054 let checked = tmp.path().join("gate-ran");
14055 runner.state.config.verify.pre_gate = vec![format!(
14056 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14057 started.display(),
14058 proceed.display()
14059 )];
14060 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14061 assert_eq!(
14062 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14063 0
14064 );
14065 let mut gate = Box::pin(runner.run_gate(&winner));
14066 assert!(
14067 tokio::time::timeout(Duration::from_millis(200), &mut gate)
14068 .await
14069 .is_err()
14070 );
14071 assert!(!started.exists());
14072 assert!(!checked.exists());
14073 assert!(git::is_clean(&winner.worktree).await.unwrap());
14074 assert_eq!(
14075 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14076 0
14077 );
14078 tokio::select! {
14079 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14080 result = tokio::time::timeout(Duration::from_secs(10), async {
14081 while !started.exists() {
14082 tokio::time::sleep(Duration::from_millis(20)).await;
14083 }
14084 }) => result.unwrap(),
14085 }
14086 assert_eq!(
14087 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14088 -1,
14089 "mechanical edits must hold the real lock"
14090 );
14091 std::fs::write(proceed, "ready").unwrap();
14092 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14093 .await
14094 .unwrap()
14095 .unwrap();
14096 drop(gate);
14097 assert!(outcomes.iter().all(CommandOutcome::ok));
14098 assert!(checked.exists());
14099 assert!(runner.state.pre_gate_commit.is_some());
14100 assert!(git::is_clean(&winner.worktree).await.unwrap());
14101 assert_eq!(
14102 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14103 0,
14104 "gate completion must release coordination"
14105 );
14106 drop(outsider);
14107 std::fs::remove_file(common.join("updater.lock")).unwrap();
14108 std::fs::create_dir(common.join("updater.lock")).unwrap();
14109 std::fs::remove_file(&started).unwrap();
14110 std::fs::remove_file(&checked).unwrap();
14111 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14112 let error = runner.run_gate(&winner).await.unwrap_err();
14113 assert!(
14114 error
14115 .to_string()
14116 .contains("acquire mandatory repository lock")
14117 );
14118 assert!(
14119 !started.exists(),
14120 "failed acquisition must prevent pre-gate"
14121 );
14122 assert!(!checked.exists(), "failed acquisition must prevent gate");
14123 assert!(git::is_clean(&winner.worktree).await.unwrap());
14124 assert_eq!(
14125 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14126 head
14127 );
14128 }
14129
14130 #[tokio::test]
14131 #[cfg(unix)]
14132 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14133 use std::os::fd::AsRawFd as _;
14134 use std::time::Duration;
14135
14136 ask_test_home();
14137 let tmp = tempfile::tempdir().unwrap();
14138 let repo = tmp.path().join("repo");
14139 std::fs::create_dir_all(&repo).unwrap();
14140 init_repo(&repo);
14141 let common = repo.join(".git");
14142 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14143 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14144 assert_eq!(
14145 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14146 0
14147 );
14148 let mut runner = runner_at(RunStatus::Prep);
14149 runner.state.repo = repo.clone();
14150 runner.state.config.coordination = crate::config::test_coordination();
14151 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14152 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14153 runner.state.config.graph.advise = true;
14154 runner.state.config.graph.advisors = 1;
14155 runner.roles.implementers = vec![spec("implement")];
14156 runner.roles.judges = vec![spec("judge")];
14157 let root = runner.state.worktree_root();
14158 let started = tmp.path().join("setup-started");
14159 let proceed = tmp.path().join("setup-proceed");
14160 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14161 run: Some(format!(
14162 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14163 started.display(),
14164 proceed.display()
14165 )),
14166 ..Default::default()
14167 }];
14168 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14169 let config = git::git(&repo, &["config", "--local", "--list"])
14170 .await
14171 .unwrap();
14172 let mut prep = Box::pin(runner.prep());
14173 assert!(
14174 tokio::time::timeout(Duration::from_millis(200), &mut prep)
14175 .await
14176 .is_err()
14177 );
14178 assert!(
14179 !root.exists(),
14180 "contention must prevent even checkout creation"
14181 );
14182 assert!(!started.exists(), "contention must prevent setup");
14183 assert_eq!(
14184 git::git(&repo, &["branch", "--list"]).await.unwrap(),
14185 branches
14186 );
14187 assert_eq!(
14188 git::git(&repo, &["config", "--local", "--list"])
14189 .await
14190 .unwrap(),
14191 config
14192 );
14193 drop(lock);
14194 tokio::select! {
14195 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14196 result = tokio::time::timeout(Duration::from_secs(10), async {
14197 while !started.exists() {
14198 tokio::time::sleep(Duration::from_millis(20)).await;
14199 }
14200 }) => result.unwrap(),
14201 }
14202 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14203 assert_eq!(
14204 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14205 -1
14206 );
14207 assert!(
14208 tokio::time::timeout(
14209 Duration::from_millis(100),
14210 crate::repo_lock::SeatLease::for_checkout(
14211 &repo,
14212 true,
14213 crate::config::test_coordination().as_ref()
14214 )
14215 )
14216 .await
14217 .is_err(),
14218 "preparation must also exclude local editors"
14219 );
14220 std::fs::write(proceed, "ready").unwrap();
14221 tokio::time::timeout(Duration::from_secs(10), &mut prep)
14222 .await
14223 .unwrap()
14224 .unwrap();
14225 drop(prep);
14226 assert_eq!(runner.state.prep_complete, Some(true));
14227 assert!(root.join("judge-1").exists());
14228 assert!(root.join("advisor-1").exists());
14229 assert_eq!(
14230 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14231 0
14232 );
14233 }
14234
14235 #[tokio::test]
14236 #[cfg(unix)]
14237 async fn preparation_lock_failure_prevents_repository_changes() {
14238 ask_test_home();
14239 let tmp = tempfile::tempdir().unwrap();
14240 let repo = tmp.path().join("repo");
14241 std::fs::create_dir_all(&repo).unwrap();
14242 init_repo(&repo);
14243 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14244 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14247 let mut runner = runner_at(RunStatus::Prep);
14248 runner.state.repo = repo.clone();
14249 runner.state.config.coordination = crate::config::test_coordination();
14250 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14251 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14252 runner.roles.implementers = vec![spec("implement")];
14253 let error = runner.prep().await.unwrap_err();
14254 assert!(
14255 error
14256 .to_string()
14257 .contains("acquire mandatory repository lock")
14258 );
14259 assert!(!runner.state.worktree_root().exists());
14260 assert!(runner.state.candidates.is_empty());
14261 assert!(!repo.join(".git/config.worktree").exists());
14262 assert_eq!(
14263 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14264 "* main"
14265 );
14266 }
14267
14268 #[tokio::test]
14269 async fn preparation_records_creation_intent_before_git_can_fail() {
14270 ask_test_home();
14271 let tmp = tempfile::tempdir().unwrap();
14272 let repo = tmp.path().join("repo");
14273 std::fs::create_dir_all(&repo).unwrap();
14274 init_repo(&repo);
14275 let mut runner = runner_at(RunStatus::Prep);
14276 runner.state.repo = repo.clone();
14277 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14278 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14279 runner.state.config.blind.commit_msg_hook = false;
14280 runner.roles.implementers = vec![spec("one")];
14281 let label = blind::assign_labels(1, runner.state.seed)[0];
14282 let branch = runner.state.branch_for(label);
14283 git::git(&repo, &["branch", &branch]).await.unwrap();
14286 assert!(runner.prep().await.is_err());
14287 let saved = RunState::load(&runner.state.id).unwrap();
14288 assert!(saved.candidates.is_empty());
14289 let event = saved
14290 .events
14291 .iter()
14292 .find(|e| e.node == "prep-intent")
14293 .unwrap();
14294 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14295 assert_eq!(intent["branch"], branch);
14296 assert_eq!(intent["start"], runner.state.base_commit);
14297 assert_eq!(
14298 intent["worktree"],
14299 runner
14300 .state
14301 .worktree_root()
14302 .join(format!("cand-{label}"))
14303 .to_string_lossy()
14304 .as_ref()
14305 );
14306 assert_eq!(intent["picks"], serde_json::json!([]));
14307 }
14308
14309 #[tokio::test]
14310 async fn invalid_pinned_sources_create_no_partial_worktrees() {
14311 ask_test_home();
14312 let tmp = tempfile::tempdir().unwrap();
14313 let repo = tmp.path().join("repo");
14314 std::fs::create_dir_all(&repo).unwrap();
14315 init_repo(&repo);
14316 let mut runner = runner_at(RunStatus::Prep);
14317 runner.state.repo = repo.clone();
14318 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14319 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14320 runner.state.sources.pick_commits.push(refs::Input {
14321 token: "missing".into(),
14322 sha: "0".repeat(40),
14323 provenance: refs::Provenance::ExplicitCli,
14324 });
14325 assert!(runner.prep().await.is_err());
14326 assert!(runner.state.candidates.is_empty());
14327 assert!(!runner.state.worktree_root().exists());
14328 }
14329
14330 #[tokio::test]
14331 async fn prose_and_legacy_inference_never_import_into_candidates() {
14332 ask_test_home();
14333 let tmp = tempfile::tempdir().unwrap();
14334 let repo = tmp.path().join("repo");
14335 std::fs::create_dir_all(&repo).unwrap();
14336 init_repo(&repo);
14337 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14338 git::git(&repo, &["checkout", "-b", "magi/history/A"])
14339 .await
14340 .unwrap();
14341 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14342 git::git(&repo, &["add", "."]).await.unwrap();
14343 git::git(&repo, &["commit", "-m", "historical implementation"])
14344 .await
14345 .unwrap();
14346 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14347 git::git(&repo, &["checkout", "main"]).await.unwrap();
14348 for instruction in [
14349 format!("Example: magi/history/A at {source}"),
14350 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14351 format!("Never import magi/history/A or {source}"),
14352 format!("Conversation answer: I mentioned {source} only as background"),
14353 ] {
14354 let mut runner = runner_at(RunStatus::Prep);
14355 runner.state.repo = repo.clone();
14356 runner.state.base_commit = base.clone();
14357 runner.state.instruction = instruction.clone();
14358 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14359 runner.state.config.blind.commit_msg_hook = false;
14360 runner.state.seeds = vec![refs::Seed {
14361 token: "magi/history/A".into(),
14362 sha: source.clone(),
14363 kind: refs::SeedKind::Unmerged,
14364 branch: true,
14365 detail: String::new(),
14366 }];
14367 runner.roles.implementers = vec![spec("one")];
14368 assert_eq!(seeded_instruction(&runner.state), instruction);
14369 let mut legacy = serde_json::to_value(&runner.state).unwrap();
14371 legacy.as_object_mut().unwrap().remove("sources");
14372 runner.state = serde_json::from_value(legacy).unwrap();
14373 assert_eq!(runner.state.sources, refs::Sources::default());
14374 runner.state.save().unwrap();
14375 runner.state = RunState::load(&runner.state.id).unwrap();
14376 runner.prep().await.unwrap();
14377 for candidate in &runner.state.candidates {
14378 assert_eq!(
14379 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14380 base
14381 );
14382 assert!(!candidate.worktree.join("unwanted").exists());
14383 }
14384 }
14385 }
14386
14387 #[tokio::test]
14388 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14389 ask_test_home();
14390 let tmp = tempfile::tempdir().unwrap();
14391 let repo = tmp.path().join("repo");
14392 std::fs::create_dir_all(&repo).unwrap();
14393 init_repo(&repo);
14394 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14395 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14396 std::fs::write(repo.join("referenced"), "required commit").unwrap();
14397 git::git(&repo, &["add", "."]).await.unwrap();
14398 git::git(&repo, &["commit", "-m", "required seed"])
14399 .await
14400 .unwrap();
14401 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14402 git::git(&repo, &["checkout", "main"]).await.unwrap();
14403 let mut runner = runner_at(RunStatus::Prep);
14404 runner.state.repo = repo.clone();
14405 runner.state.base_commit = base.clone();
14406 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14407 runner.state.config.graph.advise = false;
14408 runner.state.config.blind.commit_msg_hook = false;
14409 runner.state.sources =
14410 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14411 .await
14412 .unwrap();
14413 runner.roles.implementers = vec![spec("one"), spec("two")];
14414 runner.roles.judges = vec![spec("judge")];
14415 let label = blind::assign_labels(2, runner.state.seed)[0];
14416 let branch = runner.state.branch_for(label);
14417 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14418 git::worktree_add_branch(&repo, &worktree, &branch, &base)
14419 .await
14420 .unwrap();
14421 runner.state.candidates.push(Candidate {
14422 index: 0,
14423 label,
14424 agent: "one".into(),
14425 branch,
14426 worktree,
14427 summary: String::new(),
14428 stat: String::new(),
14429 files: 0,
14430 commits: 0,
14431 empty: false,
14432 failed: None,
14433 verified_noop: None,
14434 duration_ms: 0,
14435 folded: false,
14436 });
14437 runner.state.save().unwrap();
14438 runner.state = RunState::load(&runner.state.id).unwrap();
14440 assert_eq!(runner.state.prep_complete, Some(false));
14441 runner.prep().await.unwrap();
14442 assert_eq!(runner.state.prep_complete, Some(true));
14443 assert_eq!(runner.state.candidates.len(), 2);
14444 for candidate in &runner.state.candidates {
14445 assert_eq!(
14446 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14447 git::tree_of(&repo, &seed).await.unwrap()
14448 );
14449 assert_eq!(
14450 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14451 "required commit"
14452 );
14453 }
14454 assert!(runner.state.worktree_root().join("judge-1").exists());
14455 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14456 runner.prep().await.unwrap();
14457 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14458
14459 runner.state.prep_complete = Some(false);
14460 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14461 run: Some(format!(
14462 "test -f '{}'",
14463 tmp.path().join("dependency-ready").display()
14464 )),
14465 ..Default::default()
14466 }];
14467 assert!(runner.prep().await.is_err());
14468 assert!(runner.state.prep_failure.is_some());
14469 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14470 runner.state = RunState::load(&runner.state.id).unwrap();
14471 assert!(
14472 runner.prep().await.is_err(),
14473 "correcting the cause alone must not auto-retry"
14474 );
14475 let failed_at = runner.state.events.last().unwrap().at;
14476 assert!(
14477 runner.retry_preparation(failed_at).is_err(),
14478 "old approvals cannot unlock the hold"
14479 );
14480 let mut task = crate::queue::Task::new(
14481 "retry".into(),
14482 "retry setup".into(),
14483 repo.clone(),
14484 crate::queue::Source::Human,
14485 );
14486 task.start(runner.state.id.clone());
14487 task.hold_machine(Some("setup failed".into()));
14488 task.release_operator("cli", None);
14489 runner
14490 .retry_preparation(task.resumption.as_ref().unwrap().at)
14491 .unwrap();
14492 runner.prep().await.unwrap();
14493 assert!(runner.state.prep_failure.is_none());
14494 assert_eq!(runner.state.prep_complete, Some(true));
14495 assert_eq!(runner.state.candidates.len(), 2);
14496 assert!(runner.state.worktree_root().join("judge-1").exists());
14497 for candidate in &runner.state.candidates {
14498 assert_eq!(
14499 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14500 git::tree_of(&repo, &seed).await.unwrap()
14501 );
14502 }
14503 }
14504
14505 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14506 let mut heads = Vec::new();
14507 for candidate in candidates {
14508 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14509 }
14510 heads
14511 }
14512
14513 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14514 ask_test_home();
14515 let tmp = tempfile::tempdir().unwrap();
14516 let repo = tmp.path().join("repo");
14517 std::fs::create_dir(&repo).unwrap();
14518 init_repo(&repo);
14519 let mut runner = runner_at(RunStatus::Reviewing);
14520 runner.state.repo = repo.clone();
14521 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14522 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14523 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14524 let branch = format!("candidate-{label}");
14525 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14526 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14527 .await
14528 .unwrap();
14529 if label == 'B' {
14530 std::fs::write(worktree.join("change"), "winner").unwrap();
14531 git::rescue_commit(&worktree, "candidate change")
14532 .await
14533 .unwrap();
14534 }
14535 runner.state.candidates.push(Candidate {
14536 index,
14537 label,
14538 agent: "alpha".into(),
14539 branch,
14540 worktree,
14541 summary: "completed reply".into(),
14542 stat: String::new(),
14543 files: usize::from(label == 'B'),
14544 commits: usize::from(label == 'B'),
14545 empty: label == 'A',
14546 failed: None,
14547 verified_noop: None,
14548 duration_ms: 1,
14549 folded: false,
14550 });
14551 }
14552 runner.state.judge_skipped = true;
14553 runner.state.tally = Some(Tally {
14554 first_choice: BTreeMap::new(),
14555 borda: BTreeMap::new(),
14556 winner: 'B',
14557 rankings: 0,
14558 unanimous_initial: false,
14559 deliberated: false,
14560 changed_votes: 0,
14561 unanimous_final: false,
14562 tie_break: None,
14563 judges: 0,
14564 present: 0,
14565 quorum: 0,
14566 met_quorum: true,
14567 uncontested: Some("only B produced a change".into()),
14568 });
14569 runner.state.save().unwrap();
14570 (tmp, runner)
14571 }
14572
14573 #[tokio::test]
14574 #[cfg(unix)]
14575 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14576 for (solo, failed_checkpoint) in
14577 [(true, false), (false, false), (true, true), (false, true)]
14578 {
14579 let (_tmp, mut runner) = preservation_resume_fixture().await;
14580 if solo {
14581 runner.state.candidates.retain(|c| c.label == 'A');
14582 }
14583 runner.state.candidates[0].empty = false;
14584 runner.state.candidates[0].summary.clear();
14585 runner.state.tally = None;
14586 runner.state.judge_skipped = false;
14587 runner.state.status = RunStatus::Implementing;
14588 runner.state.config.disk.min_free_bytes = u64::MAX;
14589 let mut implementer = spec("alpha");
14590 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14591 runner.roles.implementers = vec![implementer];
14592 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14593 if failed_checkpoint {
14594 std::fs::create_dir_all(&patch).unwrap();
14595 }
14596 runner.state.save().unwrap();
14597 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14598 runner.state = RunState::load(&runner.state.id).unwrap();
14599 assert_eq!(runner.state.status, RunStatus::Blocked);
14600 assert!(runner.state.interrupted_implementations.contains(&'A'));
14601 assert_eq!(runner.state.candidates[0].commits, 0);
14602 assert!(!runner.state.candidates[0].empty);
14603 assert!(runner.state.candidates[0].failed.is_none());
14604 if failed_checkpoint {
14605 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14606 std::fs::remove_dir(patch).unwrap();
14607 }
14608 runner.state.resource_block = None;
14609 runner.state.config.disk.min_free_bytes = 0;
14610 runner.roles.implementers[0].command = vec![
14611 "sh".into(),
14612 "-c".into(),
14613 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14614 ];
14615 runner.implement().await.unwrap();
14616 assert!(runner.state.interrupted_implementations.is_empty());
14617 assert!(runner.state.candidate_preservation.is_empty());
14618 assert_eq!(runner.state.candidates[0].commits, 1);
14619 assert!(!runner.state.candidates[0].empty);
14620 assert!(
14621 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14622 .unwrap()
14623 .contains("retried empty interrupted seat")
14624 );
14625 if !solo {
14626 assert_eq!(runner.state.candidates[1].commits, 1);
14627 }
14628 }
14629 }
14630
14631 #[tokio::test]
14632 #[cfg(unix)]
14633 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14634 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14635 let (_tmp, mut runner) = preservation_resume_fixture().await;
14636 runner.state.candidates.retain(|c| c.label == 'A');
14637 runner.state.candidates[0].empty = false;
14638 runner.state.candidates[0].summary.clear();
14639 runner.state.tally = None;
14640 runner.state.judge_skipped = false;
14641 runner.state.status = RunStatus::Implementing;
14642 runner.state.config.disk.min_free_bytes = u64::MAX;
14645 let wt = runner.state.candidates[0].worktree.clone();
14646 std::fs::write(
14647 wt.join("interrupted-source"),
14648 "retain implementation work\n",
14649 )
14650 .unwrap();
14651 if committed {
14652 git::rescue_commit(&wt, "implementation checkpoint")
14653 .await
14654 .unwrap();
14655 }
14656 let lock = git::git(
14657 &wt,
14658 &[
14659 "rev-parse",
14660 "--path-format=absolute",
14661 "--git-path",
14662 "index.lock",
14663 ],
14664 )
14665 .await
14666 .unwrap();
14667 if block_rescue {
14668 std::fs::write(&lock, "fixture-owned lock").unwrap();
14669 }
14670 let mut implementer = spec("alpha");
14671 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14672 runner.roles.implementers = vec![implementer];
14673 runner.state.save().unwrap();
14674 let result = runner.implement().await;
14675 assert_eq!(result.is_err(), block_rescue);
14676 assert_eq!(
14677 runner.state.resource_block.as_ref().unwrap().kind,
14678 "disk pressure"
14679 );
14680 assert!(
14681 runner
14682 .state
14683 .events
14684 .iter()
14685 .any(|e| e.message.contains("agent execution interrupted"))
14686 );
14687 assert!(
14688 runner
14689 .state
14690 .dir()
14691 .join("artifacts/candidate-A-recovery/staged.patch")
14692 .exists()
14693 );
14694 if block_rescue {
14695 let saved = RunState::load(&runner.state.id).unwrap();
14696 assert!(saved.candidate_preservation.contains_key(&'A'));
14697 assert!(!saved.candidates[0].empty);
14698 assert_eq!(
14699 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14700 "retain implementation work\n"
14701 );
14702 std::fs::remove_file(lock).unwrap();
14703 runner.state = saved;
14704 } else {
14705 assert_eq!(runner.state.status, RunStatus::Blocked);
14706 assert_eq!(runner.state.candidates[0].commits, 1);
14707 }
14708 runner.roles.implementers.clear();
14711 runner.state.config.disk.min_free_bytes = 0;
14712 runner.state.resource_block = None;
14713 runner.implement().await.unwrap();
14714 assert!(runner.state.candidate_preservation.is_empty());
14715 assert_eq!(runner.state.candidates[0].commits, 1);
14716 assert!(!runner.state.candidates[0].empty);
14717 assert!(git::is_clean(&wt).await.unwrap());
14718 assert!(
14719 crate::run::read_artifact(&runner.state, "cand-A.patch")
14720 .unwrap()
14721 .contains("retain implementation work")
14722 );
14723 }
14724 }
14725
14726 #[tokio::test]
14727 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14728 let (_tmp, mut runner) = preservation_resume_fixture().await;
14729 runner.implement().await.unwrap();
14731 assert!(runner.state.candidate_preservation.is_empty());
14732 assert!(
14733 !runner
14734 .state
14735 .dir()
14736 .join("artifacts/candidate-A-recovery")
14737 .exists()
14738 );
14739 runner.fold_losers().await.unwrap();
14740 assert!(!runner.state.candidates[0].worktree.exists());
14741 for schema in [14, crate::run::SCHEMA] {
14742 runner.state.schema = schema;
14743 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14744 runner.state.save().unwrap();
14745 runner.state = RunState::load(&runner.state.id).unwrap();
14746 runner.implement().await.unwrap();
14747 assert!(runner.state.candidate_preservation.is_empty());
14748 assert!(runner.state.candidates[0].folded);
14749 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14750 }
14751 }
14752
14753 #[tokio::test]
14754 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14755 let (_tmp, mut runner) = preservation_resume_fixture().await;
14756 runner.state.candidates.retain(|c| c.label == 'A');
14757 runner.state.tally = None;
14758 runner.state.judge_skipped = false;
14759 let candidate = &mut runner.state.candidates[0];
14760 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14761 candidate.failed = Some("deployment validation unavailable".into());
14762 std::fs::write(
14763 candidate.worktree.join("repair"),
14764 "future launcher policy\n",
14765 )
14766 .unwrap();
14767 runner
14768 .state
14769 .candidate_preservation
14770 .insert('A', "awaiting persistence".into());
14771 runner.persist_candidates().await.unwrap();
14772 let candidate = &runner.state.candidates[0];
14773 assert!(!candidate.empty);
14774 assert_eq!(candidate.commits, 1);
14775 assert_eq!(candidate.files, 1);
14776 assert!(candidate.failed.is_none());
14777 assert!(candidate.summary.contains("EPERM"));
14778 assert!(
14779 crate::run::read_artifact(&runner.state, "cand-A.patch")
14780 .unwrap()
14781 .contains("future launcher policy")
14782 );
14783 assert!(runner.state.candidate_preservation.is_empty());
14784 }
14785
14786 #[tokio::test]
14787 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14788 let (_tmp, mut runner) = preservation_resume_fixture().await;
14789 let wt = runner.state.candidates[0].worktree.clone();
14790 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14791 runner.state.schema = 14;
14792 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14793 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14794 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14795 for _ in 0..2 {
14796 assert!(runner.implement().await.is_err());
14797 assert_eq!(runner.state.status, RunStatus::Blocked);
14798 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14799 assert!(
14800 runner.state.candidates[0].empty,
14801 "old verdict must remain unusable"
14802 );
14803 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14804 assert!(runner.state.judge_skipped);
14805 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14806 assert!(
14807 crate::run::read_artifact(&runner.state, "cand-A.patch")
14808 .unwrap()
14809 .contains("owned edits")
14810 );
14811 assert!(wt.exists());
14812 assert_eq!(
14813 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14814 .await
14815 .unwrap(),
14816 1
14817 );
14818 runner.state = RunState::load(&runner.state.id).unwrap();
14819 }
14820 }
14821
14822 #[tokio::test]
14823 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14824 let (_tmp, mut runner) = preservation_resume_fixture().await;
14825 let wt = runner.state.candidates[0].worktree.clone();
14826 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14827 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14828 let index_lock = PathBuf::from(
14829 git::git(
14830 &wt,
14831 &[
14832 "rev-parse",
14833 "--path-format=absolute",
14834 "--git-path",
14835 "index.lock",
14836 ],
14837 )
14838 .await
14839 .unwrap(),
14840 );
14841 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14842 runner.state.schema = 14;
14843 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14844 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14845 assert!(runner.implement().await.is_err());
14846 assert!(runner.after_implement().is_err());
14847 assert!(runner.fold_losers().await.is_err());
14848 assert_eq!(
14849 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14850 "failed seat edits"
14851 );
14852 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14853 runner.state = RunState::load(&runner.state.id).unwrap();
14854 std::fs::remove_file(index_lock).unwrap();
14855 assert!(runner.implement().await.is_err());
14857 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14858 assert!(runner.state.candidates[0].empty);
14859 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14860 assert!(
14861 crate::run::read_artifact(&runner.state, "cand-A.patch")
14862 .unwrap()
14863 .contains("failed seat edits")
14864 );
14865 assert!(wt.exists());
14866 }
14867
14868 #[tokio::test]
14869 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14870 let (_tmp, mut runner) = preservation_resume_fixture().await;
14871 let repo = runner.state.repo.clone();
14872 let common = git::git(
14873 &repo,
14874 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14875 )
14876 .await
14877 .unwrap();
14878 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14879 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14880 assert!(runner.fold_losers().await.is_err());
14881 assert!(!runner.state.candidates[0].worktree.exists());
14882 assert!(!runner.state.candidates[0].folded);
14883 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14884 runner.state = RunState::load(&runner.state.id).unwrap();
14885 std::fs::remove_file(branch_lock).unwrap();
14886 let judge = runner.state.worktree_root().join("judge-1");
14887 git::worktree_add_detached(&repo, &judge, "main")
14888 .await
14889 .unwrap();
14890 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14891 runner.roles.judges = vec![spec("judge")];
14892 assert!(runner.fold_losers().await.is_err());
14893 runner.state = RunState::load(&runner.state.id).unwrap();
14894 assert!(
14895 runner.state.candidates[0].folded,
14896 "candidate progress saved before judge failure"
14897 );
14898 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14899 assert!(judge.join("uncommitted").exists());
14900 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14901 runner.fold_losers().await.unwrap();
14902 assert!(!judge.exists());
14903 runner.fold_losers().await.unwrap();
14904 }
14905
14906 #[tokio::test]
14907 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14908 let (_tmp, mut runner) = preservation_resume_fixture().await;
14909 runner.state.candidates.retain(|c| c.label == 'B');
14910 runner.state.tally = None;
14911 runner.state.judge_skipped = false;
14912 runner.state.prep_complete = Some(true);
14913 runner.state.config.graph.advise = false;
14914 runner.state.config.graph.selected_implementer = Some("alpha".into());
14915 let wt = runner.state.candidates[0].worktree.clone();
14916 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14917 runner
14918 .state
14919 .candidate_preservation
14920 .insert('B', "awaiting persistence".into());
14921 let index_lock = git::git(
14922 &wt,
14923 &[
14924 "rev-parse",
14925 "--path-format=absolute",
14926 "--git-path",
14927 "index.lock",
14928 ],
14929 )
14930 .await
14931 .unwrap();
14932 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14933 runner.state.save().unwrap();
14934 assert!(runner.execute_graph().await.is_err());
14935 assert_eq!(runner.state.status, RunStatus::Blocked);
14936 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14937 std::fs::remove_file(index_lock).unwrap();
14938 for _ in 0..2 {
14941 runner.state = RunState::load(&runner.state.id).unwrap();
14942 runner.pause.park();
14943 runner.execute_graph().await.unwrap();
14944 assert_eq!(runner.state.status, RunStatus::Implementing);
14945 assert!(runner.state.candidate_preservation.is_empty());
14946 assert_eq!(runner.state.candidates[0].commits, 2);
14947 assert!(runner.state.tally.is_none());
14948 }
14949 runner.implement().await.unwrap();
14950 assert_eq!(runner.state.status, RunStatus::Judging);
14951 runner.state.status = RunStatus::Blocked;
14953 runner.state.candidates[0].failed = Some("implementation failed".into());
14954 runner.state.save().unwrap();
14955 let error = runner.execute_graph().await.unwrap_err();
14956 assert!(
14957 error
14958 .to_string()
14959 .contains("selected implementation is held")
14960 );
14961 }
14962
14963 #[tokio::test]
14964 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14965 let (_tmp, mut runner) = preservation_resume_fixture().await;
14966 let repo = runner.state.repo.clone();
14967 let candidate = runner.state.candidates[0].clone();
14968 let registered = git::worktree_holding(&repo, &candidate.branch)
14971 .await
14972 .unwrap()
14973 .unwrap();
14974 assert_eq!(
14975 registered.canonicalize().unwrap(),
14976 candidate.worktree.canonicalize().unwrap()
14977 );
14978 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14979 assert_eq!(
14980 git::worktree_holding(&repo, &candidate.branch)
14981 .await
14982 .unwrap(),
14983 Some(registered)
14984 );
14985 assert!(
14986 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14987 "the stale registration initially keeps the branch checked out"
14988 );
14989 runner.fold_losers().await.unwrap();
14990 assert!(runner.state.candidates[0].folded);
14991 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14992 assert_eq!(
14993 git::worktree_holding(&repo, &candidate.branch)
14994 .await
14995 .unwrap(),
14996 None
14997 );
14998 runner.state = RunState::load(&runner.state.id).unwrap();
14999 runner.fold_losers().await.unwrap();
15000 }
15001
15002 #[tokio::test]
15003 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15004 let (_tmp, mut runner) = preservation_resume_fixture().await;
15005 let repo = runner.state.repo.clone();
15006 let candidate = runner.state.candidates[0].clone();
15007 let winner = runner.state.candidates[1].clone();
15008 let registered = git::worktree_holding(&repo, &candidate.branch)
15009 .await
15010 .unwrap()
15011 .unwrap();
15012 assert_eq!(
15013 registered.canonicalize().unwrap(),
15014 candidate.worktree.canonicalize().unwrap()
15015 );
15016 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15017 assert_eq!(
15018 git::worktree_holding(&repo, &candidate.branch)
15019 .await
15020 .unwrap(),
15021 Some(registered)
15022 );
15023 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15024
15025 for _ in 0..2 {
15026 fold_run(&mut runner.state, false, &crate::run::home())
15027 .await
15028 .unwrap();
15029 runner.state = RunState::load(&runner.state.id).unwrap();
15030 assert!(runner.state.candidates[0].folded);
15031 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15032 assert_eq!(
15033 git::worktree_holding(&repo, &candidate.branch)
15034 .await
15035 .unwrap(),
15036 None
15037 );
15038 assert!(!runner.state.candidates[1].folded);
15039 assert!(winner.worktree.exists());
15040 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15041 }
15042 }
15043
15044 #[tokio::test]
15045 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15046 ask_test_home();
15047 let tmp = tempfile::tempdir().unwrap();
15048 let repo = tmp.path().join("repo");
15049 std::fs::create_dir(&repo).unwrap();
15050 init_repo(&repo);
15051 let mut runner = runner_at(RunStatus::Implementing);
15052 runner.state.repo = repo.clone();
15053 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15054 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15055 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15056 let branch = format!("candidate-{label}");
15057 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15058 git::worktree_add_branch(&repo, &wt, &branch, "main")
15059 .await
15060 .unwrap();
15061 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15062 git::git(&wt, &["add", "README.md"]).await.unwrap();
15063 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15064 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15065 runner.state.candidates.push(Candidate {
15066 index,
15067 label,
15068 agent: "alpha".to_owned(),
15069 branch,
15070 worktree: wt,
15071 summary: "completed implementation".to_owned(),
15072 stat: String::new(),
15073 files: 0,
15074 commits: 0,
15075 empty: false,
15076 failed: None,
15077 verified_noop: None,
15078 duration_ms: 42,
15079 folded: false,
15080 });
15081 runner
15082 .state
15083 .candidate_preservation
15084 .insert(label, "awaiting persistence".to_owned());
15085 }
15086 let wt = runner.state.candidates[0].worktree.clone();
15087 let index_lock = git::git(
15088 &wt,
15089 &[
15090 "rev-parse",
15091 "--path-format=absolute",
15092 "--git-path",
15093 "index.lock",
15094 ],
15095 )
15096 .await
15097 .unwrap();
15098 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15099 runner.state.save().unwrap();
15100 assert!(runner.persist_candidates().await.is_err());
15101 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15102 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15103 assert_eq!(
15104 runner.state.candidates[1].commits, 1,
15105 "other results are still collected"
15106 );
15107 assert!(
15108 !runner.state.candidates[0].empty,
15109 "failed rescue is not no change"
15110 );
15111 assert_eq!(
15112 runner.state.candidates[0].summary,
15113 "completed implementation"
15114 );
15115 assert!(runner.after_implement().is_err());
15116 let home = tmp.path().join("fold-home");
15117 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15118 assert!(wt.join("new-file").exists());
15119 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15120 assert!(recovery.join("staged.patch").exists());
15121 assert!(recovery.join("unstaged.patch").exists());
15122 assert_eq!(
15123 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15124 "untracked edits\n"
15125 );
15126 runner.state.begin_pass();
15127 runner.state.end_pass_parked(ParkCause::Upgrade {
15128 from: "1.0".into(),
15129 to: Some("1.1".into()),
15130 });
15131 runner.state.save().unwrap();
15132 runner.state = RunState::load(&runner.state.id).unwrap();
15133 assert!(matches!(
15134 runner.state.passes.last().unwrap().end,
15135 Some(crate::run::PassEnd::Parked {
15136 cause: ParkCause::Upgrade { .. }
15137 })
15138 ));
15139 std::fs::remove_file(index_lock).unwrap();
15140 runner.implement().await.unwrap();
15142 assert!(runner.state.candidate_preservation.is_empty());
15143 assert!(
15144 runner
15145 .state
15146 .candidates
15147 .iter()
15148 .all(|c| c.commits == 1 && !c.empty)
15149 );
15150 assert!(git::is_clean(&wt).await.unwrap());
15151 assert!(
15152 crate::run::read_artifact(&runner.state, "cand-A.patch")
15153 .unwrap()
15154 .contains("later tracked edits")
15155 );
15156 }
15157
15158 #[tokio::test]
15159 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15160 ask_test_home();
15161 let tmp = tempfile::tempdir().unwrap();
15162 let repo = tmp.path().join("repo");
15163 std::fs::create_dir(&repo).unwrap();
15164 init_repo(&repo);
15165 let wt = tmp.path().join("candidate");
15166 git::worktree_add_branch(&repo, &wt, "candidate", "main")
15167 .await
15168 .unwrap();
15169 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15170 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15171 assert!(wt.join("uncommitted").exists());
15172 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15173 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15174 let mut runner = runner_at(RunStatus::Implementing);
15175 runner.state.repo = repo.clone();
15176 runner.state.base_commit = "missing-base".to_owned();
15177 runner.state.candidates.push(Candidate {
15178 index: 0,
15179 label: 'A',
15180 agent: "alpha".into(),
15181 branch: "candidate".into(),
15182 worktree: wt.clone(),
15183 summary: "completed".into(),
15184 stat: String::new(),
15185 files: 0,
15186 commits: 0,
15187 empty: false,
15188 failed: None,
15189 verified_noop: None,
15190 duration_ms: 1,
15191 folded: false,
15192 });
15193 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15194 assert!(
15195 fold_run(&mut runner.state, true, &tmp.path().join("home"))
15196 .await
15197 .is_err()
15198 );
15199 assert!(!runner.state.candidates[0].folded);
15200 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15201 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15202 runner
15203 .state
15204 .candidate_preservation
15205 .insert('A', "pending".into());
15206 assert!(runner.persist_candidates().await.is_err());
15207 assert!(!runner.state.candidates[0].empty);
15208 assert!(!runner.state.candidates[0].folded);
15209 assert!(wt.exists());
15210 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15211 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15212 std::fs::create_dir_all(&patch_path).unwrap();
15213 assert!(
15214 runner.persist_candidates().await.is_err(),
15215 "artifact failure stays pending"
15216 );
15217 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15218 assert!(!runner.state.candidates[0].empty);
15219 std::fs::remove_dir(patch_path).unwrap();
15220 runner.persist_candidates().await.unwrap();
15221 assert!(
15222 runner.state.candidates[0].empty,
15223 "only successful inspection confirms no change"
15224 );
15225 }
15226
15227 fn init_repo(dir: &Path) {
15228 let run = |args: &[&str]| {
15229 let out = std::process::Command::new("git")
15230 .args(args)
15231 .current_dir(dir)
15232 .quiet()
15233 .output()
15234 .expect("spawn git");
15235 assert!(
15236 out.status.success(),
15237 "git {args:?} failed: {}",
15238 String::from_utf8_lossy(&out.stderr)
15239 );
15240 };
15241 run(&["init", "-b", "main"]);
15242 run(&["config", "user.name", "magi test"]);
15243 run(&["config", "user.email", "magi@example.com"]);
15244 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15245 run(&["add", "-A"]);
15246 run(&["commit", "-m", "init"]);
15247 }
15248
15249 fn ask_test_home() {
15257 crate::run::pin_test_home();
15258 }
15259
15260 #[cfg(unix)]
15261 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15262 ask_test_home();
15263 let tmp = tempfile::tempdir().unwrap();
15264 let repo = tmp.path().join("repo");
15265 std::fs::create_dir(&repo).unwrap();
15266 init_repo(&repo);
15267 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15268 let origin = tmp.path().join("origin.git");
15269 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15270 .await
15271 .unwrap();
15272 git::git(
15273 &repo,
15274 &["remote", "add", "origin", origin.to_str().unwrap()],
15275 )
15276 .await
15277 .unwrap();
15278 git::git(&repo, &["checkout", "-b", "fix/retained"])
15279 .await
15280 .unwrap();
15281 std::fs::write(repo.join("retained"), "retained change").unwrap();
15282 git::git(&repo, &["add", "."]).await.unwrap();
15283 git::git(&repo, &["commit", "-m", "retained implementation"])
15284 .await
15285 .unwrap();
15286 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15287 git::git(&repo, &["checkout", "main"]).await.unwrap();
15288 std::fs::write(repo.join("base-change"), "new base").unwrap();
15289 git::git(&repo, &["add", "."]).await.unwrap();
15290 git::git(&repo, &["commit", "-m", "base moved"])
15291 .await
15292 .unwrap();
15293 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15294 git::git(&repo, &["checkout", "fix/retained"])
15295 .await
15296 .unwrap();
15297 let mut runner = runner_at(RunStatus::Voting);
15298 runner.state.repo = repo.clone();
15299 runner.state.base_commit = base;
15300 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15301 runner.state.config.disk.min_free_bytes = 0;
15302 runner.state.config.graph.review_rounds = 1;
15303 runner.state.config.verify.e2e = vec!["true".into()];
15304 runner.state.config.verify.gate = vec!["true".into()];
15305 runner.state.candidates = vec![Candidate {
15306 index: 0,
15307 label: 'A',
15308 agent: "fixture".into(),
15309 branch: "fix/retained".into(),
15310 worktree: repo.clone(),
15311 summary: String::new(),
15312 stat: String::new(),
15313 files: 1,
15314 commits: 1,
15315 empty: false,
15316 failed: None,
15317 verified_noop: None,
15318 duration_ms: 0,
15319 folded: false,
15320 }];
15321 runner.state.tally = Some(Tally {
15322 first_choice: BTreeMap::new(),
15323 borda: BTreeMap::new(),
15324 winner: 'A',
15325 rankings: 0,
15326 unanimous_initial: true,
15327 deliberated: false,
15328 changed_votes: 0,
15329 unanimous_final: true,
15330 tie_break: None,
15331 judges: 0,
15332 present: 0,
15333 quorum: 0,
15334 met_quorum: true,
15335 uncontested: Some("review-only".into()),
15336 });
15337 let mut task = crate::queue::Task::new(
15338 "fixture".into(),
15339 "fixture".into(),
15340 repo,
15341 crate::queue::Source::Human,
15342 );
15343 task.link_run(&runner.state.id);
15344 crate::queue::Queue::open().put(&mut task).unwrap();
15345 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15346 runner.state.base_sync = Some(BaseSync {
15347 tip: String::new(),
15348 behind: 1,
15349 attempts: BASE_SYNC_ROUNDS,
15350 conflict: None,
15351 already_in: None,
15352 });
15353 runner.sync_to_base().await.unwrap();
15354 assert_eq!(runner.state.status, RunStatus::Blocked);
15355 (tmp, runner, head, task.id)
15356 }
15357
15358 #[cfg(unix)]
15359 #[tokio::test]
15360 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15361 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15362 if std::env::var_os(CHILD).is_none() {
15363 let home = tempfile::tempdir().unwrap();
15364 let output = std::process::Command::new(std::env::current_exe().unwrap())
15365 .args([
15366 "--exact",
15367 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15368 "--nocapture",
15369 ])
15370 .env(CHILD, "1")
15371 .env("MAGI_HOME", home.path())
15372 .env("MAGI_WORKER_HOME", home.path())
15373 .env("MAGI_WORKER_ID", "abcdef")
15374 .output()
15375 .unwrap();
15376 assert!(
15377 output.status.success()
15378 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15379 "{}\n{}",
15380 String::from_utf8_lossy(&output.stdout),
15381 String::from_utf8_lossy(&output.stderr)
15382 );
15383 return;
15384 }
15385 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15386 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15387 let home = crate::run::home();
15388 let queue = crate::queue::Queue::open();
15389 let mut owner = queue.get(&task).unwrap();
15390 owner.release_operator("test-release", None);
15391 owner.start(runner.state.id.clone());
15392 queue.put(&mut owner).unwrap();
15393 runner.state.driver_exited = true;
15394 runner.state.task_instruction = Some(owner.instruction.clone());
15395 runner.state.save().unwrap();
15396 let _claim = queue.claim(&task).unwrap();
15397 let reserve = |id: &str| {
15398 let directory = home.join("workers/jobs").join(id);
15399 std::fs::create_dir_all(&directory).unwrap();
15400 let job = crate::worker::Job {
15401 id: id.into(),
15402 sequence: 1,
15403 kind: "task".into(),
15404 key: format!("task:{task}"),
15405 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15406 generation: "fixture-generation".into(),
15407 version: None,
15408 status: crate::worker::Status::Running,
15409 result: None,
15410 pid: Some(std::process::id()),
15411 created_at: 0,
15412 finished_at: None,
15413 };
15414 std::fs::write(
15415 directory.join("job.json"),
15416 serde_json::to_vec(&job).unwrap(),
15417 )
15418 .unwrap();
15419 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15420 };
15421 let _own = reserve("abcdef");
15422 let other = reserve("fedcba");
15423 assert!(
15424 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15425 .await
15426 .is_err()
15427 );
15428 assert!(
15429 RunState::load(&runner.state.id)
15430 .unwrap()
15431 .base_sync_recovery
15432 .authorizations
15433 .is_empty()
15434 );
15435 drop(other);
15436 assert!(crate::daemon::is_working_on(
15437 &home,
15438 &runner.state.id,
15439 Timestamp::now()
15440 ));
15441 assert!(
15442 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15443 );
15444 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15445 .await
15446 .unwrap();
15447 assert!(
15448 RunState::load(&runner.state.id)
15449 .unwrap()
15450 .base_recovery_pending()
15451 );
15452 }
15453
15454 #[cfg(unix)]
15455 #[tokio::test]
15456 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15457 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15458 let home = crate::run::home();
15459 runner.state.driver_pid = Some(std::process::id());
15460 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15461 runner.state.driver_exited = false;
15462 runner.state.begin_pass();
15463 runner.state.save().unwrap();
15464 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15465 driver.record(&runner.state).unwrap();
15466 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15467 drop(driver);
15469 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15470 assert_eq!(
15471 runner.state.liveness_under(&home, false),
15472 crate::run::Liveness::Dead
15473 );
15474 let queue = crate::queue::Queue::open();
15475 let _claim = queue.claim(&task).unwrap();
15476 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15477 .await
15478 .unwrap();
15479 let disk = RunState::load(&runner.state.id).unwrap();
15480 assert!(!disk.driver_exited);
15481 assert!(disk.base_recovery_pending());
15482 }
15483
15484 #[cfg(unix)]
15485 #[tokio::test]
15486 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15487 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15488 if std::env::var_os(CHILD).is_none() {
15489 use std::os::unix::fs::PermissionsExt;
15490 let tmp = tempfile::tempdir().unwrap();
15491 let gh = tmp.path().join("gh");
15492 std::fs::write(&gh, r#"#!/bin/sh
15493case "$1 $2" in
15494 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15495 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15496 *) exit 2 ;;
15497esac
15498"#).unwrap();
15499 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15500 let mut paths = vec![tmp.path().to_path_buf()];
15501 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15502 let output = std::process::Command::new(std::env::current_exe().unwrap())
15503 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15504 .env(CHILD, "1")
15505 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15506 .env("PATH", std::env::join_paths(paths).unwrap())
15507 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15508 .output().unwrap();
15509 assert!(
15510 output.status.success()
15511 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15512 "{}\n{}",
15513 String::from_utf8_lossy(&output.stdout),
15514 String::from_utf8_lossy(&output.stderr)
15515 );
15516 return;
15517 }
15518 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15519 let repo = runner.state.repo.clone();
15520 git::git(&repo, &["checkout", "main"]).await.unwrap();
15521 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15522 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15523 git::git(&repo, &["checkout", "fix/retained"])
15524 .await
15525 .unwrap();
15526 runner.state.pr = Some(crate::run::PrRecord {
15527 url: "https://example.test/project/pull/1".into(),
15528 number: 1,
15529 state: "open".into(),
15530 checks: "unknown".into(),
15531 round: 0,
15532 rounds: 1,
15533 red_at_merge: Vec::new(),
15534 });
15535 runner.sync_to_base().await.unwrap();
15536 assert_eq!(runner.state.status, RunStatus::Blocked);
15537 assert_eq!(
15538 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15539 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15540 );
15541 assert!(runner.state.base_sync_closure_pending());
15542 assert!(crate::handover::can_resume(&runner.state));
15543 let queue = crate::queue::Queue::open();
15544 assert!(!crate::base_sync_recovery::held_task(
15545 &queue,
15546 &queue.get(&task).unwrap()
15547 ));
15548 assert!(runner.state.authorize_base_sync(&task).is_err());
15549 runner.state.base_sync_recovery.stop = None;
15551 runner.state.status = RunStatus::Voting;
15552 runner.state.gate_ran = true;
15553 runner.state.config.graph.review_rounds = 0;
15554 runner.state.save().unwrap();
15555 assert!(crate::handover::can_resume(&runner.state));
15556 runner.state = RunState::load(&runner.state.id).unwrap();
15557 runner.execute_graph().await.unwrap();
15558 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15559 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15560 assert_eq!(
15561 runner.state.base_sync.as_ref().unwrap().attempts,
15562 BASE_SYNC_ROUNDS
15563 );
15564 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15565 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15566 assert!(runner.state.reviews.is_empty());
15567 assert!(runner.state.gate.is_empty());
15568 }
15569
15570 #[cfg(unix)]
15571 #[tokio::test]
15572 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15573 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15574 runner.state.driver_exited = true;
15575 runner.state.save().unwrap();
15576 let home = crate::run::home();
15577 let queue = crate::queue::Queue::open();
15578 let _claim = queue.claim(&task).unwrap();
15579 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15580 assert!(
15581 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15582 .await
15583 .is_err()
15584 );
15585 drop(lease);
15586 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15587 assert!(
15588 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15589 .await
15590 .is_err()
15591 );
15592 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15593 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15594 .await
15595 .unwrap();
15596 let disk = RunState::load(&runner.state.id).unwrap();
15597 assert!(disk.base_recovery_pending());
15598 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15599 assert!(
15600 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15601 .await
15602 .is_err()
15603 );
15604 assert_eq!(
15605 RunState::load(&runner.state.id)
15606 .unwrap()
15607 .base_sync_recovery
15608 .authorizations
15609 .len(),
15610 1
15611 );
15612 let mut owner = queue.get(&task).unwrap();
15613 owner.runs.push("20260101-000000-ffff".into());
15614 queue.put(&mut owner).unwrap();
15615 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15616 .await
15617 .unwrap_err();
15618 assert!(error.to_string().contains("run task ownership changed"));
15619 }
15620
15621 #[cfg(unix)]
15622 #[tokio::test]
15623 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15624 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15625 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15626 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15627 legacy["status"] = serde_json::json!("voting");
15628 runner.state = serde_json::from_value(legacy).unwrap();
15629 runner.state.save().unwrap();
15630 for _ in 0..3 {
15631 runner.execute_graph().await.unwrap();
15632 assert_eq!(runner.state.status, RunStatus::Blocked);
15633 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15634 assert!(!crate::handover::can_resume(&runner.state));
15635 assert_eq!(
15636 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15637 head
15638 );
15639 assert!(runner.state.reviews.is_empty());
15640 assert!(!runner.state.gate_ran);
15641 }
15642 let notice = crate::notices::run_ended(&runner.state).unwrap();
15643 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15644 runner.state.begin_pass();
15645 runner.state.status = RunStatus::Voting;
15646 runner.deliberate().await.unwrap();
15647 runner.vote().await.unwrap();
15648 runner.finish_after_tally().await.unwrap();
15649 assert_eq!(runner.state.status, RunStatus::Blocked);
15650 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15651 assert_eq!(
15652 crate::notices::run_ended(&runner.state).unwrap().since,
15653 notice.since
15654 );
15655 }
15656
15657 #[cfg(unix)]
15658 #[tokio::test]
15659 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15660 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15661 for detail in [
15662 "fix/retained conflicts with origin/main and did not rebase: conflict",
15663 "rebased fix/retained locally but origin refused the push",
15664 "unknown failure",
15665 ] {
15666 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15667 runner.state.record_base_sync_stop(&head);
15668 assert!(runner.state.authorize_base_sync(&task).is_err());
15669 runner.state.status = RunStatus::Voting;
15670 runner.execute_graph().await.unwrap();
15671 assert_eq!(runner.state.status, RunStatus::Blocked);
15672 assert!(!crate::handover::can_resume(&runner.state));
15673 assert_eq!(
15674 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15675 head
15676 );
15677 }
15678 }
15679
15680 #[cfg(unix)]
15681 #[tokio::test]
15682 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15683 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15684 runner.state.authorize_base_sync(&task).unwrap();
15685 assert!(runner.state.consume_base_recovery(&head));
15686 runner.state.save().unwrap();
15687 runner.state = RunState::load(&runner.state.id).unwrap();
15688 runner.execute_graph().await.unwrap();
15689 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15690 assert_eq!(runner.state.status, RunStatus::Blocked);
15691 assert!(!runner.state.consume_base_recovery(&head));
15692 assert!(!crate::handover::can_resume(&runner.state));
15693 runner.state.authorize_base_sync(&task).unwrap();
15694 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15695 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15696 assert_eq!(
15697 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15698 head
15699 );
15700 }
15701
15702 #[cfg(unix)]
15703 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
15704 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
15705 runner.state.prep_complete = Some(true);
15707 runner.state.base_sync = None;
15708 runner.state.base_sync_recovery = Default::default();
15709 runner.state.config.graph.gate_fix_rounds = 1;
15710 let mut review = review_round(true, 0, 1, 1, false, true);
15711 review.head = head.clone();
15712 review.verified_head = Some(head.clone());
15713 runner.state.reviews = vec![review];
15714 runner.state.gate = vec![CommandOutcome {
15715 command: "old gate".into(),
15716 code: Some(1),
15717 output_tail: "old failure".into(),
15718 duration_ms: 0,
15719 resource_blocked: false,
15720 }];
15721 runner.state.gate_head = Some(head.clone());
15722 runner.state.gate_ran = true;
15723 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
15724 agent: "fixture".into(),
15725 failed: runner.state.gate.clone(),
15726 notes: "spent".into(),
15727 committed: false,
15728 error: None,
15729 }];
15730 runner.state.land_approval = Some(crate::run::LandApproval {
15731 question: "old approval".into(),
15732 head: head.clone(),
15733 });
15734 runner.state.status = RunStatus::Blocked;
15735 (tmp, runner, head)
15736 }
15737
15738 #[cfg(unix)]
15739 #[tokio::test]
15740 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
15741 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
15742 let (tmp, mut runner, old) = resumed_gate_fixture().await;
15743 let phases = tmp.path().join("phases");
15744 runner.state.config.verify.e2e = vec![format!(
15745 "echo e2e >> '{}'; {}",
15746 phases.display(),
15747 if e2e_ok {
15748 "true"
15749 } else {
15750 "echo new-e2e-failure; false"
15751 }
15752 )];
15753 runner.state.config.verify.gate = vec![format!(
15754 "echo gate >> '{}'; {}",
15755 phases.display(),
15756 if gate_ok {
15757 "true"
15758 } else {
15759 "echo new-gate-failure; false"
15760 }
15761 )];
15762 runner.state.save().unwrap();
15763 runner.state = RunState::load(&runner.state.id).unwrap();
15764 runner.sync_to_base().await.unwrap();
15765 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15766 assert_ne!(head, old);
15767 runner.state = RunState::load(&runner.state.id).unwrap();
15769 runner.execute_graph().await.unwrap();
15770 assert_eq!(runner.state.gate_fixes.len(), 1);
15771 assert_eq!(runner.state.reviews.len(), 1);
15772 assert_eq!(runner.state.reviews[0].head, old);
15773 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
15774 assert_eq!(runner.state.verification_history.len(), 1);
15775 let history = &runner.state.verification_history[0];
15776 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
15777 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15778 assert!(history.archived_at.is_some());
15779 assert_eq!(history.fixes_spent, Some(1));
15780 assert_eq!(history.gate[0].output_tail, "old failure");
15781 let archive_events: Vec<_> = runner
15782 .state
15783 .events
15784 .iter()
15785 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
15786 .collect();
15787 assert_eq!(archive_events.len(), 1);
15788 assert!(archive_events[0].message.contains(&short(&old)));
15789 assert!(archive_events[0].message.contains(&short(&head)));
15790 assert!(
15791 archive_events[0]
15792 .message
15793 .contains("1 gate-fix round(s) already spent stay spent")
15794 );
15795 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
15796 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
15797 if e2e_ok {
15798 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15799 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
15800 } else {
15801 assert!(!runner.state.gate_ran);
15802 }
15803 assert_eq!(
15804 runner.state.status,
15805 if e2e_ok && gate_ok {
15806 RunStatus::Ready
15807 } else {
15808 RunStatus::Blocked
15809 }
15810 );
15811 if !e2e_ok || !gate_ok {
15812 let cause = runner.state.verification_blocking_cause().unwrap();
15813 assert!(cause.contains(&head));
15814 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
15815 runner.state.event("land", "rebased successfully");
15817 let notice = crate::notices::run_ended(&runner.state).unwrap();
15818 assert!(!format!("{notice:?}").contains("rebased successfully"));
15819 assert!(runner.state.merge.is_none());
15820 }
15821 let before = std::fs::read_to_string(&phases).unwrap();
15822 runner.state.save().unwrap();
15823 runner.state = RunState::load(&runner.state.id).unwrap();
15824 runner.review_loop().await.unwrap();
15825 runner.gate().await.unwrap();
15826 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
15827 assert_eq!(runner.state.verification_history.len(), 1);
15828 let history = &runner.state.verification_history[0];
15829 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15830 assert!(history.archived_at.is_some());
15831 assert_eq!(history.fixes_spent, Some(1));
15832 }
15833 }
15834
15835 #[cfg(unix)]
15836 #[tokio::test]
15837 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
15838 let (tmp, mut runner, head) = resumed_gate_fixture().await;
15839 let marker = tmp.path().join("gate-ran");
15840 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
15841 runner.review_loop().await.unwrap();
15842 runner.gate().await.unwrap();
15843 assert_eq!(runner.state.status, RunStatus::Blocked);
15844 assert!(!marker.exists());
15845 assert!(runner.state.verification_history.is_empty());
15846 runner.state.gate_head = None;
15847 runner.gate().await.unwrap();
15848 assert!(marker.exists());
15849 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15850 assert_eq!(runner.state.verification_history.len(), 1);
15851 assert_eq!(runner.state.gate_fixes.len(), 1);
15852 }
15853
15854 #[cfg(unix)]
15855 #[tokio::test]
15856 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
15857 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
15858 runner.sync_to_base().await.unwrap();
15859 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15860 runner.state.reconcile_verification_head(&head);
15861 for outcomes in [
15863 Vec::new(),
15864 vec![CommandOutcome {
15865 command: "true".into(),
15866 code: None,
15867 output_tail: "cache unavailable".into(),
15868 duration_ms: 0,
15869 resource_blocked: true,
15870 }],
15871 ] {
15872 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
15873 head: head.clone(),
15874 passed: false,
15875 outcomes,
15876 });
15877 runner.state.gate_ran = false;
15878 runner.state.gate.clear();
15879 runner.state.save().unwrap();
15880 runner.state = RunState::load(&runner.state.id).unwrap();
15881 runner.merge().await.unwrap();
15882 assert!(runner.state.merge.is_none());
15883 runner.review_loop().await.unwrap();
15884 runner.gate().await.unwrap();
15885 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
15886 assert!(runner.state.gate_allows_progress());
15887 assert_eq!(runner.state.gate_fixes.len(), 1);
15888 assert!(runner.state.merge.is_none());
15889 }
15890 git::git(
15892 &runner.state.repo,
15893 &["commit", "--allow-empty", "-m", "external update"],
15894 )
15895 .await
15896 .unwrap();
15897 runner.merge().await.unwrap();
15898 assert!(runner.state.merge.is_none());
15899 assert_eq!(runner.state.status, RunStatus::Blocked);
15900 }
15901
15902 #[cfg(unix)]
15903 #[tokio::test]
15904 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
15905 for waiting_on_gate in [false, true] {
15906 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
15907 runner.sync_to_base().await.unwrap();
15908 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15909 runner.state.reconcile_verification_head(&head);
15910 if waiting_on_gate {
15911 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
15912 head: head.clone(),
15913 passed: true,
15914 outcomes: Vec::new(),
15915 });
15916 }
15917 let cache = tmp.path().join("cache");
15918 runner.state.config.verify.e2e =
15919 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15920 runner.state.config.verify.gate = vec!["true".into()];
15921 runner.state.config.graph.timeout_resource_wait = 1;
15922 let owner =
15923 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
15924 let guard =
15925 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
15926 crate::cache::AcquireOutcome::Acquired(g) => g,
15927 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
15928 };
15929 runner.review_loop().await.unwrap();
15930 runner.gate().await.unwrap();
15931 assert!(!runner.state.gate_ran);
15932 assert_eq!(runner.state.gate_fixes.len(), 1);
15933 assert!(runner.state.resource_block.is_some());
15934 runner.merge().await.unwrap();
15935 assert!(runner.state.merge.is_none());
15936 drop(guard);
15937 runner.state = RunState::load(&runner.state.id).unwrap();
15938 runner.state.resource_block = None;
15940 runner.review_loop().await.unwrap();
15941 runner.gate().await.unwrap();
15942 assert!(runner.state.gate_allows_progress());
15943 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15944 assert_eq!(runner.state.gate_fixes.len(), 1);
15945 }
15946 }
15947
15948 #[cfg(unix)]
15949 #[tokio::test]
15950 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
15951 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
15952 runner.sync_to_base().await.unwrap();
15953 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15954 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
15955 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
15957 runner.review_loop().await.unwrap();
15958 runner.gate().await.unwrap();
15959 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15960 assert_ne!(before, after);
15961 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
15962 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
15963 assert!(!runner.state.gate_allows_progress());
15964 runner.merge().await.unwrap();
15965 assert!(runner.state.merge.is_none());
15966 assert!(
15967 runner
15968 .state
15969 .verification_history
15970 .iter()
15971 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
15972 );
15973
15974 runner.state.reviews[0].clean = false;
15976 runner.state.reviews[0].answered = 0;
15977 runner.state.config.verify.e2e = vec!["true".into()];
15978 runner.review_loop().await.unwrap();
15979 runner.gate().await.unwrap();
15980 runner.merge().await.unwrap();
15981 assert_eq!(runner.state.status, RunStatus::Blocked);
15982 assert!(runner.state.merge.is_none());
15983 assert!(
15984 runner
15985 .state
15986 .verification_blocking_cause()
15987 .unwrap()
15988 .contains("review blocked")
15989 );
15990 }
15991
15992 #[cfg(unix)]
15993 #[tokio::test]
15994 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
15995 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
15996 runner.state.gate_head = None;
15997 runner.state.config.verify.gate =
15998 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
15999 runner.review_loop().await.unwrap();
16000 runner.gate().await.unwrap();
16001 assert_ne!(
16002 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16003 old
16004 );
16005 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16006 assert!(!runner.state.gate_allows_progress());
16007 assert_eq!(runner.state.status, RunStatus::Blocked);
16008 runner.merge().await.unwrap();
16009 assert!(runner.state.merge.is_none());
16010 }
16011
16012 #[cfg(unix)]
16013 #[tokio::test]
16014 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16015 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16016 let mut old_review = review_round(true, 0, 1, 1, false, true);
16017 old_review.head = head.clone();
16018 old_review.verified_head = Some(head.clone());
16019 runner.state.reviews = vec![old_review];
16020 runner.state.gate = vec![CommandOutcome {
16021 command: "old gate".into(),
16022 code: None,
16023 output_tail: "timed out after 2400s".into(),
16024 duration_ms: 2_400_000,
16025 resource_blocked: false,
16026 }];
16027 runner.state.gate_ran = true;
16028 runner.state.gate_head = Some(head.clone());
16029 runner.state.land_approval = Some(crate::run::LandApproval {
16030 question: "old-head-approval".into(),
16031 head: head.clone(),
16032 });
16033 runner.state.authorize_base_sync(&task).unwrap();
16034 runner.sync_to_base().await.unwrap();
16035 assert!(!runner.state.base_sync_blocked());
16036 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16037 assert_ne!(head, fresh);
16038 assert!(runner.state.reviews.is_empty());
16039 assert!(!runner.state.gate_ran);
16040 let saved = &runner.state.base_sync_recovery.authorizations[0];
16041 assert_eq!(saved.reviews[0].head, head);
16042 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16043 assert!(saved.gate_ran);
16044 assert!(!saved.gate[0].ok());
16045 runner.review_loop().await.unwrap();
16046 runner.gate().await.unwrap();
16047 assert_eq!(runner.state.reviews[0].head, fresh);
16048 assert_eq!(
16049 runner.state.reviews[0].verified_head.as_deref(),
16050 Some(fresh.as_str())
16051 );
16052 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16053 assert!(runner.state.gate_ran);
16054 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16055 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16056 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16057 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16058 assert_eq!(
16059 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16060 "retained change"
16061 );
16062 }
16063
16064 #[cfg(unix)]
16065 #[tokio::test]
16066 async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16067 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16068 let queue = crate::queue::Queue::open();
16069 let mut owner = queue.get(&task).unwrap();
16070 owner.release_operator("test-resume", None);
16071 queue.put(&mut owner).unwrap();
16072 let generation = owner.resumption.as_ref().unwrap().generation;
16073 runner.state.task_instruction = Some(owner.instruction.clone());
16074 runner.state.driver_exited = true;
16075 runner.state.save().unwrap();
16076 let home = crate::run::home();
16077 let _claim = queue.claim(&task).unwrap();
16078 owner.resumption.as_mut().unwrap().at =
16079 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16080 queue.put(&mut owner).unwrap();
16081 assert!(
16082 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16083 .await
16084 .is_err(),
16085 "an approval from before the stop grants nothing"
16086 );
16087 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16088 queue.put(&mut owner).unwrap();
16089 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16090 .await
16091 .unwrap();
16092 assert!(
16093 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16094 .await
16095 .is_err()
16096 );
16097 runner.state = RunState::load(&runner.state.id).unwrap();
16098 assert_eq!(
16099 runner.state.base_sync_recovery.authorizations[0].generation,
16100 Some(generation)
16101 );
16102 runner.sync_to_base().await.unwrap();
16103 let repo = runner.state.repo.clone();
16104 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16105 git::git(&repo, &["checkout", "main"]).await.unwrap();
16106 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16107 git::git(&repo, &["add", "."]).await.unwrap();
16108 git::git(&repo, &["commit", "-m", "move base again"])
16109 .await
16110 .unwrap();
16111 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16112 git::git(&repo, &["checkout", "fix/retained"])
16113 .await
16114 .unwrap();
16115 runner.finish_after_tally().await.unwrap();
16116 assert_eq!(runner.state.status, RunStatus::Blocked);
16117 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16118 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16119 assert!(!runner.state.base_recovery_pending());
16120 assert!(!crate::handover::can_resume(&runner.state));
16121 assert!(!runner.state.gate_ran);
16122 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16124 queue.put(&mut owner).unwrap();
16125 assert!(
16126 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16127 .await
16128 .is_err()
16129 );
16130 runner.state = RunState::load(&runner.state.id).unwrap();
16131 runner.sync_to_base().await.unwrap();
16132 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16133 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16134 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16135 }
16136
16137 #[cfg(unix)]
16138 #[tokio::test]
16139 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16140 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16141 runner.state.base_sync = None;
16142 runner.state.status = RunStatus::Blocked;
16143 let mut second = runner.state.candidates[0].clone();
16144 second.label = 'B';
16145 runner.state.candidates.push(second);
16146 assert!(runner.state.votes.is_empty());
16147 assert!(runner.state.deliberation.is_empty());
16148 runner.deliberate().await.unwrap();
16149 runner.vote().await.unwrap();
16150 assert_eq!(runner.state.status, RunStatus::Blocked);
16151 assert!(runner.state.votes.is_empty());
16152 assert!(runner.state.deliberation.is_empty());
16153 }
16154
16155 fn runner_at(status: RunStatus) -> Runner {
16158 let mut state = RunState::new(
16159 PathBuf::from("/nonexistent/repo"),
16160 "main".to_owned(),
16161 "deadbeef".to_owned(),
16162 "task".to_owned(),
16163 Config::default(),
16164 );
16165 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16168 state.id = format!(
16169 "20000101-000000-{:04x}",
16170 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16171 );
16172 state.status = status;
16173 Runner {
16174 state,
16175 roles: ResolvedRoles {
16176 implementers: Vec::new(),
16177 judges: Vec::new(),
16178 reviewers: Vec::new(),
16179 fixer: None,
16180 conductor: conductor(),
16181 implementer_roster: Vec::new(),
16182 judge_roster: Vec::new(),
16183 reviewer_roster: Vec::new(),
16184 },
16185 sem: Arc::new(Semaphore::new(1)),
16186 pause: Pause::new(),
16187 interrupt: Pause::new(),
16188 }
16189 }
16190
16191 #[tokio::test]
16192 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16193 crate::run::pin_test_home();
16194 let mut runner = runner_at(RunStatus::Gating);
16195 runner.state.save().unwrap();
16196 let id = runner.state.id.clone();
16197 let passes = runner.state.passes.len();
16198 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16199 let mut claiming = Box::pin(runner.claim_driver());
16200 std::future::poll_fn(|cx| {
16201 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16202 std::task::Poll::Ready(())
16203 })
16204 .await;
16205 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16206 drop(owner);
16207 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16208 .await
16209 .unwrap()
16210 .unwrap();
16211 assert_eq!(execution.runner.state.passes.len(), passes + 1);
16212 assert!(execution.runner.state.resource_block.is_none());
16213 assert_eq!(
16214 execution.runner.state.liveness(false),
16215 crate::run::Liveness::Live
16216 );
16217 drop(execution);
16218 assert!(RunState::load(&id).unwrap().driver_exited);
16219 }
16220
16221 #[tokio::test]
16222 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16223 crate::run::pin_test_home();
16224 let mut runner = runner_at(RunStatus::Gating);
16225 runner.state.save().unwrap();
16226 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16229 assert!(runner.claim_driver().await.is_err());
16230 assert!(runner.state.driver_exited);
16231 assert_eq!(runner.state.driver_pid, None);
16232 let disk = RunState::load(&runner.state.id).unwrap();
16233 assert!(disk.driver_exited);
16234 assert_eq!(disk.driver_pid, None);
16235 assert_eq!(disk.status, RunStatus::Gating);
16236 }
16237
16238 #[tokio::test]
16239 async fn an_early_graph_error_releases_its_driver() {
16240 crate::run::pin_test_home();
16241 let mut runner = runner_at(RunStatus::Blocked);
16242 runner.state.config.graph.selected_implementer = Some("selected".into());
16243 runner.state.save().unwrap();
16244 assert!(runner.execute().await.is_err());
16245 let disk = RunState::load(&runner.state.id).unwrap();
16246 assert!(disk.driver_exited);
16247 assert_eq!(disk.driver_pid, None);
16248 assert_eq!(disk.status, RunStatus::Blocked);
16249 }
16250
16251 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16252 crate::run::MergeRepair {
16253 question: "repair-question".into(),
16254 head: "old-head".into(),
16255 pr: "https://github.com/o/r/pull/42".into(),
16256 reviews: vec![review_round(false, 1, 1, 1, false, true)],
16257 gate: vec![outcome(Some(0), false)],
16258 gate_head: Some("old-head".into()),
16259 gate_fixes: Vec::new(),
16260 pre_gate: Vec::new(),
16261 phase,
16262 failure: None,
16263 }
16264 }
16265
16266 #[tokio::test]
16267 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16268 crate::run::pin_test_home();
16269 let mut runner = runner_at(RunStatus::Landing);
16270 runner.state.merge_repairs.push(merge_repair_request(
16271 crate::run::MergeRepairPhase::Requested,
16272 ));
16273 runner.state.save().unwrap();
16274 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16275 assert!(
16276 runner
16277 .execute()
16278 .await
16279 .unwrap_err()
16280 .to_string()
16281 .contains("already running")
16282 );
16283 assert_eq!(
16284 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16285 crate::run::MergeRepairPhase::Requested
16286 );
16287 drop(claim);
16288 let mut disk = runner.state.clone();
16289 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16290 disk.save().unwrap();
16291 assert!(
16292 runner
16293 .execute()
16294 .await
16295 .unwrap_err()
16296 .to_string()
16297 .contains("decision changed")
16298 );
16299 assert_eq!(
16300 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16301 crate::run::MergeRepairPhase::Complete
16302 );
16303 }
16304
16305 #[test]
16306 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16307 crate::run::pin_test_home();
16308 let mut runner = runner_at(RunStatus::Landing);
16309 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16310 runner.state.reviews = request.reviews.clone();
16311 runner.state.gate = request.gate.clone();
16312 runner.state.gate_ran = true;
16313 runner.state.land_approval = Some(crate::run::LandApproval {
16314 question: "old".into(),
16315 head: "old-head".into(),
16316 });
16317 runner.state.merge_repairs.push(request);
16318 runner.begin_merge_repair_review(0).unwrap();
16319 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16320 assert!(!runner.state.gate_status().ok());
16321 assert!(runner.state.land_approval.is_none());
16322 let disk = RunState::load(&runner.state.id).unwrap();
16323 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16324 assert!(disk.merge_repairs[0].gate[0].ok());
16325 assert_eq!(
16326 disk.merge_repairs[0].phase,
16327 crate::run::MergeRepairPhase::Reviewing
16328 );
16329 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16330 assert!(disk.reviews.is_empty());
16331 }
16332
16333 #[tokio::test]
16334 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16335 crate::run::pin_test_home();
16336 for phase in [
16337 crate::run::MergeRepairPhase::Fixing,
16338 crate::run::MergeRepairPhase::Requested,
16339 ] {
16340 let mut runner = runner_at(RunStatus::Landing);
16341 runner.state.config.graph.land_rounds = 0;
16342 runner.state.merge_repairs.push(merge_repair_request(phase));
16343 runner.state.save().unwrap();
16344 assert!(!runner.repair_merge_review().await.unwrap());
16345 assert_eq!(runner.state.status, RunStatus::Blocked);
16346 assert_eq!(
16347 runner.state.merge_repairs[0].phase,
16348 crate::run::MergeRepairPhase::Failed
16349 );
16350 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16351 assert!(
16352 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16353 "interrupted"
16354 } else {
16355 "budget exhausted"
16356 })
16357 );
16358 assert!(runner.state.merge.is_none());
16359 }
16360 }
16361
16362 #[test]
16366 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16367 crate::run::pin_test_home();
16368 let mut runner = runner_at(RunStatus::Implementing);
16369 let interrupt = Pause::new();
16370 runner.watch_interrupt(interrupt.clone());
16371
16372 interrupt.park_because("task a1b2 asked to run first");
16373
16374 assert!(runner.park_here().expect("park_here"));
16375 assert!(runner.state.parked);
16376 let last = runner.state.events.last().expect("a park event");
16377 assert_eq!(last.node, "park");
16378 assert!(
16379 last.message.contains("task a1b2 asked to run first"),
16380 "expected the interrupt reason in {:?}",
16381 last.message
16382 );
16383 }
16384
16385 #[test]
16388 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16389 crate::run::pin_test_home();
16390 let mut runner = runner_at(RunStatus::Implementing);
16391 runner.state.begin_pass();
16392 let shutdown = Pause::new();
16393 runner.on_pause(shutdown.clone());
16394 let upgrade = ParkCause::Upgrade {
16395 from: "0.121.0".to_owned(),
16396 to: Some("0.121.1".to_owned()),
16397 };
16398 shutdown.park_for(upgrade.clone());
16399 shutdown.park_for(ParkCause::Operator);
16400
16401 assert!(runner.park_here().expect("park_here"));
16402 assert_eq!(
16403 runner.state.passes[0].end,
16404 Some(crate::run::PassEnd::Parked { cause: upgrade })
16405 );
16406 let last = runner.state.events.last().expect("a park event");
16407 assert!(last.message.contains("0.121.0"), "{}", last.message);
16408 }
16409
16410 #[tokio::test]
16411 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16412 crate::run::pin_test_home();
16413 let mut runner = runner_at(RunStatus::Reviewing);
16414 runner.state.save().unwrap();
16415 let mut execution = runner.claim_driver().await.expect("first execution");
16416 assert_eq!(execution.runner.state.passes.len(), 1);
16417 execution
16418 .driver
16419 .finish(&mut execution.runner.state)
16420 .await
16421 .unwrap();
16422 }
16423
16424 #[tokio::test]
16425 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
16426 crate::run::pin_test_home();
16427 let mut runner = runner_at(RunStatus::Reviewing);
16428 runner.state.config.agents = vec![conductor()];
16429 runner.state.config.graph.timeout_resource_wait = 0;
16430 runner.state.save().unwrap();
16431 let id = runner.state.id.clone();
16432 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16433 assert!(runner.claim_driver().await.is_err());
16434 drop(owner);
16435 drop(runner);
16436 let mut resumed = Runner::resume(&id).unwrap();
16437 assert_eq!(resumed.state.driver_claimed, Some(false));
16438 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
16439 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
16440 let mut execution = resumed
16441 .claim_driver()
16442 .await
16443 .expect("retry initial claim after reload");
16444 assert_eq!(execution.runner.state.driver_claimed, Some(true));
16445 assert_eq!(execution.runner.state.passes.len(), 1);
16446 execution
16447 .driver
16448 .finish(&mut execution.runner.state)
16449 .await
16450 .unwrap();
16451 }
16452
16453 #[tokio::test]
16454 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
16455 crate::run::pin_test_home();
16456 let mut runner = runner_at(RunStatus::Reviewing);
16457 runner.state.config.agents = vec![conductor()];
16458 runner.state.save().unwrap();
16459 let id = runner.state.id.clone();
16460 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
16461 let mut resumed = Runner::resume(&id).unwrap();
16462 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
16463 assert!(resumed.claim_driver().await.is_err());
16464 assert!(RunState::load(&id).unwrap().passes.is_empty());
16465 drop(lease);
16466 let mut execution = resumed.claim_driver().await.unwrap();
16467 execution
16468 .driver
16469 .finish(&mut execution.runner.state)
16470 .await
16471 .unwrap();
16472 }
16473
16474 #[tokio::test]
16475 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
16476 crate::run::pin_test_home();
16477 let mut runner = runner_at(RunStatus::Reviewing);
16478 let state = &mut runner.state;
16479 state.config.agents = vec![conductor()];
16480 state.driver_claimed = None;
16481 state.driver_pid = Some(std::process::id());
16482 state.driver_started_at = None;
16483 state.begin_pass();
16484 state.save().unwrap();
16485 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
16486 std::fs::write(
16487 state.dir().join("driver.json"),
16488 serde_json::to_vec(&serde_json::json!({
16489 "pid": state.driver_pid, "started_at": null,
16490 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
16491 }))
16492 .unwrap(),
16493 )
16494 .unwrap();
16495 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
16496 drop(lease);
16497 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
16498 let mut resumed = Runner::resume(&state.id).unwrap();
16499 assert!(!resumed.state.driver_exited);
16500 let mut execution = resumed
16501 .claim_driver()
16502 .await
16503 .expect("lease proves previous execution ended");
16504 assert_eq!(execution.runner.state.passes.len(), 2);
16505 execution
16506 .driver
16507 .finish(&mut execution.runner.state)
16508 .await
16509 .unwrap();
16510 }
16511
16512 #[tokio::test]
16513 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16514 crate::run::pin_test_home();
16515 let mut runner = runner_at(RunStatus::Reviewing);
16516 let mut json = serde_json::to_value(&runner.state).unwrap();
16517 json.as_object_mut().unwrap().remove("driver_claimed");
16518 runner.state = serde_json::from_value(json).unwrap();
16519 assert_eq!(runner.state.driver_claimed, None);
16520 runner.state.save().unwrap();
16521 let mut execution = runner
16522 .claim_driver()
16523 .await
16524 .expect("the execution lease, not a missing marker, excludes a second driver");
16525 assert_eq!(execution.runner.state.passes.len(), 1);
16526 execution
16527 .driver
16528 .finish(&mut execution.runner.state)
16529 .await
16530 .unwrap();
16531 }
16532
16533 #[tokio::test]
16534 async fn driver_admission_refuses_a_proven_live_driver() {
16535 crate::run::pin_test_home();
16536 let mut runner = runner_at(RunStatus::Reviewing);
16537 let pid = std::process::id();
16538 runner.state.driver_pid = Some(pid);
16539 runner.state.driver_started_at = crate::proc::process_started_at(pid);
16540 assert!(runner.state.driver_started_at.is_some());
16541 runner.state.save().unwrap();
16542 let error = runner
16543 .claim_driver()
16544 .await
16545 .err()
16546 .expect("a live driver must refuse");
16547 assert!(error.to_string().contains("live driver"), "{error:#}");
16548 let saved = RunState::load(&runner.state.id).unwrap();
16549 assert_eq!(saved.driver_pid, Some(pid));
16550 assert_eq!(saved.status, RunStatus::Reviewing);
16551 assert!(saved.passes.is_empty());
16552 }
16553
16554 #[test]
16555 fn parking_resource_halt_persists_the_pass_once() {
16556 crate::run::pin_test_home();
16557 let mut runner = runner_at(RunStatus::Reviewing);
16558 runner.state.begin_pass();
16559 runner.state.resource_block = Some(crate::run::ResourceBlock {
16560 node: "fix".into(),
16561 kind: "shared build cache".into(),
16562 detail: "cancelled".into(),
16563 at: Timestamp::now(),
16564 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16565 });
16566 let cause = ParkCause::Upgrade {
16567 from: "old".into(),
16568 to: Some("new".into()),
16569 };
16570 runner.pause.park_for(cause.clone());
16571 runner.stop_for_resource().unwrap();
16574 assert!(runner.resource_halt().unwrap());
16575 assert!(runner.park_here().unwrap());
16576 let saved = RunState::load(&runner.state.id).unwrap();
16577 assert!(saved.parked);
16578 assert_eq!(saved.status, RunStatus::Reviewing);
16579 assert!(saved.passes[0].ended_at.is_some());
16580 assert_eq!(
16581 saved.passes[0].end,
16582 Some(crate::run::PassEnd::Parked {
16583 cause: cause.clone()
16584 })
16585 );
16586 assert_eq!(
16587 saved.resource_block.unwrap().cause,
16588 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16589 );
16590 assert_eq!(
16591 saved
16592 .events
16593 .iter()
16594 .filter(|event| event.node == "park")
16595 .count(),
16596 1
16597 );
16598 }
16599
16600 #[test]
16608 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16609 crate::run::pin_test_home();
16610 let mut runner = runner_at(RunStatus::Implementing);
16611 let shutdown = Pause::new();
16612 runner.on_pause(shutdown.clone());
16613 let interrupt = Pause::new();
16614 runner.watch_interrupt(interrupt.clone());
16615
16616 assert!(!runner.park_here().expect("park_here"));
16618 assert!(!runner.state.parked);
16619
16620 interrupt.park_because("test");
16622 assert!(!shutdown.parked());
16623 assert!(runner.park_here().expect("park_here"));
16624 }
16625
16626 #[tokio::test]
16640 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16641 crate::run::pin_test_home();
16642 let mut runner = runner_at(RunStatus::Implementing);
16643 let interrupt = Pause::new();
16644 runner.watch_interrupt(interrupt.clone());
16645
16646 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16647 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16648
16649 let node = async move {
16653 started_tx.send(()).expect("send started");
16654 finish_rx.await.expect("recv finish");
16655 "node finished"
16656 };
16657
16658 let interrupter = async move {
16659 started_rx.await.expect("recv started");
16660 interrupt.park_because("higher-priority task waiting");
16662 tokio::task::yield_now().await;
16666 finish_tx.send(()).expect("send finish");
16667 };
16668
16669 let (node_result, ()) = tokio::join!(node, interrupter);
16670 assert_eq!(
16671 node_result, "node finished",
16672 "the in-flight call ran to completion"
16673 );
16674
16675 assert!(runner.park_here().expect("park_here"));
16678 assert!(runner.state.parked);
16679 }
16680
16681 #[test]
16687 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16688 crate::run::pin_test_home();
16689 let mut runner = runner_at(RunStatus::Judging);
16690 runner.state.config.agents = vec![conductor()];
16694 runner.state.candidates = vec![Candidate {
16695 index: 0,
16696 label: 'A',
16697 agent: "alpha".to_owned(),
16698 branch: "magi/x/A".to_owned(),
16699 worktree: PathBuf::from("/nonexistent/worktree"),
16700 summary: "did the thing".to_owned(),
16701 stat: "1 file changed".to_owned(),
16702 files: 1,
16703 commits: 1,
16704 empty: false,
16705 failed: None,
16706 verified_noop: None,
16707 duration_ms: 1234,
16708 folded: false,
16709 }];
16710 let run_id = runner.state.id.clone();
16711
16712 let interrupt = Pause::new();
16713 runner.watch_interrupt(interrupt.clone());
16714 interrupt.park_because("task c3d4 asked to run first");
16715 assert!(runner.park_here().expect("park_here"));
16716
16717 let resumed = Runner::resume(&run_id).expect("resume");
16718 assert_eq!(resumed.state.candidates.len(), 1);
16719 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16720 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16721 assert_eq!(resumed.state.status, runner.state.status);
16722 assert!(
16723 resumed.state.parked,
16724 "still parked until `execute` actually walks the graph again"
16725 );
16726 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16727 }
16728
16729 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16731 let mut q = ask::Question::new(
16732 run.to_owned(),
16733 "implement".to_owned(),
16734 "impl-A".to_owned(),
16735 "Which storage backend should the cache use?".to_owned(),
16736 String::new(),
16737 vec!["SQLite".to_owned(), "Redis".to_owned()],
16738 );
16739 store.put(&mut q).unwrap();
16740 q
16741 }
16742
16743 #[test]
16744 fn a_failed_runs_open_question_is_abandoned() {
16745 ask_test_home();
16746 let store = ask::Questions::open();
16747 let mut runner = runner_at(RunStatus::Failed);
16748 let run = runner.state.id.clone();
16749 let q = ask_open_question(&store, &run);
16750
16751 runner.settle_questions();
16752
16753 let back = store.get(&q.id).unwrap();
16754 assert!(
16755 !back.status.open(),
16756 "the seat that asked died with the run; nobody is left to read an answer"
16757 );
16758 assert!(
16759 back.detail.contains(&run) && back.detail.contains("failed"),
16760 "the reason names what the run became, not just that it is gone: {}",
16761 back.detail
16762 );
16763 }
16764
16765 #[test]
16766 fn a_merged_runs_open_question_is_abandoned_too() {
16767 ask_test_home();
16768 let store = ask::Questions::open();
16769 for status in [RunStatus::Merged, RunStatus::Ready] {
16772 let mut runner = runner_at(status);
16773 let run = runner.state.id.clone();
16774 let q = ask_open_question(&store, &run);
16775
16776 runner.settle_questions();
16777
16778 let back = store.get(&q.id).unwrap();
16779 assert!(
16780 !back.status.open(),
16781 "{status:?} run's question must not outlive the run"
16782 );
16783 }
16784 }
16785
16786 #[test]
16787 fn a_still_resumable_runs_open_question_is_left_alone() {
16788 ask_test_home();
16789 let store = ask::Questions::open();
16790 for status in [RunStatus::Blocked, RunStatus::Stalled] {
16796 let mut runner = runner_at(status);
16797 let run = runner.state.id.clone();
16798 let q = ask_open_question(&store, &run);
16799
16800 runner.settle_questions();
16801
16802 let back = store.get(&q.id).unwrap();
16803 assert!(
16804 back.status.open(),
16805 "{status:?} is still alive; the question must still be waiting"
16806 );
16807 }
16808 }
16809
16810 #[test]
16811 fn settle_questions_never_touches_an_already_answered_question() {
16812 ask_test_home();
16813 let store = ask::Questions::open();
16814 let mut runner = runner_at(RunStatus::Failed);
16815 let run = runner.state.id.clone();
16816 let mut q = ask_open_question(&store, &run);
16817 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16818 .unwrap();
16819 store.put(&mut q).unwrap();
16820
16821 runner.settle_questions();
16826 runner.settle_questions();
16827
16828 let back = store.get(&q.id).unwrap();
16829 assert_eq!(
16830 back.status,
16831 ask::QuestionStatus::Answered,
16832 "a real answer is a decision on record, never overwritten by a sweep"
16833 );
16834 }
16835
16836 #[tokio::test]
16847 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16848 crate::run::pin_test_home();
16849 let tmp = tempfile::tempdir().expect("tempdir");
16850 let repo = tmp.path().join("repo");
16851 std::fs::create_dir_all(&repo).unwrap();
16852 init_repo(&repo);
16853
16854 let mut config = Config::default();
16855 config.graph.worktree_root = Some(tmp.path().join("wt"));
16856
16857 let mut state = RunState::new(
16858 repo.clone(),
16859 "main".to_owned(),
16860 "deadbeef".to_owned(),
16861 "task".to_owned(),
16862 config,
16863 );
16864 let root = state.worktree_root();
16865 let wt_a = root.join("cand-A");
16866 let wt_b = root.join("cand-B");
16867 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16868 .await
16869 .expect("worktree A");
16870 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16871 .await
16872 .expect("worktree B");
16873
16874 state.candidates = vec![
16875 Candidate {
16876 index: 0,
16877 label: 'A',
16878 agent: "alpha".to_owned(),
16879 branch: "magi/x/A".to_owned(),
16880 worktree: wt_a.clone(),
16881 summary: String::new(),
16882 stat: String::new(),
16883 files: 0,
16884 commits: 0,
16885 empty: false,
16886 failed: None,
16887 verified_noop: None,
16888 duration_ms: 0,
16889 folded: false,
16890 },
16891 Candidate {
16892 index: 1,
16893 label: 'B',
16894 agent: "beta".to_owned(),
16895 branch: "magi/x/B".to_owned(),
16896 worktree: wt_b.clone(),
16897 summary: String::new(),
16898 stat: String::new(),
16899 files: 0,
16900 commits: 0,
16901 empty: false,
16902 failed: None,
16903 verified_noop: None,
16904 duration_ms: 0,
16905 folded: false,
16906 },
16907 ];
16908 state.tally = Some(Tally {
16909 first_choice: BTreeMap::from([('A', 1)]),
16910 borda: BTreeMap::new(),
16911 winner: 'A',
16912 rankings: 1,
16913 unanimous_initial: true,
16914 deliberated: false,
16915 changed_votes: 0,
16916 unanimous_final: true,
16917 tie_break: None,
16918 judges: 1,
16919 present: 1,
16920 quorum: 1,
16921 met_quorum: true,
16922 uncontested: None,
16923 });
16924 state.status = RunStatus::Ready;
16925
16926 fold_run(&mut state, false, &crate::run::home())
16927 .await
16928 .expect("fold_run");
16929
16930 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
16931 assert!(
16932 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16933 "the unmerged winner's branch survives"
16934 );
16935 assert!(
16936 !state.candidates[0].folded,
16937 "the winner is not marked folded"
16938 );
16939
16940 assert!(!wt_b.exists(), "the loser's worktree is removed");
16941 assert!(
16942 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
16943 "the loser's branch is removed"
16944 );
16945 assert!(state.candidates[1].folded, "the loser is marked folded");
16946 }
16947
16948 #[tokio::test]
16949 #[cfg(unix)]
16950 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
16951 use std::os::fd::AsRawFd as _;
16952 use std::time::Duration;
16953
16954 let tmp = tempfile::tempdir().unwrap();
16955 let repo = tmp.path().join("repo");
16956 std::fs::create_dir_all(&repo).unwrap();
16957 init_repo(&repo);
16958 let common = repo.join(".git");
16959 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
16960 let home = tmp.path().join("home");
16961 let mut config = Config::default();
16962 config.coordination = crate::config::test_coordination();
16963 config.graph.worktree_root = Some(tmp.path().join("wt"));
16964 config.graph.timeout_resource_wait = 1;
16965 let mut state = RunState::new(
16966 repo.clone(),
16967 "main".into(),
16968 "deadbeef".into(),
16969 "task".into(),
16970 config,
16971 );
16972 let root = state.worktree_root();
16973 let candidate = root.join("cand-A");
16974 let reviewer = root.join("review-1");
16975 let branch = state.branch_for('A');
16976 git::worktree_add_branch(&repo, &candidate, &branch, "main")
16977 .await
16978 .unwrap();
16979 git::worktree_add_detached(&repo, &reviewer, "main")
16980 .await
16981 .unwrap();
16982 git::acquire_worktree_config(&repo).await.unwrap();
16983 state.enabled_worktree_config = true;
16984 state.status = RunStatus::Ready;
16985 state.candidates.push(Candidate {
16986 index: 0,
16987 label: 'A',
16988 agent: "implement".into(),
16989 branch: branch.clone(),
16990 worktree: candidate.clone(),
16991 summary: String::new(),
16992 stat: String::new(),
16993 files: 0,
16994 commits: 0,
16995 empty: false,
16996 failed: None,
16997 verified_noop: None,
16998 duration_ms: 0,
16999 folded: false,
17000 });
17001 state.save_under(&home).unwrap();
17002 let config_before = std::fs::read(common.join("config")).unwrap();
17003 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17004 .await
17005 .unwrap();
17006 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17009 assert_eq!(
17010 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17011 0
17012 );
17013 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17014 assert!(
17015 tokio::time::timeout(Duration::from_millis(200), &mut fold)
17016 .await
17017 .is_err()
17018 );
17019 assert!(candidate.exists());
17020 assert!(reviewer.exists());
17021 assert!(git::branch_exists(&repo, &branch).await.unwrap());
17022 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17023 assert_eq!(
17024 git::git(&repo, &["worktree", "list", "--porcelain"])
17025 .await
17026 .unwrap(),
17027 worktrees_before
17028 );
17029 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17030 .await
17031 .unwrap()
17032 .unwrap_err();
17033 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17034 drop(fold);
17035 assert!(candidate.exists() && reviewer.exists());
17036 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17037 assert_eq!(
17038 state.resource_block.as_ref().unwrap().kind,
17039 "repository coordination"
17040 );
17041 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17042 drop(external);
17043 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17044 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17045 .await
17046 .unwrap()
17047 .unwrap();
17048 drop(fold);
17049 assert!(removed.contains(&branch));
17050 assert!(!root.exists());
17051 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17052 assert!(state.candidates[0].folded);
17053 assert!(!state.enabled_worktree_config);
17054 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17055 let enabled = git::git_raw(
17056 &repo,
17057 &["config", "--local", "--get", "extensions.worktreeConfig"],
17058 )
17059 .await
17060 .unwrap();
17061 assert!(!enabled.ok(), "the last config reference was released");
17062 }
17063
17064 #[tokio::test]
17067 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17068 let tmp = tempfile::tempdir().expect("tempdir");
17069 let repo = tmp.path().join("repo");
17070 std::fs::create_dir_all(&repo).unwrap();
17071 init_repo(&repo);
17072 let home = tmp.path().join("home");
17073
17074 let mut config = Config::default();
17075 config.graph.worktree_root = Some(tmp.path().join("wt"));
17076 let mut state = RunState::new(
17077 repo.clone(),
17078 "main".to_owned(),
17079 "deadbeef".to_owned(),
17080 "task".to_owned(),
17081 config,
17082 );
17083 git::git(&repo, &["branch", "magi/x/A", "main"])
17085 .await
17086 .expect("branch");
17087 state.candidates = vec![Candidate {
17088 index: 0,
17089 label: 'A',
17090 agent: "alpha".to_owned(),
17091 branch: "magi/x/A".to_owned(),
17092 worktree: state.worktree_root().join("cand-A"),
17093 summary: String::new(),
17094 stat: String::new(),
17095 files: 0,
17096 commits: 0,
17097 empty: false,
17098 failed: None,
17099 verified_noop: None,
17100 duration_ms: 0,
17101 folded: true,
17102 }];
17103 state.released_to = Some("20260901-000000-new1".to_owned());
17104 state.released_branches = vec!["magi/x/A".to_owned()];
17105
17106 fold_run(&mut state, true, &home).await.expect("fold_run");
17107 state.status = RunStatus::Merged;
17108 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17109 state.merge = Some(crate::run::MergeOutcome {
17110 mode: MergeMode::Pr,
17111 ok: true,
17112 detail: "confirmed merged".into(),
17113 empty: false,
17114 });
17115 fold_run(&mut state, true, &home)
17116 .await
17117 .expect("merged fold");
17118
17119 assert!(
17120 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17121 "the handed-over branch survives a fold"
17122 );
17123 }
17124
17125 #[tokio::test]
17129 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17130 let tmp = tempfile::tempdir().expect("tempdir");
17131 let repo = tmp.path().join("repo");
17132 std::fs::create_dir_all(&repo).unwrap();
17133 init_repo(&repo);
17134 let run = |args: &[&str]| {
17135 let out = std::process::Command::new("git")
17136 .quiet()
17137 .args(args)
17138 .current_dir(&repo)
17139 .output()
17140 .expect("spawn git");
17141 assert!(out.status.success(), "git {args:?}");
17142 };
17143 run(&["branch", "magi/x/A"]);
17144 run(&["checkout", "-q", "-b", "magi/x/B"]);
17145 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17146 run(&["add", "-A"]);
17147 run(&["commit", "-q", "-m", "work"]);
17148 run(&["checkout", "-q", "main"]);
17149
17150 let probe = RunState::new(
17154 repo.clone(),
17155 "main".to_owned(),
17156 "deadbeef".to_owned(),
17157 "task".to_owned(),
17158 Config::default(),
17159 );
17160 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17161 let bare = tmp.path().join("origin.git");
17162 let out = std::process::Command::new("git")
17163 .quiet()
17164 .args(["init", "-q", "--bare"])
17165 .arg(&bare)
17166 .output()
17167 .expect("spawn git");
17168 assert!(out.status.success(), "git init --bare");
17169 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17170 run(&["push", "-q", "origin", "main"]);
17171
17172 let mut state = RunState::new(
17173 repo.clone(),
17174 "main".to_owned(),
17175 "deadbeef".to_owned(),
17176 "task".to_owned(),
17177 Config::default(),
17178 );
17179 state.seeds = vec![refs::Seed {
17180 token: "magi/27b2/A".to_owned(),
17181 kind: refs::SeedKind::Unresolved,
17182 sha: String::new(),
17183 branch: true,
17184 detail: "no branch or commit named magi/27b2/A".to_owned(),
17185 }];
17186
17187 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17188 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17189 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17190 let detail = empty_candidate_detail(&state, "main");
17191 assert!(detail.starts_with("empty candidate"), "{detail}");
17192 assert!(!detail.contains("magi/27b2/A"), "{detail}");
17193 assert!(detail.contains("--start-from"), "{detail}");
17194 }
17195
17196 #[tokio::test]
17205 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17206 let tmp = tempfile::tempdir().expect("tempdir");
17207 let repo = tmp.path().join("repo");
17208 std::fs::create_dir_all(&repo).unwrap();
17209 init_repo(&repo);
17210
17211 let mut config = Config::default();
17212 config.merge.mode = MergeMode::Local;
17213
17214 let mut state = RunState::new(
17215 repo.clone(),
17216 "main".to_owned(),
17217 "deadbeef".to_owned(),
17218 "task".to_owned(),
17219 config,
17220 );
17221 state.candidates = vec![Candidate {
17222 index: 0,
17223 label: 'A',
17224 agent: "alpha".to_owned(),
17225 branch: "does-not-exist".to_owned(),
17226 worktree: repo.clone(),
17227 summary: String::new(),
17228 stat: String::new(),
17229 files: 0,
17230 commits: 0,
17231 empty: false,
17232 failed: None,
17233 verified_noop: None,
17234 duration_ms: 0,
17235 folded: false,
17236 }];
17237 state.tally = Some(Tally {
17238 first_choice: BTreeMap::from([('A', 1)]),
17239 borda: BTreeMap::new(),
17240 winner: 'A',
17241 rankings: 1,
17242 unanimous_initial: true,
17243 deliberated: false,
17244 changed_votes: 0,
17245 unanimous_final: true,
17246 tie_break: None,
17247 judges: 0,
17248 present: 0,
17249 quorum: 0,
17250 met_quorum: true,
17251 uncontested: Some("only candidate A produced a change".to_owned()),
17252 });
17253 state.reviews = vec![ReviewRound {
17254 round: 1,
17255 head: "deadbeef".to_owned(),
17256 verified_head: None,
17257 verified_at: None,
17258 reviews: Vec::new(),
17259 e2e: Vec::new(),
17260 fix: None,
17261 blocking: 0,
17262 answered: 0,
17263 expected: 0,
17264 clean: true,
17265 verify_retried: false,
17266 e2e_deferred: false,
17267 e2e_defer_reason: None,
17268 progressed: false,
17269 vote_split: false,
17270 reconsideration: Vec::new(),
17271 verdict: None,
17272 }];
17273 state.gate = vec![CommandOutcome {
17274 command: "test".to_owned(),
17275 code: Some(0),
17276 output_tail: String::new(),
17277 duration_ms: 0,
17278 resource_blocked: false,
17279 }];
17280 state.gate_ran = true;
17281 state.status = RunStatus::Ready;
17286 state.merge = Some(MergeOutcome {
17287 mode: MergeMode::Local,
17288 ok: false,
17289 detail: "already concluded".to_owned(),
17290 empty: false,
17291 });
17292
17293 let mut runner = Runner {
17294 state,
17295 roles: ResolvedRoles {
17296 implementers: Vec::new(),
17297 judges: Vec::new(),
17298 reviewers: Vec::new(),
17299 fixer: None,
17300 conductor: conductor(),
17301 implementer_roster: Vec::new(),
17302 judge_roster: Vec::new(),
17303 reviewer_roster: Vec::new(),
17304 },
17305 sem: Arc::new(Semaphore::new(1)),
17306 pause: Pause::new(),
17307 interrupt: Pause::new(),
17308 };
17309
17310 runner.merge().await.expect("merge");
17311
17312 assert_eq!(
17313 runner.state.status,
17314 RunStatus::Ready,
17315 "a concluded run's status must not change on reentry"
17316 );
17317 assert_eq!(
17318 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17319 Some("already concluded"),
17320 "merge must not run again once the node already recorded an outcome"
17321 );
17322 }
17323
17324 #[tokio::test]
17333 async fn merge_refuses_a_gate_that_has_not_actually_run() {
17334 let tmp = tempfile::tempdir().expect("tempdir");
17335 let repo = tmp.path().join("repo");
17336 std::fs::create_dir_all(&repo).unwrap();
17337 init_repo(&repo);
17338
17339 let mut config = Config::default();
17340 config.merge.mode = MergeMode::Local;
17341
17342 let mut state = RunState::new(
17343 repo.clone(),
17344 "main".to_owned(),
17345 "deadbeef".to_owned(),
17346 "task".to_owned(),
17347 config,
17348 );
17349 state.candidates = vec![Candidate {
17350 index: 0,
17351 label: 'A',
17352 agent: "alpha".to_owned(),
17353 branch: "does-not-exist".to_owned(),
17354 worktree: repo.clone(),
17355 summary: String::new(),
17356 stat: String::new(),
17357 files: 0,
17358 commits: 0,
17359 empty: false,
17360 failed: None,
17361 verified_noop: None,
17362 duration_ms: 0,
17363 folded: false,
17364 }];
17365 state.tally = Some(Tally {
17366 first_choice: BTreeMap::from([('A', 1)]),
17367 borda: BTreeMap::new(),
17368 winner: 'A',
17369 rankings: 1,
17370 unanimous_initial: true,
17371 deliberated: false,
17372 changed_votes: 0,
17373 unanimous_final: true,
17374 tie_break: None,
17375 judges: 0,
17376 present: 0,
17377 quorum: 0,
17378 met_quorum: true,
17379 uncontested: Some("only candidate A produced a change".to_owned()),
17380 });
17381 state.reviews = vec![ReviewRound {
17382 round: 1,
17383 head: "deadbeef".to_owned(),
17384 verified_head: None,
17385 verified_at: None,
17386 reviews: Vec::new(),
17387 e2e: Vec::new(),
17388 fix: None,
17389 blocking: 0,
17390 answered: 0,
17391 expected: 0,
17392 clean: true,
17393 verify_retried: false,
17394 e2e_deferred: false,
17395 e2e_defer_reason: None,
17396 progressed: false,
17397 vote_split: false,
17398 reconsideration: Vec::new(),
17399 verdict: None,
17400 }];
17401 state.gate = Vec::new();
17403 state.gate_ran = false;
17404 state.status = RunStatus::Gating;
17405
17406 let mut runner = Runner {
17407 state,
17408 roles: ResolvedRoles {
17409 implementers: Vec::new(),
17410 judges: Vec::new(),
17411 reviewers: Vec::new(),
17412 fixer: None,
17413 conductor: conductor(),
17414 implementer_roster: Vec::new(),
17415 judge_roster: Vec::new(),
17416 reviewer_roster: Vec::new(),
17417 },
17418 sem: Arc::new(Semaphore::new(1)),
17419 pause: Pause::new(),
17420 interrupt: Pause::new(),
17421 };
17422
17423 runner.merge().await.expect("merge");
17424
17425 assert!(
17426 runner.state.merge.is_none(),
17427 "an empty gate must never be read as a passing one: {:?}",
17428 runner.state.merge
17429 );
17430 }
17431
17432 #[tokio::test]
17439 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
17440 let tmp = tempfile::tempdir().expect("tempdir");
17441 let repo = tmp.path().join("repo");
17442 std::fs::create_dir_all(&repo).unwrap();
17443 init_repo(&repo);
17444
17445 let config = Config::default();
17447
17448 let mut state = RunState::new(
17449 repo.clone(),
17450 "main".to_owned(),
17451 "deadbeef".to_owned(),
17452 "task".to_owned(),
17453 config,
17454 );
17455 state.candidates = vec![Candidate {
17456 index: 0,
17457 label: 'A',
17458 agent: "alpha".to_owned(),
17459 branch: "does-not-exist".to_owned(),
17460 worktree: repo.clone(),
17461 summary: String::new(),
17462 stat: String::new(),
17463 files: 0,
17464 commits: 0,
17465 empty: false,
17466 failed: None,
17467 verified_noop: None,
17468 duration_ms: 0,
17469 folded: false,
17470 }];
17471 state.tally = Some(Tally {
17472 first_choice: BTreeMap::from([('A', 1)]),
17473 borda: BTreeMap::new(),
17474 winner: 'A',
17475 rankings: 1,
17476 unanimous_initial: true,
17477 deliberated: false,
17478 changed_votes: 0,
17479 unanimous_final: true,
17480 tie_break: None,
17481 judges: 0,
17482 present: 0,
17483 quorum: 0,
17484 met_quorum: true,
17485 uncontested: Some("only candidate A produced a change".to_owned()),
17486 });
17487 state.reviews = vec![ReviewRound {
17488 round: 1,
17489 head: "deadbeef".to_owned(),
17490 verified_head: None,
17491 verified_at: None,
17492 reviews: Vec::new(),
17493 e2e: Vec::new(),
17494 fix: None,
17495 blocking: 0,
17496 answered: 0,
17497 expected: 0,
17498 clean: true,
17499 verify_retried: false,
17500 e2e_deferred: false,
17501 e2e_defer_reason: None,
17502 progressed: false,
17503 vote_split: false,
17504 reconsideration: Vec::new(),
17505 verdict: None,
17506 }];
17507
17508 let mut runner = Runner {
17509 state,
17510 roles: ResolvedRoles {
17511 implementers: Vec::new(),
17512 judges: Vec::new(),
17513 reviewers: Vec::new(),
17514 fixer: None,
17515 conductor: conductor(),
17516 implementer_roster: Vec::new(),
17517 judge_roster: Vec::new(),
17518 reviewer_roster: Vec::new(),
17519 },
17520 sem: Arc::new(Semaphore::new(1)),
17521 pause: Pause::new(),
17522 interrupt: Pause::new(),
17523 };
17524
17525 runner.gate().await.expect("gate");
17526 assert!(
17527 runner.state.gate_ran,
17528 "zero configured commands is still a real attempt, not an unrun gate"
17529 );
17530 assert!(runner.state.gate.is_empty());
17531 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17532 assert_ne!(
17533 runner.state.status,
17534 RunStatus::Blocked,
17535 "a gate with nothing to check must not read as failed"
17536 );
17537
17538 runner.merge().await.expect("merge");
17539 assert_eq!(
17540 runner.state.status,
17541 RunStatus::Ready,
17542 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17543 );
17544 }
17545
17546 #[tokio::test]
17557 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17558 crate::run::pin_test_home();
17559 let home = crate::run::home();
17560
17561 let tmp = tempfile::tempdir().expect("tempdir");
17562 let repo = tmp.path().join("repo");
17563 std::fs::create_dir_all(&repo).unwrap();
17564 init_repo(&repo);
17565 let cache_dir = tmp.path().join("target");
17568
17569 let mut config = Config::default();
17570 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17571 config.graph.timeout_verify = Some(2);
17574 config.graph.timeout_resource_wait = 2;
17575
17576 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17577 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17578 .expect("no io error acquiring directly")
17579 {
17580 crate::cache::AcquireOutcome::Acquired(g) => g,
17581 crate::cache::AcquireOutcome::Busy(b) => {
17582 panic!("expected the direct acquire to win the lease first: {b:?}")
17583 }
17584 };
17585
17586 let mut state = RunState::new(
17587 repo.clone(),
17588 "main".to_owned(),
17589 "deadbeef".to_owned(),
17590 "task".to_owned(),
17591 config,
17592 );
17593 state.candidates = vec![Candidate {
17594 index: 0,
17595 label: 'A',
17596 agent: "alpha".to_owned(),
17597 branch: "does-not-exist".to_owned(),
17598 worktree: repo.clone(),
17599 summary: String::new(),
17600 stat: String::new(),
17601 files: 0,
17602 commits: 0,
17603 empty: false,
17604 failed: None,
17605 verified_noop: None,
17606 duration_ms: 0,
17607 folded: false,
17608 }];
17609 state.tally = Some(Tally {
17610 first_choice: BTreeMap::from([('A', 1)]),
17611 borda: BTreeMap::new(),
17612 winner: 'A',
17613 rankings: 1,
17614 unanimous_initial: true,
17615 deliberated: false,
17616 changed_votes: 0,
17617 unanimous_final: true,
17618 tie_break: None,
17619 judges: 0,
17620 present: 0,
17621 quorum: 0,
17622 met_quorum: true,
17623 uncontested: Some("only candidate A produced a change".to_owned()),
17624 });
17625 state.reviews = vec![ReviewRound {
17626 round: 1,
17627 head: "deadbeef".to_owned(),
17628 verified_head: None,
17629 verified_at: None,
17630 reviews: Vec::new(),
17631 e2e: Vec::new(),
17632 fix: None,
17633 blocking: 0,
17634 answered: 0,
17635 expected: 0,
17636 clean: true,
17637 verify_retried: false,
17638 e2e_deferred: false,
17639 e2e_defer_reason: None,
17640 progressed: false,
17641 vote_split: false,
17642 reconsideration: Vec::new(),
17643 verdict: None,
17644 }];
17645
17646 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
17647 state.reviews[0].e2e = vec![CommandOutcome {
17648 command: state.config.verify.e2e[0].clone(),
17649 code: Some(0),
17650 output_tail: String::new(),
17651 duration_ms: 0,
17652 resource_blocked: false,
17653 }];
17654 let mut runner = Runner {
17655 state,
17656 roles: ResolvedRoles {
17657 implementers: Vec::new(),
17658 judges: Vec::new(),
17659 reviewers: Vec::new(),
17660 fixer: None,
17661 conductor: conductor(),
17662 implementer_roster: Vec::new(),
17663 judge_roster: Vec::new(),
17664 reviewer_roster: Vec::new(),
17665 },
17666 sem: Arc::new(Semaphore::new(1)),
17667 pause: Pause::new(),
17668 interrupt: Pause::new(),
17669 };
17670
17671 let started = std::time::Instant::now();
17672 runner.gate().await.expect("gate");
17673 assert!(
17674 started.elapsed() < Duration::from_secs(1),
17675 "a gate with nothing to run must never wait on a lease it never needed"
17676 );
17677 assert!(
17678 runner.state.gate_ran,
17679 "zero commands is still a real, immediate attempt"
17680 );
17681 assert!(runner.state.gate.is_empty());
17682 assert_ne!(
17683 runner.state.status,
17684 RunStatus::Blocked,
17685 "must not read as resource-blocked on a lease it never asked for"
17686 );
17687 }
17688
17689 #[cfg(unix)]
17692 #[tokio::test]
17693 async fn verification_is_standalone_inside_a_live_worker() {
17694 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17695 if std::env::var_os(CHILD).is_none() {
17696 let home = tempfile::tempdir().unwrap();
17697 let generation = crate::worker::pin_current(home.path()).unwrap();
17698 let output = std::process::Command::new(std::env::current_exe().unwrap())
17699 .args([
17700 "--exact",
17701 "graph::tests::verification_is_standalone_inside_a_live_worker",
17702 "--nocapture",
17703 ])
17704 .env(CHILD, "1")
17705 .env("MAGI_HOME", home.path())
17706 .env("MAGI_WORKER_HOME", home.path())
17707 .env("MAGI_WORKER_ID", "abcdef")
17708 .env("MAGI_WORKER_GENERATION", &generation.id)
17709 .env("MAGI_WORKER_EXE", &generation.executable)
17710 .env("MAGI_RUN", "verification-parent")
17711 .env("MAGI_NODE", "gate")
17712 .env("MAGI_SEAT", "worker-fixture")
17713 .env("MAGI_PROCESS_OWNER", "fixture-owner")
17714 .env("MAGI_NO_AUTOUPDATE", "0")
17715 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17716 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17717 .env(
17718 "VERIFY_EXPECT_TOOLCHAIN",
17719 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17720 )
17721 .env(
17722 "VERIFY_EXPECT_CACHE",
17723 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17724 )
17725 .output()
17726 .unwrap();
17727 assert!(
17728 output.status.success()
17729 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17730 "{}\n{}",
17731 String::from_utf8_lossy(&output.stdout),
17732 String::from_utf8_lossy(&output.stderr)
17733 );
17734 return;
17735 }
17736 let home = crate::run::home();
17737 let directory = home.join("workers/jobs/abcdef");
17738 std::fs::create_dir_all(&directory).unwrap();
17739 let job = crate::worker::Job {
17740 id: "abcdef".into(),
17741 sequence: 1,
17742 kind: "task".into(),
17743 key: "task:verification".into(),
17744 payload: serde_json::json!({}),
17745 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17746 version: Some(env!("CARGO_PKG_VERSION").into()),
17747 status: crate::worker::Status::Running,
17748 result: None,
17749 pid: Some(std::process::id()),
17750 created_at: 0,
17751 finished_at: None,
17752 };
17753 std::fs::write(
17754 directory.join("job.json"),
17755 serde_json::to_vec(&job).unwrap(),
17756 )
17757 .unwrap();
17758 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17759 let assert_worker = || {
17760 assert_eq!(
17761 crate::worker::current_job(&home).unwrap().unwrap().id,
17762 job.id
17763 );
17764 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17765 assert_eq!(
17766 crate::worker::helper_executable(&home).unwrap(),
17767 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17768 );
17769 assert!(
17770 crate::updater::Checker::new(&crate::config::Update {
17771 mode: crate::config::UpdateMode::Notify,
17772 interval: None,
17773 auto_restart: false,
17774 })
17775 .is_none(),
17776 "the driver remains pinned, even while tests are standalone"
17777 );
17778 };
17779 assert_worker();
17780 let repo = home.join("repo");
17781 std::fs::create_dir_all(&repo).unwrap();
17782 init_repo(&repo);
17783 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17786 test -z "${MAGI_WORKER_GENERATION+x}" &&
17787 test -z "${MAGI_WORKER_HOME+x}" &&
17788 test -z "${MAGI_WORKER_EXE+x}" &&
17789 test "$MAGI_HOME/repo" -ef . &&
17790 test "$MAGI_RUN" = verification-parent &&
17791 test "$MAGI_NODE" = gate &&
17792 test "$MAGI_SEAT" = worker-fixture &&
17793 test -n "$MAGI_PROCESS_OWNER" &&
17794 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17795 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17796 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17797 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17798 assert!(
17800 !std::process::Command::new("sh")
17801 .args(["-c", probe])
17802 .current_dir(&repo)
17803 .status()
17804 .unwrap()
17805 .success()
17806 );
17807 let unrelated_home = tempfile::tempdir().unwrap();
17808 assert!(
17809 crate::worker::current_job(unrelated_home.path()).is_err(),
17810 "an inherited worker id cannot be resolved in a unit fixture's home"
17811 );
17812 let mut runner = runner_at(RunStatus::Gating);
17813 runner.state.repo = repo.clone();
17814 runner.state.config.disk.min_free_bytes = 0;
17815 runner.state.config.coordination = crate::config::test_coordination();
17816 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17817 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17818 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17819 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17820 let winner = Candidate {
17821 index: 0,
17822 label: 'A',
17823 agent: "fixture".into(),
17824 branch: "main".into(),
17825 worktree: repo.clone(),
17826 summary: String::new(),
17827 stat: String::new(),
17828 files: 0,
17829 commits: 0,
17830 empty: false,
17831 failed: None,
17832 verified_noop: None,
17833 duration_ms: 0,
17834 folded: false,
17835 };
17836 let (e2e, retried, pids) = run_e2e_with_retry(
17837 &mut runner.state,
17838 &["sh".into(), "-c".into()],
17839 &[format!("{probe} && printf initial")],
17840 &repo,
17841 Duration::from_secs(10),
17842 "initial",
17843 )
17844 .await;
17845 assert!(!retried && pids.is_empty());
17846 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17847 assert_eq!(e2e[0].output_tail, "initial");
17848 let gate = runner.run_gate(&winner).await.unwrap();
17849 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17850 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17853 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17854 passed: false,
17855 outcomes: Vec::new(),
17856 });
17857 runner.state.save().unwrap();
17858 let mut resumed = RunState::load(&runner.state.id).unwrap();
17859 let wait = crate::repo_lock::WaitPolicy::default();
17860 assert!(matches!(
17861 check_gate_repair_e2e(&mut resumed, &winner, &wait)
17862 .await
17863 .unwrap(),
17864 GateFix::Retry
17865 ));
17866 let saved = RunState::load(&resumed.id).unwrap();
17867 let repair = saved.gate_repair_e2e.unwrap();
17868 assert!(repair.passed);
17869 assert_eq!(repair.outcomes.len(), 1);
17870 assert!(repair.outcomes[0].ok());
17871 let landing = run_landing_gate(&mut resumed, &winner, &wait)
17872 .await
17873 .unwrap();
17874 assert_eq!(landing.len(), 1);
17875 assert!(landing[0].ok(), "{landing:?}");
17876 assert_eq!(
17877 std::fs::read_to_string(repo.join("phases")).unwrap(),
17878 "pre\ngate\ne2e\npre\ngate\n"
17879 );
17880 assert_worker();
17881 }
17882
17883 #[cfg(unix)]
17884 #[tokio::test]
17885 async fn verification_keeps_the_panic_and_the_full_output() {
17886 crate::run::pin_test_home();
17887 let tmp = tempfile::tempdir().unwrap();
17888 let mut config = Config::default();
17889 config.disk.min_free_bytes = 0;
17890 let mut state = RunState::new(
17891 tmp.path().to_owned(),
17892 "main".into(),
17893 "base".into(),
17894 "verification evidence".into(),
17895 config,
17896 );
17897 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
17898 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
17899 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
17900 let mut names = Vec::new();
17901 for _ in 0..2 {
17902 let (outcomes, _) = run_commands(
17903 &mut state,
17904 "gate",
17905 "gate",
17906 0,
17907 &["sh".into(), "-c".into()],
17908 &[script.into()],
17909 tmp.path(),
17910 Duration::from_secs(60),
17911 )
17912 .await;
17913 let tail = &outcomes[0].output_tail;
17914 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
17915 assert!(tail.contains("boom assertion"));
17916 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
17917 let line = tail.lines().next().unwrap();
17918 names.push(line.to_owned());
17919 }
17920 assert_ne!(names[0], names[1], "the second run must not overwrite");
17921 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
17922 assert!(full.len() > 500_000 && full.contains("boom assertion"));
17923 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
17924
17925 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
17927 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
17928 let (outcomes, _) = run_commands(
17929 &mut state,
17930 "gate",
17931 "gate",
17932 0,
17933 &["sh".into(), "-c".into()],
17934 &["echo failure-output; exit 7".into()],
17935 tmp.path(),
17936 Duration::from_secs(10),
17937 )
17938 .await;
17939 assert_eq!(outcomes[0].code, Some(7));
17940 assert!(state.events.iter().any(|e| {
17941 e.node == "gate" && e.message.contains("could not save the full gate output:")
17942 }));
17943 }
17944
17945 #[cfg(unix)]
17946 #[tokio::test]
17947 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
17948 crate::run::pin_test_home();
17949 let tmp = tempfile::tempdir().unwrap();
17950 let mut config = Config::default();
17951 config.disk.min_free_bytes = 0;
17952 let mut state = RunState::new(
17953 tmp.path().to_owned(),
17954 "main".into(),
17955 "base".into(),
17956 "verification evidence".into(),
17957 config,
17958 );
17959 let (outcomes, pids) = run_commands(
17960 &mut state,
17961 "gate",
17962 "gate",
17963 1,
17964 &["sh".into(), "-c".into()],
17965 &[
17966 "printf before-timeout; exec sleep 30".into(),
17967 "kill -TERM $$".into(),
17968 "exit 7".into(),
17969 "printf success".into(),
17970 ],
17971 tmp.path(),
17972 Duration::from_secs(1),
17973 )
17974 .await;
17975 wait_for_timed_out_children_to_die(&pids).await;
17976 assert_eq!(
17977 pids.len(),
17978 1,
17979 "unleased timeout retains its death-check PID"
17980 );
17981 assert_eq!(outcomes.len(), 4);
17982 assert_eq!(outcomes[0].code, None);
17983 assert!(outcomes[0].output_tail.contains("before-timeout"));
17984 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
17985 assert_eq!(outcomes[1].code, None);
17986 assert!(outcomes[1].output_tail.contains("command terminated:"));
17987 assert!(!outcomes[1].output_tail.contains("timed out"));
17988 assert_eq!(outcomes[2].code, Some(7));
17989 assert!(!outcomes[2].output_tail.contains("timed out"));
17990 assert_eq!(outcomes[3].code, Some(0));
17991 assert_eq!(outcomes[3].output_tail, "success");
17992 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
17993 }
17994
17995 #[tokio::test]
18005 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18006 crate::run::pin_test_home();
18007
18008 let tmp = tempfile::tempdir().expect("tempdir");
18009 let repo = tmp.path().join("repo");
18010 std::fs::create_dir_all(&repo).unwrap();
18011 init_repo(&repo);
18012
18013 let mut config = Config::default();
18014 config.verify.gate = vec![
18015 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18016 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18017 .to_owned(),
18018 ];
18019
18020 let mut state = RunState::new(
18021 repo.clone(),
18022 "main".to_owned(),
18023 "deadbeef".to_owned(),
18024 "task".to_owned(),
18025 config,
18026 );
18027 let run_id = state.id.clone();
18028 state.candidates = vec![Candidate {
18029 index: 0,
18030 label: 'A',
18031 agent: "alpha".to_owned(),
18032 branch: "does-not-exist".to_owned(),
18033 worktree: repo.clone(),
18034 summary: String::new(),
18035 stat: String::new(),
18036 files: 0,
18037 commits: 0,
18038 empty: false,
18039 failed: None,
18040 verified_noop: None,
18041 duration_ms: 0,
18042 folded: false,
18043 }];
18044 state.tally = Some(Tally {
18045 first_choice: BTreeMap::from([('A', 1)]),
18046 borda: BTreeMap::new(),
18047 winner: 'A',
18048 rankings: 1,
18049 unanimous_initial: true,
18050 deliberated: false,
18051 changed_votes: 0,
18052 unanimous_final: true,
18053 tie_break: None,
18054 judges: 0,
18055 present: 0,
18056 quorum: 0,
18057 met_quorum: true,
18058 uncontested: Some("only candidate A produced a change".to_owned()),
18059 });
18060 state.reviews = vec![ReviewRound {
18061 round: 1,
18062 head: "deadbeef".to_owned(),
18063 verified_head: None,
18064 verified_at: None,
18065 reviews: Vec::new(),
18066 e2e: Vec::new(),
18067 fix: None,
18068 blocking: 0,
18069 answered: 0,
18070 expected: 0,
18071 clean: true,
18072 verify_retried: false,
18073 e2e_deferred: false,
18074 e2e_defer_reason: None,
18075 progressed: false,
18076 vote_split: false,
18077 reconsideration: Vec::new(),
18078 verdict: None,
18079 }];
18080
18081 let mut runner = Runner {
18082 state,
18083 roles: ResolvedRoles {
18084 implementers: Vec::new(),
18085 judges: Vec::new(),
18086 reviewers: Vec::new(),
18087 fixer: None,
18088 conductor: conductor(),
18089 implementer_roster: Vec::new(),
18090 judge_roster: Vec::new(),
18091 reviewer_roster: Vec::new(),
18092 },
18093 sem: Arc::new(Semaphore::new(1)),
18094 pause: Pause::new(),
18095 interrupt: Pause::new(),
18096 };
18097
18098 let started_marker = repo.join("started.marker");
18099 let release_marker = repo.join("release.marker");
18100 let poller = tokio::spawn(async move {
18101 for _ in 0..100 {
18106 if started_marker.exists()
18107 && let Ok(s) = crate::run::RunState::load(&run_id)
18108 && let Some(a) = s.active.get("gate")
18109 {
18110 std::fs::write(&release_marker, b"go").expect("release marker");
18111 return Some(a.clone());
18112 }
18113 tokio::time::sleep(Duration::from_millis(50)).await;
18114 }
18115 None
18116 });
18117
18118 runner.gate().await.expect("gate");
18119 let captured = poller.await.expect("poller task");
18120 let captured = captured.expect(
18121 "the poller never saw a `gate` task entry in run.json while the command was \
18122 still blocked on its own release marker",
18123 );
18124
18125 assert_eq!(captured.task.as_deref(), Some("gate"));
18126 assert_eq!(captured.node, "gate");
18127 assert_eq!(captured.index, Some(1));
18128 assert_eq!(captured.total, Some(1));
18129 assert!(
18130 captured
18131 .command
18132 .as_deref()
18133 .is_some_and(|c| c.contains("started.marker")),
18134 "{captured:?}"
18135 );
18136
18137 assert!(
18138 runner.state.active.is_empty(),
18139 "the entry must be cleared once the command actually finished: {:?}",
18140 runner.state.active
18141 );
18142 assert!(runner.state.gate_ran);
18143 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18144 }
18145
18146 #[tokio::test]
18149 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18150 use crate::verdict::{Finding, ReviewVote, Severity};
18151 crate::run::pin_test_home();
18152 let tmp = tempfile::tempdir().expect("tempdir");
18153 let repo = tmp.path().join("repo");
18154 std::fs::create_dir_all(&repo).unwrap();
18155 init_repo(&repo);
18156
18157 for (severity, vote, expect) in [
18158 (Severity::Major, ReviewVote::Reject, true),
18159 (Severity::Minor, ReviewVote::Reject, false),
18160 (Severity::Major, ReviewVote::Approve, false),
18161 ] {
18162 let mut round = review_round(false, 1, 1, 1, false, true);
18163 round.reviews = vec![ReviewRecord {
18164 reviewer: 1,
18165 agent: "alpha".to_owned(),
18166 summary: String::new(),
18167 findings: vec![Finding {
18168 id: "R1-1-1".to_owned(),
18169 severity,
18170 file: None,
18171 line: None,
18172 title: "t".to_owned(),
18173 detail: String::new(),
18174 }],
18175 vote: Some(vote),
18176 failed: None,
18177 duration_ms: 0,
18178 attempts: 0,
18179 }];
18180 let mut state = RunState::new(
18181 repo.clone(),
18182 "main".to_owned(),
18183 "deadbeef".to_owned(),
18184 "task".to_owned(),
18185 Config::default(),
18186 );
18187 state.reviews = vec![round];
18188 let mut runner = Runner {
18189 state,
18190 roles: ResolvedRoles {
18191 implementers: Vec::new(),
18192 judges: Vec::new(),
18193 reviewers: Vec::new(),
18194 fixer: None,
18195 conductor: conductor(),
18196 implementer_roster: Vec::new(),
18197 judge_roster: Vec::new(),
18198 reviewer_roster: Vec::new(),
18199 },
18200 sem: Arc::new(Semaphore::new(1)),
18201 pause: Pause::new(),
18202 interrupt: Pause::new(),
18203 };
18204 let shell = runner.state.config.shell();
18205 runner
18206 .stop_reviewing("round budget spent", &shell, &repo)
18207 .await
18208 .expect("stop_reviewing");
18209 assert_eq!(runner.state.status, RunStatus::Gating);
18210 assert_eq!(
18211 runner.state.contested_handoff.is_some(),
18212 expect,
18213 "{severity:?} + {vote:?}"
18214 );
18215 }
18216 }
18217
18218 #[tokio::test]
18231 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18232 crate::run::pin_test_home();
18233 let home = crate::run::home();
18234
18235 let tmp = tempfile::tempdir().expect("tempdir");
18236 let repo = tmp.path().join("repo");
18237 std::fs::create_dir_all(&repo).unwrap();
18238 init_repo(&repo);
18239 let head = crate::git::rev_parse(&repo, "HEAD")
18240 .await
18241 .expect("rev-parse");
18242 let cache_dir = tmp.path().join("target");
18245
18246 let mut config = Config::default();
18247 config.verify.e2e = vec![format!(
18248 "CARGO_TARGET_DIR='{}' test -f README.md",
18249 cache_dir.display()
18250 )];
18251 config.graph.review_rounds = 1;
18252 config.graph.timeout_verify = Some(2);
18255 config.graph.timeout_resource_wait = 2;
18256
18257 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18258 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18259 .expect("no io error acquiring directly")
18260 {
18261 crate::cache::AcquireOutcome::Acquired(g) => g,
18262 crate::cache::AcquireOutcome::Busy(b) => {
18263 panic!("expected the direct acquire to win the lease first: {b:?}")
18264 }
18265 };
18266
18267 let mut state = RunState::new(
18268 repo.clone(),
18269 "main".to_owned(),
18270 head.clone(),
18271 "task".to_owned(),
18272 config,
18273 );
18274 state.candidates = vec![Candidate {
18275 index: 0,
18276 label: 'A',
18277 agent: "alpha".to_owned(),
18278 branch: "does-not-exist".to_owned(),
18279 worktree: repo.clone(),
18280 summary: String::new(),
18281 stat: String::new(),
18282 files: 0,
18283 commits: 0,
18284 empty: false,
18285 failed: None,
18286 verified_noop: None,
18287 duration_ms: 0,
18288 folded: false,
18289 }];
18290 state.tally = Some(Tally {
18291 first_choice: BTreeMap::from([('A', 1)]),
18292 borda: BTreeMap::new(),
18293 winner: 'A',
18294 rankings: 1,
18295 unanimous_initial: true,
18296 deliberated: false,
18297 changed_votes: 0,
18298 unanimous_final: true,
18299 tie_break: None,
18300 judges: 0,
18301 present: 0,
18302 quorum: 0,
18303 met_quorum: true,
18304 uncontested: Some("only candidate A produced a change".to_owned()),
18305 });
18306 state.reviews = vec![ReviewRound {
18310 round: 1,
18311 head: head.clone(),
18312 verified_head: None,
18313 verified_at: None,
18314 reviews: Vec::new(),
18315 e2e: Vec::new(),
18316 fix: None,
18317 blocking: 1,
18318 answered: 1,
18319 expected: 1,
18320 clean: false,
18321 verify_retried: false,
18322 e2e_deferred: true,
18323 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18324 progressed: false,
18325 vote_split: false,
18326 reconsideration: Vec::new(),
18327 verdict: None,
18328 }];
18329
18330 let mut runner = Runner {
18331 state,
18332 roles: ResolvedRoles {
18333 implementers: Vec::new(),
18334 judges: Vec::new(),
18335 reviewers: Vec::new(),
18336 fixer: None,
18337 conductor: conductor(),
18338 implementer_roster: Vec::new(),
18339 judge_roster: Vec::new(),
18340 reviewer_roster: Vec::new(),
18341 },
18342 sem: Arc::new(Semaphore::new(1)),
18343 pause: Pause::new(),
18344 interrupt: Pause::new(),
18345 };
18346
18347 let shell = runner.state.config.shell();
18348 runner
18349 .stop_reviewing("round budget spent", &shell, &repo)
18350 .await
18351 .expect("stop_reviewing");
18352
18353 let last = runner.state.reviews.last().expect("round record");
18354 assert_eq!(
18355 last.e2e_status(),
18356 E2eStatus::ResourceBlocked,
18357 "the shared cache is still held; the attempt must read as blocked, not deferred or \
18358 failed: {last:?}"
18359 );
18360 assert_eq!(
18361 last.verified_head.as_deref(),
18362 Some(head.as_str()),
18363 "which commit this attempt targeted is known even though nothing finished checking \
18364 it"
18365 );
18366 let first_attempt_at = last
18367 .verified_at
18368 .expect("when this attempt ran is known too");
18369 assert_ne!(
18370 runner.state.status,
18371 RunStatus::Blocked,
18372 "contention is evidence about the machine, not the patch — it must not settle the \
18373 run as blocked: {:?}",
18374 runner.state.status
18375 );
18376 assert!(
18377 !runner
18378 .state
18379 .events
18380 .iter()
18381 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18382 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18383 runner.state.events
18384 );
18385
18386 runner
18391 .stop_reviewing("round budget spent", &shell, &repo)
18392 .await
18393 .expect("stop_reviewing retry");
18394 assert_eq!(
18395 runner.state.reviews.len(),
18396 1,
18397 "no new round was started: {:?}",
18398 runner.state.reviews
18399 );
18400 let last = runner.state.reviews.last().expect("round record");
18401 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18402 assert!(
18403 last.verified_at.expect("still known") > first_attempt_at,
18404 "a second reentry must be a fresh attempt, not a stale copy of the first"
18405 );
18406 assert_ne!(runner.state.status, RunStatus::Blocked);
18407
18408 held.release();
18409 }
18410
18411 #[cfg(unix)]
18412 #[tokio::test]
18413 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
18414 ask_test_home();
18415 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
18416 let tmp = tempfile::tempdir().unwrap();
18417 let repo = tmp.path().join("repo");
18418 std::fs::create_dir(&repo).unwrap();
18419 init_repo(&repo);
18420 let run_git = |args: &[&str]| {
18421 let out = std::process::Command::new("git")
18422 .args(args)
18423 .current_dir(&repo)
18424 .output()
18425 .unwrap();
18426 assert!(
18427 out.status.success(),
18428 "{args:?}: {}",
18429 String::from_utf8_lossy(&out.stderr)
18430 );
18431 };
18432 let origin = tmp.path().join("origin.git");
18433 run_git(&["init", "--bare", origin.to_str().unwrap()]);
18434 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
18435 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
18436 run_git(&["add", "spec.rs"]);
18437 run_git(&["commit", "-m", "initial spec"]);
18438 run_git(&["push", "origin", "main"]);
18439 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
18440 run_git(&["checkout", "-b", "fix/review"]);
18441 std::fs::write(
18442 repo.join("check.rs"),
18443 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
18444 )
18445 .unwrap();
18446 run_git(&["add", "check.rs"]);
18447 run_git(&["commit", "-m", "use spec"]);
18448 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
18449 run_git(&["checkout", "main"]);
18450 std::fs::write(
18451 repo.join("spec.rs"),
18452 "struct AgentSpec { max_turn_secs: u64 }\n",
18453 )
18454 .unwrap();
18455 run_git(&["add", "spec.rs"]);
18456 run_git(&["commit", "-m", "extend spec"]);
18457 run_git(&["push", "origin", "main"]);
18458 run_git(&["checkout", "fix/review"]);
18459
18460 let mut runner = runner_at(RunStatus::Reviewing);
18461 runner.state.repo = repo.clone();
18462 runner.state.base_commit = base;
18463 runner.state.pr_head = Some("fix/review".into());
18464 runner.state.config.graph.review_rounds = max_rounds;
18465 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
18466 runner.state.config.graph.sessions = false;
18467 runner.state.config.disk.min_free_bytes = 0;
18468 runner.state.config.verify.e2e =
18469 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
18470 runner.state.candidates = vec![Candidate {
18471 index: 0,
18472 label: 'A',
18473 agent: "fixture".into(),
18474 branch: "fix/review".into(),
18475 worktree: repo.clone(),
18476 summary: String::new(),
18477 stat: String::new(),
18478 files: 1,
18479 commits: 1,
18480 empty: false,
18481 failed: None,
18482 verified_noop: None,
18483 duration_ms: 0,
18484 folded: false,
18485 }];
18486 runner.state.tally = Some(Tally {
18487 first_choice: BTreeMap::new(),
18488 borda: BTreeMap::new(),
18489 winner: 'A',
18490 rankings: 0,
18491 unanimous_initial: true,
18492 deliberated: false,
18493 changed_votes: 0,
18494 unanimous_final: true,
18495 tie_break: None,
18496 judges: 0,
18497 present: 0,
18498 quorum: 0,
18499 met_quorum: true,
18500 uncontested: Some("review-only".into()),
18501 });
18502 let mut last = review_round(false, 0, 1, 1, false, false);
18503 last.head = reviewed.clone();
18504 last.e2e_deferred = true;
18505 last.e2e[0].resource_blocked = true;
18506 last.e2e[0].output_tail = "waiting for the shared build cache".into();
18507 if max_rounds == 1 {
18508 last.e2e.clear();
18509 }
18510 runner.state.reviews = vec![last];
18511 let calls = tmp.path().join("fix-calls");
18512 let mut fixer = spec("repair");
18513 fixer
18514 .env
18515 .insert("FIX_CALLS".into(), calls.display().to_string());
18516 fixer.command = vec!["sh".into(), "-c".into(), r#"
18517 echo fix >> "$FIX_CALLS"
18518 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18519 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18520 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18521 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18522 "#.into()];
18523 let mut reviewer = spec("reviewer");
18524 reviewer.command = vec![
18525 "sh".into(),
18526 "-c".into(),
18527 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18528 .into(),
18529 ];
18530 runner.roles.fixer = Some(vec![fixer]);
18531 runner.roles.reviewers = vec![reviewer];
18532
18533 runner.sync_to_base().await.unwrap();
18534 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18535 assert_ne!(rebased, reviewed);
18536 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18537 if reentry {
18538 let shell = runner.state.config.shell();
18541 runner
18542 .stop_unresolved_review("deferred verification", &shell, &repo)
18543 .await
18544 .unwrap();
18545 assert!(runner.catchup_repairable());
18546 assert!(!calls.exists());
18547 runner.state = RunState::load(&runner.state.id).unwrap();
18548 assert!(runner.catchup_repairable());
18549 }
18550 runner.review_loop().await.unwrap();
18551 let first = &runner.state.reviews[0];
18552 assert_eq!(first.head, reviewed);
18553 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18554 assert!(
18555 first.verify_retried,
18556 "a reproduced build failure gets exactly one retry: {:?}",
18557 first.e2e
18558 );
18559 assert!(first.e2e[0].output_tail.contains("E0063"));
18560 assert!(!runner.catchup_repairable());
18561 if max_rounds == 2 {
18562 assert_eq!(
18563 runner.state.status,
18564 RunStatus::Gating,
18565 "{}",
18566 crate::report::run(&runner.state)
18567 );
18568 assert_eq!(runner.state.reviews.len(), 2);
18569 assert!(first.fix.as_ref().unwrap().committed);
18570 assert!(runner.state.reviews[1].clean);
18571 assert_eq!(
18572 runner
18573 .state
18574 .reviews
18575 .iter()
18576 .filter(|r| r.fix.is_some())
18577 .count(),
18578 1
18579 );
18580 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18581 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18582 } else {
18583 assert_eq!(runner.state.status, RunStatus::Blocked);
18584 assert!(first.fix.is_none());
18585 assert!(!calls.exists());
18586 let recovery = crate::recovery::assess(
18587 "prior",
18588 Some(&runner.state),
18589 &runner.landing_base(),
18590 crate::recovery::Relation::Diverged,
18591 );
18592 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18593 let reasons = recovery.reasons.join("\n");
18594 assert!(
18595 reasons.contains(&reviewed) && reasons.contains(&rebased),
18596 "{reasons}"
18597 );
18598 assert!(
18599 reasons.contains("base sync to")
18600 && reasons.contains("additional review is required"),
18601 "{reasons}"
18602 );
18603 }
18604 }
18605 }
18606
18607 #[tokio::test]
18619 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18620 crate::run::pin_test_home();
18621 let home = crate::run::home();
18622
18623 let tmp = tempfile::tempdir().expect("tempdir");
18624 let repo = tmp.path().join("repo");
18625 std::fs::create_dir_all(&repo).unwrap();
18626 init_repo(&repo);
18627 let head = crate::git::rev_parse(&repo, "HEAD")
18628 .await
18629 .expect("rev-parse");
18630 let cache_dir = tmp.path().join("target");
18631
18632 let mut config = Config::default();
18633 config.verify.e2e = vec![format!(
18634 "CARGO_TARGET_DIR='{}' test -f README.md",
18635 cache_dir.display()
18636 )];
18637 config.graph.review_rounds = 1;
18638 config.graph.timeout_verify = Some(2);
18639 config.graph.timeout_resource_wait = 2;
18640
18641 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18642 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18643 .expect("no io error acquiring directly")
18644 {
18645 crate::cache::AcquireOutcome::Acquired(g) => g,
18646 crate::cache::AcquireOutcome::Busy(b) => {
18647 panic!("expected the direct acquire to win the lease first: {b:?}")
18648 }
18649 };
18650
18651 let mut state = RunState::new(
18652 repo.clone(),
18653 "main".to_owned(),
18654 head.clone(),
18655 "task".to_owned(),
18656 config,
18657 );
18658 state.candidates = vec![Candidate {
18659 index: 0,
18660 label: 'A',
18661 agent: "alpha".to_owned(),
18662 branch: "does-not-exist".to_owned(),
18663 worktree: repo.clone(),
18664 summary: String::new(),
18665 stat: String::new(),
18666 files: 0,
18667 commits: 0,
18668 empty: false,
18669 failed: None,
18670 verified_noop: None,
18671 duration_ms: 0,
18672 folded: false,
18673 }];
18674 state.tally = Some(Tally {
18675 first_choice: BTreeMap::from([('A', 1)]),
18676 borda: BTreeMap::new(),
18677 winner: 'A',
18678 rankings: 1,
18679 unanimous_initial: true,
18680 deliberated: false,
18681 changed_votes: 0,
18682 unanimous_final: true,
18683 tie_break: None,
18684 judges: 0,
18685 present: 0,
18686 quorum: 0,
18687 met_quorum: true,
18688 uncontested: Some("only candidate A produced a change".to_owned()),
18689 });
18690 state.reviews = vec![ReviewRound {
18694 round: 1,
18695 head: head.clone(),
18696 verified_head: Some(head.clone()),
18697 verified_at: Some(jiff::Timestamp::now()),
18698 reviews: Vec::new(),
18699 e2e: vec![CommandOutcome {
18700 command: format!(
18701 "CARGO_TARGET_DIR='{}' test -f README.md",
18702 cache_dir.display()
18703 ),
18704 code: None,
18705 output_tail: "waiting for the shared build cache".to_owned(),
18706 duration_ms: 0,
18707 resource_blocked: true,
18708 }],
18709 fix: None,
18710 blocking: 1,
18711 answered: 1,
18712 expected: 1,
18713 clean: false,
18714 verify_retried: false,
18715 e2e_deferred: false,
18716 e2e_defer_reason: None,
18717 progressed: false,
18718 vote_split: false,
18719 reconsideration: Vec::new(),
18720 verdict: None,
18721 }];
18722
18723 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18724 let mut runner = Runner {
18725 state,
18726 roles: ResolvedRoles {
18727 implementers: Vec::new(),
18728 judges: Vec::new(),
18729 reviewers: Vec::new(),
18730 fixer: None,
18731 conductor: conductor(),
18732 implementer_roster: Vec::new(),
18733 judge_roster: Vec::new(),
18734 reviewer_roster: Vec::new(),
18735 },
18736 sem: Arc::new(Semaphore::new(1)),
18737 pause: Pause::new(),
18738 interrupt: Pause::new(),
18739 };
18740
18741 runner.review_loop().await.expect("review_loop");
18746
18747 assert_eq!(
18748 runner.state.reviews.len(),
18749 1,
18750 "no new round was started on top of the unresolved one: {:?}",
18751 runner.state.reviews
18752 );
18753 let last = &runner.state.reviews[0];
18754 assert_eq!(
18755 last.e2e_status(),
18756 E2eStatus::ResourceBlocked,
18757 "still contended: {last:?}"
18758 );
18759 assert!(
18760 last.verified_at.expect("still known") > first_attempt_at,
18761 "review_loop must have actually retried the check, not left it exactly as found"
18762 );
18763 assert_ne!(
18764 runner.state.status,
18765 RunStatus::Blocked,
18766 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18767 runner.state.status
18768 );
18769
18770 held.release();
18771 }
18772
18773 #[tokio::test]
18774 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18775 crate::run::pin_test_home();
18776 for status in [RunStatus::Blocked, RunStatus::Gating] {
18777 let mut runner = runner_at(status);
18778 runner.state.config.graph.land = true;
18779 runner.state.gate_ran = true;
18780 runner.state.merge = Some(MergeOutcome {
18781 mode: MergeMode::Pr,
18782 ok: false,
18783 detail: "the fixer produced no commit".into(),
18784 empty: false,
18785 });
18786 let url = "https://example.invalid/o/r/pull/1";
18787 runner.state.pr = Some(crate::run::PrRecord {
18788 url: url.into(),
18789 number: 1,
18790 state: "open".into(),
18791 checks: "red".into(),
18792 round: 1,
18793 rounds: 1,
18794 red_at_merge: Vec::new(),
18795 });
18796 runner.state.save().unwrap();
18797 runner.execute_graph().await.unwrap();
18801 assert_eq!(runner.state.status, RunStatus::Blocked);
18802 assert!(runner.state.land_sync_pending);
18803 assert!(runner.state.events.iter().any(|event|
18804 event.node == "land" && event.message == format!("watching {url}")));
18805 assert_eq!(
18806 runner.state.merge.as_ref().unwrap().detail,
18807 "the fixer produced no commit"
18808 );
18809 }
18810 }
18811
18812 #[tokio::test]
18813 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18814 crate::run::pin_test_home();
18815 let tmp = tempfile::tempdir().expect("tempdir");
18816 let repo = tmp.path().join("repo");
18817 std::fs::create_dir_all(&repo).unwrap();
18818 init_repo(&repo);
18819
18820 let mut config = Config::default();
18821 config.merge.mode = MergeMode::Pr;
18822 config.graph.land = true;
18823 config.graph.land_approval = false;
18824
18825 let mut state = RunState::new(
18826 repo.clone(),
18827 "main".to_owned(),
18828 "deadbeef".to_owned(),
18829 "task".to_owned(),
18830 config,
18831 );
18832 state.candidates = vec![Candidate {
18833 index: 0,
18834 label: 'A',
18835 agent: "alpha".to_owned(),
18836 branch: "does-not-exist".to_owned(),
18837 worktree: repo.clone(),
18838 summary: String::new(),
18839 stat: String::new(),
18840 files: 0,
18841 commits: 0,
18842 empty: false,
18843 failed: None,
18844 verified_noop: None,
18845 duration_ms: 0,
18846 folded: false,
18847 }];
18848 state.tally = Some(Tally {
18849 first_choice: BTreeMap::from([('A', 1)]),
18850 borda: BTreeMap::new(),
18851 winner: 'A',
18852 rankings: 1,
18853 unanimous_initial: true,
18854 deliberated: false,
18855 changed_votes: 0,
18856 unanimous_final: true,
18857 tie_break: None,
18858 judges: 0,
18859 present: 0,
18860 quorum: 0,
18861 met_quorum: true,
18862 uncontested: Some("only candidate A produced a change".to_owned()),
18863 });
18864 state.reviews = vec![ReviewRound {
18865 round: 1,
18866 head: "deadbeef".to_owned(),
18867 verified_head: None,
18868 verified_at: None,
18869 reviews: Vec::new(),
18870 e2e: Vec::new(),
18871 fix: None,
18872 blocking: 0,
18873 answered: 0,
18874 expected: 0,
18875 clean: true,
18876 verify_retried: false,
18877 e2e_deferred: false,
18878 e2e_defer_reason: None,
18879 progressed: false,
18880 vote_split: false,
18881 reconsideration: Vec::new(),
18882 verdict: None,
18883 }];
18884 state.gate = vec![CommandOutcome {
18885 command: "test".to_owned(),
18886 code: Some(0),
18887 output_tail: String::new(),
18888 duration_ms: 0,
18889 resource_blocked: false,
18890 }];
18891 state.gate_ran = true;
18892 state.status = RunStatus::Landing;
18896 state.merge = Some(MergeOutcome {
18897 mode: MergeMode::Pr,
18898 ok: true,
18899 detail: "https://example.invalid/x/y/pull/1".to_owned(),
18900 empty: false,
18901 });
18902
18903 ask_test_home();
18907 state.save().expect("save landing run");
18910 let store = ask::Questions::open();
18911 let q = ask_open_question(&store, &state.id);
18912
18913 let mut runner = Runner {
18914 state,
18915 roles: ResolvedRoles {
18916 implementers: Vec::new(),
18917 judges: Vec::new(),
18918 reviewers: Vec::new(),
18919 fixer: None,
18920 conductor: conductor(),
18921 implementer_roster: Vec::new(),
18922 judge_roster: Vec::new(),
18923 reviewer_roster: Vec::new(),
18924 },
18925 sem: Arc::new(Semaphore::new(1)),
18926 pause: Pause::new(),
18927 interrupt: Pause::new(),
18928 };
18929
18930 runner.execute().await.expect("execute");
18935
18936 assert_eq!(
18937 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
18938 Some("https://example.invalid/x/y/pull/1"),
18939 "reentry must not push again or open a second pull request over the \
18940 one `land` is already watching"
18941 );
18942 assert_ne!(
18943 runner.state.status,
18944 RunStatus::Landing,
18945 "land could not actually reach the fake pull request, so it must \
18946 have given up rather than left the run silently parked forever"
18947 );
18948 assert_eq!(runner.state.status, RunStatus::Blocked);
18952 assert!(
18953 store.get(&q.id).unwrap().status.open(),
18954 "Blocked is still alive; settle_questions must have been a no-op here"
18955 );
18956 }
18957
18958 fn state_with_round(round: ReviewRound) -> RunState {
18959 let mut s = RunState::new(
18960 PathBuf::from("/repo"),
18961 "main".to_owned(),
18962 "abc1234".to_owned(),
18963 "add retries".to_owned(),
18964 Config::default(),
18965 );
18966 s.reviews = vec![round];
18967 s
18968 }
18969
18970 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
18971 crate::verdict::Finding {
18972 id: id.to_owned(),
18973 severity,
18974 file: None,
18975 line: None,
18976 title: title.to_owned(),
18977 detail: String::new(),
18978 }
18979 }
18980
18981 #[test]
18982 fn pr_body_names_open_findings_and_declined_ones() {
18983 let round = ReviewRound {
18984 round: 2,
18985 head: "deadbee".to_owned(),
18986 verified_head: None,
18987 verified_at: None,
18988 reviews: vec![ReviewRecord {
18989 attempts: 0,
18990 reviewer: 1,
18991 agent: "alpha".to_owned(),
18992 summary: String::new(),
18993 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
18994 vote: None,
18995 failed: None,
18996 duration_ms: 0,
18997 }],
18998 e2e: vec![CommandOutcome {
18999 command: "cargo test".to_owned(),
19000 code: Some(0),
19001 output_tail: String::new(),
19002 duration_ms: 0,
19003 resource_blocked: false,
19004 }],
19005 verify_retried: false,
19006 e2e_deferred: false,
19007 e2e_defer_reason: None,
19008 fix: Some(FixRecord {
19009 agent: "alpha".to_owned(),
19010 addressed: Vec::new(),
19011 rejected: vec![crate::verdict::Rejection {
19012 id: "R1-1-1".to_owned(),
19013 why: "not reachable from any caller".to_owned(),
19014 }],
19015 notes: String::new(),
19016 committed: true,
19017 failed: None,
19018 duration_ms: 0,
19019 continuation: None,
19020 }),
19021 blocking: 0,
19022 answered: 1,
19023 expected: 1,
19024 clean: false,
19025 progressed: true,
19026 vote_split: false,
19027 reconsideration: Vec::new(),
19028 verdict: None,
19029 };
19030 let state = state_with_round(round);
19031 let body = pr_message(&state, 'A').body;
19032
19033 assert!(body.contains("add retries"), "the task must still be there");
19034 assert!(body.contains("R2-1-1"), "{body}");
19035 assert!(!body.contains("unused import"), "{body}");
19036 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19037 assert!(
19038 !body.contains("not reachable from any caller"),
19039 "the reason it was declined: {body}"
19040 );
19041 }
19042
19043 #[test]
19044 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19045 let round = ReviewRound {
19046 round: 1,
19047 head: "deadbee".to_owned(),
19048 verified_head: None,
19049 verified_at: None,
19050 reviews: vec![ReviewRecord {
19051 attempts: 0,
19052 reviewer: 1,
19053 agent: "alpha".to_owned(),
19054 summary: String::new(),
19055 findings: Vec::new(),
19056 vote: None,
19057 failed: None,
19058 duration_ms: 0,
19059 }],
19060 e2e: Vec::new(),
19061 verify_retried: false,
19062 e2e_deferred: false,
19063 e2e_defer_reason: None,
19064 fix: None,
19065 blocking: 0,
19066 answered: 1,
19067 expected: 1,
19068 clean: true,
19069 progressed: false,
19070 vote_split: false,
19071 reconsideration: Vec::new(),
19072 verdict: None,
19073 };
19074 let state = state_with_round(round);
19075 let body = pr_message(&state, 'A').body;
19076 assert!(!body.contains("Open review findings"), "{body}");
19077 assert!(!body.contains("Declined"), "{body}");
19078 }
19079
19080 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19081 let mut state = RunState::new(
19082 PathBuf::from("/repo"),
19083 "main".to_owned(),
19084 "abc1234".to_owned(),
19085 instruction.to_owned(),
19086 Config::default(),
19087 );
19088 state.candidates.push(Candidate {
19089 index: 0,
19090 label: 'A',
19091 agent: "alpha".to_owned(),
19092 branch: "magi/x/A".to_owned(),
19093 worktree: PathBuf::from("/wt"),
19094 summary: summary.to_owned(),
19095 stat: String::new(),
19096 files: 1,
19097 commits: 1,
19098 empty: false,
19099 failed: None,
19100 verified_noop: None,
19101 folded: false,
19102 duration_ms: 0,
19103 });
19104 state
19105 }
19106
19107 fn review_state(subjects: &[&str]) -> RunState {
19108 let mut state = state_with_summary(
19109 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19110 "",
19111 );
19112 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19113 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19114 state
19115 }
19116
19117 #[tokio::test]
19120 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19121 ask_test_home();
19122 let tmp = tempfile::tempdir().unwrap();
19123 let repo = tmp.path().join("repo");
19124 std::fs::create_dir_all(&repo).unwrap();
19125 init_repo(&repo);
19126 let origin = tmp.path().join("origin.git");
19127 let g = |dir: &Path, args: &[&str]| {
19128 let out = std::process::Command::new("git")
19129 .args(args)
19130 .current_dir(dir)
19131 .quiet()
19132 .output()
19133 .expect("spawn git");
19134 assert!(
19135 out.status.success(),
19136 "git {args:?}: {}",
19137 String::from_utf8_lossy(&out.stderr)
19138 );
19139 };
19140 g(
19141 tmp.path(),
19142 &[
19143 "clone",
19144 "--bare",
19145 "-q",
19146 repo.to_str().unwrap(),
19147 origin.to_str().unwrap(),
19148 ],
19149 );
19150 g(
19151 &repo,
19152 &["remote", "add", "origin", origin.to_str().unwrap()],
19153 );
19154 let c1 = git::rev_parse(&repo, "main").await.unwrap();
19155
19156 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19158 g(&repo, &["add", "-A"]);
19159 g(
19160 &repo,
19161 &["commit", "-q", "-m", "chore(deps): update a crate"],
19162 );
19163 g(&repo, &["push", "-q", "origin", "main"]);
19164 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19165 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19166 g(&repo, &["add", "-A"]);
19167 g(
19168 &repo,
19169 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19170 );
19171 g(&repo, &["checkout", "-q", "main"]);
19172
19173 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19174 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19175 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19177 assert_eq!(fallback, c1);
19178
19179 let mut state = review_state(&[
19180 "chore(deps): update a crate",
19181 "fix(daemon): apply a chosen action",
19182 ]);
19183 state.repo = repo.clone();
19184 state.base_branch = "main".to_owned();
19185 state.base_commit = c1;
19186 refresh_reviewed_commits(&mut state, "feat/own").await;
19187 assert_eq!(
19188 state.reviewed_commits,
19189 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19190 );
19191 assert_eq!(
19192 review_title(&state).as_deref(),
19193 Some("fix(daemon): apply a chosen action")
19194 );
19195 assert_eq!(
19196 leaked_subjects(&state, "feat/own").await,
19197 Some(vec!["chore(deps): update a crate".to_owned()])
19198 );
19199 state.config.merge.remote = "nowhere".to_owned();
19202 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19203 }
19204
19205 struct SummaryFixture {
19209 _tmp: tempfile::TempDir,
19210 repo: PathBuf,
19211 reply: PathBuf,
19212 runner: Runner,
19213 }
19214
19215 fn summary_fixture() -> SummaryFixture {
19216 ask_test_home();
19217 let tmp = tempfile::tempdir().unwrap();
19218 let repo = tmp.path().join("repo");
19219 std::fs::create_dir_all(&repo).unwrap();
19220 init_repo(&repo);
19221 let origin = tmp.path().join("origin.git");
19222 let g = |args: &[&str]| {
19223 let out = std::process::Command::new("git")
19224 .args(args)
19225 .current_dir(&repo)
19226 .quiet()
19227 .output()
19228 .expect("spawn git");
19229 assert!(
19230 out.status.success(),
19231 "git {args:?}: {}",
19232 String::from_utf8_lossy(&out.stderr)
19233 );
19234 };
19235 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19236 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19237 g(&["push", "-q", "origin", "main"]);
19238 g(&["checkout", "-q", "-b", "feat/own"]);
19239 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19240 g(&["add", "-A"]);
19241 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19242 let reply = tmp.path().join("reply.txt");
19243 std::fs::write(
19244 &reply,
19245 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19246 )
19247 .unwrap();
19248 let mut runner = runner_at(RunStatus::Gating);
19249 let mut state = review_state(&["feat(own): add the own file"]);
19250 state.repo = repo.clone();
19251 state.base_branch = "main".to_owned();
19252 state.candidates[0].branch = "feat/own".to_owned();
19253 state.candidates[0].worktree = repo.clone();
19254 let mut author = spec("alpha");
19255 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19256 author
19257 .env
19258 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19259 state.config.agents = vec![author];
19260 runner.state = state;
19261 SummaryFixture {
19262 _tmp: tmp,
19263 repo,
19264 reply,
19265 runner,
19266 }
19267 }
19268
19269 async fn posted(f: &mut SummaryFixture) -> PrMessage {
19270 let winner = f.runner.state.candidates[0].clone();
19271 f.runner
19272 .guarded_pr_message(&winner, None, true)
19273 .await
19274 .unwrap()
19275 .expect("an English description never parks")
19276 }
19277
19278 fn clean_round_with_minor_findings() -> ReviewRound {
19279 let mut round = passing_round(2, None);
19280 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19281 round.reviews = vec![ReviewRecord {
19282 attempts: 0,
19283 reviewer: 1,
19284 agent: "alpha".into(),
19285 summary: "内部レビュー".into(),
19286 findings: vec![
19287 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19288 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19289 ],
19290 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19291 failed: None,
19292 duration_ms: 0,
19293 }];
19294 round
19295 }
19296
19297 #[tokio::test]
19298 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19299 for review_only in [false, true] {
19300 let mut f = summary_fixture();
19301 if !review_only {
19302 f.runner.state.candidates[0].agent = "alpha".into();
19303 }
19304 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19305 assert!(f.runner.state.reviews[0].clean);
19306 assert!(f.runner.state.reviews[0].fix.is_none());
19307 assert!(f.runner.state.open_findings().is_empty());
19308 assert_eq!(
19309 crate::pr_outcome::required_finding_ids(&f.runner.state),
19310 ["R2-1-1", "R2-1-2"]
19311 );
19312 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19313 let evidence = crate::pr_outcome::summary_evidence(
19314 &f.runner.state,
19315 crate::pr_outcome::FinalHead::At(&head),
19316 );
19317 assert!(
19318 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19319 );
19320 let omitted = posted(&mut f).await;
19322 assert!(omitted.body.contains("No description"));
19323 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19324 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19325 assert!(f.runner.state.change_summary.is_none());
19326 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19327 let described = posted(&mut f).await;
19328 assert!(described.body.contains("improve documentation for callers"));
19329 assert!(described.body.contains("remaining edge case"));
19330 assert!(described.body.contains("## Open review findings"));
19331 assert!(!described.body.contains("未修正"));
19332 }
19333 }
19334
19335 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19336 let mut f = summary_fixture();
19337 if !review_only {
19338 f.runner.state.candidates[0].agent = "alpha".into();
19339 }
19340 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19341 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19342 fixed
19343 .fix
19344 .as_mut()
19345 .unwrap()
19346 .rejected
19347 .push(crate::verdict::Rejection {
19348 id: "R1-1-1".into(),
19349 why: "呼び出されないコードのため辞退".into(),
19350 });
19351 let mut clean = clean_round_with_minor_findings();
19352 clean.verified_head = Some(head.clone());
19353 f.runner.state.reviews = vec![fixed, clean];
19354 ok_gate(&mut f.runner.state, Some(&head));
19355 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19356 f.runner.state.config.agents.push(scripted("beta", reply));
19357 f.runner.state.config.roles.language_judge =
19358 Some(crate::config::AgentChoice::One("beta".into()));
19359 f
19360 }
19361
19362 #[tokio::test]
19363 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19364 for review_only in [false, true] {
19365 let mut f = rewrite_fixture_with_findings(review_only,
19366 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19367 let m = posted(&mut f).await;
19368 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19369 let expected =
19370 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19371 assert!(m.body.contains(&expected));
19372 assert!(m.body.contains("## Review rounds"));
19373 assert!(
19374 m.body
19375 .contains("Gate (`verify.gate`): passed on the final commit")
19376 );
19377 assert!(
19378 m.body
19379 .contains("Tests (`verify.e2e`): passed on the final commit")
19380 );
19381 let c = f.runner.state.change_summary.as_ref().unwrap();
19382 assert!(
19383 m.body
19384 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19385 );
19386 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19387 assert!(crate::pr_outcome::covers_required_findings(
19388 &f.runner.state,
19389 &m.body
19390 ));
19391 assert!(m.body.contains("documentation remains incomplete"));
19392 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19393 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19394 assert!(f.runner.state.github_text.is_none());
19395 }
19396 }
19397
19398 #[tokio::test]
19399 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19400 for review_only in [false, true] {
19401 for reply in [
19402 "TITLE: fix: public description\nA short English description.",
19403 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19404 ] {
19405 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19406 let winner = f.runner.state.candidates[0].clone();
19407 let posted = f
19408 .runner
19409 .guarded_pr_message(&winner, None, true)
19410 .await
19411 .unwrap();
19412 assert!(posted.is_none());
19413 assert!(f.runner.state.parked);
19414 let withheld = f.runner.state.github_text.as_ref().unwrap();
19415 assert!(
19416 withheld.rewrite.is_none(),
19417 "an incomplete rewrite must not be selectable"
19418 );
19419 let q = ask::Questions::open()
19420 .get(withheld.question.as_ref().unwrap())
19421 .unwrap();
19422 assert!(
19423 !q.choices
19424 .contains(&crate::github_text::USE_REWRITE.to_owned())
19425 );
19426 assert!(
19427 q.detail.contains("R1-1-1")
19428 && q.detail.contains("R2-1-1")
19429 && q.detail.contains("R2-1-2")
19430 );
19431 }
19432 }
19433 }
19434
19435 #[tokio::test]
19436 async fn japanese_reports_stay_local_in_both_public_run_modes() {
19437 for review_only in [false, true] {
19438 let mut f = summary_fixture();
19439 if !review_only {
19440 f.runner.state.candidates[0].agent = "alpha".into();
19441 }
19442 f.runner.state.instruction = "内部タスクの説明".into();
19443 f.runner.state.candidates[0].summary = "内部の候補報告".into();
19444 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19445 f.runner.state.reviews = vec![
19446 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
19447 round_with_notes(2, Some("二回目の内部修正報告です。")),
19448 passing_round(3, Some(&head)),
19449 ];
19450 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
19451 ok_gate(&mut f.runner.state, Some(&head));
19452 let captured = f.repo.join("public-prompt.txt");
19453 let author = &mut f.runner.state.config.agents[0];
19454 author.command = vec![
19455 "sh".into(),
19456 "-c".into(),
19457 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19458 ];
19459 author.env.insert(
19460 "PROMPT_FILE".into(),
19461 captured.to_string_lossy().into_owned(),
19462 );
19463 let m = posted(&mut f).await;
19464 assert!(m.body.contains("Adds `own.txt`"));
19465 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
19466 assert!(m.body.contains("R1-1-1"));
19467 assert!(
19468 m.body
19469 .contains("Gate (`verify.gate`): passed on the final commit")
19470 );
19471 assert!(
19472 m.body
19473 .contains("Tests (`verify.e2e`): passed on the final commit")
19474 );
19475 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19476 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
19477 let prompt = std::fs::read_to_string(captured).unwrap();
19478 assert!(prompt.contains(&format!(
19479 "git diff {} {head}",
19480 f.runner.state.change_summary.as_ref().unwrap().base
19481 )));
19482 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
19483 assert_eq!(
19484 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
19485 "二回目の内部修正報告です。"
19486 );
19487 }
19488 }
19489
19490 #[tokio::test]
19491 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
19492 for review_only in [false, true] {
19493 let mut f = summary_fixture();
19494 if !review_only {
19495 f.runner.state.candidates[0].agent = "alpha".into();
19496 }
19497 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
19498 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
19499 std::fs::write(&f.reply, "no usable description").unwrap();
19500 let failed = posted(&mut f).await;
19501 assert!(
19502 failed
19503 .body
19504 .contains("No description of what this change does could be generated")
19505 );
19506 assert!(failed.body.is_ascii());
19507 assert!(f.runner.state.github_text.is_none());
19508 assert!(f.runner.state.change_summary.is_none());
19509 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19510 let evidence = crate::pr_outcome::summary_evidence(
19511 &f.runner.state,
19512 crate::pr_outcome::FinalHead::At(&head),
19513 );
19514 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
19515 assert!(
19516 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19517 .exists()
19518 );
19519 f.runner.state =
19520 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19521 std::fs::write(
19522 &f.reply,
19523 "TITLE: feat(own): add the own file\nAdds the committed own file.",
19524 )
19525 .unwrap();
19526 let retried = posted(&mut f).await;
19527 assert!(retried.body.contains("Adds the committed own file."));
19528 assert_eq!(
19529 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19530 "日本語の修正報告"
19531 );
19532 }
19533 }
19534
19535 #[tokio::test]
19536 async fn same_head_verification_and_findings_invalidate_public_prose() {
19537 let mut f = summary_fixture();
19538 posted(&mut f).await;
19539 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19540 let original = f
19541 .runner
19542 .state
19543 .change_summary
19544 .as_ref()
19545 .unwrap()
19546 .evidence
19547 .clone();
19548 ok_gate(&mut f.runner.state, Some(&head));
19549 std::fs::write(
19550 &f.reply,
19551 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19552 )
19553 .unwrap();
19554 let verified = posted(&mut f).await;
19555 assert!(verified.body.contains("now verified"));
19556 assert_ne!(
19557 f.runner.state.change_summary.as_ref().unwrap().evidence,
19558 original
19559 );
19560 let mut round = round_with_notes(1, Some("内部報告"));
19561 round.reviews.push(ReviewRecord {
19562 attempts: 0,
19563 reviewer: 1,
19564 agent: "alpha".into(),
19565 summary: "内部レビュー".into(),
19566 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19567 vote: None,
19568 failed: None,
19569 duration_ms: 0,
19570 });
19571 round
19572 .fix
19573 .as_mut()
19574 .unwrap()
19575 .rejected
19576 .push(crate::verdict::Rejection {
19577 id: "R1-1-1".into(),
19578 why: "呼び出し元から到達できない".into(),
19579 });
19580 f.runner.state.reviews = vec![round];
19581 let omitted = posted(&mut f).await;
19583 assert!(omitted.body.contains("No description"));
19584 assert!(omitted.body.contains("R1-1-1"));
19585 assert!(!omitted.body.contains("到達"));
19586 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19587 let described = posted(&mut f).await;
19588 assert!(described.body.contains("unreachable from callers"));
19589 assert!(described.body.contains("R1-1-1"));
19590 assert!(!described.body.contains("到達"));
19591 let evidence = f
19592 .runner
19593 .state
19594 .change_summary
19595 .as_ref()
19596 .unwrap()
19597 .evidence
19598 .clone();
19599 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19600 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19601 assert!(posted(&mut f).await.body.contains("rationale updated"));
19602 assert_ne!(
19603 f.runner.state.change_summary.as_ref().unwrap().evidence,
19604 evidence
19605 );
19606 f.runner
19608 .state
19609 .change_summary
19610 .as_mut()
19611 .unwrap()
19612 .evidence
19613 .clear();
19614 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19615 assert!(posted(&mut f).await.body.contains("current specification"));
19616 }
19617
19618 #[tokio::test]
19619 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19620 let mut f = summary_fixture();
19621 let m = posted(&mut f).await;
19622 assert_eq!(m.title, "feat(own): add the own file");
19623 assert!(
19624 m.body
19625 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19626 "{}",
19627 m.body
19628 );
19629 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19630 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19631 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19632 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19633 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19634 let kept = f.runner.state.change_summary.as_ref().unwrap();
19635 assert_eq!(
19636 (kept.head.as_str(), kept.base.as_str()),
19637 (head.as_str(), base.as_str())
19638 );
19639 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19640 }
19641
19642 #[tokio::test]
19643 async fn review_only_summary_receives_the_configured_github_language() {
19644 let mut f = summary_fixture();
19645 f.runner.state.config.graph.github_language = "ja".into();
19646 let captured = f.repo.join("summary-prompt.txt");
19647 let author = &mut f.runner.state.config.agents[0];
19648 author.command = vec![
19649 "sh".into(),
19650 "-c".into(),
19651 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19652 ];
19653 author.env.insert(
19654 "PROMPT_FILE".into(),
19655 captured.to_string_lossy().into_owned(),
19656 );
19657 std::fs::write(
19658 &f.reply,
19659 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19660 )
19661 .unwrap();
19662 let m = posted(&mut f).await;
19663 let prompt = std::fs::read_to_string(captured).unwrap();
19664 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19665 assert!(!prompt.contains("in English"));
19666 assert_eq!(m.title, "feat(own): ファイルを追加");
19667 assert!(m.body.contains("own.txt を追加します。"));
19668 }
19669
19670 #[tokio::test]
19671 async fn review_fixes_come_after_the_whole_change_summary() {
19672 let mut f = summary_fixture();
19673 f.runner.state.reviews = vec![
19674 round_with_notes(1, Some("handled the PATH case")),
19675 round_with_notes(2, None),
19676 ];
19677 let m = posted(&mut f).await;
19678 let summary = m.body.find("## Summary").expect("summary");
19679 let fixes = m.body.find("## Review rounds").expect("review rounds");
19680 assert!(summary < fixes, "{}", m.body);
19681 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19682 assert!(m.body.contains("## Final verification"), "{}", m.body);
19683 }
19684
19685 #[tokio::test]
19686 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19687 let mut f = summary_fixture();
19688 let first = posted(&mut f).await;
19689 let first_head = f.runner.state.change_summary.clone().unwrap().head;
19690 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19692 let again = posted(&mut f).await;
19693 assert_eq!((again.title, again.body), (first.title, first.body));
19694
19695 let g = |args: &[&str]| {
19696 let out = std::process::Command::new("git")
19697 .args(args)
19698 .current_dir(&f.repo)
19699 .quiet()
19700 .output()
19701 .expect("spawn git");
19702 assert!(out.status.success(), "git {args:?}");
19703 };
19704 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19705 g(&["add", "-A"]);
19706 g(&["commit", "-q", "-m", "fix(own): handle more"]);
19707 let second = posted(&mut f).await;
19708 let kept = f.runner.state.change_summary.clone().unwrap();
19709 assert_ne!(kept.head, first_head);
19710 assert!(second.body.contains("Not read yet."), "{}", second.body);
19711 assert!(!second.body.contains("so the first summary is read"));
19712 assert_eq!(second.title, "feat(own): other");
19713 }
19714
19715 #[tokio::test]
19716 async fn an_unavailable_summary_is_stated_and_never_invented() {
19717 let mut f = summary_fixture();
19718 std::fs::write(&f.reply, "no title line here").unwrap();
19719 let m = posted(&mut f).await;
19720 assert!(
19721 m.body
19722 .contains("No description of what this change does could be generated"),
19723 "{}",
19724 m.body
19725 );
19726 assert!(m.body.contains("internal reports remain in local records"));
19727 assert!(f.runner.state.change_summary.is_none());
19728 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19729 let winner = f.runner.state.candidates[0].clone();
19731 let preview = f
19732 .runner
19733 .guarded_pr_message(&winner, None, false)
19734 .await
19735 .unwrap()
19736 .unwrap();
19737 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19738 }
19739
19740 #[test]
19741 fn pr_message_review_single_commit_uses_its_subject() {
19742 let state = review_state(&["feat(nats): per-role user"]);
19743 let m = pr_message(&state, 'A');
19744 assert_eq!(m.title, "feat(nats): per-role user");
19745 assert!(!m.body.contains("Review the work already"), "{}", m.body);
19746 assert!(m.body.contains("## Commits under review"), "{}", m.body);
19747 }
19748
19749 #[test]
19750 fn pr_message_review_multi_commit_takes_the_oldest() {
19751 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19752 let m = pr_message(&state, 'A');
19753 assert_eq!(m.title, "feat: the change");
19754 for s in ["feat: the change", "fix: typo", "fix: again"] {
19755 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19756 }
19757 }
19758
19759 #[test]
19760 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19761 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19762 let mut state = review_state(&[first, "fix: later fixup"]);
19763 state.candidates[0].branch = "機能/ログイン".to_owned();
19764 let m = pr_message(&state, 'A');
19765 assert!(
19766 m.title.starts_with("chore: land candidate A of run"),
19767 "{}",
19768 m.title
19769 );
19770 }
19771 }
19772
19773 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19774 BranchFacts {
19775 commits: commits
19776 .iter()
19777 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19778 .collect(),
19779 stat: stat.to_owned(),
19780 }
19781 }
19782
19783 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19784 let mut r = review_round(true, 0, 1, 1, true, true);
19785 r.round = round;
19786 r.fix = notes.map(|n| FixRecord {
19787 agent: "fixer".to_owned(),
19788 addressed: Vec::new(),
19789 rejected: Vec::new(),
19790 notes: n.to_owned(),
19791 committed: true,
19792 failed: None,
19793 duration_ms: 0,
19794 continuation: None,
19795 });
19796 r
19797 }
19798
19799 #[test]
19800 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19801 let state = review_state(&["ignored"]);
19802 let f = facts(
19803 &[
19804 (
19805 "fix(login): resolve PATH on macOS",
19806 "Login shells skip rc files.",
19807 ),
19808 ("fix: address review", ""),
19809 ],
19810 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19811 );
19812 let m = pr_message_with(&state, 'A', Some(&f));
19813 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19814 assert!(
19815 m.body
19816 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
19817 "{}",
19818 m.body
19819 );
19820 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19821 assert!(m.body.contains("## Diff stat"), "{}", m.body);
19822 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19823 for banned in [
19824 "Review the work already",
19825 "no task statement",
19826 "Original task",
19827 ] {
19828 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19829 }
19830 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19831 }
19832
19833 #[test]
19834 fn pr_message_review_truncates_a_huge_first_commit_body() {
19835 let state = review_state(&["ignored"]);
19836 let f = facts(
19837 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19838 "s",
19839 );
19840 let m = pr_message_with(&state, 'A', Some(&f));
19841 assert!(m.body.len() < 30_000, "{}", m.body.len());
19842 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19843 assert!(
19844 m.body.contains("- ... 1 more commit(s)"),
19845 "{}",
19846 m.body.len()
19847 );
19848 assert!(m.body.ends_with("magi:candidate-a\n"));
19849 }
19850
19851 #[test]
19852 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19853 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19854 assert_eq!(m.title, "feat: x");
19855 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19856 assert!(!m.body.contains("Diff stat"), "{}", m.body);
19857 assert!(!m.body.contains("no task statement"), "{}", m.body);
19858 }
19859
19860 #[test]
19861 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19862 let mut state = review_state(&["日本語の件名"]);
19863 state.candidates[0].branch = "fix/macos-login-path".to_owned();
19864 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19865 state.candidates[0].branch = "機能/ログイン".to_owned();
19866 assert!(
19867 pr_message(&state, 'A')
19868 .title
19869 .starts_with("chore: land candidate A")
19870 );
19871 }
19872
19873 #[test]
19874 fn pr_message_review_fixes_survive_a_clean_final_round() {
19875 let mut state = review_state(&["feat: x"]);
19876 state.reviews = vec![
19877 round_with_notes(1, Some("handled the PATH case")),
19878 round_with_notes(2, None),
19879 ];
19880 let body = pr_message(&state, 'A').body;
19881 assert!(
19882 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19883 "{body}"
19884 );
19885 assert!(body.contains("### Round 1"), "{body}");
19886
19887 state.reviews = vec![
19888 round_with_notes(1, Some("first fix")),
19889 round_with_notes(2, Some("")),
19890 round_with_notes(3, Some("second fix")),
19891 round_with_notes(4, None),
19892 ];
19893 let body = pr_message(&state, 'A').body;
19894 assert!(
19895 body.contains("### Round 1") && !body.contains("first fix"),
19896 "{body}"
19897 );
19898 assert!(
19899 body.contains("### Round 3") && !body.contains("second fix"),
19900 "{body}"
19901 );
19902 assert!(body.contains("### Round 2"), "{body}");
19903 }
19904
19905 const FINAL: &str = "abcdef1234567890";
19906
19907 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
19908 let mut r = round_with_notes(round, None);
19909 r.verified_head = verified.map(str::to_owned);
19910 r
19911 }
19912
19913 fn ok_gate(state: &mut RunState, head: Option<&str>) {
19914 state.gate = vec![CommandOutcome {
19915 command: "gate".to_owned(),
19916 code: Some(0),
19917 output_tail: String::new(),
19918 duration_ms: 0,
19919 resource_blocked: false,
19920 }];
19921 state.gate_ran = true;
19922 state.gate_head = head.map(str::to_owned);
19923 }
19924
19925 fn body_at(state: &RunState) -> String {
19926 pr_message_raw(
19927 state,
19928 'A',
19929 None,
19930 crate::pr_outcome::FinalHead::At(FINAL),
19931 ChangeNote::NotAsked,
19932 )
19933 .body
19934 }
19935
19936 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
19937
19938 #[test]
19939 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
19940 let mut state = review_state(&["feat: x"]);
19941 state.reviews = vec![
19942 round_with_notes(1, Some(SANDBOXED)),
19943 passing_round(2, Some(FINAL)),
19944 ];
19945 ok_gate(&mut state, Some(FINAL));
19946 let body = body_at(&state);
19947 assert!(!body.contains("Handled the PATH case."), "{body}");
19948 assert!(!body.contains("could not commit"), "{body}");
19949 assert!(!body.contains("could not run"), "{body}");
19950 assert!(
19951 body.contains("The fix is committed on the branch"),
19952 "{body}"
19953 );
19954 assert!(!body.contains("sandbox"), "{body}");
19955 assert!(
19956 body.contains("Tests (`verify.e2e`): passed on the final commit"),
19957 "{body}"
19958 );
19959 assert!(
19960 body.contains("Gate (`verify.gate`): passed on the final commit"),
19961 "{body}"
19962 );
19963 assert!(
19965 state.reviews[0]
19966 .fix
19967 .as_ref()
19968 .unwrap()
19969 .notes
19970 .contains("could not commit")
19971 );
19972 }
19973
19974 #[test]
19975 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
19976 let mut state = review_state(&["feat: x"]);
19977 let mut r1 = round_with_notes(1, Some(SANDBOXED));
19978 r1.fix.as_mut().unwrap().committed = false;
19979 let mut r2 = passing_round(2, Some(FINAL));
19980 r2.e2e[0].code = Some(1);
19981 state.reviews = vec![r1, r2];
19982 let body = body_at(&state);
19983 assert!(!body.contains("could not commit"), "{body}");
19984 assert!(!body.contains("could not run"), "{body}");
19985 assert!(body.contains("This round produced no commit"), "{body}");
19986 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
19987 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
19988 assert!(!body.contains("passed"), "{body}");
19989 }
19990
19991 #[test]
19992 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
19993 let mut state = review_state(&["feat: x"]);
19994 state.reviews = vec![
19995 round_with_notes(1, Some(SANDBOXED)),
19996 passing_round(2, Some("0123456789abcdef")),
19997 ];
19998 ok_gate(&mut state, Some("0123456789abcdef"));
19999 let body = body_at(&state);
20000 assert!(!body.contains("HTTP tests could not run"), "{body}");
20001 assert!(!body.contains("could not commit"), "{body}");
20003 assert!(
20004 body.contains("passed on an earlier commit (`0123456`)"),
20005 "{body}"
20006 );
20007 assert!(!body.contains("passed on the final commit"), "{body}");
20008
20009 ok_gate(&mut state, None);
20011 let body = body_at(&state);
20012 assert!(
20013 body.contains("passed on a commit that was not recorded"),
20014 "{body}"
20015 );
20016 }
20017
20018 #[test]
20019 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20020 let state = state_with_summary(
20021 "add retries to the client",
20022 "TITLE: feat: retries\n\nDid it.",
20023 );
20024 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20025 assert_eq!(m.title, "feat: retries");
20026 assert!(
20027 m.body.contains("<summary>Original task</summary>"),
20028 "{}",
20029 m.body
20030 );
20031 assert!(!m.body.contains("Commits under review"), "{}", m.body);
20032 assert!(
20033 m.body
20034 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20035 "{}",
20036 m.body
20037 );
20038 }
20039
20040 #[test]
20041 fn pr_message_review_bounds_a_long_english_subject() {
20042 let long = format!("feat: {}", "word ".repeat(100));
20043 let m = pr_message(&review_state(&[&long]), 'A');
20044 assert!(m.title.starts_with("feat: word"), "{}", m.title);
20045 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20046 }
20047
20048 #[test]
20049 fn pr_message_implementation_run_is_unchanged_by_review_support() {
20050 let state = state_with_summary("add retries\n\ndetails", "- did some things");
20051 let m = pr_message(&state, 'A');
20052 assert_eq!(m.title, "add retries");
20053 assert!(m.body.contains("<summary>Original task</summary>"));
20054 assert!(!m.body.contains("Commits under review"));
20055 assert_eq!(landing_subject_source(&state), state.instruction);
20056 }
20057
20058 #[test]
20059 fn review_run_squash_subject_is_the_change_not_the_prompt() {
20060 let state = review_state(&["feat: the change", "fix: typo"]);
20061 let source = landing_subject_source(&state);
20062 assert_eq!(land::merge_subject("", &source), "feat: the change");
20063 assert_eq!(
20064 land::merge_subject("magi: candidate A (uncommitted work)", &source),
20065 "feat: the change"
20066 );
20067 assert_eq!(
20069 land::merge_subject("feat: renamed by hand", &source),
20070 "feat: renamed by hand"
20071 );
20072 let blank = review_state(&["日本語"]);
20073 assert!(
20074 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20075 );
20076 }
20077
20078 #[test]
20079 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20080 let state = review_state(&["feat: the change"]);
20081 let source = landing_subject_source(&state);
20082 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20083 assert_eq!(
20084 land::merge_subject(landing_title(&state, old), &source),
20085 "feat: the change"
20086 );
20087 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20088 let task = state_with_summary("add retries", "");
20089 assert_eq!(landing_title(&task, old), old);
20090 }
20091
20092 #[test]
20093 fn pr_message_describes_the_change_not_the_task() {
20094 let state = state_with_summary(
20095 "今回やってほしいこと: results projector を直す",
20096 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20097 );
20098 let m = pr_message(&state, 'A');
20099 assert_eq!(m.title, "fix(web): batch the runs list reads");
20100 assert!(
20101 m.body.starts_with("## Summary\n\n- reads run.json once"),
20102 "{}",
20103 m.body
20104 );
20105 assert!(!m.body.contains("TITLE:"), "{}", m.body);
20106 let task_at = m.body.find("今回やってほしいこと").unwrap();
20107 let details_at = m.body.find("<details>").unwrap();
20108 assert!(
20109 details_at < task_at,
20110 "the task lives inside <details>: {}",
20111 m.body
20112 );
20113 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20114 assert!(m.body.contains("magi:candidate-a"));
20115 }
20116
20117 #[test]
20118 fn pr_message_falls_back_to_the_task_without_a_title_line() {
20119 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20120 let m = pr_message(&state, 'A');
20121 assert_eq!(m.title, "add retries");
20122 assert!(
20123 m.body.contains("## Summary\n\n- did some things"),
20124 "{}",
20125 m.body
20126 );
20127
20128 let none = RunState::new(
20129 PathBuf::from("/repo"),
20130 "main".to_owned(),
20131 "abc1234".to_owned(),
20132 "add retries".to_owned(),
20133 Config::default(),
20134 );
20135 let m = pr_message(&none, 'A');
20136 assert_eq!(m.title, "add retries");
20137 assert!(!m.body.contains("## Summary"), "{}", m.body);
20138 }
20139
20140 #[test]
20141 fn pr_message_refuses_the_candidate_commit_subject() {
20142 for bad in [
20143 "TITLE: magi: candidate A (uncommitted work)",
20144 "TITLE: chore: stuff (uncommitted work)",
20145 "TITLE: ",
20146 ] {
20147 let state = state_with_summary("add retries", bad);
20148 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20149 }
20150 }
20151
20152 #[test]
20153 fn pr_message_bounds_a_very_long_task_and_title() {
20154 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20155 let state = state_with_summary(&long, "- nothing");
20156 let m = pr_message(&state, 'A');
20157 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20158 assert!(!m.title.contains('\n'));
20159
20160 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20161 let m = pr_message(&state, 'A');
20162 assert!(m.title.starts_with("feat: "));
20163 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20164 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20165 }
20166
20167 fn long_title_of(instruction: &str) -> String {
20168 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20169 }
20170
20171 #[test]
20172 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20173 let first = "Make the landing path keep a readable title for long tasks";
20174 let line = format!(
20175 "{first}. {}",
20176 "And then keep going with more words ".repeat(20)
20177 );
20178 let t = long_title_of(&line);
20179 assert_eq!(t, first);
20180 assert!(!t.starts_with("chore: land"));
20181 }
20182
20183 #[test]
20184 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20185 let line = "word ".repeat(200);
20186 let t = long_title_of(&line);
20187 assert!(t.ends_with("word..."), "{t}");
20188 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20189 }
20190
20191 #[test]
20192 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20193 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20194 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20195 }
20196 }
20197
20198 #[test]
20199 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20200 let mut state = state_with_summary("再試行を追加する", "- nothing");
20201 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20202 state.config.graph.github_language = "ja".into();
20203 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20204 }
20205
20206 #[test]
20207 fn pr_message_title_limit_is_exact() {
20208 let at = "a".repeat(PR_TITLE_MAX);
20209 assert_eq!(long_title_of(&at), at);
20210 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20211 assert!(over.ends_with("..."), "{over}");
20212 assert_eq!(over.chars().count(), PR_TITLE_MAX);
20213 }
20214
20215 #[test]
20216 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20217 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20218 let t = long_title_of(&line);
20219 assert!(t.ends_with("..."), "{t}");
20220 assert!(t.is_ascii(), "{t}");
20221 }
20222
20223 #[test]
20224 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20225 let line = format!(
20226 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20227 "plus more filler words ".repeat(20)
20228 );
20229 let t = long_title_of(&line);
20230 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20231 }
20232
20233 #[test]
20234 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20235 let line = format!("\n\n# {}", "title words ".repeat(40));
20236 let state = state_with_summary(&line, "- nothing");
20237 let m = pr_message(&state, 'A');
20238 assert!(m.title.starts_with("title words"), "{}", m.title);
20239 assert_eq!(
20240 land::merge_subject(&m.title, &landing_subject_source(&state)),
20241 m.title
20242 );
20243 assert_eq!(
20245 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20246 "feat: renamed by hand"
20247 );
20248 }
20249
20250 #[tokio::test]
20251 async fn github_text_rewrite_is_bounded_and_falls_back() {
20252 crate::run::pin_test_home();
20253 for (reply, accepted) in [
20254 (
20255 "TITLE: fix: retries\nAdd retries for failed requests.",
20256 true,
20257 ),
20258 (
20259 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20260 false,
20261 ),
20262 (
20263 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20264 true,
20265 ),
20266 (
20267 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20268 true,
20269 ),
20270 (
20271 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20272 true,
20273 ),
20274 (
20275 "TITLE: fix: retries\n日本語の説明です。 token=secret",
20276 false,
20277 ),
20278 ("TITLE: fix: retries\n", false),
20279 ] {
20280 let dir = tempfile::tempdir().unwrap();
20281 let mut runner = runner_at(RunStatus::Gating);
20282 runner.state = state_with_summary(
20283 "add retries",
20284 "TITLE: fix: retries\n日本語の説明を書きます。",
20285 );
20286 runner.state.repo = dir.path().to_owned();
20287 runner.state.candidates[0].worktree = dir.path().to_owned();
20288 let mut author = spec("alpha");
20289 author.command = vec![
20290 "sh".into(),
20291 "-c".into(),
20292 "printf '%s' \"$REWRITE_REPLY\"".into(),
20293 ];
20294 author.env.insert("REWRITE_REPLY".into(), reply.into());
20295 runner.state.config.agents = vec![author];
20296 seed_public_summary(&mut runner);
20297 let winner = runner.state.candidates[0].clone();
20298 let first = runner
20299 .guarded_pr_message(&winner, None, true)
20300 .await
20301 .unwrap();
20302 let message = if accepted {
20303 first.expect("a passing rewrite never asks")
20304 } else {
20305 assert!(first.is_none());
20307 assert!(runner.state.parked);
20308 let store = ask::Questions::open();
20309 let asked: Vec<_> = store
20310 .list()
20311 .into_iter()
20312 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20313 .collect();
20314 assert_eq!(asked.len(), 1);
20315 assert!(!asked[0].detail.contains("secret"));
20316 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20317 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20318 assert_eq!(
20319 asked[0]
20320 .choices
20321 .contains(&crate::github_text::USE_REWRITE.to_owned()),
20322 rewrite.is_some()
20323 );
20324 store
20327 .update(&asked[0].id, |q| {
20328 q.abandon("test");
20329 Ok(())
20330 })
20331 .unwrap();
20332 let again = runner
20333 .guarded_pr_message(&winner, None, true)
20334 .await
20335 .unwrap()
20336 .expect("an abandoned question falls back");
20337 let n = store
20338 .list()
20339 .into_iter()
20340 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20341 .count();
20342 assert_eq!(n, 1, "asked once per rejected text");
20343 again
20344 };
20345 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20346 assert!(!message.body.contains("secret"));
20347 assert!(!message.body.contains("prefix"));
20348 assert!(!message.title.contains("secret"));
20349 assert_eq!(
20350 message.body.contains("Add retries for failed requests."),
20351 accepted
20352 );
20353 assert_eq!(
20354 runner.state.seats["github-text-rewrite"].turns, 1,
20355 "only one rewrite invocation"
20356 );
20357 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20358 if accepted {
20359 assert!(runner.state.github_text.is_none());
20360 assert!(!runner.state.parked);
20361 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20362 assert!(message.body.contains("## Final verification"));
20363 assert!(message.body.contains("Evidence: final diff"));
20364 assert!(
20365 message
20366 .body
20367 .contains(&format!("magi:run/{}", runner.state.id))
20368 );
20369 }
20370 if !accepted {
20371 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20372 assert!(
20373 message
20374 .body
20375 .contains(&format!("magi:run/{}", runner.state.id))
20376 );
20377 }
20378 }
20379 }
20380
20381 fn scripted(id: &str, reply: &str) -> AgentSpec {
20382 let mut a = spec(id);
20383 a.command = vec![
20384 "sh".into(),
20385 "-c".into(),
20386 "printf '%s' \"$REWRITE_REPLY\"".into(),
20387 ];
20388 a.env.insert("REWRITE_REPLY".into(), reply.into());
20389 a
20390 }
20391
20392 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20393 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20394
20395 fn seed_public_summary(runner: &mut Runner) {
20397 let repo = &runner.state.repo;
20398 let g = |args: &[&str]| {
20399 let out = std::process::Command::new("git")
20400 .args(args)
20401 .current_dir(repo)
20402 .quiet()
20403 .output()
20404 .unwrap();
20405 assert!(out.status.success(), "git {args:?}");
20406 String::from_utf8_lossy(&out.stdout).trim().to_owned()
20407 };
20408 g(&["init", "-q", "-b", "main"]);
20409 g(&[
20410 "-c",
20411 "user.name=Test",
20412 "-c",
20413 "user.email=test@example.invalid",
20414 "commit",
20415 "-q",
20416 "--allow-empty",
20417 "-m",
20418 "feat: fixture",
20419 ]);
20420 let head = g(&["rev-parse", "HEAD"]);
20421 g(&["update-ref", "refs/remotes/origin/main", &head]);
20422 let evidence = crate::pr_outcome::summary_evidence(
20423 &runner.state,
20424 crate::pr_outcome::FinalHead::At(&head),
20425 );
20426 let generated = &runner.state.candidates[0].summary;
20427 runner.state.change_summary = Some(crate::run::ChangeSummary {
20428 base: head.clone(),
20429 head,
20430 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
20431 title: summary_title(generated).unwrap(),
20432 body: summary_without_title(generated),
20433 });
20434 }
20435
20436 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
20437 let mut runner = runner_at(RunStatus::Gating);
20438 runner.state = state_with_summary("add retries", JA_SUMMARY);
20439 if review_only {
20440 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
20441 }
20442 runner.state.repo = dir.to_owned();
20443 runner.state.candidates[0].worktree = dir.to_owned();
20444 runner.state.config.agents = agents;
20445 seed_public_summary(&mut runner);
20446 runner
20447 }
20448
20449 fn gate_questions(runner: &Runner) -> usize {
20450 ask::Questions::open()
20451 .list()
20452 .into_iter()
20453 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20454 .count()
20455 }
20456
20457 #[tokio::test]
20458 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
20459 crate::run::pin_test_home();
20460 let dir = tempfile::tempdir().unwrap();
20461 let mut runner = rewrite_runner(
20462 dir.path(),
20463 true,
20464 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20465 );
20466 runner.state.config.roles.language_judge =
20467 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
20468 let winner = runner.state.candidates[0].clone();
20469 let msg = runner
20470 .guarded_pr_message(&winner, None, true)
20471 .await
20472 .unwrap()
20473 .expect("an English rewrite posts without asking");
20474 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
20475 assert!(msg.body.contains("<details>kept</details>"));
20476 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
20477 assert_eq!(gate_questions(&runner), 0);
20478 assert!(runner.state.github_text.is_none());
20479 assert!(!runner.state.seats.contains_key("impl-A"));
20480 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20481 }
20482
20483 #[tokio::test]
20484 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
20485 crate::run::pin_test_home();
20486 let dir = tempfile::tempdir().unwrap();
20487 let mut runner = rewrite_runner(
20488 dir.path(),
20489 false,
20490 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20491 );
20492 runner.state.config.roles.language_judge =
20493 Some(crate::config::AgentChoice::One("beta".into()));
20494 runner.state.seats.insert(
20495 "impl-A".into(),
20496 SeatState::new("impl-A", "alpha", runner.state.seed),
20497 );
20498 let winner = runner.state.candidates[0].clone();
20499 let msg = runner
20500 .guarded_pr_message(&winner, None, true)
20501 .await
20502 .unwrap()
20503 .expect("rewritten");
20504 assert!(msg.body.contains("<details>kept</details>"));
20505 assert_eq!(runner.state.seats["impl-A"].turns, 0);
20506 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20507 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20508 }
20509
20510 #[tokio::test]
20511 async fn rewrite_reuses_a_resumable_implementer_session() {
20512 crate::run::pin_test_home();
20513 let dir = tempfile::tempdir().unwrap();
20514 let mut runner = rewrite_runner(
20515 dir.path(),
20516 false,
20517 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20518 );
20519 runner.state.config.roles.language_judge =
20520 Some(crate::config::AgentChoice::One("beta".into()));
20521 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20522 seat.turns = 1;
20523 runner.state.seats.insert("impl-A".into(), seat);
20524 let winner = runner.state.candidates[0].clone();
20525 runner
20526 .guarded_pr_message(&winner, None, true)
20527 .await
20528 .unwrap()
20529 .expect("rewritten");
20530 assert_eq!(runner.state.seats["impl-A"].turns, 2);
20531 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20532 }
20533
20534 #[tokio::test]
20535 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20536 crate::run::pin_test_home();
20539 let dir = tempfile::tempdir().unwrap();
20540 let mut exhausted = spec("alpha");
20541 exhausted.command = vec!["false".into()];
20542 let mut runner = rewrite_runner(
20543 dir.path(),
20544 false,
20545 vec![exhausted, scripted("beta", EN_REWRITE)],
20546 );
20547 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20549 "alpha".into(),
20550 "beta".into(),
20551 ]));
20552 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20553 seat.turns = 1;
20554 runner.state.seats.insert("impl-A".into(), seat);
20555 let winner = runner.state.candidates[0].clone();
20556 let msg = runner
20557 .guarded_pr_message(&winner, None, true)
20558 .await
20559 .unwrap()
20560 .expect("the role rewrites after the implementer fails");
20561 assert!(msg.body.contains("`src/net.rs`"));
20562 assert_eq!(
20563 runner.state.seats["impl-A"].turns, 2,
20564 "the implementer was tried once"
20565 );
20566 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20567 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20568 }
20569
20570 #[tokio::test]
20571 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20572 crate::run::pin_test_home();
20573 let dir = tempfile::tempdir().unwrap();
20574 let mut broken = spec("bad");
20575 broken.command = vec!["false".into()];
20576 let mut runner = rewrite_runner(
20577 dir.path(),
20578 true,
20579 vec![
20580 broken,
20581 scripted("noformat", "Add retries."),
20582 scripted("good", EN_REWRITE),
20583 ],
20584 );
20585 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20586 "bad".into(),
20587 "noformat".into(),
20588 "good".into(),
20589 ]));
20590 let winner = runner.state.candidates[0].clone();
20591 let msg = runner
20592 .guarded_pr_message(&winner, None, true)
20593 .await
20594 .unwrap()
20595 .expect("the third agent rewrites");
20596 assert!(msg.body.contains("`src/net.rs`"));
20597 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20598 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20599 }
20600
20601 #[tokio::test]
20602 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20603 crate::run::pin_test_home();
20604 for (agents, role, saved) in [
20606 (
20607 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20608 Some("beta"),
20609 true,
20610 ),
20611 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20612 (Vec::new(), None, false),
20613 ] {
20614 let dir = tempfile::tempdir().unwrap();
20615 let mut runner = rewrite_runner(dir.path(), true, agents);
20616 runner.state.config.roles.language_judge =
20617 role.map(|r| crate::config::AgentChoice::One(r.into()));
20618 let winner = runner.state.candidates[0].clone();
20619 let first = runner
20620 .guarded_pr_message(&winner, None, true)
20621 .await
20622 .unwrap();
20623 assert!(first.is_none(), "withheld text parks the run");
20624 assert!(runner.state.parked);
20625 assert_eq!(gate_questions(&runner), 1);
20626 let g = runner.state.github_text.as_ref().unwrap();
20627 assert_eq!(g.rewrite.is_some(), saved);
20628 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20629 }
20630 }
20631
20632 #[tokio::test]
20633 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20634 crate::run::pin_test_home();
20635 let dir = tempfile::tempdir().unwrap();
20636 let mut runner = runner_at(RunStatus::Gating);
20637 runner.state =
20638 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20639 runner.state.repo = dir.path().to_owned();
20640 runner.state.candidates[0].worktree = dir.path().to_owned();
20641 let mut author = spec("alpha");
20642 author.command = vec![
20643 "sh".into(),
20644 "-c".into(),
20645 "printf '%s' \"$REWRITE_REPLY\"".into(),
20646 ];
20647 author.env.insert(
20648 "REWRITE_REPLY".into(),
20649 format!(
20650 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20651 "English sentence. ".repeat(600)
20652 ),
20653 );
20654 runner.state.config.agents = vec![author];
20655 seed_public_summary(&mut runner);
20656 let winner = runner.state.candidates[0].clone();
20657 assert!(
20658 runner
20659 .guarded_pr_message(&winner, None, true)
20660 .await
20661 .unwrap()
20662 .is_none()
20663 );
20664 let record = runner.state.github_text.clone().unwrap();
20665 let rewrite = record.rewrite.clone().unwrap();
20666 assert_eq!(rewrite.failed, ["body-language"]);
20667 assert!(
20668 rewrite
20669 .body
20670 .ends_with(&format!("magi:run/{}", runner.state.id))
20671 );
20672 let store = ask::Questions::open();
20673 let q = store.get(record.question.as_ref().unwrap()).unwrap();
20674 assert!(
20675 q.choices
20676 .contains(&crate::github_text::USE_REWRITE.to_owned())
20677 );
20678 for label in [
20679 "ORIGINAL",
20680 "FALLBACK",
20681 "REWRITE",
20682 "Rules still failed: body-language",
20683 "truncated",
20684 ] {
20685 assert!(q.detail.contains(label), "{label}");
20686 }
20687 let artifact = std::fs::read_to_string(crate::run::artifact_path(
20688 &runner.state,
20689 &crate::github_text::artifact_name(&record.fingerprint),
20690 ))
20691 .unwrap();
20692 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20693 runner.state =
20695 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20696 store
20697 .update(&q.id, |q| {
20698 q.say("まだ日本語")?;
20699 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20700 })
20701 .unwrap();
20702 assert!(
20703 runner
20704 .guarded_pr_message(&winner, None, true)
20705 .await
20706 .unwrap()
20707 .is_none()
20708 );
20709 let retry = runner.state.github_text.as_ref().unwrap();
20710 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20711 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20712 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20713 assert!(
20714 q.posting_comparison
20715 .as_ref()
20716 .unwrap()
20717 .matches_detail(&q.detail)
20718 );
20719 store
20720 .update(&q.id, |q| {
20721 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20722 })
20723 .unwrap();
20724 let message = runner
20725 .guarded_pr_message(&winner, None, true)
20726 .await
20727 .unwrap()
20728 .unwrap();
20729 assert_eq!(message.title, rewrite.title);
20730 assert_eq!(message.body, rewrite.body);
20731 assert_eq!(
20732 runner.state.github_text.as_ref().unwrap().choice,
20733 Some(GithubTextChoice::Rewrite)
20734 );
20735 let again = runner
20736 .guarded_pr_message(&winner, None, true)
20737 .await
20738 .unwrap()
20739 .unwrap();
20740 assert_eq!(again.title, message.title);
20741 assert_eq!(again.body, message.body);
20742 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20743 assert!(
20744 !runner
20745 .state
20746 .events
20747 .iter()
20748 .any(|e| e.message.contains(&rewrite.title)
20749 || e.message.contains("English sentence."))
20750 );
20751 }
20752
20753 #[tokio::test]
20754 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20755 crate::run::pin_test_home();
20756 let dir = tempfile::tempdir().unwrap();
20757 let mut runner = runner_at(RunStatus::Gating);
20758 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20759 runner.state.repo = dir.path().to_owned();
20760 runner.state.candidates[0].worktree = dir.path().to_owned();
20761 seed_public_summary(&mut runner);
20762 let winner = runner.state.candidates[0].clone();
20763 assert!(
20764 runner
20765 .guarded_pr_message(&winner, None, true)
20766 .await
20767 .unwrap()
20768 .is_none()
20769 );
20770 let record = runner.state.github_text.as_ref().unwrap();
20771 assert!(record.rewrite.is_none());
20772 let mut old = serde_json::to_value(record).unwrap();
20774 old.as_object_mut().unwrap().remove("rewrite");
20775 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20776 assert!(old.rewrite.is_none());
20777 let id = record.question.clone().unwrap();
20778 ask::Questions::open()
20779 .update(&id, |q| {
20780 q.choices.push(crate::github_text::USE_REWRITE.into());
20781 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20782 })
20783 .unwrap();
20784 let message = runner
20785 .guarded_pr_message(&winner, None, true)
20786 .await
20787 .unwrap()
20788 .unwrap();
20789 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20790 assert_eq!(
20791 runner.state.github_text.as_ref().unwrap().choice,
20792 Some(GithubTextChoice::Fallback)
20793 );
20794 }
20795
20796 #[tokio::test]
20797 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20798 crate::run::pin_test_home();
20799 let dir = tempfile::tempdir().unwrap();
20800 let mut runner = runner_at(RunStatus::Gating);
20801 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20802 runner.state.repo = dir.path().to_owned();
20803 runner.state.candidates[0].worktree = dir.path().to_owned();
20804 seed_public_summary(&mut runner);
20805 let winner = runner.state.candidates[0].clone();
20806 assert!(
20807 runner
20808 .guarded_pr_message(&winner, None, true)
20809 .await
20810 .unwrap()
20811 .is_none()
20812 );
20813 let store = ask::Questions::open();
20814 let find = |run: &str| {
20815 store
20816 .list()
20817 .into_iter()
20818 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20819 .unwrap()
20820 };
20821 let run = runner.state.id.clone();
20822 let q = find(&run);
20823 let fp = runner
20824 .state
20825 .github_text
20826 .as_ref()
20827 .unwrap()
20828 .fingerprint
20829 .clone();
20830 let name = crate::github_text::artifact_name(&fp);
20831 let path = crate::run::artifact_path(&runner.state, &name);
20832 assert!(q.detail.contains(&path.display().to_string()));
20833 assert!(q.detail.contains("title-language: title"));
20834 let saved = std::fs::read_to_string(&path).expect("artifact written");
20835 assert!(saved.contains("日本語のタイトル"));
20836 assert!(q.cwd.is_none());
20837 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20838 let say = |id: &str, text: &str, choice: &str| {
20840 store
20841 .update(id, |q| {
20842 q.thread.push(ask::Turn {
20843 who: ask::Who::Operator,
20844 body: text.to_owned(),
20845 at: jiff::Timestamp::now(),
20846 note: None,
20847 });
20848 q.answer(ask::Answer::Choice(choice.to_owned()))
20849 })
20850 .unwrap();
20851 };
20852 say(&q.id, "まだ日本語", "use my text");
20853 assert!(
20854 runner
20855 .guarded_pr_message(&winner, None, true)
20856 .await
20857 .unwrap()
20858 .is_none()
20859 );
20860 let second = find(&run);
20861 assert_ne!(second.id, q.id);
20862 assert!(second.detail.contains("did not pass"));
20863 assert!(second.detail.contains("日本語のタイトル"));
20866 assert!(!second.detail.contains("まだ日本語"));
20867 assert!(second.detail.contains(&path.display().to_string()));
20868 say(&second.id, "fix: retry failed requests", "use my text");
20869 let message = runner
20870 .guarded_pr_message(&winner, None, true)
20871 .await
20872 .unwrap()
20873 .expect("a vetted replacement is posted");
20874 assert_eq!(message.title, "fix: retry failed requests");
20875 assert_eq!(
20876 runner
20877 .state
20878 .github_text
20879 .as_ref()
20880 .unwrap()
20881 .chosen_title
20882 .as_deref(),
20883 Some("fix: retry failed requests")
20884 );
20885 assert!(runner.state.github_text.as_ref().unwrap().resolved);
20886 let again = runner
20888 .guarded_pr_message(&winner, None, true)
20889 .await
20890 .unwrap()
20891 .unwrap();
20892 assert_eq!(again.title, message.title);
20893 assert_eq!(
20894 store
20895 .list()
20896 .into_iter()
20897 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
20898 .count(),
20899 2
20900 );
20901 }
20902
20903 #[test]
20904 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
20905 let mut state = state_with_summary(
20909 "add retries",
20910 "TITLE: fix(web): batch reads\n- reads run.json once",
20911 );
20912 state.config.graph.language = "ja".to_owned();
20913 let m = pr_message(&state, 'A');
20914 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
20915 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
20916
20917 let task = "今回やってほしいこと: results projector を直す";
20920 let mut state = state_with_summary(task, "- no title line");
20921 state.config.graph.language = "ja".to_owned();
20922 let m = pr_message(&state, 'A');
20923 assert_eq!(
20924 m.title,
20925 format!("chore: land candidate A of run {}", state.id)
20926 );
20927 assert!(
20928 m.body.contains(&format!(
20929 "<summary>Original task</summary>\n\n{task}\n\n</details>"
20930 )),
20931 "{}",
20932 m.body
20933 );
20934 }
20935
20936 #[test]
20937 fn pr_message_scrubs_home_paths_and_addresses() {
20938 let state = state_with_summary(
20939 "fix it in /Users/someone/src/x",
20940 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
20941 );
20942 let m = pr_message(&state, 'A');
20943 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
20944 assert!(!m.body.contains(leak), "{}", m.body);
20945 }
20946 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
20947 }
20948
20949 #[test]
20950 fn pr_message_survives_a_task_that_closes_details() {
20951 let state = state_with_summary("a </details> b", "TITLE: fix: x");
20952 let m = pr_message(&state, 'A');
20953 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
20954 }
20955
20956 #[test]
20957 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
20958 let cmd = manual_merge_command(
20959 MergeStyle::Squash,
20960 Path::new("/repo"),
20961 "b",
20962 "fix: \"quoted\" $(x) `y`\n\nbody",
20963 );
20964 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
20965 }
20966
20967 #[test]
20968 fn manual_merge_command_matches_the_configured_style() {
20969 let repo = Path::new("/repo");
20970 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
20971
20972 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
20973 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
20974
20975 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
20976 assert_eq!(
20977 squash,
20978 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
20979 \"Merge magi run 0832 (candidate A)\""
20980 );
20981
20982 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
20983 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
20984 }
20985
20986 #[test]
20987 fn a_nudge_gets_a_quarter_of_the_budget() {
20988 assert_eq!(retry_budget(secs(1200), true), secs(300));
20990 assert_eq!(retry_budget(secs(3600), true), secs(900));
20991 }
20992
20993 #[test]
20994 fn a_resent_prompt_keeps_the_whole_budget() {
20995 assert_eq!(retry_budget(secs(1200), false), secs(1200));
20998 assert_eq!(retry_budget(secs(60), false), secs(60));
20999 }
21000
21001 #[test]
21002 fn the_floor_never_exceeds_the_original_budget() {
21003 assert_eq!(retry_budget(secs(60), true), secs(60));
21007 assert_eq!(retry_budget(secs(480), true), secs(120));
21008 assert_eq!(retry_budget(secs(0), true), secs(0));
21009 }
21010
21011 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21012 agent::CommandEvidence {
21013 id: "item1".to_owned(),
21014 description: "cargo test".to_owned(),
21015 exit_code,
21016 result_summary: String::new(),
21017 source: "codex".to_owned(),
21018 }
21019 }
21020
21021 #[test]
21022 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21023 assert!(!has_unconfirmed_command(&[]));
21027 }
21028
21029 #[test]
21030 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21031 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21035 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21036 assert!(!has_unconfirmed_command(&[
21037 evidence(Some(0)),
21038 evidence(Some(101))
21039 ]));
21040 }
21041
21042 #[test]
21043 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21044 assert!(has_unconfirmed_command(&[
21045 evidence(Some(0)),
21046 evidence(None)
21047 ]));
21048 }
21049
21050 #[test]
21051 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21052 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21053 assert_eq!(
21054 verified_noop_claim(true, &[], text).as_deref(),
21055 Some("already fixed by b32cfc4, on main.")
21056 );
21057 }
21058
21059 #[test]
21060 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21061 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21064 assert!(verified_noop_claim(false, &[], text).is_none());
21065 }
21066
21067 #[test]
21068 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21069 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21070 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21071 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21073 }
21074
21075 #[test]
21076 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21077 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21078 }
21079
21080 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21083 runner.state.candidates = shape
21084 .iter()
21085 .enumerate()
21086 .map(|(i, &(empty, verified))| Candidate {
21087 index: i,
21088 label: (b'A' + i as u8) as char,
21089 agent: "sonnet".to_owned(),
21090 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21091 worktree: PathBuf::from(format!("/wt/{i}")),
21092 summary: String::new(),
21093 stat: String::new(),
21094 files: 0,
21095 commits: 0,
21096 empty,
21097 failed: None,
21098 verified_noop: verified.map(str::to_owned),
21099 duration_ms: 0,
21100 folded: false,
21101 })
21102 .collect();
21103 }
21104
21105 #[test]
21106 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21107 ask_test_home();
21108 let mut runner = runner_at(RunStatus::Implementing);
21109 set_candidates(
21110 &mut runner,
21111 &[
21112 (true, Some("already on main at b32cfc4")),
21113 (true, Some("same fix, see the existing test")),
21114 ],
21115 );
21116
21117 runner
21118 .after_implement()
21119 .expect("a verified no-op is not an error");
21120
21121 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21122 }
21123
21124 #[test]
21125 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21126 ask_test_home();
21127 let mut runner = runner_at(RunStatus::Implementing);
21128 set_candidates(
21132 &mut runner,
21133 &[(true, Some("already on main at b32cfc4")), (true, None)],
21134 );
21135
21136 let err = runner
21137 .after_implement()
21138 .expect_err("an unverified empty candidate must still fail the run");
21139
21140 assert!(
21141 err.to_string().contains("no candidate produced a change"),
21142 "{err}"
21143 );
21144 assert_eq!(runner.state.status, RunStatus::Failed);
21145 }
21146
21147 #[test]
21148 fn after_implement_still_fails_an_ordinary_all_empty_run() {
21149 ask_test_home();
21150 let mut runner = runner_at(RunStatus::Implementing);
21151 set_candidates(&mut runner, &[(true, None), (true, None)]);
21152
21153 let err = runner
21154 .after_implement()
21155 .expect_err("no candidate declared anything; this is an ordinary failure");
21156
21157 assert!(
21158 err.to_string().contains("no candidate produced a change"),
21159 "{err}"
21160 );
21161 assert_eq!(runner.state.status, RunStatus::Failed);
21162 }
21163}
21164
21165#[derive(Debug)]
21168struct GraftRestoreFailed(String);
21169
21170impl std::fmt::Display for GraftRestoreFailed {
21171 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21172 write!(
21173 f,
21174 "could not restore the winner to its pre-graft commit: {}",
21175 self.0
21176 )
21177 }
21178}
21179
21180impl std::error::Error for GraftRestoreFailed {}
21181
21182#[cfg(test)]
21186mod resource_wait_tests {
21187 use super::*;
21188 use std::collections::BTreeMap;
21189
21190 fn sh(id: &str, script: &str) -> AgentSpec {
21191 AgentSpec {
21192 id: id.to_owned(),
21193 kind: crate::config::AgentKind::Command,
21194 model: None,
21195 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21196 extra_args: Vec::new(),
21197 env: BTreeMap::new(),
21198 prompt_delivery: None,
21199 max_turn_secs: None,
21200 }
21201 }
21202
21203 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21204 SeatJob {
21205 seat: SeatState::new("impl-A", &spec.id, 1),
21206 spec,
21207 cwd: dir.to_path_buf(),
21208 prompt: "do it".to_owned(),
21209 timeout,
21210 allow_write: true,
21211 sessions: false,
21212 artifacts: dir.join("art"),
21213 stem: "impl-A".to_owned(),
21214 handover: None,
21215 }
21216 }
21217
21218 fn state(resource_wait_secs: u64) -> RunState {
21219 let mut config = Config::default();
21220 config.graph.timeout_resource_wait = resource_wait_secs;
21221 RunState::new(
21222 PathBuf::from("/repo"),
21223 "main".to_owned(),
21224 "abc1234".to_owned(),
21225 "task".to_owned(),
21226 config,
21227 )
21228 }
21229
21230 fn hold(cache: &Path) -> crate::cache::Guard {
21231 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21232 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21233 crate::cache::AcquireOutcome::Acquired(g) => g,
21234 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21235 }
21236 }
21237
21238 async fn run(
21239 jobs: Vec<SeatJob>,
21240 sem: Arc<Semaphore>,
21241 cache: Option<&Path>,
21242 st: &mut RunState,
21243 ) -> Vec<(usize, SeatState, AgentOutcome)> {
21244 let prompts = Prompts::default();
21245 let ctx = WaveCtx {
21246 run: "r",
21247 node: "implement",
21248 prompts: &prompts,
21249 cache,
21250 round: None,
21251 carry_seats: false,
21252 pauses: None,
21253 };
21254 wave(jobs, sem, &ctx, st, 0).await
21255 }
21256
21257 #[tokio::test]
21258 async fn verification_cache_wait_preserves_the_command_budget() {
21259 crate::run::pin_test_home();
21260 let tmp = tempfile::tempdir().unwrap();
21261 let cache = tmp.path().join("target");
21262 let guard = hold(&cache);
21263 let release = tokio::spawn(async move {
21264 tokio::time::sleep(Duration::from_millis(100)).await;
21265 guard.release();
21266 });
21267 let mut st = state(10);
21268 let budget = Duration::from_millis(50);
21269 let wait = repository_wait(&st, None);
21270 let (outcomes, _) = with_cache_lease(
21271 &mut st,
21272 Some(&cache),
21273 "gate",
21274 "gate",
21275 tmp.path(),
21276 "abc1234",
21277 budget,
21278 "test gate",
21279 &wait,
21280 |_, available| async move {
21281 assert_eq!(
21282 available, budget,
21283 "contention must not shorten verification"
21284 );
21285 (Vec::new(), false, Vec::new())
21286 },
21287 )
21288 .await
21289 .unwrap();
21290 release.await.unwrap();
21291 assert!(outcomes.is_empty(), "{outcomes:?}");
21292 assert!(
21293 st.events
21294 .iter()
21295 .any(|event| event.message.contains("waiting for"))
21296 );
21297 assert!(st.resource_wait.is_none());
21298 hold(&cache).release();
21300 }
21301
21302 #[tokio::test]
21303 async fn verification_cache_wait_expires_without_starting_commands() {
21304 crate::run::pin_test_home();
21305 let tmp = tempfile::tempdir().unwrap();
21306 let cache = tmp.path().join("target");
21307 let _held = hold(&cache);
21308 let mut st = state(0);
21309 let wait = repository_wait(&st, None);
21310 let (outcomes, _) = with_cache_lease(
21311 &mut st,
21312 Some(&cache),
21313 "gate",
21314 "gate",
21315 tmp.path(),
21316 "abc1234",
21317 Duration::from_secs(60),
21318 "test gate",
21319 &wait,
21320 |_, _| async { panic!("commands must not start without the cache lease") },
21321 )
21322 .await
21323 .unwrap();
21324 assert_eq!(outcomes.len(), 1);
21325 assert!(outcomes[0].resource_blocked);
21326 assert!(st.resource_wait.is_none());
21327 assert!(matches!(
21328 st.resource_block.unwrap().cause,
21329 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21330 ));
21331 }
21332
21333 #[tokio::test]
21334 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21335 crate::run::pin_test_home();
21336 let tmp = tempfile::tempdir().unwrap();
21337 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21338 let cache = tmp.path().join("cache");
21339 let mut state = state(21600);
21340 let wait = repository_wait(&state, None);
21341 let (outcomes, _) = with_cache_lease(
21342 &mut state,
21343 Some(&cache),
21344 "gate",
21345 "gate",
21346 tmp.path(),
21347 "head",
21348 Duration::from_secs(60),
21349 "gate",
21350 &wait,
21351 |_, _| async { panic!("freshness failure must not start verification") },
21352 )
21353 .await
21354 .unwrap();
21355 assert_eq!(outcomes.len(), 1);
21356 assert!(outcomes[0].resource_blocked);
21357 let block = state.resource_block.unwrap();
21358 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21359 assert!(!block.description().contains("timed out"));
21360 }
21361
21362 #[tokio::test]
21363 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21364 crate::run::pin_test_home();
21365 let tmp = tempfile::tempdir().unwrap();
21366 let cache = tmp.path().join("cache");
21367 let _held = hold(&cache);
21368 let mut state = state(21600);
21369 let pause = Pause::new();
21370 let interrupt = Pause::new();
21371 let cause = ParkCause::Operator;
21372 pause.park_for(cause.clone());
21373 let prompts = Prompts::default();
21374 let ctx = WaveCtx {
21375 run: "test",
21376 node: "fix",
21377 prompts: &prompts,
21378 cache: Some(&cache),
21379 round: None,
21380 carry_seats: false,
21381 pauses: Some((&pause, &interrupt)),
21382 };
21383 let result = wave(
21384 vec![job(
21385 sh("writer", "exit 99"),
21386 tmp.path(),
21387 Duration::from_secs(1),
21388 )],
21389 Arc::new(Semaphore::new(1)),
21390 &ctx,
21391 &mut state,
21392 0,
21393 )
21394 .await;
21395 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21396 assert_eq!(result[0].1.turns, 0);
21397 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21398 assert_eq!(
21399 state.resource_block.unwrap().cause,
21400 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21401 );
21402 assert!(state.active.is_empty());
21403 }
21404
21405 #[tokio::test]
21406 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21407 crate::run::pin_test_home();
21408 let tmp = tempfile::tempdir().unwrap();
21409 let cache = tmp.path().join("target-a");
21410 let guard = hold(&cache);
21411 tokio::spawn(async move {
21413 tokio::time::sleep(Duration::from_millis(1500)).await;
21414 guard.release();
21415 });
21416 let mut st = state(60);
21417 let results = run(
21418 vec![job(
21419 sh("alpha", "echo ok"),
21420 tmp.path(),
21421 Duration::from_secs(1),
21422 )],
21423 Arc::new(Semaphore::new(1)),
21424 Some(&cache),
21425 &mut st,
21426 )
21427 .await;
21428 assert!(
21429 matches!(results[0].2, AgentOutcome::Ok(_)),
21430 "the wait must not be charged to the agent"
21431 );
21432 assert!(st.resource_block.is_none());
21433 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21434 assert!(st.resource_wait.is_none() && st.active.is_empty());
21435 }
21436
21437 #[tokio::test]
21438 #[cfg(unix)]
21439 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
21440 use std::os::fd::AsRawFd as _;
21441 crate::run::pin_test_home();
21442 let tmp = tempfile::tempdir().unwrap();
21443 crate::git::git(tmp.path(), &["init"]).await.unwrap();
21444 let common = tmp.path().join(".git");
21445 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
21446 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
21447 assert_eq!(
21448 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
21449 0
21450 );
21451 let mut st = state(1);
21452 st.config.coordination = crate::config::test_coordination();
21453 let mut reader = job(
21454 sh("reader", "echo must-not-run"),
21455 tmp.path(),
21456 Duration::from_secs(30),
21457 );
21458 reader.allow_write = false;
21459 let results = run(
21460 vec![reader.clone()],
21461 Arc::new(Semaphore::new(1)),
21462 None,
21463 &mut st,
21464 )
21465 .await;
21466 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21467 assert_eq!(results[0].1.turns, 0);
21468 assert!(FailClass::of(&results[0].2).is_none());
21469 assert_eq!(
21470 st.resource_block.as_ref().unwrap().kind,
21471 "repository coordination"
21472 );
21473 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21474 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21475 drop(external);
21476 let editor = crate::repo_lock::SeatLease::for_checkout(
21477 tmp.path(),
21478 true,
21479 crate::config::test_coordination().as_ref(),
21480 )
21481 .await
21482 .unwrap();
21483 let pause = Pause::new();
21484 let interrupt = Pause::new();
21485 let prompts = Prompts::default();
21486 let ctx = WaveCtx {
21487 run: "r",
21488 node: "review",
21489 prompts: &prompts,
21490 cache: None,
21491 round: None,
21492 carry_seats: false,
21493 pauses: Some((&pause, &interrupt)),
21494 };
21495 let (results, ()) = tokio::join!(
21496 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
21497 async {
21498 tokio::time::sleep(Duration::from_millis(50)).await;
21499 pause.park();
21500 }
21501 );
21502 assert!(
21503 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
21504 );
21505 assert_eq!(results[0].1.turns, 0);
21506 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21507 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21508 drop(editor);
21509 }
21510
21511 #[tokio::test]
21512 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
21513 crate::run::pin_test_home();
21514 let tmp = tempfile::tempdir().unwrap();
21515 let cache = tmp.path().join("target-b");
21516 let _held = hold(&cache);
21517 let mut st = state(1);
21518 let results = run(
21519 vec![job(
21520 sh("alpha", "echo ok"),
21521 tmp.path(),
21522 Duration::from_secs(1),
21523 )],
21524 Arc::new(Semaphore::new(1)),
21525 Some(&cache),
21526 &mut st,
21527 )
21528 .await;
21529 let (_, seat, out) = &results[0];
21530 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21531 assert!(wave_blocked(&results));
21532 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21533 assert_eq!(seat.turns, 0, "nothing was asked");
21534 assert!(
21535 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21536 "no agent was spawned"
21537 );
21538 assert!(FailClass::of(out).is_none(), "not a failure class");
21539 let block = st.resource_block.as_ref().expect("recorded for the resume");
21540 assert_eq!(block.kind, "shared build cache");
21541 assert!(matches!(
21542 block.cause,
21543 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21544 ));
21545 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21546 assert!(st.active.is_empty());
21547 }
21548
21549 #[tokio::test]
21550 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21551 crate::run::pin_test_home();
21552 let tmp = tempfile::tempdir().unwrap();
21553 let mut st = state(1);
21554 let results = run(
21555 vec![job(
21556 sh("alpha", "echo ok"),
21557 tmp.path(),
21558 Duration::from_secs(1),
21559 )],
21560 Arc::new(Semaphore::new(0)),
21561 None,
21562 &mut st,
21563 )
21564 .await;
21565 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21566 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21567 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21568 }
21569
21570 #[tokio::test]
21571 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21572 crate::run::pin_test_home();
21573 let tmp = tempfile::tempdir().unwrap();
21574 let mut st = state(60);
21575 let results = run(
21576 vec![job(
21577 sh("alpha", "sleep 30"),
21578 tmp.path(),
21579 Duration::from_secs(1),
21580 )],
21581 Arc::new(Semaphore::new(1)),
21582 None,
21583 &mut st,
21584 )
21585 .await;
21586 let out = &results[0].2;
21587 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21588 let class = FailClass::of(out).expect("a failure");
21589 assert_eq!(class, FailClass::Timeout);
21590 assert!(should_hand_over(None, &class), "existing handover policy");
21591 assert!(st.resource_block.is_none());
21592 }
21593}