1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false, None).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true, None).await
1357 }
1358
1359 pub(crate) async fn authorize_base_recovery_control(
1360 home: &Path,
1361 task: &str,
1362 control: &mut crate::base_recovery_control::Transaction,
1363 ) -> Result<()> {
1364 let id = control.request.run.clone();
1365 Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366 }
1367
1368 async fn authorize_base_recovery_request(
1369 home: &Path,
1370 id: &str,
1371 task: &str,
1372 resumption: bool,
1373 mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374 ) -> Result<()> {
1375 let _owner = crate::pr_head::owner_lock(home, id)?;
1376 let mut state = RunState::load_under(id, home)?;
1377 crate::pr_head::ensure_owner(&state)?;
1378 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379 anyhow::ensure!(
1380 queued.execution_runs().last().is_some_and(|run| run == id)
1381 && queued.status != crate::queue::TaskStatus::Done
1382 && !queued.fresh_start
1383 && queued.review_branch.is_none(),
1384 "run task ownership changed"
1385 );
1386 anyhow::ensure!(
1387 state
1388 .origin
1389 .as_ref()
1390 .and_then(|o| o.task.as_deref())
1391 .is_none_or(|owner| owner == task),
1392 "run belongs to another task"
1393 );
1394 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395 anyhow::ensure!(
1396 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397 "run is live or its ownership is unknown"
1398 );
1399 let _execution = crate::driver::maintenance(home, id)?;
1402 anyhow::ensure!(
1403 state.status.resumable()
1404 && state.released_to.is_none()
1405 && state.retired_to.is_none()
1406 && state.pr_taken_over_by.is_none(),
1407 "run ownership is final"
1408 );
1409 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410 anyhow::ensure!(
1411 !state.pr_publication_pending
1412 && !crate::handover::landing_checkpoint(&state)
1413 && state.merge_decision.is_none(),
1414 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415 );
1416 let winner = state.winner().context("run has no retained winner")?;
1417 anyhow::ensure!(
1418 git::is_clean(&winner.worktree).await?,
1419 "retained worktree has uncommitted changes"
1420 );
1421 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422 anyhow::ensure!(
1423 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424 "retained worktree does not match the winner branch"
1425 );
1426 if let Some(control) = control.as_ref() {
1427 anyhow::ensure!(
1428 head.starts_with(&control.request.head),
1429 "retained HEAD changed since the owner request"
1430 );
1431 anyhow::ensure!(
1432 state
1433 .base_sync_recovery
1434 .stop
1435 .as_ref()
1436 .is_none_or(|stop| stop.head == head),
1437 "retained HEAD differs from the recorded stop head; inspect it first"
1438 );
1439 }
1440 state.ensure_base_episode();
1441 state.record_base_sync_stop(&head);
1442 anyhow::ensure!(
1443 state
1444 .base_sync_recovery
1445 .stop
1446 .as_ref()
1447 .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448 "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449 );
1450 let fetched =
1451 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452 anyhow::ensure!(
1453 fetched.ok(),
1454 "cannot authorize base recovery without a fetched base"
1455 );
1456 let approved_tip = git::rev_parse(
1457 &state.repo,
1458 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459 )
1460 .await?;
1461 if let Some(control) = control.as_deref_mut() {
1462 let queue = crate::queue::Queue::at(home.join("queue"));
1463 control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464 if let Some(prior) = state
1465 .base_sync_recovery
1466 .authorizations
1467 .iter()
1468 .find(|a| a.request.as_deref() == Some(&control.key()))
1469 {
1470 anyhow::ensure!(
1471 !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472 "recovery request was consumed or invalidated; it cannot queue again"
1473 );
1474 return control.enqueue(&queue);
1475 }
1476 }
1477 if resumption {
1478 anyhow::ensure!(
1479 crate::daemon::compatible_run_instruction(&state, &queued),
1480 "resumption instruction does not match the retained run"
1481 );
1482 let approval = queued
1483 .resumption
1484 .as_ref()
1485 .context("no operator resumption")?;
1486 anyhow::ensure!(
1487 approval.instruction == queued.instruction,
1488 "resumption approval does not match the current task instruction"
1489 );
1490 state.authorize_base_sync_resumption(task, approval)?;
1491 } else {
1492 state.authorize_base_sync(task)?;
1493 }
1494 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495 state
1496 .base_sync_recovery
1497 .authorizations
1498 .last_mut()
1499 .unwrap()
1500 .scope = Some(crate::base_sync_recovery::Scope {
1501 tip: approved_tip,
1502 instruction: queued.instruction.clone(),
1503 episode,
1504 });
1505 if let Some(control) = control.as_deref_mut() {
1506 let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507 authorization.request = Some(control.key());
1508 authorization.generation = control.generation();
1509 authorization.decision = control.decision();
1510 authorization.question = control
1511 .request
1512 .decision
1513 .as_ref()
1514 .map(|_| control.request.request.clone());
1515 }
1516 state.event(
1517 "base-recovery",
1518 "owner authorized a bounded base synchronization interval; fresh verification required",
1519 );
1520 state.save_under(home)?;
1521 if let Some(control) = control {
1522 control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523 }
1524 Ok(())
1525 }
1526
1527 pub fn resume(id: &str) -> Result<Self> {
1529 let state = RunState::load(id)?;
1530 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531 if state.startup.as_ref().is_some_and(|s| !s.complete)
1532 || (state.status == RunStatus::Prep
1533 && state.instruction.trim().is_empty()
1534 && state.candidates.is_empty()
1535 && state.prep_complete == Some(false))
1536 {
1537 bail!(
1538 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539 );
1540 }
1541 crate::pr_head::ensure_owner(&state)?;
1542 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543 bail!(
1544 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545 );
1546 }
1547 if let Some(to) = &state.released_to {
1548 bail!(
1549 "run {} cannot be resumed: its worktree was released to run {}",
1550 state.short(),
1551 crate::run::short_of(to)
1552 );
1553 }
1554 let roles = state.config.resolve_roles()?;
1555 if let Some(id) = &state.config.graph.selected_implementer
1556 && state.candidates.iter().any(|c| c.agent != *id)
1557 {
1558 bail!(
1559 "replacement implementer pin does not match the saved candidates; task must stay held"
1560 );
1561 }
1562 let max_parallel = state.config.graph.max_parallel.max(1);
1563 Ok(Self {
1564 state,
1565 roles,
1566 sem: Arc::new(Semaphore::new(max_parallel)),
1567 pause: Pause::new(),
1568 interrupt: Pause::new(),
1569 })
1570 }
1571
1572 pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579 Box::pin(self.execute_inner())
1582 }
1583
1584 async fn execute_inner(&mut self) -> Result<()> {
1585 anyhow::ensure!(
1586 self.state.startup.as_ref().is_none_or(|s| s.complete),
1587 "startup incomplete; inspect retained evidence before explicit recovery"
1588 );
1589 if let Some(selection) = &self.state.pr_recovery
1590 && (selection.source_proof.is_some() || selection.source_run.is_some())
1591 {
1592 Box::pin(selection.verify_source(&self.state.repo)).await?;
1593 }
1594 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597 || self.state.land_approval.is_some()
1598 || self.state.merge_decision.is_some()
1599 || !self.state.merge_repairs.is_empty()
1600 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601 {
1602 let claim = FixClaim::acquire(&self.state.dir())?;
1603 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604 let disk = RunState::load(&self.state.id)?;
1605 let checkpoint = |s: &RunState| {
1606 (
1607 s.merge_repairs.len(),
1608 s.merge_repairs
1609 .last()
1610 .map(|r| (r.question.clone(), r.phase)),
1611 s.land_approval
1612 .as_ref()
1613 .map(|a| (a.question.clone(), a.head.clone())),
1614 )
1615 };
1616 if checkpoint(&disk) != checkpoint(&self.state) {
1617 bail!(
1618 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619 );
1620 }
1621 Some(claim)
1622 } else {
1623 None
1624 };
1625 let mut execution = self.claim_driver().await?;
1626 let runner = &mut *execution.runner;
1627 let wait = runner.repository_wait();
1628 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631 let result = match result {
1632 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633 record_repository_block(&mut runner.state, &e);
1634 runner.park_here()?;
1635 Ok(())
1636 }
1637 other => other,
1638 };
1639 execution.driver.finish(&mut runner.state).await?;
1640 let ended = if result.is_err() {
1641 Some({
1642 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643 if let Err(error) = &result {
1644 notice.message = format!(
1645 "Run {} stopped with an error: {}",
1646 runner.state.id,
1647 error.to_string().lines().next().unwrap_or("unknown error")
1648 );
1649 }
1650 notice
1651 })
1652 } else {
1653 crate::notices::run_ended(&runner.state)
1654 };
1655 if let Some(notice) = ended {
1656 crate::notices::raise(notice);
1657 }
1658 result
1659 }
1660
1661 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663 }
1664
1665 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666 let wait = self.repository_wait();
1667 coordinate_operation(&mut self.state, cwd, &wait).await
1668 }
1669
1670 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672 let owner = match crate::pr_head::owner_lock_wait(
1673 &crate::run::home(),
1674 &self.state.id,
1675 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676 )
1677 .await
1678 {
1679 Ok(owner) => owner,
1680 Err(error) => {
1681 let detail = format!("worktree maintenance or ownership update: {error:#}");
1682 self.state.resource_block = Some(crate::run::ResourceBlock {
1686 cause: crate::run::ResourceBlockCause::Other,
1687 node: "start".into(),
1688 kind: "worktree maintenance".into(),
1689 detail: detail.clone(),
1690 at: Timestamp::now(),
1691 });
1692 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693 }
1694 };
1695 let disk = RunState::load(&self.state.id)?;
1696 crate::pr_head::ensure_owner(&disk)?;
1697 let claimed =
1698 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699 anyhow::ensure!(
1704 disk.liveness(claimed) != crate::run::Liveness::Live,
1705 "run already has a live driver; wait for its execution to end"
1706 );
1707 if let Some(to) = &disk.released_to {
1708 bail!(
1709 "run {} cannot continue: its worktree was released to run {}",
1710 self.state.short(),
1711 crate::run::short_of(to)
1712 );
1713 }
1714 anyhow::ensure!(
1715 serde_json::to_value(&disk.base_sync_recovery)?
1716 == serde_json::to_value(&self.state.base_sync_recovery)?,
1717 "base recovery changed since this run was loaded; reload the saved run"
1718 );
1719 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720 let mut execution = DriverExecution {
1721 runner: self,
1722 driver,
1723 };
1724 let _owner = owner;
1727 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728 .await?;
1729 let state = &mut execution.runner.state;
1730 state.parked = false;
1735 state.resource_wait = None;
1736 state.resource_block = None;
1737 state.clear_active();
1744 let pid = std::process::id();
1759 state.driver_pid = Some(pid);
1760 state.driver_started_at = crate::proc::process_started_at(pid);
1761 state.driver_exited = false;
1762 state.driver_claimed = Some(true);
1763 state.begin_pass();
1764 execution.driver.bind(state);
1765 state.save()?;
1766 execution.driver.record(state)?;
1767 Ok(execution)
1768 }
1769
1770 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774 if let Err(error) = &result
1775 && !error.is::<crate::repo_lock::ResourceBlocked>()
1776 && self
1777 .state
1778 .repair_grants
1779 .last()
1780 .is_some_and(|g| g.conflict.is_none())
1781 {
1782 self.state.repair_grants.last_mut().unwrap().phase =
1783 crate::repair_budget::Phase::Complete;
1784 self.state.status = RunStatus::Blocked;
1785 self.state.save()?;
1786 bail!(
1787 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788 self.state.passes.last().map_or(0, |p| p.index)
1789 );
1790 }
1791 result
1792 }
1793
1794 async fn execute_granted_repair(&mut self) -> Result<()> {
1797 use crate::repair_budget::{Kind, Phase};
1798 let index = self.state.repair_grants.len() - 1;
1799 let winner = self
1800 .state
1801 .winner()
1802 .cloned()
1803 .context("repair winner missing")?;
1804 loop {
1805 let grant = self.state.repair_grants[index].clone();
1806 match grant.phase {
1807 Phase::Pending => {
1808 if grant.spent >= grant.receipt.request.rounds {
1809 self.state.repair_grants[index].phase = Phase::Complete;
1810 self.state.status = RunStatus::Blocked;
1811 self.state.save()?;
1812 return Ok(());
1813 }
1814 self.state.repair_grants[index].spent += 1;
1817 self.state.repair_grants[index].phase = Phase::Fixing;
1818 self.state.save()?;
1819 match grant.receipt.request.kind {
1820 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821 Kind::Gate => {
1822 let failed = self.state.gate.clone();
1823 let _ = self.gate_fix_round(&winner, &failed).await?;
1824 }
1825 Kind::Review => {
1826 let mut record = self
1827 .state
1828 .reviews
1829 .last()
1830 .cloned()
1831 .context("review failure missing")?;
1832 let round = record.round;
1833 let step = self.run_fix_round(&winner, &mut record).await?;
1834 self.state.reviews[round - 1] = record;
1835 if matches!(step, FixStep::Recorded { report_lost: true }) {
1836 self.state.repair_grants[index].phase = Phase::Complete;
1837 self.state.status = RunStatus::Blocked;
1838 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839 self.state.save()?;
1840 return Ok(());
1841 }
1842 if matches!(step, FixStep::Resource) {
1843 self.stop_for_resource()?;
1844 return Ok(());
1845 }
1846 self.state.save()?;
1847 }
1848 }
1849 if self.resource_halt()? {
1850 return Ok(());
1851 }
1852 }
1853 Phase::Fixing => {
1854 if grant.receipt.request.kind == Kind::BaseConflict {
1855 self.repair_base_conflict(false).await?;
1856 }
1857 let _coordination = self.coordinate(&winner.worktree).await?;
1860 let rescued = git::rescue_commit(
1861 &winner.worktree,
1862 "magi: preserve interrupted granted repair",
1863 )
1864 .await?;
1865 self.state.note_withheld("repair-grant", &rescued.withheld);
1866 self.state
1869 .verification_history
1870 .push(crate::run::VerificationEvidence {
1871 superseded_by: None,
1872 archived_at: Some(Timestamp::now()),
1873 fixes_spent: Some(self.state.gate_fixes.len()),
1874 gate_head: self.state.gate_head.take(),
1875 gate_ran: std::mem::take(&mut self.state.gate_ran),
1876 gate: std::mem::take(&mut self.state.gate),
1877 e2e: self.state.head_e2e.take(),
1878 });
1879 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880 self.state
1881 .verification_history
1882 .push(crate::run::VerificationEvidence {
1883 superseded_by: None,
1884 archived_at: Some(Timestamp::now()),
1885 fixes_spent: Some(self.state.gate_fixes.len()),
1886 gate_head: None,
1887 gate_ran: false,
1888 gate: Vec::new(),
1889 e2e: Some(e2e),
1890 });
1891 }
1892 self.state.repair_grants[index].review_start = self.state.reviews.len();
1893 self.state.repair_grants[index].head =
1894 git::rev_parse(&winner.worktree, "HEAD").await?;
1895 self.state.repair_grants[index].phase = Phase::Reviewing;
1896 if grant.receipt.request.kind == Kind::BaseConflict {
1897 self.state.base_sync_recovery.stop = None;
1898 if let Some(sync) = &mut self.state.base_sync {
1899 sync.tip = grant
1900 .receipt
1901 .request
1902 .tip
1903 .clone()
1904 .context("repair base tip missing")?;
1905 sync.conflict = None;
1906 sync.behind = 0;
1907 }
1908 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909 }
1910 self.state.save()?;
1911 }
1912 Phase::Reviewing => {
1913 self.review_loop_pass().await?;
1914 if self.resource_halt()? {
1915 return Ok(());
1916 }
1917 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918 let clean = self.state.reviews.len() > grant.review_start
1919 && self.state.reviews.last().is_some_and(|r| {
1920 r.head == head
1921 && r.clean
1922 && r.expected > 0
1923 && r.answered == r.expected
1924 && matches!(
1925 r.e2e_status(),
1926 E2eStatus::Passed | E2eStatus::NotConfigured
1927 )
1928 });
1929 if !clean {
1930 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932 == Kind::Review
1933 && grant.spent < grant.receipt.request.rounds
1934 && self.state.reviews.last().is_some_and(|r| {
1935 r.fix.is_none()
1936 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937 }) {
1938 Phase::Pending
1939 } else {
1940 Phase::Complete
1941 };
1942 self.state.status = RunStatus::Blocked;
1943 self.state.event(
1944 "repair-grant",
1945 "fresh review did not clear the repaired head; no publication",
1946 );
1947 self.state.save()?;
1948 if self.state.repair_grants[index].phase == Phase::Pending {
1949 continue;
1950 }
1951 return Ok(());
1952 }
1953 self.state.repair_grants[index].phase = Phase::Gating;
1954 self.state.save()?;
1955 }
1956 Phase::Gating => {
1957 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958 self.state.reconcile_verification_head(&head);
1959 let wait = self.repository_wait();
1960 if self
1961 .state
1962 .reviews
1963 .last()
1964 .is_none_or(|r| r.head != head || !r.clean)
1965 {
1966 self.state.status = RunStatus::Blocked;
1967 self.state.repair_grants[index].phase = Phase::Complete;
1968 self.state.event(
1969 "repair-grant",
1970 "head changed since fresh review; repair held",
1971 );
1972 self.state.save()?;
1973 return Ok(());
1974 }
1975 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976 return Ok(());
1977 }
1978 let outcomes = self.run_gate(&winner).await?;
1979 if verify_inconclusive(&outcomes) {
1980 return Ok(());
1981 }
1982 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983 if after != head {
1984 self.state.repair_grants[index].review_start = self.state.reviews.len();
1985 self.state.repair_grants[index].head = after;
1986 self.state.repair_grants[index].phase = Phase::Reviewing;
1987 self.state.status = RunStatus::Blocked;
1988 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989 self.state.save()?;
1990 return Ok(());
1991 }
1992 let passed = outcomes.iter().all(CommandOutcome::ok);
1993 if passed
1996 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997 {
1998 return Ok(());
1999 }
2000 self.state.repair_grants[index].succeeded = passed;
2001 self.state.gate = outcomes;
2002 self.state.gate_ran = true;
2003 self.state.status = if passed {
2004 RunStatus::Gating
2005 } else {
2006 RunStatus::Blocked
2007 };
2008 self.state.repair_grants[index].phase = if !passed
2009 && grant.receipt.request.kind == Kind::Gate
2010 && grant.spent < grant.receipt.request.rounds
2011 {
2012 Phase::Pending
2013 } else {
2014 Phase::Complete
2015 };
2016 if passed && self.state.config.merge.mode == MergeMode::Pr {
2017 self.state.pr_publication_pending = true;
2019 } else if passed {
2020 self.state.status = RunStatus::Ready;
2021 }
2022 self.state.save()?;
2023 if !passed {
2024 continue;
2025 }
2026 if self.state.config.merge.mode != MergeMode::Pr {
2030 self.state.status = RunStatus::Ready;
2031 self.state.event(
2032 "repair-grant",
2033 "repair verified locally; grant carries no merge authorization",
2034 );
2035 self.state.save()?;
2036 return Ok(());
2037 }
2038 return self.merge().await;
2039 }
2040 Phase::Complete => return Ok(()),
2041 }
2042 }
2043 }
2044
2045 async fn execute_graph(&mut self) -> Result<()> {
2046 let archive_home = crate::run::home().canonicalize()?;
2050 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051 let repo = self.state.repo.clone();
2052 let _coordination = self.coordinate(&repo).await?;
2053 ensure_setup_config(&mut self.state, &repo).await?;
2054 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055 }
2056 self.state.configure_recovery_resume()?;
2057 if self
2058 .state
2059 .repair_grants
2060 .last()
2061 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062 {
2063 return self.execute_granted_repair().await;
2064 }
2065 if self
2066 .state
2067 .repair_grants
2068 .last()
2069 .is_some_and(|g| !g.succeeded)
2070 {
2071 self.state.status = RunStatus::Blocked;
2072 self.state.save()?;
2073 return Ok(());
2074 }
2075 self.persist_candidates().await?;
2078 if self.state.base_sync_blocked() {
2081 self.sync_to_base().await?;
2082 if !self.state.base_sync_blocked()
2083 && self.state.status != RunStatus::AlreadyInBase
2084 && !self.resource_halt()?
2085 {
2086 self.finish_after_tally().await?;
2087 }
2088 return Ok(());
2089 }
2090 if self
2091 .state
2092 .merge_decision
2093 .as_ref()
2094 .is_some_and(|decision| !decision.closed)
2095 && self.state.status != RunStatus::Landing
2096 {
2097 if crate::merge_decision::resumable_hold(&self.state) {
2098 self.state.merge = Some(MergeOutcome {
2099 mode: MergeMode::Pr,
2100 ok: false,
2101 detail: self
2102 .state
2103 .merge_decision
2104 .as_ref()
2105 .unwrap()
2106 .target
2107 .pr
2108 .clone(),
2109 empty: false,
2110 });
2111 self.run_land().await?;
2112 self.settle_questions();
2113 return Ok(());
2114 }
2115 bail!(
2116 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117 );
2118 }
2119 if self.state.config.graph.selected_implementer.is_some()
2120 && self.state.status == RunStatus::Blocked
2121 && self.state.tally.is_none()
2122 && self.state.candidate_preservation.is_empty()
2123 && self.state.interrupted_implementations.is_empty()
2124 {
2125 bail!(
2126 "selected implementation is held; explicitly release the owning task to retry it"
2127 );
2128 }
2129
2130 if self.state.merge_repairs.last().is_some_and(|r| {
2131 matches!(
2132 r.phase,
2133 crate::run::MergeRepairPhase::Requested
2134 | crate::run::MergeRepairPhase::Fixing
2135 | crate::run::MergeRepairPhase::Reviewing
2136 )
2137 }) {
2138 if self.repair_merge_review().await? {
2139 self.run_land().await?;
2140 }
2141 self.settle_questions();
2142 return Ok(());
2143 }
2144 if self.state.pr_publication_pending {
2147 self.merge().await?;
2148 if self.resource_halt()? {
2149 return Ok(());
2150 }
2151 self.state.save()?;
2152 self.settle_questions();
2153 return Ok(());
2154 }
2155 if self.state.status == RunStatus::Stalled {
2168 if self.recover_stall().await? {
2169 self.finish_after_tally().await?;
2170 } else {
2171 self.state.save()?;
2173 }
2174 return Ok(());
2175 }
2176 if self.state.status == RunStatus::Landing
2186 || crate::handover::landing_checkpoint(&self.state)
2187 {
2188 self.state.land_sync_pending = true;
2189 self.state.save()?;
2190 self.run_land().await?;
2191 self.settle_questions();
2196 return Ok(());
2197 }
2198 if self.state.github_text.is_some()
2203 && self.state.merge.is_none()
2204 && self.state.status == RunStatus::Gating
2205 {
2206 self.merge().await?;
2207 if self.resource_halt()? {
2208 return Ok(());
2209 }
2210 self.state.save()?;
2211 return Ok(());
2212 }
2213 if let Err(e) = self.prep().await {
2214 if e.is::<crate::repo_lock::ResourceBlocked>() {
2215 return Err(e);
2216 }
2217 self.state.status = RunStatus::Blocked;
2218 self.state
2219 .prep_failure
2220 .get_or_insert_with(|| format!("{e:#}"));
2221 self.state
2222 .event("prep", format!("preparation stopped: {e:#}"));
2223 self.state.save()?;
2224 return Err(e);
2225 }
2226 if self.park_here()? {
2227 return Ok(());
2228 }
2229 self.advise().await?;
2230 if self.park_here()? {
2231 return Ok(());
2232 }
2233 self.implement().await?;
2234 if self.park_here()? {
2235 return Ok(());
2236 }
2237 if self.state.status == RunStatus::VerifiedNoop {
2241 return Ok(());
2242 }
2243 self.judge().await?;
2244 if self.park_here()? {
2245 return Ok(());
2246 }
2247 self.deliberate().await?;
2248 if self.park_here()? {
2249 return Ok(());
2250 }
2251 self.vote().await?;
2252 if self.park_here()? {
2253 return Ok(());
2254 }
2255 self.tally()?;
2256 if self.state.status == RunStatus::Stalled {
2261 self.state.save()?;
2265 return Ok(());
2266 }
2267 self.finish_after_tally().await?;
2268 Ok(())
2269 }
2270
2271 fn stop_for_resource(&mut self) -> Result<()> {
2276 if self.pause.parked() || self.interrupt.parked() {
2279 self.park_here()?;
2280 return Ok(());
2281 }
2282 if self.state.resource_block.is_some() {
2283 self.state.status = RunStatus::Blocked;
2284 }
2285 self.state.save()
2286 }
2287
2288 fn resource_halt(&mut self) -> Result<bool> {
2290 if self.state.resource_block.is_none() {
2291 return Ok(false);
2292 }
2293 self.stop_for_resource()?;
2294 Ok(true)
2295 }
2296
2297 fn park_here(&mut self) -> Result<bool> {
2304 let parking = self.pause.parked() || self.interrupt.parked();
2305 if !parking && self.state.resource_block.is_some() {
2306 self.stop_for_resource()?;
2307 return Ok(true);
2308 }
2309 if !self.pause.parked() && !self.interrupt.parked() {
2314 return Ok(false);
2315 }
2316 if self.state.parked {
2317 return Ok(true);
2318 }
2319 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320 if let Some(block) = &mut self.state.resource_block
2321 && matches!(
2322 block.cause,
2323 crate::run::ResourceBlockCause::Cancelled { .. }
2324 )
2325 {
2326 block.cause = crate::run::ResourceBlockCause::Cancelled {
2327 park: cause.clone(),
2328 };
2329 }
2330 let why = match cause.as_ref().map(ParkCause::label) {
2331 Some(reason) => format!(
2332 "parked after `{}` ({reason}) — resume to carry on from here",
2333 self.state.status.as_str()
2334 ),
2335 None => format!(
2336 "parked after `{}` — resume to carry on from here",
2337 self.state.status.as_str()
2338 ),
2339 };
2340 self.state.event("park", why);
2341 self.state
2342 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343 self.state.parked = true;
2344 self.state.save()?;
2345 Ok(true)
2346 }
2347
2348 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350 if self.state.prep_failure.is_none() {
2351 return Ok(());
2352 }
2353 let failed_at = self
2354 .state
2355 .events
2356 .iter()
2357 .rev()
2358 .find(|event| event.node == "prep")
2359 .map_or(self.state.updated_at, |event| event.at);
2360 anyhow::ensure!(
2361 approval_at > failed_at,
2362 "preparation retry requires a new explicit release after the failure"
2363 );
2364 self.state.prep_failure = None;
2365 self.state.prep_complete = Some(false);
2366 self.state
2367 .event("prep-retry", "operator approved retry of preparation");
2368 Ok(())
2369 }
2370
2371 pub fn on_pause(&mut self, pause: Pause) {
2373 self.pause = pause;
2374 }
2375
2376 pub fn watch_interrupt(&mut self, pause: Pause) {
2382 self.interrupt = pause;
2383 }
2384
2385 fn settle_questions(&mut self) {
2405 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407 }
2408 }
2409
2410 fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413 Box::pin(self.finish_after_tally_inner())
2414 }
2415
2416 async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417 self.graft().await?;
2420 if self.resource_halt()? {
2421 return Ok(());
2422 }
2423 self.fold_losers().await?;
2424 self.sync_to_base().await?;
2429 if self.hold_base_sync().await? || self.resource_halt()? {
2430 return Ok(());
2431 }
2432 if self.state.status == RunStatus::AlreadyInBase {
2433 return Ok(());
2434 }
2435 self.review_loop().await?;
2436 if self.resource_halt()? {
2437 return Ok(());
2438 }
2439 self.sync_to_base().await?;
2440 if self.hold_base_sync().await? || self.resource_halt()? {
2441 return Ok(());
2442 }
2443 if self.state.status == RunStatus::AlreadyInBase {
2444 return Ok(());
2445 }
2446 self.gate().await?;
2447 self.merge().await?;
2448 if self.resource_halt()? {
2449 return Ok(());
2450 }
2451 self.state.save()?;
2452 Ok(())
2453 }
2454
2455 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462 let result = match ensure_setup_config(&mut self.state, repo).await {
2463 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464 Err(e) => Err(e),
2465 };
2466 if let Err(e) = result {
2467 git::worktree_remove_clean(repo, wt).await.ok();
2468 if in_prep {
2469 self.rollback_prep(repo).await;
2470 }
2471 self.state.status = RunStatus::Blocked;
2472 self.state.event(
2473 "setup",
2474 format!("worktree setup failed in {}: {e:#}", wt.display()),
2475 );
2476 self.state.save()?;
2477 return Err(e);
2478 }
2479 Ok(())
2480 }
2481
2482 async fn rollback_prep(&mut self, repo: &Path) {
2484 let root = self.state.worktree_root();
2485 for c in std::mem::take(&mut self.state.candidates) {
2486 if matches!(
2487 git::worktree_remove_clean(repo, &c.worktree).await,
2488 Ok(true)
2489 ) {
2490 git::branch_delete(repo, &c.branch).await.ok();
2491 } else {
2492 self.state.candidates.push(c);
2493 }
2494 }
2495 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496 for j in 1..=seats {
2497 remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2498 .await
2499 .ok();
2500 }
2501 for k in 1..=self.state.config.graph.advisors {
2502 remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2503 .await
2504 .ok();
2505 }
2506 }
2507
2508 async fn prep(&mut self) -> Result<()> {
2509 if let Some(reason) = &self.state.prep_failure {
2510 bail!(
2511 "candidate preparation previously failed: {reason}; correct the source before retrying"
2512 );
2513 }
2514 if self
2515 .state
2516 .prep_complete
2517 .unwrap_or(!self.state.candidates.is_empty())
2518 {
2519 return Ok(());
2520 }
2521 let repo = self.state.repo.clone();
2522 let _coordination = self.coordinate(&repo).await?;
2525 let base = self.state.base_commit.clone();
2526 let plan = self.state.sources.plan(&repo, &base).await?;
2527 self.rollback_prep(&repo).await;
2528 if !self.state.candidates.is_empty() {
2529 bail!(
2530 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531 );
2532 }
2533 self.state.prep_complete = Some(false);
2534 self.state.status = RunStatus::Prep;
2535 self.state.save()?;
2536 let repo = self.state.repo.clone();
2537 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538 let root = self.state.worktree_root();
2539 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541 let hooks_dir = self.state.dir().join("hooks");
2544 if self.state.config.blind.commit_msg_hook {
2545 std::fs::create_dir_all(&hooks_dir)
2546 .with_context(|| format!("create {}", hooks_dir.display()))?;
2547 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548 let path = hooks_dir.join("commit-msg");
2549 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550 make_executable(&path)?;
2551 git::acquire_worktree_config(&repo).await?;
2559 self.state.enabled_worktree_config = true;
2560 }
2561
2562 ensure_setup_config(&mut self.state, &repo).await?;
2565
2566 for (index, (spec, label)) in self
2567 .roles
2568 .implementers
2569 .clone()
2570 .into_iter()
2571 .zip(labels)
2572 .enumerate()
2573 {
2574 let branch = self.state.branch_for(label);
2575 let worktree = root.join(format!("cand-{label}"));
2576 self.state.event(
2580 "prep-intent",
2581 serde_json::json!({
2582 "branch": branch,
2583 "worktree": worktree,
2584 "start": start,
2585 "picks": plan.picks,
2586 })
2587 .to_string(),
2588 );
2589 self.state.save()?;
2590 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591 if self.state.config.blind.commit_msg_hook {
2592 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593 }
2594 git::local_exclude(&worktree, "/.magi/").await?;
2595 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596 git::worktree_remove_clean(&repo, &worktree).await.ok();
2599 git::branch_delete(&repo, &branch).await.ok();
2600 self.rollback_prep(&repo).await;
2601 self.state.status = RunStatus::Blocked;
2602 self.state
2603 .event("prep", format!("worktree setup failed: {e:#}"));
2604 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605 self.state.save()?;
2606 return Err(e);
2607 }
2608 self.state.candidates.push(Candidate {
2609 index,
2610 label,
2611 agent: spec.id.clone(),
2612 branch,
2613 worktree,
2614 summary: String::new(),
2615 stat: String::new(),
2616 files: 0,
2617 commits: 0,
2618 empty: false,
2619 failed: None,
2620 verified_noop: None,
2621 duration_ms: 0,
2622 folded: false,
2623 });
2624 self.state.save()?;
2625 for pick in &plan.picks {
2626 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627 {
2628 self.state.status = RunStatus::Blocked;
2629 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630 self.state
2631 .event("prep", format!("cannot apply referenced commit: {e}"));
2632 self.state.save()?;
2633 return Err(e);
2634 }
2635 }
2636 }
2637
2638 for j in 1..=self.roles.judges.len() {
2639 let wt = root.join(format!("judge-{j}"));
2640 if !wt.exists() {
2641 git::worktree_add_detached(&repo, &wt, &base).await?;
2642 self.setup_seat_worktree(&repo, &wt, true).await?;
2643 }
2644 }
2645
2646 if self.state.config.graph.advise {
2654 for k in 1..=self.state.config.graph.advisors {
2655 let wt = root.join(format!("advisor-{k}"));
2656 if !wt.exists() {
2657 git::worktree_add_detached(&repo, &wt, &base).await?;
2658 self.setup_seat_worktree(&repo, &wt, true).await?;
2659 }
2660 }
2661 }
2662
2663 let authors: Vec<&str> = self
2668 .roles
2669 .implementers
2670 .iter()
2671 .map(|a| a.id.as_str())
2672 .collect();
2673 let overlap: Vec<String> = self
2674 .roles
2675 .judges
2676 .iter()
2677 .enumerate()
2678 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680 .collect();
2681 if !overlap.is_empty() {
2682 let note = format!(
2683 "{} also authored a candidate; blind, but the panel is less \
2684 independent than {} distinct agents would be",
2685 overlap.join(", "),
2686 self.roles.judges.len()
2687 );
2688 self.state.event("prep", note);
2689 }
2690
2691 self.state.event(
2692 "prep",
2693 format!(
2694 "{} candidates, {} judges, base {} ({})",
2695 self.state.candidates.len(),
2696 self.roles.judges.len(),
2697 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698 self.state.base_branch
2699 ),
2700 );
2701 self.state.prep_complete = Some(true);
2702 self.state.status = RunStatus::Implementing;
2703 self.state.save()?;
2704 Ok(())
2705 }
2706
2707 async fn advise(&mut self) -> Result<()> {
2740 let implement_untouched = self
2741 .state
2742 .candidates
2743 .iter()
2744 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745 if !self.state.config.graph.advise || self.state.advise_attempted {
2746 return Ok(());
2747 }
2748 if !implement_untouched {
2749 self.state.event(
2750 "advise",
2751 "skipping the design-deliberation stage: at least one \
2752 candidate already shows implementation progress, so this \
2753 run is past the point the stage exists to run before"
2754 .to_owned(),
2755 );
2756 self.state.advise_attempted = true;
2757 self.state.save()?;
2758 return Ok(());
2759 }
2760 let run_id = self.state.id.clone();
2761 let prompts = self.state.config.prompts.clone();
2762 let instruction = self.state.instruction.clone();
2763 let language = self.state.config.graph.language.clone();
2764 let root = self.state.worktree_root();
2765 let n = self.state.config.graph.advisors;
2766 let where_recorded = self.state.dir().join("run.json");
2767
2768 let seats = match self.state.config.advisors() {
2769 Ok(seats) if !seats.is_empty() => seats,
2770 Ok(_) => {
2771 self.state.event(
2772 "advise",
2773 format!(
2774 "[graph] advisors is 0; skipping the design-deliberation \
2775 stage and continuing without a synthesis brief (see {})",
2776 where_recorded.display()
2777 ),
2778 );
2779 self.state.advise_attempted = true;
2780 self.state.save()?;
2781 return Ok(());
2782 }
2783 Err(e) => {
2784 self.state.event(
2785 "advise",
2786 format!(
2787 "could not resolve advisor seats ({e:#}); continuing \
2788 without a design-deliberation brief (see {})",
2789 where_recorded.display()
2790 ),
2791 );
2792 self.state.advise_attempted = true;
2793 self.state.save()?;
2794 return Ok(());
2795 }
2796 };
2797
2798 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799 let artifacts = agent::artifacts_dir(&self.state.dir());
2800 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802 let mut jobs = Vec::new();
2803 for (i, spec) in seats.iter().cloned().enumerate() {
2804 let seat_key = format!("advisor-{}", i + 1);
2805 let seat = self.seat(&seat_key, &spec.id);
2806 jobs.push(SeatJob {
2807 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808 spec,
2809 seat,
2810 cwd: worktrees[i % worktrees.len()].clone(),
2811 timeout,
2812 allow_write: false,
2813 sessions: false,
2814 artifacts: artifacts.clone(),
2815 stem: seat_key,
2816 handover: None,
2817 });
2818 }
2819
2820 self.state.event(
2821 "advise",
2822 format!(
2823 "{} advisor seat(s) sketching a design in parallel",
2824 jobs.len()
2825 ),
2826 );
2827 let mut quota_losses = Vec::new();
2828 let cache = self.state.config.cache_dir();
2829 let ctx = WaveCtx {
2830 carry_seats: false,
2831 pauses: Some((&self.pause, &self.interrupt)),
2832 run: &run_id,
2833 node: "advise",
2834 prompts: &prompts,
2835 cache: cache.as_deref(),
2836 round: None,
2837 };
2838 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839 let results = ask_json_wave::<Proposal>(
2840 jobs,
2841 Arc::clone(&self.sem),
2842 self.state.config.graph.retries,
2843 &advisor_roster,
2844 &ctx,
2845 &mut quota_losses,
2846 &mut self.state,
2847 &|p: &Proposal| p.validate(),
2848 )
2849 .await;
2850 if self.state.resource_block.is_some() {
2851 return self.stop_for_resource();
2852 }
2853 self.state.quota.extend(quota_losses);
2854
2855 let mut records = Vec::with_capacity(results.len());
2856 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857 let agent_id = seat.agent.clone();
2858 self.state.seats.insert(seat.key.clone(), seat);
2859 match res {
2860 Ok((proposal, out)) => {
2861 self.state
2862 .event("advise", format!("advisor-{} proposed a design", i + 1));
2863 records.push(advise::AdvisorRecord::proposed(
2864 i + 1,
2865 agent_id,
2866 proposal,
2867 out.duration_ms,
2868 ));
2869 }
2870 Err(e) => {
2871 self.state.event(
2872 "advise",
2873 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874 );
2875 records.push(advise::AdvisorRecord::failed(
2876 i + 1,
2877 agent_id,
2878 e.to_string(),
2879 ));
2880 }
2881 }
2882 }
2883
2884 let mut advice = advise::Advice {
2885 records,
2886 synthesis: None,
2887 };
2888 if advice.proposals().is_empty() {
2889 self.state.event(
2890 "advise",
2891 "no advisor produced a usable proposal; continuing without a \
2892 synthesis brief"
2893 .to_owned(),
2894 );
2895 } else {
2896 match self
2897 .synthesize_brief(
2898 &advice,
2899 &instruction,
2900 &language,
2901 &worktrees[0],
2902 &artifacts,
2903 &run_id,
2904 &prompts,
2905 cache.as_deref(),
2906 )
2907 .await
2908 {
2909 Ok(Some(text)) => {
2910 self.state.event(
2911 "advise",
2912 "synthesized a design brief for the implementer".to_owned(),
2913 );
2914 advice.synthesis = Some(text);
2915 }
2916 Ok(None) => {
2917 self.state.event(
2918 "advise",
2919 "the synthesis seat produced nothing usable; continuing \
2920 without a design brief"
2921 .to_owned(),
2922 );
2923 }
2924 Err(e) => {
2925 if e.is::<crate::repo_lock::ResourceBlocked>() {
2926 return Err(e);
2927 }
2928 self.state.event(
2929 "advise",
2930 format!("could not synthesize a design brief: {e:#}"),
2931 );
2932 }
2933 }
2934 }
2935 advise::apply_reflection(&mut advice);
2936
2937 self.state.advice = Some(advice);
2938 self.state.advise_attempted = true;
2939 self.state.save()?;
2940 Ok(())
2941 }
2942
2943 #[allow(clippy::too_many_arguments)]
2955 async fn synthesize_brief(
2956 &mut self,
2957 advice: &advise::Advice,
2958 instruction: &str,
2959 language: &str,
2960 cwd: &Path,
2961 artifacts: &Path,
2962 run_id: &str,
2963 prompts: &Prompts,
2964 cache: Option<&Path>,
2965 ) -> Result<Option<String>> {
2966 let chain = agent::pick_chain(
2967 &self.state.config.agents,
2968 self.state.config.roles.synthesizer.as_ref(),
2969 &agent::installed,
2970 "synthesizer",
2971 )?;
2972 let proposals = advice.proposals();
2973 let mut prompt = prompt::with_overlay(
2974 prompt::synthesize_brief(instruction, &proposals, language),
2975 prompts.overlay("advise"),
2976 );
2977 if cache.is_some() {
2978 prompt.push('\n');
2983 prompt.push_str(&prompt::build_cache_note("advise", false));
2984 }
2985 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986 let mut last = None;
2991 for (n, spec) in chain.iter().enumerate() {
2992 if n > 0 {
2993 self.state
2994 .event("advise", format!("synthesis falling back to {}", spec.id));
2995 }
2996 let mut seat = self.seat("advise-synthesis", &spec.id);
2997 let wait = self.repository_wait();
2998 let outcome = wait
2999 .scope(agent::invoke(
3000 spec,
3001 &mut seat,
3002 &Invocation {
3003 coordination: self.state.config.coordination.as_ref(),
3004 fetched_base: None,
3005 cwd,
3006 prompt: &prompt,
3007 timeout,
3008 allow_write: false,
3009 sessions: false,
3010 artifacts,
3011 stem: &if n == 0 {
3012 "advise-synthesis".to_owned()
3013 } else {
3014 format!("advise-synthesis-{}", spec.id)
3015 },
3016 run: run_id,
3017 node: "advise",
3018 cache_dir: None,
3019 attachments: &[],
3020 writable: &[],
3021 },
3022 ))
3023 .await;
3024 if outcome.is_ok() {
3025 self.state.seats.insert(seat.key.clone(), seat);
3026 }
3027 let advance = agent::chain_advances(&outcome);
3028 last = Some(outcome);
3029 if !advance {
3030 break;
3031 }
3032 }
3033 let out = last.expect("a chain holds at least one agent")?;
3036 if !out.usable() {
3037 return Ok(None);
3038 }
3039 let text =
3040 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041 Ok((!text.trim().is_empty()).then_some(text))
3042 }
3043
3044 async fn implement(&mut self) -> Result<()> {
3047 self.persist_candidates().await?;
3048 let run_id = self.state.id.clone();
3053 let prompts = self.state.config.prompts.clone();
3054 let todo: Vec<usize> = self
3055 .state
3056 .candidates
3057 .iter()
3058 .enumerate()
3059 .filter(|(_, c)| {
3060 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061 && c.failed.is_none()
3062 && !c.empty
3063 })
3064 .map(|(i, _)| i)
3065 .collect();
3066 if todo.is_empty() {
3067 return self.after_implement();
3068 }
3069 self.state.status = RunStatus::Implementing;
3070
3071 let language = self.state.config.graph.language.clone();
3072 let github_language = self.state.config.graph.github_language.clone();
3073 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074 let sessions = self.state.config.graph.sessions;
3075 let artifacts = agent::artifacts_dir(&self.state.dir());
3076 let brief = self
3080 .state
3081 .advice
3082 .as_ref()
3083 .and_then(|a| a.synthesis.as_deref())
3084 .map(str::to_owned);
3085 let attachments = self.state.attachments.clone();
3086
3087 let mut jobs = Vec::new();
3088 for &i in &todo {
3089 let (index, label, worktree) = {
3090 let c = &self.state.candidates[i];
3091 (c.index, c.label, c.worktree.clone())
3092 };
3093 let spec = self.roles.implementers[index].clone();
3094 let seat_key = format!("impl-{label}");
3095 let seat = self.seat(&seat_key, &spec.id);
3096 let instruction = seeded_instruction(&self.state);
3097 jobs.push(SeatJob {
3098 spec,
3099 seat,
3100 prompt: prompt::implement_in(
3101 &instruction,
3102 &worktree.to_string_lossy(),
3103 &language,
3104 &github_language,
3105 brief.as_deref(),
3106 &attachments,
3107 ),
3108 cwd: worktree,
3109 timeout,
3110 allow_write: true,
3111 sessions,
3112 artifacts: artifacts.clone(),
3113 stem: format!("impl-{label}"),
3114 handover: None,
3115 });
3116 }
3117
3118 self.state.event(
3119 "implement",
3120 format!("{} candidates in parallel", jobs.len()),
3121 );
3122 let mut sent = jobs.clone();
3128 let cache = self.state.config.cache_dir();
3129 let ctx = WaveCtx {
3130 carry_seats: false,
3131 pauses: Some((&self.pause, &self.interrupt)),
3132 run: &run_id,
3133 node: "implement",
3134 prompts: &prompts,
3135 cache: cache.as_deref(),
3136 round: None,
3137 };
3138 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139 if !wave_blocked(&results) {
3140 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141 .await;
3142 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143 .await?;
3144 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145 .await;
3146 }
3147 if !results.is_empty()
3150 && results
3151 .iter()
3152 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153 {
3154 let losses: Vec<_> = results
3155 .iter()
3156 .map(|(_, seat, out)| {
3157 let AgentOutcome::Quota(o) = out else {
3158 unreachable!()
3159 };
3160 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161 })
3162 .collect();
3163 self.state.quota.extend(losses.clone());
3164 for (_, seat, _) in results {
3165 self.state.seats.insert(seat.key.clone(), seat);
3166 }
3167 for &i in &todo {
3168 let label = self.state.candidates[i].label;
3169 self.state.interrupted_implementations.insert(label);
3170 self.state
3171 .candidate_preservation
3172 .insert(label, "machine outage awaiting persistence".into());
3173 }
3174 block_machine_outage(&mut self.state, "implement", &losses);
3177 self.state.save()?;
3178 self.persist_candidates().await?;
3179 for &i in &todo {
3180 self.state
3181 .interrupted_implementations
3182 .insert(self.state.candidates[i].label);
3183 }
3184 return self.stop_for_resource();
3185 }
3186 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189 && results
3190 .iter()
3191 .any(|(_, _, out)| FailClass::of(out).is_some());
3192 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194 if self
3198 .state
3199 .candidate_preservation
3200 .contains_key(&self.state.candidates[i].label)
3201 {
3202 self.state.seats.insert(seat.key.clone(), seat);
3203 }
3204 continue;
3205 }
3206 let seat_key = seat.key.clone();
3207 let agent = seat.agent.clone();
3216 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217 self.state.seats.insert(seat.key.clone(), seat);
3218 let label = self.state.candidates[i].label;
3219 self.state.interrupted_implementations.remove(&label);
3220
3221 let (summary, duration, failed, verified_claim) = match out {
3222 AgentOutcome::Ok(o) => {
3223 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224 let failed = (!o.usable()).then(|| {
3225 if o.timed_out {
3226 "agent timed out".to_owned()
3227 } else {
3228 format!(
3229 "agent exited with {}",
3230 crate::notices::exit_status(o.exit_code)
3231 )
3232 }
3233 });
3234 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235 (text, o.duration_ms, failed, verified_claim)
3236 }
3237 AgentOutcome::Dropped(o) => {
3243 let why = o
3244 .dropped
3245 .as_ref()
3246 .map(|d| d.why.as_str())
3247 .unwrap_or("the CLI ended the stream without delivering its answer");
3248 (
3249 String::new(),
3250 o.duration_ms,
3251 Some(format!("the CLI dropped the stream ({why})")),
3252 None,
3253 )
3254 }
3255 AgentOutcome::Quota(o) => {
3256 self.state.quota.push(QuotaLoss::from_report(
3257 seat_key,
3258 "implement",
3259 o.quota.as_ref(),
3260 ));
3261 (
3262 String::new(),
3263 o.duration_ms,
3264 Some("rate limited (quota); produced no change".to_owned()),
3265 None,
3266 )
3267 }
3268 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269 (String::new(), 0, Some(e), None)
3270 }
3271 };
3272
3273 let c = &mut self.state.candidates[i];
3276 if !exhausted_the_fallback_chain {
3277 c.agent = agent;
3278 }
3279 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280 c.duration_ms = duration;
3281 c.failed = failed;
3282 c.verified_noop = verified_claim;
3283 self.state
3284 .candidate_preservation
3285 .insert(label, "awaiting persistence".to_owned());
3286 }
3287 self.state.save()?;
3288 self.persist_candidates().await?;
3289
3290 if resource_blocked {
3291 return self.stop_for_resource();
3292 }
3293 if selected_failed {
3294 self.state.status = RunStatus::Blocked;
3295 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296 self.state.save()?;
3297 bail!(
3298 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299 );
3300 }
3301 self.after_implement()
3302 }
3303
3304 async fn persist_candidates(&mut self) -> Result<()> {
3306 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307 for label in labels {
3308 let i = self
3309 .state
3310 .candidates
3311 .iter()
3312 .position(|c| c.label == label)
3313 .context("pending candidate is missing")?;
3314 let c = self.state.candidates[i].clone();
3315 let interrupted = self.state.interrupted_implementations.contains(&label);
3316 let result = async {
3317 let wait = crate::repo_lock::WaitPolicy::new(
3318 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319 || false,
3320 );
3321 let _coordination =
3322 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323 if let Err(e) = git::snapshot_uncommitted(
3326 &c.worktree,
3327 &self
3328 .state
3329 .dir()
3330 .join("artifacts")
3331 .join(format!("candidate-{label}-recovery")),
3332 )
3333 .await
3334 {
3335 self.state.event(
3336 "preserve",
3337 format!("candidate {label} snapshot incomplete: {e:#}"),
3338 );
3339 }
3340 let rescued = git::rescue_commit(
3341 &c.worktree,
3342 &format!("magi: candidate {label} (uncommitted work)"),
3343 )
3344 .await?;
3345 self.state.note_withheld("implement", &rescued.withheld);
3346 let base = &self.state.base_commit;
3347 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352 if c.empty && commits > 0 && !patch.trim().is_empty()
3353 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355 || self.state.tally.is_some())
3356 {
3357 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361 }
3362 let c = &mut self.state.candidates[i];
3363 c.commits = commits;
3364 c.stat = stat;
3365 c.files = files;
3366 let no_changes = commits == 0 || patch.trim().is_empty();
3367 c.empty = no_changes && !interrupted;
3368 if interrupted && no_changes {
3369 c.failed = None;
3370 c.verified_noop = None;
3371 }
3372 if !c.empty {
3373 c.failed = None;
3374 c.verified_noop = None;
3375 }
3376 self.state.event(
3377 "preserve",
3378 format!("candidate {label}: {files} files, {commits} commits"),
3379 );
3380 Ok::<_, anyhow::Error>(())
3381 }
3382 .await;
3383 match result {
3384 Ok(()) => {
3385 self.state.candidate_preservation.remove(&label);
3386 if !interrupted || self.state.candidates[i].files > 0 {
3387 self.state.interrupted_implementations.remove(&label);
3388 }
3389 }
3390 Err(e) => {
3391 let reason = format!("candidate {label} preservation failed: {e:#}");
3392 self.state.event("preserve", &reason);
3393 self.state.candidate_preservation.insert(label, reason);
3394 }
3395 }
3396 if self.state.candidate_preservation.is_empty()
3397 && self.state.status == RunStatus::Blocked
3398 && self.state.candidates.iter().all(|c| c.failed.is_none())
3399 {
3400 self.state.status = RunStatus::Implementing;
3404 self.state.event(
3405 "preserve",
3406 "candidate persistence recovered; implementation can continue",
3407 );
3408 }
3409 self.state.save()?;
3410 }
3411 if !self.state.candidate_preservation.is_empty() {
3412 self.state.status = RunStatus::Blocked;
3413 self.state.save()?;
3414 bail!(
3415 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416 );
3417 }
3418 Ok(())
3419 }
3420
3421 async fn resume_undelivered(
3449 &mut self,
3450 results: &mut [(usize, SeatState, AgentOutcome)],
3451 sent: &[SeatJob],
3452 prompts: &Prompts,
3453 run_id: &str,
3454 ) {
3455 for (wi, seat, out) in results.iter_mut() {
3456 let Some(dropped) = (match &*out {
3457 AgentOutcome::Dropped(o) => o.dropped.clone(),
3458 _ => None,
3459 }) else {
3460 continue;
3461 };
3462 let Some(job) = sent.get(*wi) else { continue };
3463 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465 self.state.event(
3466 "implement",
3467 format!(
3468 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469 work is in the tree",
3470 seat.key, dropped.output_tokens, dropped.why
3471 ),
3472 );
3473 continue;
3474 }
3475 if !has_context(&job.spec, seat, job.sessions) {
3483 self.state.event(
3484 "implement",
3485 format!(
3486 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487 is no session left to resume",
3488 seat.key, dropped.output_tokens, dropped.why
3489 ),
3490 );
3491 continue;
3492 }
3493 self.state.event(
3494 "implement",
3495 format!(
3496 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497 conversation",
3498 seat.key, dropped.output_tokens, dropped.why
3499 ),
3500 );
3501 let mut retry = job.clone();
3502 retry.seat = seat.clone();
3503 retry.prompt = prompt::resume_after_drop(&dropped.why);
3504 retry.timeout = retry_budget(job.timeout, true);
3505 retry.stem = format!("{}-resume", job.stem);
3506 let cache = self.state.config.cache_dir();
3507 let ctx = WaveCtx {
3508 carry_seats: false,
3509 pauses: Some((&self.pause, &self.interrupt)),
3510 run: run_id,
3511 node: "implement",
3512 prompts,
3513 cache: cache.as_deref(),
3514 round: None,
3515 };
3516 let (resumed_seat, resumed) =
3517 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518 *seat = resumed_seat;
3519 *out = resumed;
3520 }
3521 }
3522
3523 async fn resume_seat_handovers(
3588 &mut self,
3589 results: &mut [(usize, SeatState, AgentOutcome)],
3590 sent: &mut [SeatJob],
3591 prompts: &Prompts,
3592 run_id: &str,
3593 ) -> Result<()> {
3594 let instruction = seeded_instruction(&self.state);
3595 let language = self.state.config.graph.language.clone();
3596 let github_language = self.state.config.graph.github_language.clone();
3597 let brief = self
3598 .state
3599 .advice
3600 .as_ref()
3601 .and_then(|a| a.synthesis.as_deref())
3602 .map(str::to_owned);
3603 let attachments = self.state.attachments.clone();
3604 for (wi, seat, out) in results.iter_mut() {
3605 let others: BTreeSet<String> = sent
3608 .iter()
3609 .enumerate()
3610 .filter(|(j, _)| j != wi)
3611 .map(|(_, j)| j.spec.id.clone())
3612 .collect();
3613 let Some(job) = sent.get_mut(*wi) else {
3614 continue;
3615 };
3616 let start = self
3621 .roles
3622 .implementer_roster
3623 .iter()
3624 .position(|s| s.id == job.spec.id)
3625 .unwrap_or(0);
3626 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627 let mut fallback_attempt = 0usize;
3628 let mut prev: Option<FailClass> = None;
3629 while let Some(cur) = FailClass::of(&*out) {
3630 if !should_hand_over(prev.as_ref(), &cur) {
3631 break;
3632 }
3633 let Some(next) =
3634 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635 .cloned()
3636 else {
3637 break;
3638 };
3639 tried.insert(next.id.clone());
3640 fallback_attempt += 1;
3641
3642 let coordination = self.coordinate(&job.cwd).await?;
3643 if let Ok(r) = git::rescue_commit(
3644 &job.cwd,
3645 &format!(
3646 "magi: candidate {} (uncommitted work before {} fallback)",
3647 seat.key,
3648 if cur == FailClass::Quota {
3649 "quota"
3650 } else {
3651 "handover"
3652 }
3653 ),
3654 )
3655 .await
3656 {
3657 self.state.note_withheld("implement", &r.withheld);
3658 }
3659 drop(coordination);
3660
3661 record_handover(
3662 &mut self.state,
3663 "implement",
3664 &seat.key,
3665 &seat.agent,
3666 &next.id,
3667 &cur,
3668 &fail_reason(&*out),
3669 );
3670 prev = Some(cur.clone());
3671
3672 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674 job.spec = next.clone();
3682 let mut retry = job.clone();
3683 retry.seat = new_seat;
3684 retry.prompt = prompt::implement_in(
3685 &instruction,
3686 &job.cwd.to_string_lossy(),
3687 &language,
3688 &github_language,
3689 brief.as_deref(),
3690 &attachments,
3691 );
3692 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693 let cache = self.state.config.cache_dir();
3694 let ctx = WaveCtx {
3695 carry_seats: false,
3696 pauses: Some((&self.pause, &self.interrupt)),
3697 run: run_id,
3698 node: "implement",
3699 prompts,
3700 cache: cache.as_deref(),
3701 round: None,
3702 };
3703 let (fallback_seat, fallback_out) = run_one(
3704 retry,
3705 Arc::clone(&self.sem),
3706 &ctx,
3707 &mut self.state,
3708 fallback_attempt,
3709 )
3710 .await;
3711 *seat = fallback_seat;
3712 *out = fallback_out;
3713 }
3714 }
3715 Ok(())
3716 }
3717
3718 async fn resume_unconfirmed_commands(
3742 &mut self,
3743 results: &mut [(usize, SeatState, AgentOutcome)],
3744 sent: &[SeatJob],
3745 prompts: &Prompts,
3746 run_id: &str,
3747 ) {
3748 for (wi, seat, out) in results.iter_mut() {
3749 let AgentOutcome::Ok(o) = &*out else {
3750 continue;
3751 };
3752 if !has_unconfirmed_command(&o.commands) {
3753 continue;
3754 }
3755 let Some(job) = sent.get(*wi) else { continue };
3756 if !has_context(&job.spec, seat, job.sessions) {
3757 self.state.event(
3758 "implement",
3759 format!(
3760 "{}: the reply named a command whose own CLI never confirmed the exit \
3761 status of, but there is no session left to resume",
3762 seat.key
3763 ),
3764 );
3765 continue;
3766 }
3767 self.state.event(
3768 "implement",
3769 format!(
3770 "{}: the reply named a command whose own CLI never confirmed the exit \
3771 status of; resuming the conversation",
3772 seat.key
3773 ),
3774 );
3775 let mut retry = job.clone();
3776 retry.seat = seat.clone();
3777 retry.prompt = prompt::resume_incomplete(
3778 "a command in your last reply had no confirmed exit status",
3779 );
3780 retry.timeout = retry_budget(job.timeout, true);
3781 retry.stem = format!("{}-confirm", job.stem);
3782 let cache = self.state.config.cache_dir();
3783 let ctx = WaveCtx {
3784 carry_seats: false,
3785 pauses: Some((&self.pause, &self.interrupt)),
3786 run: run_id,
3787 node: "implement",
3788 prompts,
3789 cache: cache.as_deref(),
3790 round: None,
3791 };
3792 let (resumed_seat, resumed) =
3793 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794 *seat = resumed_seat;
3795 *out = resumed;
3796 }
3797 }
3798
3799 async fn continue_fix_report(
3820 &mut self,
3821 mut seat: SeatState,
3822 parse_err: String,
3823 job: &SeatJob,
3824 prompts: &Prompts,
3825 run_id: &str,
3826 round: usize,
3827 ) -> (
3828 SeatState,
3829 Option<FixReport>,
3830 Option<String>,
3831 ContinuationRecord,
3832 ) {
3833 let mut last_err = parse_err;
3834 let mut cumulative_wait_ms = 0u64;
3835 let mut attempts = 0usize;
3836 loop {
3837 if !has_context(&job.spec, &seat, job.sessions) {
3838 self.state.event(
3839 "fix",
3840 format!(
3841 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842 session left to resume into"
3843 ),
3844 );
3845 let outcome = if attempts == 0 {
3846 ContinuationOutcome::NoSession
3847 } else {
3848 ContinuationOutcome::Exhausted
3849 };
3850 return (
3851 seat,
3852 None,
3853 Some(format!("unparsable fix report: {last_err}")),
3854 ContinuationRecord {
3855 attempts,
3856 cumulative_wait_ms,
3857 outcome,
3858 },
3859 );
3860 }
3861 if attempts >= MAX_FIX_CONTINUATIONS {
3862 self.state.event(
3863 "fix",
3864 format!(
3865 "round {round}: fixer's reply still had no adoption report after \
3866 {attempts} continuation(s) ({last_err}); giving up"
3867 ),
3868 );
3869 return (
3870 seat,
3871 None,
3872 Some(format!(
3873 "unparsable fix report after {attempts} continuation(s): {last_err}"
3874 )),
3875 ContinuationRecord {
3876 attempts,
3877 cumulative_wait_ms,
3878 outcome: ContinuationOutcome::Exhausted,
3879 },
3880 );
3881 }
3882 attempts += 1;
3883 self.state.event(
3884 "fix",
3885 format!(
3886 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888 ),
3889 );
3890 let mut retry = job.clone();
3891 retry.seat = seat.clone();
3892 retry.prompt = prompt::resume_incomplete(&last_err);
3893 retry.timeout = retry_budget(job.timeout, true);
3894 retry.stem = format!("{}-continue{attempts}", job.stem);
3895 let cache = self.state.config.cache_dir();
3896 let ctx = WaveCtx {
3897 carry_seats: false,
3898 pauses: Some((&self.pause, &self.interrupt)),
3899 run: run_id,
3900 node: "fix",
3901 prompts,
3902 cache: cache.as_deref(),
3903 round: Some(round),
3904 };
3905 let (resumed_seat, resumed_out) = run_one(
3906 retry,
3907 Arc::clone(&self.sem),
3908 &ctx,
3909 &mut self.state,
3910 attempts,
3911 )
3912 .await;
3913 seat = resumed_seat;
3914 match resumed_out {
3915 AgentOutcome::Ok(o) => {
3916 cumulative_wait_ms += o.duration_ms;
3917 match verdict::extract_json::<FixReport>(&o.text) {
3918 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919 self.state.event(
3920 "fix",
3921 format!(
3922 "round {round}: fixer's adoption report recovered after \
3923 {attempts} continuation(s)"
3924 ),
3925 );
3926 return (
3927 seat,
3928 Some(report),
3929 None,
3930 ContinuationRecord {
3931 attempts,
3932 cumulative_wait_ms,
3933 outcome: ContinuationOutcome::Resumed,
3934 },
3935 );
3936 }
3937 Ok(_) => {
3945 last_err = "the reply parsed, but it reported a command whose own CLI \
3946 never confirmed an exit status"
3947 .to_owned();
3948 }
3949 Err(e) => last_err = e.to_string(),
3950 }
3951 }
3952 AgentOutcome::Quota(o) => {
3953 cumulative_wait_ms += o.duration_ms;
3954 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956 self.state.quota.push(loss);
3957 self.state.event(
3958 "fix",
3959 format!(
3960 "round {round}: continuation rate limited (quota); not retrying now"
3961 ),
3962 );
3963 return (
3964 seat,
3965 None,
3966 Some("rate limited (quota) while recovering the fix report".to_owned()),
3967 ContinuationRecord {
3968 attempts,
3969 cumulative_wait_ms,
3970 outcome: ContinuationOutcome::QuotaLost,
3971 },
3972 );
3973 }
3974 AgentOutcome::Dropped(o) => {
3975 cumulative_wait_ms += o.duration_ms;
3976 let why = o
3977 .dropped
3978 .as_ref()
3979 .map(|d| d.why.as_str())
3980 .unwrap_or("the CLI ended the stream without delivering its answer");
3981 last_err = format!("the CLI dropped the stream ({why})");
3982 }
3983 AgentOutcome::Failed(e) => last_err = e,
3984 AgentOutcome::ResourceBlocked(e) => {
3987 return (
3988 seat,
3989 None,
3990 Some(format!("the continuation was not started: {e}")),
3991 ContinuationRecord {
3992 attempts: attempts.saturating_sub(1),
3993 cumulative_wait_ms,
3994 outcome: ContinuationOutcome::NoSession,
3995 },
3996 );
3997 }
3998 }
3999 }
4000 }
4001
4002 fn after_implement(&mut self) -> Result<()> {
4003 if !self.state.candidate_preservation.is_empty() {
4004 bail!("candidate preservation incomplete; judging is unsafe");
4005 }
4006 if self.state.leaks.is_empty() {
4008 let cfg = self.state.config.blind.clone();
4009 let mut leaks = Vec::new();
4010 for c in &self.state.candidates {
4011 let Some(patch) =
4012 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013 else {
4014 continue;
4015 };
4016 leaks.extend(blind::scan(
4017 &format!("candidate {} patch", c.label),
4018 &patch,
4019 &cfg.vendor_tokens,
4020 ));
4021 }
4022 if !leaks.is_empty() {
4023 let summary = leaks
4024 .iter()
4025 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026 .collect::<Vec<_>>()
4027 .join(", ");
4028 match cfg.on_leak {
4029 LeakPolicy::Fail => {
4030 self.state.status = RunStatus::Failed;
4031 self.state
4032 .event("blind", format!("vendor text in a patch: {summary}"));
4033 self.state.leaks = leaks;
4034 self.state.save()?;
4035 self.settle_questions();
4036 bail!(
4037 "blind.on_leak = \"fail\" and vendor text reached a \
4038 judged patch: {summary}"
4039 );
4040 }
4041 LeakPolicy::Redact => self.state.event(
4042 "blind",
4043 format!("redacting vendor text for judging: {summary}"),
4044 ),
4045 LeakPolicy::Warn => self.state.event(
4046 "blind",
4047 format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048 ),
4049 }
4050 self.state.leaks = leaks;
4051 }
4052 }
4053
4054 if self.state.viable().is_empty() {
4055 if self.state.all_candidates_verified_noop() {
4056 self.state.status = RunStatus::VerifiedNoop;
4067 self.state.save()?;
4068 self.settle_questions();
4069 return Ok(());
4070 }
4071 self.state.status = RunStatus::Failed;
4072 self.state.save()?;
4073 self.settle_questions();
4074 bail!("no candidate produced a change; nothing to judge");
4075 }
4076 self.state.status = RunStatus::Judging;
4077 self.state.save()?;
4078 Ok(())
4079 }
4080
4081 async fn judge(&mut self) -> Result<()> {
4084 let run_id = self.state.id.clone();
4089 let prompts = self.state.config.prompts.clone();
4090 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091 return Ok(());
4092 }
4093 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094 if viable.len() == 1 {
4095 self.state.judge_skipped = true;
4102 self.state.event(
4103 "judge",
4104 format!(
4105 "only candidate {} produced a change; judging skipped",
4106 viable[0].label
4107 ),
4108 );
4109 self.state.save()?;
4110 return Ok(());
4111 }
4112 self.state.status = RunStatus::Judging;
4113
4114 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115 let language = self.state.config.graph.language.clone();
4116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117 let sessions = self.state.config.graph.sessions;
4118 let artifacts = agent::artifacts_dir(&self.state.dir());
4119 let root = self.state.worktree_root();
4120 let base_short = short(&self.state.base_commit);
4121
4122 let mut jobs = Vec::new();
4123 let mut orders = Vec::new();
4124 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128 let seat_key = format!("judge-{}", j + 1);
4129 let seat = self.seat(&seat_key, &spec.id);
4130 jobs.push(SeatJob {
4131 prompt: prompt::judge(
4132 &self.state.instruction,
4133 &views,
4134 self.roles.judges.len(),
4135 &base_short,
4136 &language,
4137 ),
4138 spec,
4139 seat,
4140 cwd: root.join(format!("judge-{}", j + 1)),
4141 timeout,
4142 allow_write: false,
4143 sessions,
4144 artifacts: artifacts.clone(),
4145 stem: format!("judge-{}", j + 1),
4146 handover: None,
4147 });
4148 }
4149
4150 self.state.event(
4151 "judge",
4152 format!(
4153 "{} judges ranking {} candidates blind",
4154 jobs.len(),
4155 viable.len()
4156 ),
4157 );
4158 let labels_for_check = labels.clone();
4159 let mut quota_losses = Vec::new();
4160 let cache = self.state.config.cache_dir();
4161 let ctx = WaveCtx {
4162 carry_seats: false,
4163 pauses: Some((&self.pause, &self.interrupt)),
4164 run: &run_id,
4165 node: "judge",
4166 prompts: &prompts,
4167 cache: cache.as_deref(),
4168 round: None,
4169 };
4170 let results = ask_json_wave::<Ranking>(
4171 jobs,
4172 Arc::clone(&self.sem),
4173 self.state.config.graph.retries,
4174 &self.roles.judge_roster,
4175 &ctx,
4176 &mut quota_losses,
4177 &mut self.state,
4178 &move |r: &Ranking| r.validate(&labels_for_check),
4179 )
4180 .await;
4181 if self.state.resource_block.is_some() {
4182 return self.stop_for_resource();
4183 }
4184 self.state.quota.extend(quota_losses);
4185
4186 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187 let agent_id = seat.agent.clone();
4188 self.state.seats.insert(seat.key.clone(), seat);
4189 let mut record = Judgement {
4190 judge: j + 1,
4191 seat: format!("judge-{}", j + 1),
4192 agent: agent_id,
4193 ranking: Vec::new(),
4194 reasons: BTreeMap::new(),
4195 confidence: None,
4196 strengths: BTreeMap::new(),
4197 order: orders[j].clone(),
4198 failed: None,
4199 duration_ms: 0,
4200 };
4201 match res {
4202 Ok((ranking, out)) => {
4203 record.ranking = ranking.normalized();
4204 record.reasons = ranking.reasons;
4205 record.strengths = ranking.strengths;
4206 record.confidence = ranking.confidence;
4207 record.duration_ms = out.duration_ms;
4208 self.state.event(
4209 "judge",
4210 format!(
4211 "judge {} ranked {}",
4212 j + 1,
4213 record.ranking.iter().collect::<String>()
4214 ),
4215 );
4216 }
4217 Err(e) => {
4218 record.failed = Some(e.to_string());
4219 self.state
4220 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221 }
4222 }
4223 self.state.judgements.push(record);
4224 self.state.save()?;
4225 }
4226 Ok(())
4227 }
4228
4229 async fn deliberate(&mut self) -> Result<()> {
4232 if self.hold_base_sync().await? {
4233 return Ok(());
4234 }
4235 let run_id = self.state.id.clone();
4240 let prompts = self.state.config.prompts.clone();
4241 let rounds = self.state.config.graph.deliberate_rounds;
4242 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244 return Ok(());
4245 }
4246 let tops: Vec<char> = self
4247 .state
4248 .judgements
4249 .iter()
4250 .filter_map(|j| j.ranking.first().copied())
4251 .collect();
4252 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254 self.state.event(
4255 "deliberate",
4256 format!("judges agreed on {} outright; no deliberation", tops[0]),
4257 );
4258 }
4259 self.state.status = RunStatus::Voting;
4260 self.state.save()?;
4261 return Ok(());
4262 }
4263
4264 self.state.status = RunStatus::Deliberating;
4265 self.state.event(
4266 "deliberate",
4267 format!(
4268 "split: first choices were {} — opening {rounds} round(s)",
4269 tops.iter().collect::<String>()
4270 ),
4271 );
4272
4273 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274 let language = self.state.config.graph.language.clone();
4275 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276 let sessions = self.state.config.graph.sessions;
4277 let artifacts = agent::artifacts_dir(&self.state.dir());
4278 let root = self.state.worktree_root();
4279 let base_short = short(&self.state.base_commit);
4280
4281 for round in self.state.deliberation.len() + 1..=rounds {
4285 let mut turns: Vec<DeliberationTurn> = Vec::new();
4286 let mut round_losses = Vec::new();
4287 let mut asked = 0;
4288 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289 if self.state.judgements[j].failed.is_some() {
4290 continue;
4291 }
4292 let seat_key = format!("judge-{}", j + 1);
4293 let spec = self.occupant(&seat_key, spec);
4294 let mut seat = self.seat(&seat_key, &spec.id);
4295 let transcript = self.transcript(&turns, j);
4296 let build = |context: Option<&str>| {
4297 prompt::deliberate(
4298 &self.state.instruction,
4299 context,
4300 &transcript,
4301 round,
4302 rounds,
4303 &language,
4304 )
4305 };
4306 let block = self.candidate_block(&viable, &base_short);
4307 let full = build(Some(&block));
4308 let text = if has_context(&spec, &seat, sessions) {
4309 build(None)
4310 } else {
4311 full.clone()
4312 };
4313 let job = SeatJob {
4314 spec,
4315 seat: seat.clone(),
4316 prompt: text,
4317 cwd: root.join(format!("judge-{}", j + 1)),
4318 timeout,
4319 allow_write: false,
4320 sessions,
4321 artifacts: artifacts.clone(),
4322 stem: format!("delib-{round}-judge-{}", j + 1),
4323 handover: Some(full),
4324 };
4325 let cache = self.state.config.cache_dir();
4326 let ctx = WaveCtx {
4327 carry_seats: false,
4328 pauses: Some((&self.pause, &self.interrupt)),
4329 run: &run_id,
4330 node: "deliberate",
4331 prompts: &prompts,
4332 cache: cache.as_deref(),
4333 round: None,
4334 };
4335 let mut losses = Vec::new();
4339 asked += 1;
4340 let mut results = ask_wave_with::<String>(
4341 vec![job],
4342 Arc::clone(&self.sem),
4343 0,
4344 &self.roles.judge_roster,
4345 &ctx,
4346 &mut losses,
4347 &mut self.state,
4348 &|text: &str| {
4349 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350 },
4351 )
4352 .await;
4353 if self.state.resource_block.is_some() {
4354 return self.stop_for_resource();
4355 }
4356 round_losses.extend(losses.iter().cloned());
4357 self.state.quota.extend(losses);
4358 let (updated, res, _) = results.pop().expect("one job in, one result out");
4359 seat = updated;
4360 let agent_id = seat.agent.clone();
4361 self.state.seats.insert(seat.key.clone(), seat);
4362 let body = match res {
4363 Ok((body, _)) => body,
4364 Err(e) => {
4367 self.state
4368 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369 continue;
4370 }
4371 };
4372 let tentative = verdict::extract_json::<Position>(&body)
4373 .ok()
4374 .and_then(|p| p.tentative)
4375 .and_then(|s| s.trim().chars().next())
4376 .map(|c| c.to_ascii_uppercase());
4377 self.state.event(
4378 "deliberate",
4379 format!(
4380 "round {round}: judge {} now favours {}",
4381 j + 1,
4382 tentative.map_or("—".to_owned(), |c| c.to_string())
4383 ),
4384 );
4385 turns.push(DeliberationTurn {
4386 judge: j + 1,
4387 agent: agent_id,
4388 body: blind::sanitize_prose(&body, &self.state.config.blind),
4389 tentative,
4390 });
4391 }
4392 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394 return self.stop_for_resource();
4395 }
4396 self.state
4397 .deliberation
4398 .push(DeliberationRound { round, turns });
4399 self.state.save()?;
4400 }
4401
4402 self.state.status = RunStatus::Voting;
4403 self.state.save()?;
4404 Ok(())
4405 }
4406
4407 async fn vote(&mut self) -> Result<()> {
4410 if self.hold_base_sync().await? {
4411 return Ok(());
4412 }
4413 let run_id = self.state.id.clone();
4418 let prompts = self.state.config.prompts.clone();
4419 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420 return Ok(());
4421 }
4422 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423 if viable.len() == 1 {
4424 return Ok(());
4425 }
4426 self.state.status = RunStatus::Voting;
4427
4428 let language = self.state.config.graph.language.clone();
4429 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430 let sessions = self.state.config.graph.sessions;
4431 let artifacts = agent::artifacts_dir(&self.state.dir());
4432 let root = self.state.worktree_root();
4433 let base_short = short(&self.state.base_commit);
4434 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436 let mut jobs = Vec::new();
4437 let mut seats_at = Vec::new();
4438 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439 if self
4440 .state
4441 .judgements
4442 .get(j)
4443 .is_some_and(|r| r.failed.is_some())
4444 {
4445 continue;
4446 }
4447 let seat_key = format!("judge-{}", j + 1);
4448 let spec = self.occupant(&seat_key, spec);
4449 let seat = self.seat(&seat_key, &spec.id);
4450 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451 let text = if has_context(&spec, &seat, sessions) {
4452 prompt::final_vote(&viable, &language)
4453 } else {
4454 full.clone()
4455 };
4456 jobs.push(SeatJob {
4457 spec,
4458 seat,
4459 prompt: text,
4460 cwd: root.join(format!("judge-{}", j + 1)),
4461 timeout,
4462 allow_write: false,
4463 sessions,
4464 artifacts: artifacts.clone(),
4465 stem: format!("vote-judge-{}", j + 1),
4466 handover: Some(full),
4467 });
4468 seats_at.push(j);
4469 }
4470
4471 self.state.event(
4472 "vote",
4473 format!(
4474 "collecting {} final votes one by one, privately",
4475 jobs.len()
4476 ),
4477 );
4478 let allowed = viable.clone();
4479 let mut quota_losses = Vec::new();
4480 let cache = self.state.config.cache_dir();
4481 let ctx = WaveCtx {
4482 carry_seats: false,
4483 pauses: Some((&self.pause, &self.interrupt)),
4484 run: &run_id,
4485 node: "vote",
4486 prompts: &prompts,
4487 cache: cache.as_deref(),
4488 round: None,
4489 };
4490 let results = ask_json_wave::<FinalVote>(
4491 jobs,
4492 Arc::clone(&self.sem),
4493 self.state.config.graph.retries,
4494 &self.roles.judge_roster,
4495 &ctx,
4496 &mut quota_losses,
4497 &mut self.state,
4498 &move |v: &FinalVote| match v.label() {
4499 Some(c) if allowed.contains(&c) => Ok(()),
4500 other => bail!("vote {other:?} is not one of {allowed:?}"),
4501 },
4502 )
4503 .await;
4504 if self.state.resource_block.is_some() {
4505 return self.stop_for_resource();
4506 }
4507 self.state.quota.extend(quota_losses);
4508
4509 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510 let agent_id = seat.agent.clone();
4511 self.state.seats.insert(seat.key.clone(), seat);
4512 let initial = self
4513 .state
4514 .judgements
4515 .get(j)
4516 .and_then(|r| r.ranking.first().copied());
4517 let mut record = VoteRecord {
4518 judge: j + 1,
4519 agent: agent_id,
4520 vote: None,
4521 reason: String::new(),
4522 changed: false,
4523 };
4524 match res {
4525 Ok((v, _)) => {
4526 record.vote = v.label();
4527 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529 self.state.event(
4530 "vote",
4531 format!(
4532 "judge {} voted {}{}",
4533 j + 1,
4534 record.vote.unwrap_or('?'),
4535 if record.changed { " (changed)" } else { "" }
4536 ),
4537 );
4538 }
4539 Err(e) => {
4540 self.state
4541 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542 }
4543 }
4544 self.state.votes.push(record);
4545 self.state.save()?;
4546 }
4547 Ok(())
4548 }
4549
4550 fn tally(&mut self) -> Result<()> {
4553 if self.state.tally.is_some() {
4554 return Ok(());
4555 }
4556 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557 let tops: Vec<char> = self
4558 .state
4559 .judgements
4560 .iter()
4561 .filter_map(|j| j.ranking.first().copied())
4562 .collect();
4563 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568 let mut cast: Vec<char> = Vec::new();
4569 for (i, j) in self.state.judgements.iter().enumerate() {
4570 let vote = self
4571 .state
4572 .votes
4573 .iter()
4574 .find(|v| v.judge == i + 1)
4575 .and_then(|v| v.vote)
4576 .or_else(|| j.ranking.first().copied());
4577 if let Some(v) = vote {
4578 *first_choice.entry(v).or_insert(0) += 1;
4579 cast.push(v);
4580 }
4581 }
4582
4583 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584 for j in &self.state.judgements {
4585 let n = j.ranking.len();
4586 for (pos, label) in j.ranking.iter().enumerate() {
4587 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588 }
4589 }
4590
4591 let best = first_choice.values().copied().max().unwrap_or(0);
4592 let mut leaders: Vec<char> = first_choice
4593 .iter()
4594 .filter(|(_, v)| **v == best)
4595 .map(|(k, _)| *k)
4596 .collect();
4597 let mut tie_break = None;
4598 if leaders.len() > 1 {
4599 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600 let borda_leaders: Vec<char> = leaders
4601 .iter()
4602 .copied()
4603 .filter(|l| borda[l] == top_borda)
4604 .collect();
4605 tie_break = Some(if borda_leaders.len() == 1 {
4606 format!(
4607 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608 leaders.len()
4609 )
4610 } else {
4611 format!(
4612 "{} way tie on both first-choice votes and Borda points, broken by label order",
4613 leaders.len()
4614 )
4615 });
4616 leaders = borda_leaders;
4617 leaders.sort_unstable();
4618 }
4619 let winner = *leaders
4620 .first()
4621 .or(viable.first())
4622 .context("no candidate to declare a winner from")?;
4623
4624 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626 let deliberated = !self.state.deliberation.is_empty();
4627
4628 let quota_seats: std::collections::BTreeSet<&str> =
4632 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633 let mut present = 0usize;
4634 for (i, j) in self.state.judgements.iter().enumerate() {
4635 if quota_seats.contains(j.seat.as_str()) {
4636 continue;
4637 }
4638 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639 let voted = self
4640 .state
4641 .votes
4642 .iter()
4643 .any(|v| v.judge == i + 1 && v.vote.is_some());
4644 if ranked || voted {
4645 present += 1;
4646 }
4647 }
4648 let needs_quorum = viable.len() > 1;
4654 let judges_total = if needs_quorum {
4655 self.roles.judges.len()
4656 } else {
4657 0
4658 };
4659 let quorum = if needs_quorum {
4660 judges_total / 2 + 1
4661 } else {
4662 0
4663 };
4664 let met_quorum = !needs_quorum || present >= quorum;
4665 let uncontested = (!needs_quorum).then(|| {
4666 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667 });
4668
4669 self.state.event(
4670 "tally",
4671 match &uncontested {
4672 Some(reason) => format!("winner {winner} — {reason}"),
4673 None => format!(
4674 "winner {winner} — votes {} | initial {} | {} changed | \
4675 {present}/{judges_total} judges{}",
4676 first_choice
4677 .iter()
4678 .map(|(k, v)| format!("{k}:{v}"))
4679 .collect::<Vec<_>>()
4680 .join(" "),
4681 if unanimous_initial {
4682 "unanimous"
4683 } else {
4684 "split"
4685 },
4686 changed_votes,
4687 if met_quorum {
4688 String::new()
4689 } else {
4690 format!(" — below quorum ({quorum} required)")
4691 },
4692 ),
4693 },
4694 );
4695 if !met_quorum {
4696 self.state.event(
4697 "stall",
4698 format!(
4699 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700 the run stops here, resumable"
4701 ),
4702 );
4703 }
4704 self.state.tally = Some(Tally {
4705 first_choice,
4706 borda,
4707 winner,
4708 rankings: tops.len(),
4709 unanimous_initial,
4710 deliberated,
4711 changed_votes,
4712 unanimous_final,
4713 tie_break,
4714 judges: judges_total,
4715 present,
4716 quorum,
4717 met_quorum,
4718 uncontested,
4719 });
4720 self.state.status = if met_quorum {
4721 RunStatus::Reviewing
4722 } else {
4723 RunStatus::Stalled
4724 };
4725 self.state.save()?;
4726 Ok(())
4727 }
4728
4729 #[allow(clippy::too_many_lines)]
4750 async fn recover_stall(&mut self) -> Result<bool> {
4751 let run_id = self.state.id.clone();
4756 let prompts = self.state.config.prompts.clone();
4757 let quota_seats: BTreeSet<&str> =
4762 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763 let absent: Vec<String> = self
4764 .state
4765 .judgements
4766 .iter()
4767 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768 .map(|j| j.seat.clone())
4769 .collect();
4770 if absent.is_empty() {
4771 return Ok(false);
4772 }
4773 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774 if viable.len() <= 1 {
4775 return Ok(false);
4776 }
4777 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778 let language = self.state.config.graph.language.clone();
4779 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780 let sessions = self.state.config.graph.sessions;
4781 let artifacts = agent::artifacts_dir(&self.state.dir());
4782 let root = self.state.worktree_root();
4783 let base_short = short(&self.state.base_commit);
4784 let candidates: Vec<Candidate> = viable.clone();
4785
4786 let mut positions: Vec<usize> = absent
4788 .iter()
4789 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790 .collect();
4791 if positions.is_empty() {
4792 return Ok(false);
4793 }
4794 positions.sort_unstable();
4795 positions.dedup();
4796
4797 let mut judge_jobs = Vec::new();
4799 for &j in &positions {
4800 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802 let seat_key = format!("judge-{}", j + 1);
4803 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804 let seat = self.seat(&seat_key, &spec.id);
4805 judge_jobs.push(SeatJob {
4806 spec,
4807 seat,
4808 prompt: prompt::judge(
4809 &self.state.instruction,
4810 &views,
4811 self.roles.judges.len(),
4812 &base_short,
4813 &language,
4814 ),
4815 cwd: root.join(seat_key),
4816 timeout,
4817 allow_write: false,
4818 sessions,
4819 artifacts: artifacts.clone(),
4820 stem: format!("judge-{}-recover", j + 1),
4821 handover: None,
4822 });
4823 }
4824
4825 let labels_for_check = labels.clone();
4826 let mut judge_losses = Vec::new();
4827 let retries = self.state.config.graph.retries;
4828 let cache = self.state.config.cache_dir();
4829 let ctx = WaveCtx {
4830 carry_seats: false,
4831 pauses: Some((&self.pause, &self.interrupt)),
4832 run: &run_id,
4833 node: "judge",
4834 prompts: &prompts,
4835 cache: cache.as_deref(),
4836 round: None,
4837 };
4838 let results = ask_json_wave::<Ranking>(
4839 judge_jobs,
4840 Arc::clone(&self.sem),
4841 retries,
4842 &self.roles.judge_roster,
4843 &ctx,
4844 &mut judge_losses,
4845 &mut self.state,
4846 &move |r: &Ranking| r.validate(&labels_for_check),
4847 )
4848 .await;
4849 if self.state.resource_block.is_some() {
4850 self.stop_for_resource()?;
4851 return Ok(false);
4852 }
4853
4854 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857 let agent_id = seat.agent.clone();
4858 self.state.seats.insert(seat.key.clone(), seat);
4859 let record = &mut self.state.judgements[j];
4860 match res {
4861 Ok((ranking, out)) => {
4862 record.agent = agent_id;
4863 record.ranking = ranking.normalized();
4864 record.reasons = ranking.reasons;
4865 record.strengths = ranking.strengths;
4866 record.confidence = ranking.confidence;
4867 record.failed = None;
4868 record.duration_ms = out.duration_ms;
4869 recovered.insert(j);
4870 self.state.event(
4871 "recover",
4872 format!("judge {} ranked again after the limit", j + 1),
4873 );
4874 }
4875 Err(e) => {
4876 self.state
4877 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878 }
4879 }
4880 }
4881
4882 let mut vote_jobs = Vec::new();
4884 let mut vote_pos: Vec<usize> = Vec::new();
4885 for &j in &recovered {
4886 let seat_key = format!("judge-{}", j + 1);
4887 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888 let seat = self.seat(&seat_key, &spec.id);
4889 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890 let text = if has_context(&spec, &seat, sessions) {
4891 prompt::final_vote(&labels, &language)
4892 } else {
4893 full.clone()
4894 };
4895 vote_jobs.push(SeatJob {
4896 spec,
4897 seat,
4898 prompt: text,
4899 cwd: root.join(seat_key),
4900 timeout,
4901 allow_write: false,
4902 sessions,
4903 artifacts: artifacts.clone(),
4904 stem: format!("vote-judge-{}-recover", j + 1),
4905 handover: Some(full),
4906 });
4907 vote_pos.push(j);
4908 }
4909 let allowed = labels.clone();
4910 let mut vote_losses = Vec::new();
4911 let vote_retries = self.state.config.graph.retries;
4912 let vote_cache = self.state.config.cache_dir();
4913 let ctx = WaveCtx {
4914 carry_seats: false,
4915 pauses: Some((&self.pause, &self.interrupt)),
4916 run: &run_id,
4917 node: "vote",
4918 prompts: &prompts,
4919 cache: vote_cache.as_deref(),
4920 round: None,
4921 };
4922 let votes = ask_json_wave::<FinalVote>(
4923 vote_jobs,
4924 Arc::clone(&self.sem),
4925 vote_retries,
4926 &self.roles.judge_roster,
4927 &ctx,
4928 &mut vote_losses,
4929 &mut self.state,
4930 &move |v: &FinalVote| match v.label() {
4931 Some(c) if allowed.contains(&c) => Ok(()),
4932 other => bail!("vote {other:?} is not one of {allowed:?}"),
4933 },
4934 )
4935 .await;
4936 if self.state.resource_block.is_some() {
4937 self.stop_for_resource()?;
4938 return Ok(false);
4939 }
4940 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941 let agent_id = seat.agent.clone();
4942 self.state.seats.insert(seat.key.clone(), seat);
4943 match res {
4944 Ok((v, _)) => {
4945 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946 rec.vote = v.label();
4947 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948 } else {
4949 self.state.votes.push(VoteRecord {
4950 judge: j + 1,
4951 agent: agent_id,
4952 vote: v.label(),
4953 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954 changed: false,
4955 });
4956 }
4957 self.state.event(
4958 "recover",
4959 format!("judge {} voted again after the limit", j + 1),
4960 );
4961 }
4962 Err(e) => {
4963 self.state
4964 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965 }
4966 }
4967 }
4968
4969 let recovered_keys: BTreeSet<String> = recovered
4973 .iter()
4974 .map(|&j| format!("judge-{}", j + 1))
4975 .collect();
4976 self.state
4977 .quota
4978 .retain(|q| !recovered_keys.contains(&q.seat));
4979 for loss in judge_losses.into_iter().chain(vote_losses) {
4983 if recovered_keys.contains(&loss.seat) {
4984 continue;
4985 }
4986 self.state.quota.retain(|q| q.seat != loss.seat);
4987 self.state.quota.push(loss);
4988 }
4989
4990 self.state.tally = None;
4992 self.tally()?;
4993 Ok(self
4994 .state
4995 .tally
4996 .as_ref()
4997 .map(|t| t.met_quorum)
4998 .unwrap_or(false))
4999 }
5000
5001 async fn graft(&mut self) -> Result<()> {
5017 let Err(e) = self.graft_inner().await else {
5018 return Ok(());
5019 };
5020 if e.is::<crate::repo_lock::ResourceBlocked>() {
5021 return Err(e);
5022 }
5023 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024 self.state.status = RunStatus::Blocked;
5025 self.state.event("graft", format!("blocked: {e:#}"));
5026 self.state.save()?;
5027 return Err(e);
5028 }
5029 if let Some(rec) = self.state.graft.clone()
5034 && !rec.outcome.settled()
5035 {
5036 if let Some(winner) = self.state.winner().cloned() {
5037 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038 if re.is::<crate::repo_lock::ResourceBlocked>() {
5039 return Err(re);
5040 }
5041 self.state.status = RunStatus::Blocked;
5042 self.state.event("graft", format!("blocked: {re:#}"));
5043 self.state.save()?;
5044 return Err(re);
5045 }
5046 }
5047 if let Some(g) = self.state.graft.as_mut() {
5048 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049 }
5050 }
5051 self.state.event("graft", format!("skipped: {e:#}"));
5052 if let Err(e) = self.state.save() {
5053 tracing::warn!("save after graft: {e:#}");
5054 }
5055 Ok(())
5056 }
5057
5058 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061 let _coordination = self.coordinate(wt).await?;
5062 crate::recovery::restore_repair(wt, pre)
5063 .await
5064 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065 }
5066
5067 fn graft_skipped(&mut self, why: &str) {
5069 let message = format!("skipped: {why}");
5070 if !self
5071 .state
5072 .events
5073 .iter()
5074 .any(|e| e.node == "graft" && e.message == message)
5075 {
5076 self.state.event("graft", message);
5077 }
5078 }
5079
5080 async fn graft_inner(&mut self) -> Result<()> {
5081 if let Some(rec) = self.state.graft.clone()
5084 && !rec.outcome.settled()
5085 {
5086 if self.state.reviews.is_empty()
5090 && let Some(winner) = self.state.winner().cloned()
5091 {
5092 self.graft_restore(&winner.worktree, &rec.pre).await?;
5093 }
5094 if rec.outcome == GraftOutcome::WaitingForRestore {
5095 self.state.graft = None;
5096 self.state.save()?;
5097 } else {
5098 if let Some(g) = self.state.graft.as_mut() {
5099 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100 }
5101 self.state.event(
5102 "graft",
5103 "rolled back: the previous round was interrupted; the winner is untouched",
5104 );
5105 self.state.save()?;
5106 return Ok(());
5107 }
5108 }
5109 let plan = match graft::plan(&self.state) {
5110 Ok(plan) => plan,
5111 Err(skip) => {
5112 if skip != graft::Skip::AlreadyDone {
5113 self.graft_skipped(skip.reason());
5114 self.state.save()?;
5115 }
5116 return Ok(());
5117 }
5118 };
5119 let Some(winner) = self.state.winner().cloned() else {
5120 return Ok(());
5121 };
5122 let wt = winner.worktree.clone();
5123 if !git::is_clean(&wt).await? {
5124 self.graft_skipped("the winner's worktree has uncommitted changes");
5125 self.state.save()?;
5126 return Ok(());
5127 }
5128 let pre = git::rev_parse(&wt, "HEAD").await?;
5129 self.state.graft = Some(GraftRecord {
5132 pre: pre.clone(),
5133 post: None,
5134 lines: None,
5135 limit: None,
5136 agent: None,
5137 outcome: GraftOutcome::Running,
5138 requested: graft::requested(&plan),
5139 adoption: None,
5140 adoption_error: None,
5141 });
5142 self.state.event(
5143 "graft",
5144 format!(
5145 "porting strengths of candidate(s) {} onto {}",
5146 plan.strengths
5147 .iter()
5148 .map(|(l, _, _)| l.to_string())
5149 .collect::<Vec<_>>()
5150 .join(", "),
5151 plan.winner
5152 ),
5153 );
5154 self.state.save()?;
5155
5156 let fix_prompt = prompt::graft(
5157 &self.state.instruction,
5158 plan.winner,
5159 &plan.strengths,
5160 &self.state.config.graph.language,
5161 );
5162 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163 let sessions = self.state.config.graph.sessions;
5164 let artifacts = agent::artifacts_dir(&self.state.dir());
5165 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166 let (_, seat, out) = self
5167 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168 SeatJob {
5169 prompt: fix_prompt.clone(),
5170 spec,
5171 seat,
5172 cwd: wt.clone(),
5173 timeout,
5174 allow_write: true,
5175 sessions,
5176 artifacts: artifacts.clone(),
5177 stem: "graft".to_owned(),
5178 handover: None,
5179 }
5180 })
5181 .await;
5182 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183 self.state.graft.as_mut().expect("graft record").outcome =
5186 GraftOutcome::WaitingForRestore;
5187 return self.stop_for_resource();
5188 }
5189 let mut why: Option<String> = None;
5190 match &out {
5191 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192 why = Some("the fixer produced nothing usable".to_owned());
5193 }
5194 AgentOutcome::Ok(_) => {}
5195 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196 AgentOutcome::Quota(o) => {
5197 self.state.quota.push(QuotaLoss::from_report(
5198 seat.key.clone(),
5199 "graft",
5200 o.quota.as_ref(),
5201 ));
5202 why = Some("rate limited (quota)".to_owned());
5203 }
5204 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205 why = Some(format!(
5206 "the fixer failed: {}",
5207 e.lines()
5208 .next()
5209 .unwrap_or("")
5210 .chars()
5211 .take(160)
5212 .collect::<String>()
5213 ));
5214 }
5215 }
5216 let agent_id = seat.agent.clone();
5217 self.state.seats.insert(seat.key.clone(), seat);
5218
5219 let mut post = None;
5220 let mut lines = None;
5221 let _coordination = self.coordinate(&wt).await?;
5222 if why.is_none() {
5223 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224 if let Ok(r) = &rescued {
5225 self.state.note_withheld("graft", &r.withheld);
5226 }
5227 let head = git::rev_parse(&wt, "HEAD").await?;
5228 let dirty = match &rescued {
5232 Ok(r) => git::status_porcelain(&wt)
5233 .await
5234 .map(|st| {
5235 st.lines().any(|l| {
5236 !r.withheld
5237 .iter()
5238 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239 })
5240 })
5241 .unwrap_or(true),
5242 Err(_) => true,
5243 };
5244 if dirty {
5245 why = Some("the fixer left changes that could not be committed".to_owned());
5246 } else if head == pre {
5247 why = Some("the fixer changed nothing".to_owned());
5248 } else {
5249 lines = git::changed_lines(&wt, &pre, &head).await;
5250 post = Some(head);
5251 }
5252 }
5253 let (adoption, adoption_error) = match &out {
5254 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255 _ => (None, None),
5256 };
5257 let requested = graft::requested(&plan);
5258 let outcome = match why {
5259 None => {
5260 self.state.event(
5261 "graft",
5262 format!(
5263 "applied: {}",
5264 lines.map_or_else(
5265 || "changed lines unavailable".to_owned(),
5266 |n| format!("{n} changed lines")
5267 )
5268 ),
5269 );
5270 GraftOutcome::Applied
5271 }
5272 Some(why) => {
5273 crate::recovery::restore_repair(&wt, &pre)
5276 .await
5277 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278 self.state.event(
5279 "graft",
5280 format!("rolled back: {why}; the winner is untouched"),
5281 );
5282 GraftOutcome::RolledBack(why)
5283 }
5284 };
5285 self.state.graft = Some(GraftRecord {
5286 pre,
5287 post,
5288 lines,
5289 limit: None,
5290 agent: Some(agent_id),
5291 outcome,
5292 requested,
5293 adoption,
5294 adoption_error,
5295 });
5296 self.state.save()?;
5297 Ok(())
5298 }
5299
5300 async fn fold_losers(&mut self) -> Result<()> {
5303 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304 return Ok(());
5305 };
5306 if !self.state.candidate_preservation.is_empty() {
5307 bail!("candidate preservation incomplete; refusing cleanup");
5308 }
5309 let repo = self.state.repo.clone();
5310 let _coordination = self.coordinate(&repo).await?;
5311 let mut folded = Vec::new();
5312 for i in 0..self.state.candidates.len() {
5313 let c = &self.state.candidates[i];
5314 if c.label == winner || c.folded {
5315 continue;
5316 }
5317 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318 remove_preserved_worktree(&repo, &wt).await?;
5319 if git::branch_exists(&repo, &branch).await?
5320 && !git::branch_delete(&repo, &branch).await?
5321 {
5322 bail!("candidate branch removal refused");
5323 }
5324 self.state.candidates[i].folded = true;
5325 self.state.save()?;
5326 folded.push(label.to_string());
5327 }
5328 let root = self.state.worktree_root();
5330 for j in 1..=self.roles.judges.len() {
5331 let wt = root.join(format!("judge-{j}"));
5332 if wt.exists() {
5333 remove_read_seat_worktree(&mut self.state, &wt).await?;
5334 self.state.save()?;
5335 }
5336 }
5337 if self.state.config.graph.advise {
5340 for k in 1..=self.state.config.graph.advisors {
5341 let wt = root.join(format!("advisor-{k}"));
5342 if wt.exists() {
5343 remove_read_seat_worktree(&mut self.state, &wt).await?;
5344 self.state.save()?;
5345 }
5346 }
5347 }
5348 if !folded.is_empty() {
5349 self.state
5350 .event("fold", format!("folded candidates {}", folded.join(", ")));
5351 self.state.save()?;
5352 }
5353 Ok(())
5354 }
5355
5356 async fn hold_base_sync(&mut self) -> Result<bool> {
5359 self.hold_base_sync_as(None).await
5360 }
5361
5362 async fn hold_base_sync_kind(
5363 &mut self,
5364 kind: crate::base_sync_recovery::StopKind,
5365 ) -> Result<()> {
5366 self.hold_base_sync_as(Some(kind)).await?;
5367 Ok(())
5368 }
5369
5370 async fn hold_base_sync_as(
5371 &mut self,
5372 kind: Option<crate::base_sync_recovery::StopKind>,
5373 ) -> Result<bool> {
5374 if !self.state.base_sync_blocked() {
5375 self.state.base_sync_recovery.stop = None;
5376 return Ok(false);
5377 }
5378 let head = match self.state.winner() {
5379 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5380 .await
5381 .unwrap_or_default(),
5382 None => String::new(),
5383 };
5384 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5385 && !authorization.consumed
5386 && authorization.stop.head != head
5387 {
5388 authorization.refuse(format!(
5389 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5390 authorization.stop.head
5391 ));
5392 }
5393 self.state.record_base_sync_stop(&head);
5394 if let Some(kind) = kind {
5395 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5396 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5397 stop.kind = kind;
5398 }
5399 }
5400 self.state.save()?;
5401 Ok(true)
5402 }
5403
5404 async fn sync_to_base(&mut self) -> Result<()> {
5446 if self.state.status == RunStatus::AlreadyInBase {
5447 return Ok(());
5448 }
5449 self.hold_base_sync().await?;
5450 let conflicted = self
5451 .state
5452 .base_sync
5453 .as_ref()
5454 .is_some_and(|s| s.conflict.is_some());
5455 let Some(winner) = self.state.winner().cloned() else {
5456 return Ok(());
5457 };
5458
5459 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5460
5461 let repo = self.state.repo.clone();
5462 let remote = self.state.config.merge.remote.clone();
5463 let base_branch = self.state.base_branch.clone();
5464 let tracking = format!("{remote}/{base_branch}");
5465
5466 let fetched_base =
5467 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5468 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5473 return Ok(());
5474 };
5475
5476 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5477 self.state.ensure_base_episode();
5478 let mut pending = self
5479 .state
5480 .base_sync_recovery
5481 .operations
5482 .last()
5483 .filter(|o| !o.finished)
5484 .cloned();
5485 let scratch = self.state.dir().join("base-sync");
5486 if git::rebase_in_progress(&scratch).await {
5487 if pending.is_none()
5490 && self.state.base_sync_recovery.operations.is_empty()
5491 && !conflicted
5492 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5493 .await
5494 .ok()
5495 .as_deref()
5496 == Some(head.as_str())
5497 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5498 && git::is_clean(&winner.worktree).await?
5499 {
5500 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5501 if self.state.base_sync.is_none() {
5502 self.state.base_sync = Some(BaseSync {
5503 tip: saved_tip.clone(),
5504 behind: 0,
5505 attempts: 0,
5506 conflict: None,
5507 already_in: None,
5508 });
5509 }
5510 self.state.reserve_base_operation(&head, &saved_tip);
5511 self.state.event(
5512 "base-recovery",
5513 "imported standing legacy rebase as a spent synchronization reservation",
5514 );
5515 self.state.save()?;
5516 pending = self.state.base_sync_recovery.operations.last().cloned();
5517 }
5518 let expected = pending.as_ref().map(|op| op.tip.as_str());
5519 if expected.is_none()
5520 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5521 {
5522 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5523 if self.state.base_sync.is_none() {
5524 self.state.base_sync = Some(BaseSync {
5525 tip: tip.clone(),
5526 behind: 0,
5527 attempts: 0,
5528 conflict: None,
5529 already_in: None,
5530 });
5531 }
5532 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5533 self.state.event("land", why);
5534 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535 .await?;
5536 return Ok(());
5537 }
5538 }
5539 if let Some(op) = &pending {
5540 let mut replay_proven =
5542 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5543 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5544 let wait = repository_wait(&self.state, None);
5545 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5546 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5547 Ok(true) => {
5548 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5549 self.state.save()?;
5550 git::worktree_remove(&repo, &scratch).await.ok();
5551 replay_proven = true;
5552 }
5553 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5554 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5555 }
5556 }
5557 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5558 if self.state.foreign_checkout.is_some() {
5559 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5560 .await
5561 .ok()
5562 .as_deref()
5563 == Some(op.head.as_str())
5564 && git::rev_parse(&repo, &format!("{head}^2"))
5565 .await
5566 .ok()
5567 .as_deref()
5568 == Some(op.tip.as_str());
5569 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5570 .await
5571 .is_ok_and(|m| m.is_empty())
5572 && let Ok(added) =
5573 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5574 {
5575 replay_proven = true;
5576 for commit in added {
5577 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5578 replay_proven = false;
5579 break;
5580 }
5581 }
5582 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5583 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5584 }
5585 }
5586 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590 || (head != op.head
5591 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5592 .await?
5593 .is_empty())
5594 || (!git::is_clean(&winner.worktree).await?
5595 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5596 .await
5597 .is_ok_and(|o| o.ok()))
5598 || !replay_proven
5599 {
5600 let why = format!(
5601 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5602 op.id
5603 );
5604 let sync = self
5605 .state
5606 .base_sync
5607 .as_mut()
5608 .context("reserved synchronization has no lifetime accounting")?;
5609 sync.conflict = Some(why.clone());
5610 self.state.event("land", why);
5611 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5612 .await?;
5613 return Ok(());
5614 }
5615 if head != op.head {
5616 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5617 }
5618 tip = op.tip.clone();
5619 }
5620 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5621 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5622 if pending.is_none()
5623 && !conflicted
5624 && fetched_base
5625 && self
5626 .state
5627 .base_sync_recovery
5628 .episodes
5629 .last()
5630 .is_some_and(|e| e.boundary.is_some())
5631 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5632 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5633 && owner.execution_runs().last() == Some(&self.state.id)
5634 && owner.status != crate::queue::TaskStatus::Done
5635 && !owner.fresh_start
5636 && owner.review_branch.is_none()
5637 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5638 && let Some(approval) = owner.resumption.as_ref()
5639 && approval.instruction == owner.instruction
5640 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5641 && git::is_clean(&winner.worktree).await?
5642 && self.state.renew_base_episode(&task, approval, &head, &tip)
5643 {
5644 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5645 self.state.save()?;
5646 }
5647
5648 if (behind > 0 || conflicted || head == tip)
5657 && self
5658 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5659 .await?
5660 {
5661 return Ok(());
5662 }
5663 if conflicted && self.state.base_recovery_pending() {
5664 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5665 let owner =
5666 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5667 let owner_valid = owner.as_ref().is_ok_and(|t| {
5668 t.execution_runs().last() == Some(&self.state.id)
5669 && t.status != crate::queue::TaskStatus::Done
5670 && !t.fresh_start
5671 && t.review_branch.is_none()
5672 });
5673 let invalidation = if !owner_valid {
5674 Some("latest task ownership changed")
5675 } else if !owner.as_ref().is_ok_and(|t| {
5676 authorization
5677 .generation
5678 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5679 }) {
5680 Some("owner approval generation changed")
5681 } else if authorization.question.as_ref().is_some_and(|question| {
5682 !crate::base_recovery_control::decision_current(
5683 &crate::queue::Queue::at(crate::run::home().join("queue")),
5684 &self.state,
5685 &authorization.task,
5686 question,
5687 )
5688 }) {
5689 Some("recovery decision ownership, routing or stop evidence changed")
5690 } else if authorization.request.is_some()
5691 && !owner
5692 .as_ref()
5693 .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5694 {
5695 Some("owner decision changed")
5696 } else if let Some(scope) = &authorization.scope {
5697 if scope.tip != tip {
5698 Some("fetched base changed")
5699 } else if !self
5700 .state
5701 .base_sync_recovery
5702 .episodes
5703 .last()
5704 .is_some_and(|e| e.id == scope.episode)
5705 {
5706 Some("active synchronization interval changed")
5707 } else if !owner
5708 .as_ref()
5709 .is_ok_and(|t| t.instruction == scope.instruction)
5710 {
5711 Some("task instruction changed")
5712 } else {
5713 None
5714 }
5715 } else {
5716 None
5717 };
5718 if let Some(reason) = invalidation {
5719 let authorization = self
5720 .state
5721 .base_sync_recovery
5722 .authorizations
5723 .last_mut()
5724 .unwrap();
5725 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5726 self.state.save()?;
5727 return Ok(());
5728 }
5729 if !fetched_base
5730 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5731 || !git::is_clean(&winner.worktree).await?
5732 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5733 {
5734 return Ok(());
5735 }
5736 }
5737 let recovery = conflicted && self.state.consume_base_recovery(&head);
5738 if conflicted {
5739 self.state.save()?;
5741 if !recovery {
5742 return Ok(());
5743 }
5744 }
5745
5746 if behind == 0 {
5747 if let Some(from) = self
5756 .state
5757 .rebase_fixes
5758 .iter()
5759 .rev()
5760 .find_map(|r| r.from.clone())
5761 && from != head
5762 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5763 .await
5764 .is_ok_and(|o| o.ok())
5765 {
5766 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5767 }
5768 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5774 if let Some(why) = &conflict {
5775 self.state.status = RunStatus::Blocked;
5776 self.state.event("land", why.clone());
5777 }
5778 self.state.base_sync = Some(BaseSync {
5779 tip,
5780 behind: 0,
5781 attempts,
5782 conflict,
5783 already_in: None,
5784 });
5785 if pending.is_some() {
5786 self.state.finish_base_operation(Some(&head));
5787 }
5788 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5789 .await?;
5790 self.state.save()?;
5791 return Ok(());
5792 }
5793
5794 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5795 let why = format!(
5796 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5797 synchronization(s); synchronizing again would only race it",
5798 winner.branch
5799 );
5800 self.state.status = RunStatus::Blocked;
5801 self.state.base_sync = Some(BaseSync {
5802 tip,
5803 behind,
5804 attempts,
5805 conflict: Some(why.clone()),
5806 already_in: None,
5807 });
5808 self.state.event("land", why);
5809 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5810 .await?;
5811 self.state.save()?;
5812 return Ok(());
5813 }
5814
5815 if self.state.foreign_checkout.is_some() {
5816 let wait = repository_wait(&self.state, None);
5819 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5820 if self.state.base_sync.is_none() {
5821 self.state.base_sync = Some(BaseSync {
5822 tip: tip.clone(),
5823 behind,
5824 attempts: 0,
5825 conflict: None,
5826 already_in: None,
5827 });
5828 }
5829 self.state.reserve_base_operation(&head, &tip);
5830 self.state.save()?;
5831 let merged = git::git_raw(
5832 &winner.worktree,
5833 &[
5834 "-c",
5835 "core.hooksPath=/dev/null",
5836 "merge",
5837 "--no-ff",
5838 "--no-autostash",
5839 "--no-edit",
5840 "-m",
5841 &format!("Merge {tracking} before verifying"),
5842 &tip,
5843 ],
5844 )
5845 .await?;
5846 let conflict = if merged.ok() {
5847 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5848 self.state.event(
5849 "land",
5850 format!("merged {tracking} into {} before verifying", winner.branch),
5851 );
5852 None
5853 } else {
5854 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5855 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5856 }
5857 let why = format!(
5858 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5859 winner.branch,
5860 merged.stderr.chars().take(600).collect::<String>()
5861 );
5862 self.state.status = RunStatus::Blocked;
5863 self.state.event("land", &why);
5864 Some(why)
5865 };
5866 let synced = conflict.is_none();
5867 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5868 self.state
5869 .finish_base_operation(synced.then_some(fresh.as_str()));
5870 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5871 self.state.base_sync = Some(BaseSync {
5872 tip,
5873 behind: if synced { 0 } else { behind },
5874 attempts,
5875 conflict,
5876 already_in: None,
5877 });
5878 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5879 .await?;
5880 self.state.save()?;
5881 return Ok(());
5882 }
5883
5884 self.state.event(
5885 "land",
5886 format!(
5887 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5888 winner.branch
5889 ),
5890 );
5891 self.state.save()?;
5892
5893 let branch_tracking = format!("{remote}/{}", winner.branch);
5899 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5900 let remote_tip = if self.state.task_pr_branch().is_none()
5901 && matches!(&fetched_branch, Ok(o) if o.ok())
5902 {
5903 git::rev_parse(&repo, &branch_tracking).await.ok()
5904 } else {
5905 None
5906 };
5907 if let Some(theirs) = &remote_tip
5911 && !git::is_ancestor(&repo, theirs, &head).await
5912 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5913 .await
5914 .is_ok_and(|missing| missing.is_empty())
5915 {
5916 let why = format!(
5917 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5918 them",
5919 short(theirs),
5920 winner.branch
5921 );
5922 self.state.status = RunStatus::Blocked;
5923 self.state.base_sync = Some(BaseSync {
5924 tip,
5925 behind,
5926 attempts,
5927 conflict: Some(why.clone()),
5928 already_in: None,
5929 });
5930 self.state.event("land", why);
5931 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5932 .await?;
5933 self.state.save()?;
5934 return Ok(());
5935 }
5936
5937 if self.state.base_sync.is_none() {
5938 self.state.base_sync = Some(BaseSync {
5939 tip: tip.clone(),
5940 behind,
5941 attempts: 0,
5942 conflict: None,
5943 already_in: None,
5944 });
5945 }
5946 let scratch = self.state.dir().join("base-sync");
5947 if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5948 {
5949 if error.is::<crate::repo_lock::ResourceBlocked>() {
5950 return Ok(());
5951 }
5952 return Err(error);
5953 }
5954 self.state.reserve_base_operation(&head, &tip);
5955 self.state.save()?;
5956 let rebased =
5957 match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
5958 .await
5959 {
5960 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5961 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5962 Err(e) => Err(e),
5963 };
5964 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5965 let stop_kind = match &rebased {
5966 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5967 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5968 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5969 };
5970 match rebased {
5971 Ok(None) => {
5972 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5976 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5977 self.state.finish_base_operation(Some(&fresh));
5978 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5979 let mut conflict = None;
5980 if let Some(pinned) = &remote_tip {
5981 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5982 match pushed {
5983 Ok(o) if o.ok() => self.state.event(
5984 "land",
5985 format!("pushed rebased {} to {remote}", winner.branch),
5986 ),
5987 Ok(o) => {
5988 conflict = Some(format!(
5989 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5990 winner.branch,
5991 short(pinned),
5992 o.stderr.chars().take(600).collect::<String>()
5993 ));
5994 }
5995 Err(e) => {
5996 conflict = Some(format!(
5997 "rebased {} locally but could not push it: {e:#}",
5998 winner.branch
5999 ));
6000 }
6001 }
6002 }
6003 if let Some(why) = &conflict {
6004 self.state.status = RunStatus::Blocked;
6005 self.state.event("land", why.clone());
6006 }
6007 self.state.base_sync = Some(BaseSync {
6008 tip: tip.clone(),
6009 behind: 0,
6010 attempts,
6011 conflict,
6012 already_in: None,
6013 });
6014 self.state
6015 .event("land", format!("rebased {} onto {tracking}", winner.branch));
6016 }
6017 Ok(Some(conflict)) => {
6018 self.state.finish_base_operation(None);
6019 let why = format!(
6020 "{} conflicts with {tracking} and did not rebase: {}",
6021 winner.branch,
6022 conflict.chars().take(600).collect::<String>()
6023 );
6024 self.state.status = RunStatus::Blocked;
6025 self.state.base_sync = Some(BaseSync {
6026 tip,
6027 behind,
6028 attempts,
6029 conflict: Some(why.clone()),
6030 already_in: None,
6031 });
6032 self.state.event("land", why);
6033 }
6034 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6038 self.state.save()?;
6039 return Ok(());
6040 }
6041 Err(e) => {
6042 self.state.finish_base_operation(None);
6043 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6044 self.state.status = RunStatus::Blocked;
6045 self.state.base_sync = Some(BaseSync {
6046 tip,
6047 behind,
6048 attempts,
6049 conflict: Some(why.clone()),
6050 already_in: None,
6051 });
6052 self.state.event("land", why);
6053 }
6054 }
6055 if !self.state.base_sync_blocked() {
6056 self.state.base_sync_recovery.stop = None;
6057 }
6058 self.hold_base_sync_kind(stop_kind).await?;
6059 self.state.save()?;
6060 Ok(())
6061 }
6062
6063 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6069 if self.state.task_pr_branch().is_some() {
6072 return None;
6073 }
6074 let repo = self.state.repo.clone();
6075 let remote = self.state.config.merge.remote.clone();
6076 let fetched = git::fetch(&repo, &remote, branch).await;
6077 if !matches!(&fetched, Ok(o) if o.ok()) {
6078 return None;
6079 }
6080 let branch_tracking = format!("{remote}/{branch}");
6081 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6082 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6083 return None;
6084 }
6085 if !crate::reconcile::origin_missing(&repo, head, &theirs)
6086 .await
6087 .is_ok_and(|missing| missing.is_empty())
6088 {
6089 return Some(format!(
6090 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6091 them",
6092 short(&theirs)
6093 ));
6094 }
6095 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6096 Ok(o) if o.ok() => {
6097 self.state
6098 .event("land", format!("pushed rebased {branch} to {remote}"));
6099 None
6100 }
6101 Ok(o) => Some(format!(
6102 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6103 someone may have pushed): {}",
6104 short(&theirs),
6105 o.stderr.chars().take(600).collect::<String>()
6106 )),
6107 Err(e) => Some(format!(
6108 "{branch} is rebased locally but could not be pushed: {e:#}"
6109 )),
6110 }
6111 }
6112
6113 async fn settle_already_in(
6130 &mut self,
6131 branch: &str,
6132 tip: &str,
6133 head: &str,
6134 attempts: usize,
6135 behind: usize,
6136 ) -> Result<bool> {
6137 let repo = self.state.repo.clone();
6138 let remote = self.state.config.merge.remote.clone();
6139 let start = self.state.base_commit.clone();
6140 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6141 Ok(Some(e)) => e,
6142 Ok(None) => return Ok(false),
6143 Err(e) => {
6144 tracing::warn!("already-in-base check for {branch}: {e:#}");
6145 return Ok(false);
6146 }
6147 };
6148 let mut verified = vec![head.to_owned()];
6149 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6150 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6151 if matches!(&fetched, Ok(o) if o.ok())
6152 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6153 && theirs != head
6154 {
6155 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6156 Ok(Some(_)) => verified.push(theirs),
6157 _ => return Ok(false),
6158 }
6159 }
6160 let base_branch = self.state.base_branch.clone();
6161 let message = format!(
6162 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6163 land",
6164 evidence.names(),
6165 evidence.proof.as_str()
6166 );
6167 let closed =
6168 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6169 .await;
6170 match closed {
6171 Ok(Ok(url)) => self
6172 .state
6173 .event("land", format!("closed {url}: superseded on {base_branch}")),
6174 Ok(Err(why)) => self
6175 .state
6176 .event("land", format!("did not close a pull request: {why}")),
6177 Err(e) => {
6178 let why = format!(
6179 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6180 not be closed ({e:#}); resume to retry"
6181 );
6182 self.state.status = RunStatus::Blocked;
6183 self.state.base_sync = Some(BaseSync {
6184 tip: tip.to_owned(),
6185 behind,
6186 attempts,
6187 conflict: Some(why.clone()),
6188 already_in: None,
6189 });
6190 self.state.event("land", why);
6191 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6192 .await?;
6193 self.state.save()?;
6194 return Ok(true);
6195 }
6196 }
6197 self.state.status = RunStatus::AlreadyInBase;
6198 self.state.base_sync_recovery.stop = None;
6199 self.state.base_sync = Some(BaseSync {
6200 tip: tip.to_owned(),
6201 behind,
6202 attempts,
6203 conflict: None,
6204 already_in: Some(evidence),
6205 });
6206 self.state.event("land", message);
6207 self.state.save()?;
6208 self.settle_questions();
6209 Ok(true)
6210 }
6211
6212 fn landing_base(&self) -> String {
6222 self.state
6223 .base_sync
6224 .as_ref()
6225 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6226 }
6227
6228 pub async fn fix_selected(
6261 &mut self,
6262 ids: &[String],
6263 reason: &str,
6264 allow_stale: bool,
6265 ) -> Result<()> {
6266 let reason = reason.trim();
6267 if reason.is_empty() {
6268 bail!("a fix request needs a reason — that is the operator's own record of why");
6269 }
6270 if ids.is_empty() {
6271 bail!("no finding id given");
6272 }
6273 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6274 bail!(
6275 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6276 has already concluded — can be given a targeted fix. A run still \
6277 in progress should simply be resumed; a `merged` run's branch has \
6278 already landed, so its answer is a fresh `magi review <branch>`, \
6279 not reopening this run's own record",
6280 self.state.id,
6281 self.state.status.as_str()
6282 );
6283 }
6284 let _claim = FixClaim::acquire(&self.state.dir())?;
6287 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6288 let Some(winner) = self.state.winner().cloned() else {
6289 bail!("run {} has no winning candidate to fix", self.state.id);
6290 };
6291 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6292 bail!(
6293 "branch `{}` no longer exists; this run cannot be extended",
6294 winner.branch
6295 );
6296 }
6297 let home = crate::run::home();
6298 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6299 bail!(
6300 "run {} is currently being worked on by another magi process",
6301 self.state.id
6302 );
6303 }
6304 let mut seen = BTreeSet::new();
6308 let mut findings = Vec::new();
6309 let mut missing = Vec::new();
6310 for id in ids {
6311 if !seen.insert(id.clone()) {
6312 continue;
6313 }
6314 match self.state.finding(id) {
6315 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6316 id: f.id.clone(),
6317 severity: f.severity,
6318 reviewer_vote: rec.vote,
6319 round: round.round,
6320 round_head: round.head.clone(),
6321 reviewer: rec.reviewer,
6322 agent: rec.agent.clone(),
6323 file: f.file.clone(),
6324 line: f.line,
6325 title: f.title.clone(),
6326 detail: f.detail.clone(),
6327 outcome: OperatorFixOutcome::Pending,
6328 }),
6329 None => missing.push(id.clone()),
6330 }
6331 }
6332 if !missing.is_empty() {
6333 bail!(
6334 "unknown finding id(s): {}; nothing was changed",
6335 missing.join(", ")
6336 );
6337 }
6338
6339 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6340 let stale_details: Vec<(String, String)> = findings
6341 .iter()
6342 .filter(|f| f.round_head != head_at_request)
6343 .map(|f| (f.id.clone(), f.round_head.clone()))
6344 .collect();
6345 let stale = !stale_details.is_empty();
6346 if stale && !allow_stale {
6347 bail!(
6348 "the branch has moved since some finding(s) were raised — {} — now \
6349 at {}; pass --allow-stale to fix anyway, or re-run review first",
6350 stale_details
6351 .iter()
6352 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6353 .collect::<Vec<_>>()
6354 .join(", "),
6355 short(&head_at_request)
6356 );
6357 }
6358
6359 let request = OperatorFixRequest {
6360 requested_at: Timestamp::now(),
6361 reason: reason.to_owned(),
6362 findings,
6363 head_at_request: head_at_request.clone(),
6364 allow_stale,
6365 stale,
6366 fix: None,
6367 result_head: None,
6368 follow_up_review_run: None,
6369 };
6370 self.state.event(
6371 "fix",
6372 format!(
6373 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6374 request.findings.len(),
6375 request
6376 .findings
6377 .iter()
6378 .map(|f| f.id.as_str())
6379 .collect::<Vec<_>>()
6380 .join(", "),
6381 ),
6382 );
6383 self.state.operator_fixes.push(request);
6390 self.state.save()?;
6391 let request_index = self.state.operator_fixes.len() - 1;
6392
6393 let repo = self.state.repo.clone();
6402 let coordination = self.coordinate(&repo).await?;
6403 if winner.worktree.exists() {
6404 let dirty = git::git(
6407 &winner.worktree,
6408 &["status", "--porcelain", "--untracked-files=all"],
6409 )
6410 .await?;
6411 let only_withheld = dirty.lines().all(|l| {
6412 l.strip_prefix("?? ")
6413 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6414 });
6415 if !only_withheld {
6416 bail!(
6417 "`{}` has uncommitted changes; refusing to touch it — commit or \
6418 discard them first",
6419 winner.worktree.display()
6420 );
6421 }
6422 git::worktree_remove(&self.state.repo, &winner.worktree)
6423 .await
6424 .ok();
6425 }
6426 let fix_worktree = self.state.worktree_root().join("operator-fix");
6427 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6428 git::git(
6429 &self.state.repo,
6430 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6431 )
6432 .await
6433 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6434 if !git::is_clean(&fix_worktree).await? {
6435 git::worktree_remove(&self.state.repo, &fix_worktree)
6436 .await
6437 .ok();
6438 bail!(
6439 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6440 winner.branch
6441 );
6442 }
6443 let repo_for_setup = self.state.repo.clone();
6444 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6445 Ok(()) => {
6446 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6447 }
6448 Err(e) => Err(e),
6449 };
6450 if let Err(e) = setup {
6451 git::worktree_remove(&self.state.repo, &fix_worktree)
6452 .await
6453 .ok();
6454 return Err(e);
6455 }
6456
6457 self.state.resource_block = None;
6460
6461 let run_id = self.state.id.clone();
6462 let prompts = self.state.config.prompts.clone();
6463 let language = self.state.config.graph.language.clone();
6464 let sessions = self.state.config.graph.sessions;
6465 let artifacts = agent::artifacts_dir(&self.state.dir());
6466 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6467 .findings
6468 .iter()
6469 .map(|f| Finding {
6470 id: f.id.clone(),
6471 severity: f.severity,
6472 file: f.file.clone(),
6473 line: f.line,
6474 title: f.title.clone(),
6475 detail: f.detail.clone(),
6476 })
6477 .collect();
6478 let fix_prompt = prompt::operator_fix_in(
6479 &self.state.instruction,
6480 &finding_list,
6481 reason,
6482 &stale_details,
6483 &head_at_request,
6484 &language,
6485 &self.state.config.graph.github_language,
6486 );
6487 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6488 drop(coordination);
6489 let (job, seat, out) = self
6490 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6491 prompt: fix_prompt.clone(),
6492 spec,
6493 seat,
6494 cwd: fix_worktree.clone(),
6495 timeout,
6496 allow_write: true,
6497 sessions,
6498 artifacts: artifacts.clone(),
6499 stem: "operator-fix".to_owned(),
6500 handover: None,
6501 })
6502 .await;
6503 let agent_id = seat.agent.clone();
6504
6505 let mut fix = FixRecord {
6506 agent: agent_id,
6507 addressed: Vec::new(),
6508 rejected: Vec::new(),
6509 notes: String::new(),
6510 committed: false,
6511 failed: None,
6512 duration_ms: 0,
6513 continuation: None,
6514 };
6515 let mut final_seat = seat.clone();
6516 match out {
6517 AgentOutcome::Ok(o) => {
6518 fix.duration_ms = o.duration_ms;
6519 let parsed = verdict::extract_json::<FixReport>(&o.text);
6520 let incomplete_reason = match &parsed {
6521 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6522 "the reply parsed, but it reported a command whose own CLI \
6523 never confirmed an exit status"
6524 .to_owned(),
6525 ),
6526 Ok(_) => None,
6527 Err(e) => Some(e.to_string()),
6528 };
6529 match incomplete_reason {
6530 None => {
6531 let report = parsed.expect("checked Ok above");
6532 fix.addressed = report.addressed;
6533 fix.rejected = report.rejected;
6534 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6535 }
6536 Some(reason) => {
6537 let (resumed_seat, resolved, failure, cont) = self
6538 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6539 .await;
6540 fix.duration_ms += cont.cumulative_wait_ms;
6541 fix.continuation = Some(cont);
6542 final_seat = resumed_seat;
6543 match resolved {
6544 Some(report) => {
6545 fix.addressed = report.addressed;
6546 fix.rejected = report.rejected;
6547 fix.notes =
6548 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6549 }
6550 None => fix.failed = failure,
6551 }
6552 }
6553 }
6554 }
6555 AgentOutcome::Dropped(o) => {
6556 fix.duration_ms = o.duration_ms;
6557 let why = o
6558 .dropped
6559 .as_ref()
6560 .map(|d| d.why.as_str())
6561 .unwrap_or("the CLI ended the stream without delivering its answer");
6562 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6563 }
6564 AgentOutcome::Quota(o) => {
6565 self.state.quota.push(QuotaLoss::from_report(
6566 final_seat.key.clone(),
6567 "fix",
6568 o.quota.as_ref(),
6569 ));
6570 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6571 }
6572 AgentOutcome::Failed(e) => fix.failed = Some(e),
6573 AgentOutcome::ResourceBlocked(e) => {
6574 fix.failed = Some(format!("the fixer was not started: {e}"));
6575 }
6576 }
6577 if fix.continuation.is_none() {
6580 fix.continuation = Some(ContinuationRecord::not_needed());
6581 }
6582 self.state.seats.insert(final_seat.key.clone(), final_seat);
6583
6584 let rescue_message = format!(
6585 "magi: operator-selected fix ({}) (uncommitted work)",
6586 self.state.operator_fixes[request_index]
6587 .findings
6588 .iter()
6589 .map(|f| f.id.as_str())
6590 .collect::<Vec<_>>()
6591 .join(", ")
6592 );
6593 let repo = self.state.repo.clone();
6594 let coordination = self.coordinate(&repo).await?;
6595 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6596 self.state.note_withheld("fix", &r.withheld);
6597 }
6598 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6599 fix.committed = after != head_at_request;
6600 git::worktree_remove(&self.state.repo, &fix_worktree)
6601 .await
6602 .ok();
6603 drop(coordination);
6604
6605 self.state.event(
6606 "fix",
6607 match &fix.failed {
6608 Some(reason) => format!(
6609 "operator fix: adoption report was lost ({reason}); {}",
6610 if fix.committed {
6611 "committed"
6612 } else {
6613 "NO new commit"
6614 }
6615 ),
6616 None => format!(
6617 "operator fix: {} addressed, {} rejected, {}",
6618 fix.addressed.len(),
6619 fix.rejected.len(),
6620 if fix.committed {
6621 "committed"
6622 } else {
6623 "NO new commit"
6624 }
6625 ),
6626 },
6627 );
6628
6629 for f in &mut self.state.operator_fixes[request_index].findings {
6636 f.outcome = if fix.failed.is_some() {
6637 OperatorFixOutcome::Unreported
6638 } else if fix.addressed.contains(&f.id) {
6639 OperatorFixOutcome::Addressed
6640 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6641 OperatorFixOutcome::Rejected { why: r.why.clone() }
6642 } else {
6643 OperatorFixOutcome::Unreported
6644 };
6645 }
6646
6647 let committed = fix.committed;
6648 if committed {
6649 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6650 }
6651 self.state.operator_fixes[request_index].fix = Some(fix);
6652 self.state.save()?;
6655 if self.state.resource_block.is_some() {
6656 return self.stop_for_resource();
6657 }
6658
6659 if committed {
6660 self.state.event(
6661 "fix",
6662 format!(
6663 "operator fix committed {}; opening a follow-up review-only run",
6664 short(&after)
6665 ),
6666 );
6667 let origin =
6670 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6671 match Self::review(
6672 &self.state.repo,
6673 &winner.branch,
6674 self.state.config.clone(),
6675 origin,
6676 )
6677 .await
6678 {
6679 Ok(mut follow_up) => {
6680 follow_up.state.event(
6681 "start",
6682 format!(
6683 "requested by an operator fix on run {} for finding(s) {}",
6684 self.state.id,
6685 self.state.operator_fixes[request_index]
6686 .findings
6687 .iter()
6688 .map(|f| f.id.as_str())
6689 .collect::<Vec<_>>()
6690 .join(", "),
6691 ),
6692 );
6693 follow_up.state.save()?;
6694 let follow_up_id = follow_up.state.id.clone();
6695 let adopted = match crate::direct::adopt(
6698 &follow_up.state,
6699 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6700 ) {
6701 Ok(a) => a,
6702 Err(e) => {
6703 self.state.event(
6706 "fix",
6707 format!(
6708 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6709 ),
6710 );
6711 self.state.save()?;
6712 return Ok(());
6713 }
6714 };
6715 let executed = follow_up.execute().await;
6716 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6717 if let Some(a) = adopted {
6718 a.finish(&follow_up.state, executed);
6719 }
6720 if let Some(e) = failure {
6721 self.state.event(
6722 "fix",
6723 format!(
6724 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6725 ),
6726 );
6727 }
6728 self.state.operator_fixes[request_index].follow_up_review_run =
6729 Some(follow_up_id);
6730 }
6731 Err(e) => {
6732 self.state.event(
6733 "fix",
6734 format!("committed the fix but could not open a follow-up review: {e:#}"),
6735 );
6736 }
6737 }
6738 self.state.save()?;
6739 }
6740
6741 Ok(())
6742 }
6743
6744 async fn ask_fixer(
6764 &mut self,
6765 winner: &Candidate,
6766 node: &'static str,
6767 round: Option<usize>,
6768 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6769 ) -> (SeatJob, SeatState, AgentOutcome) {
6770 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6771 self.ask_fixer_among(attempts, node, round, None, build)
6772 .await
6773 }
6774
6775 async fn ask_fixer_among(
6780 &mut self,
6781 attempts: Vec<(AgentSpec, String)>,
6782 node: &'static str,
6783 round: Option<usize>,
6784 restore: Option<(&Path, &str)>,
6785 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6786 ) -> (SeatJob, SeatState, AgentOutcome) {
6787 let run_id = self.state.id.clone();
6788 let prompts = self.state.config.prompts.clone();
6789 let cache = self.state.config.cache_dir();
6790 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6791 let mut last = None;
6792 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6793 let seat = self.seat(&key, &spec.id);
6794 let mut job = build(spec.clone(), seat);
6795 if i > 0 {
6796 job.stem = format!("{}-{}", job.stem, spec.id);
6797 }
6798 let ctx = WaveCtx {
6799 carry_seats: false,
6800 pauses: Some((&self.pause, &self.interrupt)),
6801 run: &run_id,
6802 node,
6803 prompts: &prompts,
6804 cache: cache.as_deref(),
6805 round,
6806 };
6807 let (seat, out) =
6808 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6809 let advances = match &out {
6812 AgentOutcome::Ok(o) => agent::output_advances(o),
6813 AgentOutcome::ResourceBlocked(_) => false,
6814 _ => true,
6815 };
6816 if let Some(next) = ids.get(i + 1)
6817 && advances
6818 {
6819 self.state.seats.insert(seat.key.clone(), seat.clone());
6820 let class =
6821 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6822 let reason = match &out {
6823 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6824 other => fail_reason(other),
6825 };
6826 if let Some((wt, rev)) = restore
6830 && let Err(e) = self.graft_restore(wt, rev).await
6831 {
6832 let why =
6833 format!("could not restore the worktree before the next fixer: {e:#}");
6834 tracing::warn!("{why}");
6835 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6836 AgentOutcome::ResourceBlocked(why)
6837 } else {
6838 AgentOutcome::Failed(why)
6839 };
6840 last = Some((job, seat, out));
6841 break;
6842 }
6843 record_handover(
6844 &mut self.state,
6845 node,
6846 &seat.key,
6847 &spec.id,
6848 next,
6849 &class,
6850 &reason,
6851 );
6852 continue;
6853 }
6854 last = Some((job, seat, out));
6855 break;
6856 }
6857 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6858 if let AgentOutcome::Quota(output) = &out
6859 && node != "graft"
6860 {
6861 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6866 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6867 self.state.quota.push(loss);
6868 self.state.seats.insert(seat.key.clone(), seat.clone());
6869 self.state.status = RunStatus::Blocked;
6870 return (
6871 job,
6872 seat,
6873 AgentOutcome::ResourceBlocked(
6874 "fixer unavailable; restore agent availability and resume".into(),
6875 ),
6876 );
6877 }
6878 (job, seat, out)
6879 }
6880
6881 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6892 use crate::recovery::{Mode, Repair};
6893 if !self.state.reviews.is_empty() {
6894 return Ok(());
6895 }
6896 let Some(mut rec) = self.state.recovery.clone() else {
6897 return Ok(());
6898 };
6899 if rec.mode != Mode::VerifyRepair {
6900 return Ok(());
6901 }
6902 let wt = winner.worktree.clone();
6903 let mut head = git::rev_parse(&wt, "HEAD").await?;
6904 match rec.repair.clone() {
6905 Repair::Done { .. } => return Ok(()),
6906 Repair::WaitingForRestore { before } => {
6907 self.graft_restore(&wt, &before).await?;
6908 head = before;
6909 rec.repair = Repair::NotStarted;
6910 self.state.recovery = Some(rec.clone());
6911 self.state.save()?;
6912 }
6913 Repair::Started { before } => {
6914 if let Err(e) = self.graft_restore(&wt, &before).await {
6918 if e.is::<crate::repo_lock::ResourceBlocked>() {
6919 return Err(e);
6920 }
6921 return self
6922 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6923 .await;
6924 }
6925 rec.repair = Repair::Done {
6926 before: before.clone(),
6927 after: before,
6928 committed: false,
6929 failed: Some("interrupted; rolled back".to_owned()),
6930 };
6931 self.state
6932 .event("recovery", "the repair was interrupted and rolled back");
6933 self.state.recovery = Some(rec);
6934 self.state.save()?;
6935 return Ok(());
6936 }
6937 Repair::NotStarted => {}
6938 }
6939 if crate::recovery::recheck(&mut rec, &head) {
6940 self.state.event("recovery", rec.summary());
6941 self.state.recovery = Some(rec);
6942 self.state.save()?;
6943 return Ok(());
6944 }
6945 rec.repair = Repair::Started {
6946 before: head.clone(),
6947 };
6948 self.state.recovery = Some(rec.clone());
6949 self.state.save()?;
6950
6951 let language = self.state.config.graph.language.clone();
6952 let sessions = self.state.config.graph.sessions;
6953 let artifacts = agent::artifacts_dir(&self.state.dir());
6954 let fix_prompt = prompt::recovery_fix(
6955 &self.state.instruction,
6956 &rec,
6957 self.state.config.graph.review_rounds,
6958 &language,
6959 );
6960 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6961 self.state.event(
6962 "recovery",
6963 format!(
6964 "repair: the fixer starts from the failed verification on {}",
6965 short(&head)
6966 ),
6967 );
6968 let (_job, seat, out) = self
6969 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6970 prompt: fix_prompt.clone(),
6971 spec,
6972 seat,
6973 cwd: wt.clone(),
6974 timeout,
6975 allow_write: true,
6976 sessions,
6977 artifacts: artifacts.clone(),
6978 stem: "fix-recovery".to_owned(),
6979 handover: None,
6980 })
6981 .await;
6982 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6983 rec.repair = Repair::WaitingForRestore { before: head };
6984 self.state.recovery = Some(rec);
6985 return self.stop_for_resource();
6986 }
6987 let failed = match &out {
6988 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6989 AgentOutcome::Quota(o) => {
6990 self.state.quota.push(QuotaLoss::from_report(
6991 seat.key.clone(),
6992 "fix",
6993 o.quota.as_ref(),
6994 ));
6995 Some("rate limited (quota); the fixer could not run".to_owned())
6996 }
6997 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6998 AgentOutcome::Failed(e) => Some(e.clone()),
6999 };
7000 self.state.seats.insert(seat.key.clone(), seat);
7001 let coordination = self.coordinate(&wt).await?;
7002 let failed = match failed {
7003 Some(why) => {
7005 if let Err(e) = git::restore_to(&wt, &head).await {
7006 return self
7007 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7008 .await;
7009 }
7010 Some(why)
7011 }
7012 None => {
7013 if let Ok(r) = git::rescue_commit(
7014 &wt,
7015 "magi: recovery repair of the failed verification (uncommitted work)",
7016 )
7017 .await
7018 {
7019 self.state.note_withheld("fix", &r.withheld);
7020 }
7021 None
7022 }
7023 };
7024 let after = git::rev_parse(&wt, "HEAD").await?;
7025 drop(coordination);
7026 self.state.event(
7027 "recovery",
7028 match &failed {
7029 Some(why) => format!("repair: nothing kept ({why})"),
7030 None if after != head => {
7031 format!("repair: committed {}", short(&after))
7032 }
7033 None => "repair: the fixer made no commit".to_owned(),
7034 },
7035 );
7036 rec.repair = Repair::Done {
7037 before: head.clone(),
7038 committed: after != head,
7039 after,
7040 failed,
7041 };
7042 self.state.recovery = Some(rec);
7043 self.state.save()?;
7044 Ok(())
7045 }
7046
7047 async fn stop_recovery(
7050 &mut self,
7051 mut rec: crate::recovery::RecoveryRecord,
7052 head: &str,
7053 why: String,
7054 ) -> Result<()> {
7055 rec.repair = crate::recovery::Repair::Started {
7057 before: head.to_owned(),
7058 };
7059 self.state.recovery = Some(rec);
7060 self.state
7061 .event("recovery", format!("repair could not be undone: {why}"));
7062 self.state.status = RunStatus::Blocked;
7063 self.state.save()?;
7064 bail!("recovery repair could not be rolled back: {why}")
7065 }
7066
7067 async fn review_loop(&mut self) -> Result<()> {
7068 if self.hold_base_sync().await? {
7069 return Ok(());
7070 }
7071 loop {
7072 if self
7073 .state
7074 .base_sync
7075 .as_ref()
7076 .is_some_and(|s| s.conflict.is_some())
7077 {
7078 return Ok(());
7079 }
7080 if self.catchup_repairable() {
7081 let record = self.state.reviews.last().expect("repairable round").clone();
7082 let winner = self
7083 .state
7084 .winner()
7085 .cloned()
7086 .context("catch-up repair has no winner")?;
7087 if !self.fix_review_round(&winner, record).await? {
7088 return Ok(());
7089 }
7090 }
7091 self.review_loop_pass().await?;
7092 if !self.catchup_repairable() {
7093 return Ok(());
7094 }
7095 }
7096 }
7097
7098 fn catchup_repairable(&self) -> bool {
7101 self.state.reviews.len() < self.state.config.graph.review_rounds
7102 && self.state.reviews.last().is_some_and(|r| {
7103 !r.clean
7104 && r.blocking == 0
7105 && r.fix.is_none()
7106 && r.e2e_status() == E2eStatus::Failed
7107 })
7108 }
7109
7110 async fn review_loop_pass(&mut self) -> Result<()> {
7111 if self
7116 .state
7117 .base_sync
7118 .as_ref()
7119 .is_some_and(|s| s.conflict.is_some())
7120 {
7121 return Ok(());
7122 }
7123 let run_id = self.state.id.clone();
7128 let prompts = self.state.config.prompts.clone();
7129 let Some(winner) = self.state.winner().cloned() else {
7130 return Ok(());
7131 };
7132 let granted_review = self
7133 .state
7134 .repair_grants
7135 .last()
7136 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7137 .cloned();
7138 let fresh_review = granted_review
7139 .as_ref()
7140 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7141 let max_rounds = granted_review
7142 .as_ref()
7143 .map_or(self.state.config.graph.review_rounds, |g| {
7144 g.review_start.saturating_add(1)
7145 });
7146 if let Some(status) =
7156 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7157 {
7158 if status == RunStatus::Gating {
7161 self.record_contested_handoff();
7162 }
7163 self.state.status = status;
7164 self.state.save()?;
7165 return Ok(());
7166 }
7167 self.state.status = RunStatus::Reviewing;
7168 if !fresh_review
7178 && self.state.reviews.last().is_some_and(|r| {
7179 r.e2e_status() == E2eStatus::ResourceBlocked
7180 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7181 })
7182 {
7183 let shell = self.state.config.shell();
7184 return self
7185 .stop_unresolved_review(
7186 "the last round's own verification never resolved",
7187 &shell,
7188 &winner.worktree,
7189 )
7190 .await;
7191 }
7192
7193 if granted_review.is_none() {
7194 self.recovery_repair(&winner).await?;
7195 }
7196 if self.resource_halt()? {
7197 return Ok(());
7198 }
7199
7200 let repo = self.state.repo.clone();
7201 let root = self.state.worktree_root();
7202 let language = self.state.config.graph.language.clone();
7203 let sessions = self.state.config.graph.sessions;
7204 let artifacts = self.merge_repair_artifacts();
7205 let base = self.landing_base();
7206 let base_short = short(&base);
7207 let reviewers = self.roles.reviewers.clone();
7208 let shell = self.state.config.shell();
7209
7210 for round in (self.state.reviews.len() + 1)..=max_rounds {
7211 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7212 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7213 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7214 let prev_verification = self
7223 .state
7224 .reviews
7225 .last()
7226 .and_then(|r| r.verification_summary(&head));
7227 let prior_section = match (&self.state.recovery, round) {
7230 (Some(rec), 1) if self.state.reviews.is_empty() => {
7231 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7232 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7233 Some(crate::recovery::head_of(
7234 &git::diff(&winner.worktree, reviewed, "HEAD")
7235 .await
7236 .unwrap_or_default(),
7237 crate::recovery::SINCE_DIFF_MAX,
7238 ))
7239 }
7240 _ => None,
7241 };
7242 Some(prompt::recovery_review(rec, since.as_deref()))
7243 }
7244 _ => None,
7245 };
7246
7247 let mut jobs = Vec::new();
7251 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7252 self.coordinate(&repo).await?
7253 } else {
7254 None
7255 };
7256 for (r, spec) in reviewers.iter().cloned().enumerate() {
7257 let wt = root.join(format!("review-{}", r + 1));
7258 if wt.exists() {
7261 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7262 } else {
7263 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7264 None
7265 } else {
7266 self.coordinate(&repo).await?
7267 };
7268 git::worktree_add_detached(&repo, &wt, &head).await?;
7269 }
7270 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7271 None
7272 } else {
7273 self.coordinate(&repo).await?
7274 };
7275 self.setup_seat_worktree(&repo, &wt, false).await?;
7276 let seat_key = format!("review-{}", r + 1);
7277 let spec = pick_start_spec(
7280 &self.roles.reviewer_roster,
7281 spec,
7282 self.state.seat_history.get(&seat_key),
7283 );
7284 let seat = self.seat(&seat_key, &spec.id);
7285 jobs.push(SeatJob {
7286 prompt: prompt::review(&prompt::ReviewCtx {
7287 instruction: &self.state.instruction,
7288 branch: &winner.branch,
7289 base_short: &base_short,
7290 stat: &stat,
7291 patch: &patch,
7292 verification: prev_verification.as_ref(),
7293 prior: prior_section.as_deref(),
7294 reviewers: reviewers.len(),
7295 round,
7296 rounds: max_rounds,
7297 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7300 lens: Lens::for_seat(r),
7301 language: &language,
7302 github_language: &self.state.config.graph.github_language,
7303 }),
7304 spec,
7305 seat,
7306 cwd: wt,
7307 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7308 allow_write: false,
7309 sessions,
7310 artifacts: artifacts.clone(),
7311 stem: format!("review-{round}-{}", r + 1),
7312 handover: None,
7313 });
7314 }
7315
7316 drop(coordination);
7317 self.state.event(
7318 "review",
7319 format!(
7320 "round {round}: {} reviewers on {}",
7321 jobs.len(),
7322 short(&head)
7323 ),
7324 );
7325 let mut quota_losses = Vec::new();
7326 let review_retries = self.state.config.graph.retries;
7327 let review_cache = self.state.config.cache_dir();
7328 let ctx = WaveCtx {
7329 carry_seats: true,
7330 pauses: Some((&self.pause, &self.interrupt)),
7331 run: &run_id,
7332 node: "review",
7333 prompts: &prompts,
7334 cache: review_cache.as_deref(),
7335 round: Some(round),
7336 };
7337 let results = ask_json_wave::<Review>(
7338 jobs,
7339 Arc::clone(&self.sem),
7340 review_retries,
7341 &self.roles.reviewer_roster,
7342 &ctx,
7343 &mut quota_losses,
7344 &mut self.state,
7345 &|_: &Review| Ok(()),
7346 )
7347 .await;
7348 if self.state.resource_block.is_some() {
7349 return self.stop_for_resource();
7350 }
7351 let round_quota_missing = quota_losses.len();
7355 self.state.quota.extend(quota_losses);
7356
7357 let mut records = Vec::new();
7358 let mut all_findings = Vec::new();
7359 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7360 let agent_id = seat.agent.clone();
7361 self.state.seats.insert(seat.key.clone(), seat);
7362 let mut record = ReviewRecord {
7363 reviewer: r + 1,
7364 agent: agent_id,
7365 summary: String::new(),
7366 findings: Vec::new(),
7367 vote: None,
7368 failed: None,
7369 duration_ms: 0,
7370 attempts,
7376 };
7377 match res {
7378 Ok((review, out)) => {
7379 record.summary =
7387 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7388 record.vote = Some(review.vote);
7389 record.duration_ms = out.duration_ms;
7390 for (n, mut f) in review.findings.into_iter().enumerate() {
7391 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7394 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7395 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7396 f.file = f
7402 .file
7403 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7404 all_findings.push(f.clone());
7405 record.findings.push(f);
7406 }
7407 self.state.event(
7408 "review",
7409 format!(
7410 "round {round}: reviewer {} voted {} with {} finding(s)",
7411 r + 1,
7412 review.vote.label(),
7413 record.findings.len()
7414 ),
7415 );
7416 }
7417 Err(e) => {
7418 record.failed = Some(e.to_string());
7419 self.state.event(
7420 "review",
7421 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7422 );
7423 }
7424 }
7425 records.push(record);
7426 }
7427
7428 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7435 let vote_split =
7436 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7437 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7438 if vote_split {
7439 self.state.event(
7440 "review",
7441 format!(
7442 "round {round}: votes split ({}) — one round of reconsideration",
7443 initial_votes
7444 .iter()
7445 .map(|v| v.label())
7446 .collect::<Vec<_>>()
7447 .join(", ")
7448 ),
7449 );
7450 let panel: Vec<ReviewSeatReport<'_>> = records
7453 .iter()
7454 .filter_map(|r| {
7455 r.vote.map(|vote| ReviewSeatReport {
7456 reviewer: r.reviewer,
7457 vote,
7458 summary: &r.summary,
7459 findings: &r.findings,
7460 })
7461 })
7462 .collect();
7463
7464 let mut jobs = Vec::new();
7465 let mut seats_at = Vec::new();
7466 for (r, spec) in reviewers.iter().cloned().enumerate() {
7467 if records[r].vote.is_none() {
7471 continue;
7472 }
7473 let wt = root.join(format!("review-{}", r + 1));
7474 let seat_key = format!("review-{}", r + 1);
7475 let spec = self.occupant(&seat_key, spec);
7476 let seat = self.seat(&seat_key, &spec.id);
7477 let build = |with_patch: bool| {
7485 prompt::review_reconsider(&ReviewReconsiderCtx {
7486 instruction: &self.state.instruction,
7487 reviewer: r + 1,
7488 lens: Lens::for_seat(r),
7489 panel: &panel,
7490 patch: with_patch.then_some(ReviewPatch {
7491 branch: &winner.branch,
7492 base_short: &base_short,
7493 stat: &stat,
7494 patch: &patch,
7495 }),
7496 round,
7497 rounds: max_rounds,
7498 language: &language,
7499 })
7500 };
7501 let full = build(true);
7502 let prompt = if has_context(&spec, &seat, sessions) {
7503 build(false)
7504 } else {
7505 full.clone()
7506 };
7507 jobs.push(SeatJob {
7508 prompt,
7509 spec,
7510 seat,
7511 cwd: wt,
7512 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7513 allow_write: false,
7514 sessions,
7515 artifacts: artifacts.clone(),
7516 stem: format!("review-{round}-reconsider-{}", r + 1),
7517 handover: Some(full),
7518 });
7519 seats_at.push(r);
7520 }
7521
7522 let mut recon_quota_losses = Vec::new();
7523 let recon_cache = self.state.config.cache_dir();
7524 let recon_ctx = WaveCtx {
7525 carry_seats: true,
7526 pauses: Some((&self.pause, &self.interrupt)),
7527 run: &run_id,
7528 node: "review",
7529 prompts: &prompts,
7530 cache: recon_cache.as_deref(),
7531 round: Some(round),
7532 };
7533 let recon_results = ask_json_wave::<ReviewRevote>(
7534 jobs,
7535 Arc::clone(&self.sem),
7536 review_retries,
7537 &self.roles.reviewer_roster,
7538 &recon_ctx,
7539 &mut recon_quota_losses,
7540 &mut self.state,
7541 &|_: &ReviewRevote| Ok(()),
7542 )
7543 .await;
7544 if self.state.resource_block.is_some() {
7545 return self.stop_for_resource();
7546 }
7547 self.state.quota.extend(recon_quota_losses);
7548
7549 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7550 let agent_id = seat.agent.clone();
7551 self.state.seats.insert(seat.key.clone(), seat);
7552 let mut rec = ReviewRevoteRecord {
7553 reviewer: r + 1,
7554 agent: agent_id,
7555 vote: None,
7556 reason: String::new(),
7557 failed: None,
7558 };
7559 match res {
7560 Ok((rv, _)) => {
7561 rec.vote = Some(rv.vote);
7562 rec.reason =
7563 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7564 self.state.event(
7565 "review",
7566 format!(
7567 "round {round}: reviewer {} revoted {}",
7568 r + 1,
7569 rv.vote.label()
7570 ),
7571 );
7572 }
7573 Err(e) => {
7574 rec.failed = Some(e.to_string());
7575 self.state.event(
7576 "review",
7577 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7578 );
7579 }
7580 }
7581 reconsideration.push(rec);
7582 }
7583 } else if initial_votes.len() > 1 {
7584 self.state.event(
7585 "review",
7586 format!(
7587 "round {round}: votes agreed ({}) — no reconsideration",
7588 initial_votes[0].label()
7589 ),
7590 );
7591 }
7592
7593 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7594 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7595 let defer_e2e =
7606 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7607 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7608 let reason =
7609 format!("{blocking} blocking finding(s) already required a fix this round");
7610 self.state.event(
7611 "verify",
7612 format!(
7613 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7614 {}); it will run once a round has none left",
7615 short(&head)
7616 ),
7617 );
7618 (Vec::new(), false, true, Some(reason))
7619 } else {
7620 let e2e_commands = self.state.config.verify.e2e.clone();
7621 let cache_dir = self.state.config.cache_dir();
7622 let context = format!("round {round}");
7623 let wait = self.repository_wait();
7624 let (e2e, verify_retried) = with_cache_lease(
7625 &mut self.state,
7626 cache_dir.as_deref(),
7627 "e2e",
7628 "e2e",
7629 &winner.worktree,
7630 &head,
7631 verify_timeout,
7632 &context,
7633 &wait,
7634 |state, budget| {
7635 let shell = shell.clone();
7636 let e2e_commands = e2e_commands.clone();
7637 let worktree = winner.worktree.clone();
7638 let context = context.clone();
7639 let winner = winner.clone();
7640 async move {
7641 if !e2e_commands.is_empty() {
7642 Self::run_pre_gate(state, &winner).await;
7646 }
7647 run_e2e_with_retry(
7648 state,
7649 &shell,
7650 &e2e_commands,
7651 &worktree,
7652 budget,
7653 &context,
7654 Some(round),
7655 )
7656 .await
7657 }
7658 },
7659 )
7660 .await?;
7661 (e2e, verify_retried, false, None)
7662 };
7663
7664 let expected = records.len();
7665 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7666 let incomplete = answered < expected;
7667 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7668 let policy = self.state.config.graph.incomplete_review;
7669 let clean = round_is_clean(
7670 blocking,
7671 e2e_ok,
7672 answered,
7673 expected,
7674 round_quota_missing,
7675 policy,
7676 );
7677
7678 let mut round_record = ReviewRound {
7679 round,
7680 head: head.clone(),
7681 verified_head: None,
7682 verified_at: None,
7683 reviews: records,
7684 e2e,
7685 verify_retried,
7686 e2e_deferred,
7687 e2e_defer_reason,
7688 fix: None,
7689 blocking,
7690 answered,
7691 expected,
7692 clean,
7693 progressed: false,
7694 vote_split,
7695 reconsideration,
7696 verdict: None,
7697 };
7698 round_record.verdict = ReviewVote::worst(
7702 round_record
7703 .final_votes()
7704 .into_iter()
7705 .map(|(_, _, vote)| vote),
7706 );
7707 if !matches!(
7718 round_record.e2e_status(),
7719 E2eStatus::Deferred | E2eStatus::NotConfigured
7720 ) {
7721 round_record.verified_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
7724 round_record.verified_at = Some(Timestamp::now());
7725 }
7726
7727 if incomplete {
7728 let missing: Vec<String> = round_record
7729 .reviews
7730 .iter()
7731 .filter(|r| r.failed.is_some())
7732 .map(|r| format!("review-{}", r.reviewer))
7733 .collect();
7734 self.state.event(
7735 "review",
7736 format!(
7737 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7738 missing.join(", ")
7739 ),
7740 );
7741 }
7742
7743 if clean {
7744 self.state.event(
7745 "review",
7746 if incomplete && policy == IncompleteReviewPolicy::Warn {
7747 format!(
7748 "round {round}: clean (warn policy, incomplete panel) — no \
7749 blocking findings from the seats that answered, verification green"
7750 )
7751 } else if incomplete {
7752 format!(
7753 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7754 quorum) — no blocking findings from the seats that answered, \
7755 verification green",
7756 expected - answered
7757 )
7758 } else {
7759 format!("round {round}: clean — no blocking findings, verification green")
7760 },
7761 );
7762 self.state.reviews.push(round_record);
7763 self.state.status = RunStatus::Gating;
7764 self.state.save()?;
7765 return Ok(());
7766 }
7767
7768 if incomplete && blocking == 0 && e2e_ok {
7776 self.state.reviews.push(round_record);
7777 self.state.save()?;
7778 if round == max_rounds {
7779 self.state.status = RunStatus::Blocked;
7780 self.state.event(
7781 "review",
7782 format!(
7783 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7784 refusing to call it clean",
7785 expected - answered
7786 ),
7787 );
7788 return Ok(());
7789 }
7790 continue;
7791 }
7792
7793 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7805 self.state.reviews.push(round_record);
7806 return self
7807 .stop_unresolved_review(
7808 "the round's own verification could not run",
7809 &shell,
7810 &winner.worktree,
7811 )
7812 .await;
7813 }
7814
7815 if round == max_rounds {
7816 self.state.reviews.push(round_record);
7817 return self
7818 .stop_reviewing(
7819 &format!(
7820 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7821 ),
7822 &shell,
7823 &winner.worktree,
7824 )
7825 .await;
7826 }
7827
7828 if !self.fix_review_round(&winner, round_record).await? {
7829 return Ok(());
7830 }
7831 }
7832 Ok(())
7833 }
7834
7835 async fn fix_review_round(
7838 &mut self,
7839 winner: &Candidate,
7840 mut round_record: ReviewRound,
7841 ) -> Result<bool> {
7842 let round = round_record.round;
7843 let shell = self.state.config.shell();
7844 let step = self.run_fix_round(winner, &mut round_record).await?;
7845 if matches!(step, FixStep::Resource) {
7846 self.stop_for_resource()?;
7847 return Ok(false);
7848 }
7849 if self.state.reviews.len() < round {
7850 self.state.reviews.push(round_record);
7851 } else {
7852 self.state.reviews[round - 1] = round_record;
7853 }
7854 self.state.save()?;
7855
7856 if matches!(step, FixStep::Recorded { report_lost: true }) {
7869 self.stop_reviewing(
7870 "the fixer's adoption report never came back, even after resuming its \
7871 own seat; refusing to start another round against the same worktree \
7872 while that is unresolved",
7873 &shell,
7874 &winner.worktree,
7875 )
7876 .await?;
7877 return Ok(false);
7878 }
7879
7880 let streak = self
7881 .state
7882 .reviews
7883 .iter()
7884 .rev()
7885 .take_while(|r| !r.progressed)
7886 .count();
7887 if streak >= STAGNANT_LIMIT {
7888 self.stop_reviewing(
7889 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7890 &shell,
7891 &winner.worktree,
7892 )
7893 .await?;
7894 return Ok(false);
7895 }
7896 Ok(true)
7897 }
7898
7899 async fn run_fix_round(
7901 &mut self,
7902 winner: &Candidate,
7903 round_record: &mut ReviewRound,
7904 ) -> Result<FixStep> {
7905 let round = round_record.round;
7906 let max_rounds = self.state.config.graph.review_rounds;
7907 let run_id = self.state.id.clone();
7908 let prompts = self.state.config.prompts.clone();
7909 let language = self.state.config.graph.language.clone();
7910 let sessions = self.state.config.graph.sessions;
7911 let artifacts = self.merge_repair_artifacts();
7912 let base = self.landing_base();
7913 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7914 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7915 let this_round_verification = round_record.verification_summary(&head);
7916 let all_findings: Vec<_> = round_record
7917 .reviews
7918 .iter()
7919 .flat_map(|r| r.findings.iter().cloned())
7920 .collect();
7921 let blocking_findings: Vec<_> = all_findings
7924 .iter()
7925 .filter(|f| f.severity.blocks())
7926 .cloned()
7927 .collect();
7928 let fix_prompt = prompt::fix_in(
7929 &self.state.instruction,
7930 &blocking_findings,
7931 this_round_verification.as_ref(),
7932 round,
7933 max_rounds,
7934 &language,
7935 &self.state.config.graph.github_language,
7936 );
7937 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7938 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7939 let (job, seat, out) = self
7940 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7941 prompt: fix_prompt.clone(),
7942 spec,
7943 seat,
7944 cwd: winner.worktree.clone(),
7945 timeout,
7946 allow_write: true,
7947 sessions,
7948 artifacts: artifacts.clone(),
7949 stem: format!("fix-{round}"),
7950 handover: None,
7951 })
7952 .await;
7953 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7956 return Ok(FixStep::Resource);
7957 }
7958 let agent_id = seat.agent.clone();
7959
7960 let mut fix = FixRecord {
7961 agent: agent_id,
7962 addressed: Vec::new(),
7963 rejected: Vec::new(),
7964 notes: String::new(),
7965 committed: false,
7966 failed: None,
7967 duration_ms: 0,
7968 continuation: None,
7969 };
7970 let mut continuation = ContinuationRecord::not_needed();
7971 let mut final_seat = seat.clone();
7972 match out {
7973 AgentOutcome::Ok(o) => {
7974 fix.duration_ms = o.duration_ms;
7975 let parsed = verdict::extract_json::<FixReport>(&o.text);
7976 let incomplete_reason = match &parsed {
7983 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7984 "the reply parsed, but it reported a command whose own CLI never \
7985 confirmed an exit status"
7986 .to_owned(),
7987 ),
7988 Ok(_) => None,
7989 Err(e) => Some(e.to_string()),
7990 };
7991 match incomplete_reason {
7992 None => {
7993 let report = parsed.expect("checked Ok above");
7994 fix.addressed = report.addressed;
7995 fix.rejected = report.rejected;
7996 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7997 }
7998 Some(reason) => {
7999 let (resumed_seat, resolved, failure, cont) = self
8000 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
8001 .await;
8002 fix.duration_ms += cont.cumulative_wait_ms;
8003 continuation = cont;
8004 final_seat = resumed_seat;
8005 match resolved {
8006 Some(report) => {
8007 fix.addressed = report.addressed;
8008 fix.rejected = report.rejected;
8009 fix.notes =
8010 blind::sanitize_prose(&report.notes, &self.state.config.blind);
8011 }
8012 None => fix.failed = failure,
8013 }
8014 }
8015 }
8016 }
8017 AgentOutcome::Dropped(o) => {
8019 fix.duration_ms = o.duration_ms;
8020 let why = o
8021 .dropped
8022 .as_ref()
8023 .map(|d| d.why.as_str())
8024 .unwrap_or("the CLI ended the stream without delivering its answer");
8025 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8026 }
8027 AgentOutcome::Quota(o) => {
8028 self.state.quota.push(QuotaLoss::from_report(
8029 final_seat.key.clone(),
8030 "fix",
8031 o.quota.as_ref(),
8032 ));
8033 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8034 }
8035 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8036 }
8037 if self.state.resource_block.is_some() {
8038 self.state.seats.insert(final_seat.key.clone(), final_seat);
8039 return Ok(FixStep::Resource);
8040 }
8041 fix.continuation = Some(continuation);
8042 self.state.seats.insert(final_seat.key.clone(), final_seat);
8043 let coordination = self.coordinate(&winner.worktree).await?;
8044 if let Ok(r) = git::rescue_commit(
8045 &winner.worktree,
8046 &format!("magi: review round {round} fixes (uncommitted work)"),
8047 )
8048 .await
8049 {
8050 self.state.note_withheld("fix", &r.withheld);
8051 }
8052 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8053 fix.committed = after != before;
8054 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8062 drop(coordination);
8063 let progressed = diff_after != patch;
8064 let commit_note = if fix.committed {
8065 "committed"
8066 } else {
8067 "NO new commit"
8068 };
8069 let tree_note = if progressed {
8070 "changed vs base"
8071 } else {
8072 "unchanged vs base"
8073 };
8074 self.state.event(
8075 "fix",
8076 match &fix.failed {
8077 Some(reason) => {
8083 format!(
8084 "round {round}: fixer's adoption report was lost ({reason}); \
8085 {commit_note}, tree {tree_note}"
8086 )
8087 }
8088 None => format!(
8089 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8090 {tree_note}{}",
8091 fix.addressed.len(),
8092 fix.rejected.len(),
8093 if continuation.outcome == ContinuationOutcome::Resumed {
8094 format!(
8095 " (adoption report recovered after {} continuation(s))",
8096 continuation.attempts
8097 )
8098 } else {
8099 String::new()
8100 },
8101 ),
8102 },
8103 );
8104 round_record.fix = Some(fix);
8105 round_record.progressed = progressed;
8106 Ok(FixStep::Recorded {
8107 report_lost: matches!(
8108 continuation.outcome,
8109 ContinuationOutcome::Exhausted
8110 | ContinuationOutcome::QuotaLost
8111 | ContinuationOutcome::NoSession
8112 ),
8113 })
8114 }
8115
8116 fn record_contested_handoff(&mut self) {
8150 if self.state.contested_handoff.is_some() {
8151 return;
8152 }
8153 let Some(contested) = self
8154 .state
8155 .reviews
8156 .last()
8157 .and_then(ReviewRound::contested_handoff)
8158 else {
8159 return;
8160 };
8161 self.state.event(
8162 "review",
8163 format!(
8164 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8165 wait for the owner's approval",
8166 contested.findings.len(),
8167 contested.rejecters.len()
8168 ),
8169 );
8170 self.state.contested_handoff = Some(contested);
8171 }
8172
8173 async fn stop_unresolved_review(
8174 &mut self,
8175 why: &str,
8176 shell: &[String],
8177 worktree: &Path,
8178 ) -> Result<()> {
8179 self.stop_reviewing_inner(why, shell, worktree, true).await
8180 }
8181
8182 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8183 self.stop_reviewing_inner(why, shell, worktree, false).await
8184 }
8185
8186 async fn stop_reviewing_inner(
8187 &mut self,
8188 why: &str,
8189 shell: &[String],
8190 worktree: &Path,
8191 repairable: bool,
8192 ) -> Result<()> {
8193 let round_idx = self.state.reviews.len() - 1;
8194 let needs_catchup_run = matches!(
8202 self.state.reviews[round_idx].e2e_status(),
8203 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8204 );
8205 if needs_catchup_run {
8206 let round = self.state.reviews[round_idx].round;
8207 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8208 let commands = self.state.config.verify.e2e.clone();
8209 let before_head = git::rev_parse(worktree, "HEAD").await?;
8210 let winner = self.state.winner().cloned();
8211 let cache_dir = self.state.config.cache_dir();
8212 let context = format!(
8213 "round {round}: verification unresolved, catching up before the final decision"
8214 );
8215 let wait = self.repository_wait();
8216 let (outcomes, verify_retried) = with_cache_lease(
8217 &mut self.state,
8218 cache_dir.as_deref(),
8219 "e2e",
8220 "e2e",
8221 worktree,
8222 &before_head,
8223 timeout,
8224 &context,
8225 &wait,
8226 |state, budget| {
8227 let shell = shell.to_vec();
8228 let commands = commands.clone();
8229 let context = context.clone();
8230 async move {
8231 if let Some(winner) = winner.filter(|_| !commands.is_empty()) {
8232 Self::run_pre_gate(state, &winner).await;
8235 }
8236 run_e2e_with_retry(
8237 state,
8238 &shell,
8239 &commands,
8240 worktree,
8241 budget,
8242 &context,
8243 Some(round),
8244 )
8245 .await
8246 }
8247 },
8248 )
8249 .await?;
8250 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8252 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8253 self.state.record_base_progress(&attempted_head);
8254 }
8255 let last = &mut self.state.reviews[round_idx];
8256 last.e2e = outcomes;
8257 last.verify_retried = verify_retried;
8258 last.verified_head = Some(attempted_head);
8265 last.verified_at = Some(Timestamp::now());
8266 if verify_inconclusive(&last.e2e) {
8267 self.state.save()?;
8274 return Ok(());
8275 }
8276 last.e2e_deferred = false;
8277 if last.answered == last.expected {
8282 last.clean = round_is_clean(
8283 last.blocking,
8284 last.e2e.iter().all(CommandOutcome::ok),
8285 last.answered,
8286 last.expected,
8287 0,
8288 self.state.config.graph.incomplete_review,
8289 );
8290 }
8291 }
8292 let last = &self.state.reviews[round_idx];
8293 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8294
8295 match last.e2e_status() {
8296 E2eStatus::Failed => {
8297 let red: Vec<String> = last
8298 .e2e
8299 .iter()
8300 .filter(|o| !o.ok())
8301 .map(|o| {
8302 format!(
8303 "`{}` -> {:?}\n{}",
8304 o.command,
8305 o.code,
8306 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8307 )
8308 })
8309 .collect();
8310 let can_repair = last.blocking == 0 && last.fix.is_none();
8311 self.state.reviews[round_idx].clean = false;
8312 self.state
8313 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8314 self.state.status = RunStatus::Blocked;
8315 if repairable
8316 && can_repair
8317 && self.state.reviews.len() < self.state.config.graph.review_rounds
8318 {
8319 self.state.status = RunStatus::Reviewing;
8320 }
8321 }
8322 E2eStatus::ResourceBlocked => {
8327 self.state.event(
8328 "review",
8329 format!(
8330 "{why}; e2e could not run (resource unavailable); not \
8331 deciding yet"
8332 ),
8333 );
8334 }
8335 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8336 self.state.event(
8337 "review",
8338 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8339 );
8340 self.record_contested_handoff();
8341 self.state.status = RunStatus::Gating;
8342 }
8343 }
8344 self.state.save()?;
8345 Ok(())
8346 }
8347
8348 async fn gate(&mut self) -> Result<()> {
8351 if self.hold_base_sync().await? {
8352 return Ok(());
8353 }
8354 if self.state.status == RunStatus::Failed
8366 || self
8367 .state
8368 .base_sync
8369 .as_ref()
8370 .is_some_and(|s| s.conflict.is_some())
8371 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8372 != Some(RunStatus::Gating)
8373 {
8374 return Ok(());
8375 }
8376 let Some(winner) = self.state.winner().cloned() else {
8377 return Ok(());
8378 };
8379 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8380 self.state.reconcile_verification_head(&head);
8381 self.state.save()?;
8382 let wait = self.repository_wait();
8383 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8384 return Ok(());
8385 }
8386 if self.state.gate_ran {
8387 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8398 self.state.status = RunStatus::Blocked;
8399 self.state.save()?;
8400 }
8401 return Ok(());
8402 }
8403 self.state.status = RunStatus::Gating;
8404 let mut outcomes = self.run_gate(&winner).await?;
8405 let mut attempted_head = self.state.gate_head.clone();
8406 loop {
8407 if verify_inconclusive(&outcomes) {
8418 self.state.save()?;
8419 return Ok(());
8420 }
8421 if outcomes.iter().all(CommandOutcome::ok) {
8422 break;
8423 }
8424 let fixes_before = self.state.gate_fixes.len();
8425 match self.gate_fix_round(&winner, &outcomes).await? {
8426 GateFix::Retry => {
8427 outcomes = self.run_gate(&winner).await?;
8428 attempted_head = self.state.gate_head.clone();
8429 }
8430 GateFix::Stop => {
8431 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8432 if self.state.gate_fixes.len() > fixes_before
8433 && attempted_head.as_deref() != Some(current.as_str())
8434 && verify_current_head(&mut self.state, &winner, &wait).await?
8435 {
8436 outcomes = self.run_gate(&winner).await?;
8439 attempted_head = self.state.gate_head.clone();
8440 } else {
8441 break;
8442 }
8443 }
8444 GateFix::Defer => {
8445 self.state.save()?;
8446 return Ok(());
8447 }
8448 }
8449 }
8450 let passed = outcomes.iter().all(CommandOutcome::ok);
8451 self.state.gate = outcomes;
8452 self.state.gate_ran = true;
8453 self.state.gate_head = attempted_head;
8454 if !self.state.gate.is_empty()
8455 && !verify_inconclusive(&self.state.gate)
8456 && let Some(head) = self.state.gate_head.clone()
8457 {
8458 self.state.record_base_progress(&head);
8459 }
8460 self.state.save()?;
8461 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8462 return Ok(());
8463 }
8464 if !passed {
8465 self.state.status = RunStatus::Blocked;
8466 let spent = self.state.gate_fixes.len();
8467 self.state.event(
8468 "gate",
8469 if spent == 0 {
8470 "gate failed; not merging".to_owned()
8471 } else {
8472 format!("gate failed after {spent} gate-fix round(s); not merging")
8473 },
8474 );
8475 }
8476 self.state.save()?;
8477 Ok(())
8478 }
8479
8480 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8498 let commands = state.config.verify.pre_gate.clone();
8499 if commands.is_empty() {
8500 return;
8501 }
8502 let shell = state.config.shell();
8503 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8504 let (outcomes, _) = run_commands(
8505 state,
8506 "pre_gate",
8507 "pre_gate",
8508 0,
8509 &shell,
8510 &commands,
8511 &winner.worktree,
8512 timeout,
8513 )
8514 .await;
8515 for o in &outcomes {
8516 if !o.ok() {
8517 tracing::warn!(
8518 "pre_gate `{}` failed ({:?}); the gate decides",
8519 o.command,
8520 o.code
8521 );
8522 }
8523 state.event(
8524 "pre_gate",
8525 format!(
8526 "`{}` -> {}",
8527 o.command,
8528 if o.ok() {
8529 "pass".to_owned()
8530 } else {
8531 format!(
8532 "FAIL ({:?})\n{}",
8533 o.code,
8534 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8535 )
8536 }
8537 ),
8538 );
8539 }
8540 state.pre_gate = outcomes;
8541 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8542 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8543 Ok(head) => {
8544 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8545 state.pre_gate_commit = Some(head);
8546 }
8547 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8548 },
8549 Ok(false) => {}
8550 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8551 }
8552 if let Err(e) = state.save() {
8553 tracing::warn!("could not persist the pre_gate record: {e:#}");
8554 }
8555 }
8556
8557 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8560 let wait = self.repository_wait();
8561 run_landing_gate(&mut self.state, winner, &wait).await
8562 }
8563
8564 async fn gate_fix_round(
8577 &mut self,
8578 winner: &Candidate,
8579 outcomes: &[CommandOutcome],
8580 ) -> Result<GateFix> {
8581 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8582 g.phase == crate::repair_budget::Phase::Fixing
8583 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8584 }) {
8585 self.state.gate_fixes.len().saturating_add(1)
8586 } else {
8587 self.state.config.graph.gate_fix_rounds
8588 };
8589 let spent = self.state.gate_fixes.len();
8590 if spent >= cap {
8591 if cap > 0 {
8592 self.state.event(
8593 "gate",
8594 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8595 );
8596 }
8597 return Ok(GateFix::Stop);
8598 }
8599 if !gate_fixable(outcomes) {
8600 self.state.event(
8601 "gate",
8602 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8603 command or similar); not spending a fix round on it",
8604 );
8605 return Ok(GateFix::Stop);
8606 }
8607 let min_free = self.state.config.disk.min_free_bytes;
8608 if min_free > 0 {
8609 match crate::disk::free_bytes(&winner.worktree) {
8610 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8611 Ok(free) => {
8612 self.state.event(
8613 "gate",
8614 format!(
8615 "only {free} bytes free ({min_free} required by `[disk] \
8616 min_free_bytes`); not spending a fix round on a failure the disk \
8617 may explain"
8618 ),
8619 );
8620 return Ok(GateFix::Stop);
8621 }
8622 Err(e) => {
8623 self.state.event(
8624 "gate",
8625 format!("free disk space could not be measured ({e:#}); no fix round"),
8626 );
8627 return Ok(GateFix::Stop);
8628 }
8629 }
8630 }
8631
8632 let attempt = spent + 1;
8633 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8634 let base = self.landing_base();
8635 let fix_prompt = prompt::gate_fix_in(
8636 &self.state.instruction,
8637 &failed,
8638 attempt,
8639 cap,
8640 &self.state.config.graph.language,
8641 &self.state.config.graph.github_language,
8642 );
8643 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8644 let sessions = self.state.config.graph.sessions;
8645 let artifacts = self.merge_repair_artifacts();
8646 self.state.event(
8647 "gate",
8648 format!("gate failed; gate-fix round {attempt} of {cap}"),
8649 );
8650 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8651 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8652 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8655 head: before.clone(),
8656 passed: false,
8657 outcomes: Vec::new(),
8658 });
8659 self.state.save()?;
8660 let (_, seat, out) = self
8661 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8662 prompt: fix_prompt.clone(),
8663 spec,
8664 seat,
8665 cwd: winner.worktree.clone(),
8666 timeout,
8667 allow_write: true,
8668 sessions,
8669 artifacts: artifacts.clone(),
8670 stem: format!("gate-fix-{attempt}"),
8671 handover: None,
8672 })
8673 .await;
8674 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8677 return Ok(GateFix::Defer);
8678 }
8679 let mut record = GateFixRecord {
8680 agent: seat.agent.clone(),
8681 failed,
8682 notes: String::new(),
8683 committed: false,
8684 error: None,
8685 };
8686 match out {
8687 AgentOutcome::Ok(o) => {
8688 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8691 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8692 }
8693 }
8694 AgentOutcome::Dropped(_) => {
8695 record.error = Some("the CLI dropped the stream".to_owned());
8696 }
8697 AgentOutcome::Quota(o) => {
8698 self.state.quota.push(QuotaLoss::from_report(
8699 seat.key.clone(),
8700 "gate-fix",
8701 o.quota.as_ref(),
8702 ));
8703 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8704 }
8705 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8706 }
8707 self.state.seats.insert(seat.key.clone(), seat);
8708 let coordination = self.coordinate(&winner.worktree).await?;
8709 if let Ok(r) = git::rescue_commit(
8710 &winner.worktree,
8711 &format!("magi: gate fix {attempt} (uncommitted work)"),
8712 )
8713 .await
8714 {
8715 self.state.note_withheld("gate-fix", &r.withheld);
8716 }
8717 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8718 record.committed = after != before;
8719 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8720 let note = record.error.clone();
8721 self.state.gate_fixes.push(record);
8722 self.state.save()?;
8723 if !changed {
8724 self.state.event(
8725 "gate",
8726 match note {
8727 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8728 None => format!("gate-fix round {attempt}: the tree did not change"),
8729 },
8730 );
8731 return Ok(GateFix::Stop);
8732 }
8733 self.state.event(
8734 "gate",
8735 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8736 );
8737
8738 drop(coordination);
8739 let wait = self.repository_wait();
8740 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8741 }
8742
8743 async fn guarded_pr_message(
8750 &mut self,
8751 winner: &Candidate,
8752 facts: Option<&BranchFacts>,
8753 posting: bool,
8754 ) -> Result<Option<PrMessage>> {
8755 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8756 let head = match final_head.as_deref() {
8757 Some(h) => crate::pr_outcome::FinalHead::At(h),
8758 None => crate::pr_outcome::FinalHead::Unknown,
8759 };
8760 let summarized = if posting {
8762 Some(self.summarize_change(winner, final_head.as_deref()).await)
8763 } else {
8764 None
8765 };
8766 if self.state.resource_block.is_some() {
8771 self.state.event(
8772 "pr-summary",
8773 "a shared resource was busy; nothing was published, resume to summarize again",
8774 );
8775 return Ok(None);
8776 }
8777 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8778 anyhow::bail!(
8779 "the final commit moved during public description generation; retry before publishing"
8780 );
8781 }
8782 let note = match (&summarized, &self.state.change_summary) {
8783 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8784 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8785 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8786 (None, _) => ChangeNote::NotAsked,
8787 };
8788 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8789 if !posting {
8790 let identity = crate::scrub::Identity::current();
8791 return Ok(Some(PrMessage {
8792 title: crate::scrub::scrub(&pr.title, &identity),
8793 body: crate::scrub::scrub(&pr.body, &identity),
8794 }));
8795 }
8796 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8799 if let Some(known) = self
8800 .state
8801 .github_text
8802 .clone()
8803 .filter(|g| g.fingerprint == raw_fp)
8804 {
8805 return self.settle_withheld(winner, &pr, known).await;
8806 }
8807 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8808 let mut violations =
8809 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8810 let identity = crate::scrub::Identity::current();
8811 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8812 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8813 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8814 {
8815 violations.push(crate::github_text::Violation::SensitiveData);
8816 }
8817 let has_language = violations
8821 .iter()
8822 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8823 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8824 self.state.event(
8825 "github-text",
8826 if has_language {
8827 format!("description rejected: {violations:?}; requesting one rewrite")
8828 } else {
8829 format!("description rejected: {violations:?}; asking the owner")
8830 },
8831 );
8832 let mut rewritten = false;
8833 let mut rejected_rewrite = None;
8834 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8835 {
8836 let output = text;
8837 if let Some(title) = summary_title(&output) {
8838 let raw_body = summary_without_title(&output);
8841 let clean_title = crate::scrub::scrub(&title, &identity);
8842 let mut body = crate::scrub::scrub(&raw_body, &identity);
8843 let redacted = clean_title != title || body != raw_body;
8844 let title = clean_title;
8845 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8846 if !complete {
8847 self.state.event(
8848 "github-text",
8849 "description rewrite omitted required findings",
8850 );
8851 }
8852 if !body.trim().is_empty() {
8853 body.push_str("\n\n");
8854 body.push_str(&public_pr_evidence(&self.state, head));
8855 }
8856 let marker = format!("magi:run/{}", self.state.id);
8857 if !body.trim().is_empty() && !body.contains(&marker) {
8858 body.push_str(&format!("\n\n{marker}"));
8859 }
8860 let rewrite = PrMessage {
8861 title: title.clone(),
8862 body: body.clone(),
8863 };
8864 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8865 let mut failed = crate::github_text::check_for(
8866 &self.state.config.graph,
8867 &title,
8868 &body,
8869 redecision,
8870 );
8871 if (!crate::github_text::shareable(&title)
8872 || !crate::github_text::shareable(&body))
8873 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8874 {
8875 failed.push(crate::github_text::Violation::SensitiveData);
8876 }
8877 if complete
8878 && !title.trim().is_empty()
8879 && !body.trim().is_empty()
8880 && !failed.is_empty()
8881 {
8882 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8883 title: title.clone(),
8884 body: body.clone(),
8885 failed: failed
8886 .iter()
8887 .map(|v| crate::github_text::category(*v).to_owned())
8888 .collect(),
8889 });
8890 }
8891 if complete
8892 && !title.trim().is_empty()
8893 && !raw_body.trim().is_empty()
8894 && failed.is_empty()
8895 {
8896 decision = redecision;
8897 pr = PrMessage { title, body };
8898 rewritten = true;
8899 self.state
8900 .event("github-text", "description rewrite passed");
8901 if redacted {
8902 self.state.event(
8903 "github-text",
8904 "sensitive data removed from description rewrite",
8905 );
8906 }
8907 }
8908 }
8909 }
8910 if !rewritten {
8911 self.state.event(
8912 "github-text",
8913 "description rewrite unavailable or rejected; asking the owner",
8914 );
8915 if let Some(w) =
8916 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8917 {
8918 self.retire_github_text_question("the withheld text changed");
8921 let record = crate::run::GithubTextAsk {
8922 rewrite: rejected_rewrite,
8923 fingerprint: raw_fp,
8924 title: w.title,
8925 body: w.body,
8926 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8927 question: None,
8928 asks: 0,
8929 resolved: false,
8930 chosen_title: None,
8931 choice: None,
8932 publication: None,
8933 };
8934 return self.settle_withheld(winner, &pr, record).await;
8935 }
8936 }
8937 }
8938 let (title, body) =
8939 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8940 Ok(Some(PrMessage { title, body }))
8941 }
8942
8943 async fn summarize_change(
8954 &mut self,
8955 winner: &Candidate,
8956 head: Option<&str>,
8957 ) -> std::result::Result<(), String> {
8958 let Some(head) = head else {
8959 return Err("the final commit could not be read".to_owned());
8960 };
8961 let remote = self.state.config.merge.remote.clone();
8962 let tracking = format!("{remote}/{}", self.state.base_branch);
8963 if !git::rev_exists(&winner.worktree, &tracking).await {
8964 return Err(format!("the remote base `{tracking}` could not be read"));
8965 }
8966 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8967 Ok(b) if !b.is_empty() => b,
8968 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8969 };
8970 let evidence = crate::pr_outcome::summary_evidence(
8971 &self.state,
8972 crate::pr_outcome::FinalHead::At(head),
8973 );
8974 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8975 if self
8976 .state
8977 .change_summary
8978 .as_ref()
8979 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8980 {
8981 return Ok(());
8982 }
8983 if self.state.change_summary.take().is_some() {
8984 self.state.event(
8985 "pr-summary",
8986 "the final diff or evidence changed; the earlier change summary was discarded",
8987 );
8988 }
8989 if let Err(e) = write_artifact(
8991 &self.state,
8992 &format!("public-description-{fingerprint}.md"),
8993 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8994 ) {
8995 tracing::warn!("could not save public description evidence: {e:#}");
8996 }
8997 let chain = match agent::pick_chain(
8998 &self.state.config.agents,
8999 self.state.config.roles.synthesizer.as_ref(),
9000 &agent::installed,
9001 "synthesizer",
9002 ) {
9003 Ok(chain) => chain,
9004 Err(e) => {
9005 tracing::warn!("no agent for the change summary: {e:#}");
9006 self.state.event("pr-summary", "no summary agent available");
9007 return Err("no agent was available to summarize it".to_owned());
9008 }
9009 };
9010 let subjects = git::subjects(&winner.worktree, &base, head)
9011 .await
9012 .unwrap_or_default();
9013 let stat = git::diff_stat(&winner.worktree, &base, head)
9014 .await
9015 .unwrap_or_default();
9016 let stat: String = stat.chars().take(6_000).collect();
9017 let mut prompt = crate::prompt::change_summary_in(
9018 &base,
9019 head,
9020 &subjects,
9021 stat.trim(),
9022 &self.state.config.graph.github_language,
9023 );
9024 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
9025 let sessions = self.state.config.graph.sessions;
9026 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
9027 let prompts = self.state.config.prompts.clone();
9028 let cache = self.state.config.cache_dir();
9029 let run = self.state.id.clone();
9030 for (i, spec) in chain.into_iter().enumerate() {
9031 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
9032 let github_spec = match agent::for_github_text(
9033 &spec,
9034 &self.state.config.graph.github_language,
9035 &winner.worktree,
9036 ) {
9037 Ok(spec) => spec,
9038 Err(e) => {
9039 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9040 continue;
9041 }
9042 };
9043 let job = SeatJob {
9044 spec: github_spec,
9045 seat,
9046 cwd: winner.worktree.clone(),
9047 prompt: prompt.clone(),
9048 timeout,
9049 allow_write: false,
9050 sessions,
9051 artifacts: agent::artifacts_dir(&self.state.dir()),
9052 stem: if i == 0 {
9053 "change-summary".to_owned()
9054 } else {
9055 format!("change-summary-{}", spec.id)
9056 },
9057 handover: None,
9058 };
9059 let ctx = WaveCtx {
9060 run: &run,
9061 node: "pr-summary",
9062 prompts: &prompts,
9063 cache: cache.as_deref(),
9064 round: None,
9065 carry_seats: false,
9066 pauses: Some((&self.pause, &self.interrupt)),
9067 };
9068 let (seat, outcome) =
9069 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9070 self.state.seats.insert(seat.key.clone(), seat);
9071 let text = match outcome {
9072 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9073 AgentOutcome::ResourceBlocked(why) => {
9074 if self.state.resource_block.is_none() {
9077 self.state.resource_block = Some(crate::run::ResourceBlock {
9078 cause: crate::run::ResourceBlockCause::Other,
9079 node: "pr-summary".to_owned(),
9080 kind: "shared resource".to_owned(),
9081 detail: why,
9082 at: Timestamp::now(),
9083 });
9084 }
9085 return Err("a shared resource was busy".to_owned());
9086 }
9087 _ => continue,
9088 };
9089 let blind = &self.state.config.blind;
9090 let Some(title) = summary_title(&text) else {
9091 continue;
9092 };
9093 let body: String = summary_without_title(&text)
9094 .lines()
9095 .filter(|l| !l.contains("magi:run/"))
9096 .collect::<Vec<_>>()
9097 .join("\n");
9098 let body = blind::sanitize_prose(body.trim(), blind);
9099 let title = blind::sanitize_prose(&title, blind);
9100 if body.trim().is_empty() || title.trim().is_empty() {
9101 continue;
9102 }
9103 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9104 continue;
9105 }
9106 if git::rev_parse(&winner.worktree, "HEAD")
9108 .await
9109 .ok()
9110 .as_deref()
9111 != Some(head)
9112 {
9113 return Err("the branch moved while the summary was written".to_owned());
9114 }
9115 if git::merge_base(&winner.worktree, &tracking, head)
9116 .await
9117 .ok()
9118 .as_deref()
9119 != Some(base.as_str())
9120 || crate::pr_outcome::summary_evidence(
9121 &self.state,
9122 crate::pr_outcome::FinalHead::At(head),
9123 ) != evidence
9124 {
9125 return Err("the summary evidence changed while it was written".to_owned());
9126 }
9127 self.state.change_summary = Some(crate::run::ChangeSummary {
9128 head: head.to_owned(),
9129 base,
9130 evidence: fingerprint,
9131 title,
9132 body,
9133 });
9134 self.state.event(
9135 "pr-summary",
9136 format!("change summary written by {} for the final diff", spec.id),
9137 );
9138 if let Err(e) = self.state.save() {
9139 tracing::warn!("could not save the change summary: {e:#}");
9140 }
9141 return Ok(());
9142 }
9143 self.state
9144 .event("pr-summary", "no agent produced a change summary");
9145 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9146 }
9147
9148 async fn rewrite_pr_text(
9158 &mut self,
9159 winner: &Candidate,
9160 pr: &PrMessage,
9161 violations: &[crate::github_text::Violation],
9162 ) -> Option<String> {
9163 let prompt = format!(
9164 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9165 pr.title, pr.body
9166 );
9167 let sessions = self.state.config.graph.sessions;
9168 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9169 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9172 let key = format!("impl-{}", winner.label);
9173 if let Some(spec) = self
9174 .state
9175 .config
9176 .agents
9177 .iter()
9178 .find(|a| a.id == winner.agent)
9179 .cloned()
9180 && let Some(existing) = self.state.seats.get(&key)
9181 && existing.agent == spec.id
9182 && agent::has_session(spec.kind, existing, sessions)
9183 {
9184 attempts.push((spec, Some(existing.clone())));
9185 }
9186 match agent::pick_chain(
9187 &self.state.config.agents,
9188 self.state.config.roles.language_judge.as_ref(),
9189 &agent::installed,
9190 "language_judge",
9191 ) {
9192 Ok(chain) => {
9193 for spec in chain {
9194 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9195 continue;
9196 }
9197 attempts.push((spec, None));
9198 }
9199 }
9200 Err(e) if attempts.is_empty() => {
9201 tracing::warn!("no agent for the description rewrite: {e:#}");
9202 self.state
9203 .event("github-text", "no rewrite agent available");
9204 return None;
9205 }
9206 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9207 }
9208 let prompts = self.state.config.prompts.clone();
9209 let cache = self.state.config.cache_dir();
9210 let run = self.state.id.clone();
9211 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9212 let resumed = resumed_seat.is_some();
9213 let seat = resumed_seat.unwrap_or_else(|| {
9214 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9215 });
9216 let github_spec = match agent::for_github_text(
9217 &spec,
9218 &self.state.config.graph.github_language,
9219 &winner.worktree,
9220 ) {
9221 Ok(spec) => spec,
9222 Err(e) => {
9223 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9224 continue;
9225 }
9226 };
9227 let job = SeatJob {
9228 spec: github_spec,
9229 seat,
9230 cwd: winner.worktree.clone(),
9231 prompt: prompt.clone(),
9232 timeout: retry_budget(timeout_full, resumed),
9233 allow_write: false,
9234 sessions,
9235 artifacts: agent::artifacts_dir(&self.state.dir()),
9236 stem: if i == 0 {
9237 "github-text-rewrite".to_owned()
9238 } else {
9239 format!("github-text-rewrite-{}", spec.id)
9240 },
9241 handover: None,
9242 };
9243 let ctx = WaveCtx {
9244 run: &run,
9245 node: "github-text",
9246 prompts: &prompts,
9247 cache: cache.as_deref(),
9248 round: None,
9249 carry_seats: false,
9250 pauses: Some((&self.pause, &self.interrupt)),
9251 };
9252 let (seat, outcome) =
9253 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9254 self.state.seats.insert(seat.key.clone(), seat);
9255 match outcome {
9256 AgentOutcome::Ok(o)
9257 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9258 {
9259 return Some(o.text);
9260 }
9261 AgentOutcome::ResourceBlocked(_) => return None,
9262 _ => {}
9263 }
9264 }
9265 None
9266 }
9267
9268 fn retire_github_text_question(&mut self, why: &str) {
9270 let Some(id) = self
9271 .state
9272 .github_text
9273 .as_ref()
9274 .and_then(|g| g.question.clone())
9275 else {
9276 return;
9277 };
9278 let _ = ask::Questions::open().update(&id, |q| {
9279 q.abandon(why);
9280 Ok(())
9281 });
9282 }
9283
9284 async fn settle_withheld(
9288 &mut self,
9289 winner: &Candidate,
9290 pr: &PrMessage,
9291 mut g: crate::run::GithubTextAsk,
9292 ) -> Result<Option<PrMessage>> {
9293 use crate::github_text as gt;
9294 let store = ask::Questions::open();
9295 let timeout = self.state.config.graph.answer_timeout;
9296 if !g.resolved {
9297 let mut standing = g
9298 .question
9299 .as_ref()
9300 .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9301 if let Some(q) = standing.as_ref()
9302 && q.status.open()
9303 && !store.held_reply(&q.id)
9304 && gt::expired(q, timeout)
9305 {
9306 let why = format!("no answer within {}s of asking", timeout.max(1));
9307 standing = store
9308 .update(&q.id, |q| {
9309 q.abandon(&why);
9310 Ok(())
9311 })
9312 .ok()
9313 .map(|(q, ())| q);
9314 }
9315 let mut ask_again: Option<Option<Vec<String>>> = None;
9316 match standing {
9317 None if g.asks <= 1 => {
9320 g.asks = 0;
9321 ask_again = Some(None);
9322 }
9323 None => {
9324 g.resolved = true;
9325 g.chosen_title = None;
9326 g.choice = Some(GithubTextChoice::Fallback);
9327 }
9328 Some(q) if q.status.open() => {
9329 self.park_on_github_text(&g)?;
9330 return Ok(None);
9331 }
9332 Some(q) => match gt::read_reply(&q) {
9333 gt::Reply::Fallback => {
9334 g.resolved = true;
9335 g.chosen_title = None;
9336 g.choice = Some(GithubTextChoice::Fallback);
9337 }
9338 gt::Reply::Original => {
9339 g.resolved = true;
9340 g.chosen_title = None;
9341 g.choice = Some(GithubTextChoice::Original);
9342 }
9343 gt::Reply::Rewrite => {
9344 g.resolved = true;
9345 g.chosen_title = None;
9346 g.choice = Some(if g.rewrite.is_some() {
9347 GithubTextChoice::Rewrite
9348 } else {
9349 GithubTextChoice::Fallback
9350 });
9351 }
9352 gt::Reply::Title(text) => {
9353 let first = text
9354 .lines()
9355 .map(str::trim)
9356 .find(|l| !l.is_empty())
9357 .unwrap_or("");
9358 let decision = if first.is_empty() {
9359 None
9360 } else {
9361 let candidate = PrMessage {
9362 title: first.to_owned(),
9363 body: String::new(),
9364 };
9365 self.judge_pr_language(&winner.worktree, &candidate).await
9366 };
9367 match gt::vet_title(
9368 &text,
9369 decision,
9370 self.state.config.graph.github_english(),
9371 ) {
9372 Ok(title) => {
9373 g.resolved = true;
9374 g.chosen_title = Some(title);
9375 g.choice = Some(GithubTextChoice::MyTitle);
9376 }
9377 Err(cats) if g.asks < 2 => {
9378 ask_again = Some(Some(
9379 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9380 ))
9381 }
9382 Err(_) => {
9383 g.resolved = true;
9384 g.chosen_title = None;
9385 g.choice = Some(GithubTextChoice::Fallback);
9386 }
9387 }
9388 }
9389 },
9390 }
9391 if let Some(rejected) = ask_again {
9392 let w = gt::Withheld {
9395 title: g.title,
9396 body: g.body,
9397 categories: g.categories.clone(),
9398 };
9399 let artifact = match crate::run::write_artifact(
9402 &self.state,
9403 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9404 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9405 ) {
9406 Ok(p) => Some(p),
9407 Err(e) => {
9408 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9409 None
9410 }
9411 };
9412 let (detail, comparison) = gt::question_content(
9413 &w,
9414 &pr.title,
9415 &pr.body,
9416 &self.state.id,
9417 rejected.as_deref(),
9418 artifact.as_deref(),
9419 g.rewrite.as_ref(),
9420 );
9421 let mut q = ask::Question::new(
9422 self.state.id.clone(),
9423 gt::ASK_NODE.to_owned(),
9424 gt::ASK_SEAT.to_owned(),
9425 gt::question_summary(&self.state.config.graph.language, &w),
9426 detail,
9427 gt::question_choices(&w, g.rewrite.as_ref()),
9428 );
9429 q.posting_comparison = Some(comparison);
9430 q.answer_timeout = timeout;
9431 g.question = Some(q.id.clone());
9434 g.asks += 1;
9435 self.state.github_text = Some(g.clone());
9436 self.state.event(
9437 "github-text",
9438 format!("asking the owner about the withheld text ({})", q.short()),
9439 );
9440 self.state.save()?;
9441 store
9442 .put_executable(&mut q)
9443 .context("file the posting-gate question")?;
9444 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9445 tracing::warn!(
9446 "could not notify about posting-gate question {}: {e:#}",
9447 q.short()
9448 );
9449 }
9450 self.park_on_github_text(&g)?;
9451 return Ok(None);
9452 }
9453 }
9454 let resolved = g.chosen_title.clone();
9455 let original = g.choice == Some(GithubTextChoice::Original);
9456 self.state.event(
9458 "github-text",
9459 if original {
9460 format!(
9461 "posting the original text for: {}",
9462 match (g.title, g.body) {
9463 (true, true) => "title, description",
9464 (true, false) => "title",
9465 _ => "description",
9466 }
9467 )
9468 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9469 "posting the saved rewrite title and description".to_owned()
9470 } else if resolved.is_some() {
9471 "posting the owner's replacement title".to_owned()
9472 } else {
9473 "posting the neutral text for the withheld fields".to_owned()
9474 },
9475 );
9476 self.state.github_text = Some(g.clone());
9477 self.state.save()?;
9478 if g.choice == Some(GithubTextChoice::Rewrite)
9479 && let Some(r) = g.rewrite.as_ref()
9480 {
9481 return Ok(Some(PrMessage {
9482 title: r.title.clone(),
9483 body: r.body.clone(),
9484 }));
9485 }
9486 let w = gt::Withheld {
9487 title: g.title,
9488 body: g.body,
9489 categories: g.categories.clone(),
9490 };
9491 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9492 if let Some(t) = resolved {
9493 title = t;
9494 }
9495 let english = gt::LanguageDecision {
9497 title_english: true,
9498 body_english: true,
9499 body_complete: true,
9500 };
9501 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9502 if original {
9503 if g.title {
9506 title = pr.title.clone();
9507 }
9508 if g.body {
9509 body = pr.body.clone();
9510 }
9511 }
9512 Ok(Some(PrMessage { title, body }))
9513 }
9514
9515 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9516 self.state.github_text = Some(g.clone());
9517 self.state.end_pass_parked(ParkCause::Question);
9518 self.state.parked = true;
9519 self.state.event(
9520 "github-text",
9521 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9522 );
9523 self.state.save()
9524 }
9525
9526 async fn judge_pr_language(
9529 &mut self,
9530 cwd: &Path,
9531 pr: &PrMessage,
9532 ) -> Option<crate::github_text::LanguageDecision> {
9533 self.state.config.roles.language_judge.as_ref()?;
9534 if !self.state.config.graph.language_gate() {
9535 return None;
9536 }
9537 let decision =
9538 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9539 self.state.event(
9540 "github-text",
9541 if decision.is_some() {
9542 "language decided by the language judge"
9543 } else {
9544 "language judge unavailable; using built-in heuristics"
9545 },
9546 );
9547 decision
9548 }
9549
9550 fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9551 Box::pin(self.merge_inner())
9552 }
9553
9554 async fn merge_inner(&mut self) -> Result<()> {
9555 if self.resource_halt()? || self.hold_base_sync().await? {
9556 return Ok(());
9557 }
9558 if self
9573 .state
9574 .base_sync
9575 .as_ref()
9576 .is_some_and(|s| s.conflict.is_some())
9577 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9578 != Some(RunStatus::Gating)
9579 || !self.state.gate_allows_progress()
9587 {
9588 return Ok(());
9589 }
9590 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9599 return Ok(());
9600 }
9601 let Some(winner) = self.state.winner().cloned() else {
9602 return Ok(());
9603 };
9604 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9605 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9606 self.state.status = RunStatus::Blocked;
9607 self.state.event(
9608 "gate",
9609 "winner changed after gate; fresh verification required",
9610 );
9611 self.state.save()?;
9612 return Ok(());
9613 }
9614 let wait = self.repository_wait();
9615 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9616 return Ok(());
9617 }
9618 let repo = self.state.repo.clone();
9619 let base = self.state.base_branch.clone();
9620 let mode = self.state.config.merge.mode;
9621 let style = self.state.config.merge.style;
9622 let facts = if is_review_run(&self.state) {
9623 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9624 let start = review_base(
9625 &repo,
9626 &self.state.config.merge.remote,
9627 &base,
9628 &self.state.base_commit,
9629 &winner.branch,
9630 )
9631 .await;
9632 branch_facts(&repo, &start, &winner.branch).await
9633 } else {
9634 None
9635 };
9636 let leaked = if is_review_run(&self.state) {
9639 leaked_subjects(&self.state, &winner.branch).await
9640 } else {
9641 Some(Vec::new())
9642 };
9643 let Some(pr) = self
9647 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9648 .await?
9649 else {
9650 return Ok(());
9651 };
9652 let message = pr.commit_message();
9653
9654 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9655 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9656 return Ok(());
9657 }
9658 let outcome = match mode {
9659 MergeMode::None => MergeOutcome {
9660 mode,
9661 ok: true,
9662 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9663 empty: false,
9664 },
9665 MergeMode::Pr | MergeMode::Local
9666 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9667 {
9668 MergeOutcome {
9669 mode,
9670 ok: false,
9671 detail: empty_candidate_detail(&self.state, &base),
9672 empty: true,
9673 }
9674 }
9675 MergeMode::Local => {
9676 let on = git::current_branch(&repo).await?;
9677 if on.as_deref() != Some(base.as_str()) {
9678 MergeOutcome {
9679 mode,
9680 ok: false,
9681 detail: format!(
9682 "{} has {} checked out, not the base branch {base}",
9683 repo.display(),
9684 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9685 ),
9686 empty: false,
9687 }
9688 } else if !git::is_clean(&repo).await? {
9689 MergeOutcome {
9690 mode,
9691 ok: false,
9692 detail: format!("{} is dirty; refusing to merge", repo.display()),
9693 empty: false,
9694 }
9695 } else {
9696 let out = match style {
9697 MergeStyle::Merge => {
9698 git::merge_no_ff(&repo, &winner.branch, &message).await?
9699 }
9700 MergeStyle::Squash => {
9701 git::merge_squash(&repo, &winner.branch, &message).await?
9702 }
9703 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9704 };
9705 MergeOutcome {
9706 mode,
9707 ok: out.ok(),
9708 detail: if out.ok() { out.stdout } else { out.stderr },
9709 empty: false,
9710 }
9711 }
9712 }
9713 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9714 self.state.pr_publication_pending = true;
9715 self.state.save()?;
9716 let out =
9717 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9718 self.state.pr_publication_pending = out.is_err();
9719 MergeOutcome {
9720 mode,
9721 ok: out.is_ok(),
9722 detail: match out {
9723 Ok(url) => url,
9724 Err(e) => format!("{e:#}"),
9725 },
9726 empty: false,
9727 }
9728 }
9729 MergeMode::Pr => {
9730 let remote = self.state.config.merge.remote.clone();
9731 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9732 let pushed = if self.state.foreign_checkout.is_some() {
9733 let refspec = format!("{delivery_head}:refs/heads/{public}");
9734 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9735 } else {
9736 git::push(&winner.worktree, &remote, &winner.branch).await?
9737 };
9738 if !pushed.ok() {
9739 MergeOutcome {
9740 mode,
9741 ok: false,
9742 detail: pushed.stderr,
9743 empty: false,
9744 }
9745 } else {
9746 if self.state.foreign_checkout.is_some() {
9747 self.state.pr_pushed_head = Some(delivery_head.clone());
9748 }
9749 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9755 let out = match pr_merge_plan(found) {
9756 PrPlan::Create => {
9757 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9758 .await
9759 }
9760 PrPlan::Adopt { url, title } => {
9761 self.state
9762 .event("merge", format!("Pr: adopted open pull request {url}"));
9763 if self.state.explicit_github_text_selection() {
9764 land::set_pr_text_verbatim(
9767 &winner.worktree,
9768 &url,
9769 &pr.title,
9770 &pr.body,
9771 )
9772 .await
9773 .map(|()| url)
9774 } else {
9775 if title != pr.title
9776 && (!is_review_run(&self.state)
9777 || leaked
9778 .as_deref()
9779 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9780 && let Err(e) =
9781 if self.state.github_text.as_ref().is_some_and(|g| {
9782 g.resolved
9783 && g.title
9784 && g.choice == Some(GithubTextChoice::Original)
9785 }) {
9786 land::set_pr_title_verbatim(
9788 &winner.worktree,
9789 &url,
9790 &pr.title,
9791 )
9792 .await
9793 } else {
9794 land::set_pr_title(
9795 &mut self.state,
9796 &winner.worktree,
9797 &url,
9798 &pr.title,
9799 )
9800 .await
9801 }
9802 {
9803 tracing::warn!("could not refresh title of {url}: {e:#}");
9804 self.state
9805 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9806 }
9807 Ok(url)
9808 }
9809 }
9810 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9811 };
9812 match out {
9813 Ok(url) => MergeOutcome {
9814 mode,
9815 ok: true,
9816 detail: url,
9817 empty: false,
9818 },
9819 Err(e) => MergeOutcome {
9820 mode,
9821 ok: false,
9822 detail: e.to_string(),
9823 empty: false,
9824 },
9825 }
9826 }
9827 }
9828 };
9829
9830 self.state.status = match (mode, outcome.ok) {
9831 (MergeMode::None, _) => RunStatus::Ready,
9832 (_, true) if outcome.empty => RunStatus::Ready,
9833 (MergeMode::Pr, true) => RunStatus::Ready,
9834 (_, true) => RunStatus::Merged,
9835 (_, false) => RunStatus::Blocked,
9836 };
9837 self.state.event(
9838 "merge",
9839 format!(
9840 "{:?}: {}",
9841 mode,
9842 outcome.detail.lines().next().unwrap_or("")
9843 ),
9844 );
9845 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9846 self.state.record_github_text_publication(&outcome.detail);
9847 }
9848 self.state.merge = Some(outcome);
9849 self.state.save()?;
9850
9851 if self.state.config.graph.land
9857 && mode == MergeMode::Pr
9858 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9859 {
9860 self.run_land().await?;
9861 }
9862 self.settle_questions();
9867 Ok(())
9868 }
9869
9870 fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9881 Box::pin(self.run_land_inner())
9882 }
9883
9884 async fn run_land_inner(&mut self) -> Result<()> {
9885 crate::pr_head::ensure_owner(&self.state)?;
9886 let url = self
9887 .state
9888 .pr
9889 .as_ref()
9890 .map(|pr| pr.url.clone())
9891 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9892 .unwrap_or_default();
9893 let url = url.lines().next().unwrap_or("").trim().to_owned();
9894 if !url.starts_with("http") {
9895 return Ok(());
9896 }
9897 loop {
9900 match land::land(&mut self.state, &url).await {
9901 Ok(_)
9902 if self
9903 .state
9904 .merge_repairs
9905 .last()
9906 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9907 {
9908 if self.repair_merge_review().await? {
9909 continue;
9910 }
9911 }
9912 Ok(_) if self.state.resource_block.is_some() => {
9913 return self.stop_for_resource();
9914 }
9915 Ok(pr) if self.state.parked => {
9916 let _ = pr;
9920 }
9921 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9922 self.state.status = RunStatus::Superseded;
9923 self.state.save()?;
9924 }
9925 Ok(pr) => {
9926 self.state.status = match pr.state {
9927 land::PrLifecycle::Merged => RunStatus::Merged,
9928 _ => RunStatus::Blocked,
9929 };
9930 if bump::should_release_bump(self.state.status)
9937 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9938 {
9939 self.state
9945 .event("bump", format!("release bump skipped: {e:#}"));
9946 }
9947 if self.state.status == RunStatus::Merged {
9950 crate::followup::after_merge(&mut self.state, &pr.url).await;
9951 }
9952 self.state.save()?;
9953 }
9954 Err(e) => {
9955 if e.is::<crate::repo_lock::ResourceBlocked>() {
9956 return Err(e);
9957 }
9958 self.state.status = RunStatus::Blocked;
9959 self.state.event("land", format!("gave up: {e}"));
9960 self.state.save()?;
9961 }
9962 }
9963 break;
9964 }
9965 Ok(())
9966 }
9967
9968 async fn repair_merge_review(&mut self) -> Result<bool> {
9971 use crate::run::MergeRepairPhase as Phase;
9972 crate::pr_head::ensure_owner(&self.state)?;
9973 let i = self.state.merge_repairs.len() - 1;
9974 let request = self.state.merge_repairs[i].clone();
9975 if request.phase == Phase::Fixing {
9976 let restored = match self.state.winner().cloned() {
9977 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9978 None => Err(anyhow::anyhow!("repair has no winning branch")),
9979 };
9980 let why = match restored {
9981 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9982 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9983 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9984 };
9985 self.hold_merge_repair(i, why)?;
9986 return Ok(false);
9987 }
9988 if request.phase == Phase::Reviewing && self.state.gate_ran {
9993 let winner = self
9994 .state
9995 .winner()
9996 .context("repair has no winning branch")?;
9997 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9998 if self.state.gate_head.as_deref() != Some(head.as_str())
9999 || !git::is_clean(&winner.worktree).await?
10000 {
10001 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10002 return Ok(false);
10003 }
10004 }
10005 let failure = if i >= self.state.config.graph.land_rounds
10006 || self.state.config.graph.review_rounds == 0
10007 {
10008 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
10009 } else {
10010 None
10011 };
10012 if let Some(why) = failure {
10013 self.hold_merge_repair(i, why)?;
10014 return Ok(false);
10015 }
10016 if request.phase == Phase::Requested {
10017 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
10018 Ok(pr) => pr,
10019 Err(e) => {
10020 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
10021 return Ok(false);
10022 }
10023 };
10024 self.state.merge_repairs[i].phase = Phase::Fixing;
10025 self.state.land_approval = None;
10026 self.state.land_armed_head = None;
10027 self.state.save()?;
10028 let evidence = serde_json::to_string_pretty(&request)?;
10031 let reason = format!(
10032 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
10033 );
10034 let budget = self.state.config.graph.land_rounds;
10035 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10036 match result {
10037 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10038 Ok(land::Fixed::Deferred) => {
10039 let worktree = self
10043 .state
10044 .winner()
10045 .context("repair has no winning branch")?
10046 .worktree
10047 .clone();
10048 self.graft_restore(&worktree, &request.head).await?;
10049 self.state.merge_repairs[i].phase = Phase::Requested;
10050 self.stop_for_resource()?;
10051 return Ok(false);
10052 }
10053 Ok(land::Fixed::Failed(why)) => {
10054 self.hold_merge_repair(
10055 i,
10056 format!(
10057 "repair failed: {why}; inspect the branch and run an explicit review"
10058 ),
10059 )?;
10060 return Ok(false);
10061 }
10062 Err(e) => {
10063 self.hold_merge_repair(
10064 i,
10065 format!(
10066 "repair failed: {e:#}; inspect the branch and run an explicit review"
10067 ),
10068 )?;
10069 return Ok(false);
10070 }
10071 }
10072 self.begin_merge_repair_review(i)?;
10076 }
10077 self.review_loop().await?;
10078 self.gate().await?;
10079 if self.resource_halt()? || self.state.parked {
10080 return Ok(false);
10081 }
10082 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10083 != Some(RunStatus::Gating)
10084 || !self.state.gate_allows_progress()
10085 {
10086 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10087 return Ok(false);
10088 }
10089 let winner = self
10092 .state
10093 .winner()
10094 .cloned()
10095 .context("repair has no winning branch")?;
10096 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10097 if self.state.gate_head.as_deref() != Some(after.as_str())
10098 || !git::is_clean(&winner.worktree).await?
10099 {
10100 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10101 return Ok(false);
10102 }
10103 let expected = self
10104 .state
10105 .pr_pushed_head
10106 .as_deref()
10107 .unwrap_or(&request.head)
10108 .to_owned();
10109 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10110 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10113 return Ok(false);
10114 }
10115 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10116 let refspec = format!("{after}:refs/heads/{public}");
10117 let push = git::git_raw(
10118 &winner.worktree,
10119 &["push", &lease, &self.state.config.merge.remote, &refspec],
10120 )
10121 .await;
10122 match push {
10123 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10124 Ok(out) => {
10125 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10126 return Ok(false);
10127 }
10128 Err(e) => {
10129 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10130 return Ok(false);
10131 }
10132 }
10133 self.state.merge_repairs[i].phase = Phase::Complete;
10134 self.state.status = RunStatus::Landing;
10135 self.state.event(
10136 "repair",
10137 "new review and gate completed; fresh owner merge decision required",
10138 );
10139 self.state.save()?;
10140 Ok(true)
10141 }
10142
10143 fn merge_repair_artifacts(&self) -> PathBuf {
10144 let artifacts = agent::artifacts_dir(&self.state.dir());
10145 match self
10146 .state
10147 .merge_repairs
10148 .last()
10149 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10150 {
10151 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10152 None => artifacts,
10153 }
10154 }
10155
10156 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10157 if let Some(e2e) = self.state.head_e2e.take() {
10158 self.state
10159 .verification_history
10160 .push(crate::run::VerificationEvidence {
10161 superseded_by: None,
10162 archived_at: Some(Timestamp::now()),
10163 fixes_spent: Some(self.state.gate_fixes.len()),
10164 gate_head: None,
10165 gate_ran: false,
10166 gate: Vec::new(),
10167 e2e: Some(e2e),
10168 });
10169 }
10170 self.state.reviews.clear();
10171 self.state.gate.clear();
10172 self.state.gate_ran = false;
10173 self.state.gate_head = None;
10174 self.state.gate_fixes.clear();
10175 self.state.gate_repair_e2e = None;
10176 self.state.pre_gate.clear();
10177 self.state.pre_gate_commit = None;
10178 self.state.contested_handoff = None;
10179 self.state.land_reruns = None;
10180 self.state.land_approval = None;
10181 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10182 self.state.status = RunStatus::Reviewing;
10183 self.state.save()
10184 }
10185
10186 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10187 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10188 self.state.merge_repairs[index].failure = Some(why.clone());
10189 self.state.status = RunStatus::Blocked;
10190 self.state.event("repair", why);
10191 self.state.save()
10192 }
10193
10194 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10198 if let Some(existing) = self.state.seats.get(key)
10199 && existing.agent == agent
10200 {
10201 return existing.clone();
10202 }
10203 let fresh = if self.state.seats.contains_key(key) {
10206 handover_seat(key, agent, self.state.next_seat_seed())
10207 } else {
10208 SeatState::new(key, agent, self.state.seed)
10209 };
10210 self.state.seats.insert(key.to_owned(), fresh.clone());
10211 fresh
10212 }
10213
10214 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10220 match self.state.seats.get(key) {
10221 Some(s) if s.agent != spec.id => {
10222 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10223 }
10224 _ => spec,
10225 }
10226 }
10227
10228 fn view(&self, c: &Candidate) -> CandidateView {
10230 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10231 .unwrap_or_default();
10232 let (patch, _) = blind::sanitize_patch(
10233 &format!("candidate {} patch", c.label),
10234 &raw,
10235 &self.state.config.blind,
10236 );
10237 CandidateView {
10238 label: c.label,
10239 branch: c.branch.clone(),
10240 summary: c.summary.clone(),
10241 stat: c.stat.clone(),
10242 patch,
10243 }
10244 }
10245
10246 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10248 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10249 prompt::judge(
10250 "(see above)",
10251 &views,
10252 self.roles.judges.len(),
10253 base_short,
10254 "en",
10255 )
10256 }
10257
10258 fn vote_prompt_full(
10264 &self,
10265 j: usize,
10266 labels: &[char],
10267 language: &str,
10268 candidates: &[Candidate],
10269 base_short: &str,
10270 ) -> String {
10271 let mut text = format!(
10272 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10273 prompt::final_vote(labels, language),
10274 self.state.instruction,
10275 self.candidate_block(candidates, base_short)
10276 );
10277 if let Some(own) = self
10278 .state
10279 .judgements
10280 .get(j)
10281 .filter(|r| !r.ranking.is_empty())
10282 {
10283 let reasons = own
10284 .reasons
10285 .iter()
10286 .map(|(k, v)| format!("- {k}: {v}"))
10287 .collect::<Vec<_>>()
10288 .join("\n");
10289 text.push_str(&format!(
10290 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10291 own.ranking.iter().collect::<String>(),
10292 if reasons.is_empty() {
10293 ""
10294 } else {
10295 ", because:\n"
10296 }
10297 ));
10298 }
10299 if !self.state.deliberation.is_empty() {
10300 text.push_str("\n# What was argued before this vote\n");
10301 for t in self.transcript(&[], j) {
10302 text.push_str(&format!(
10303 "\n## {}{}\n\n{}\n",
10304 t.who,
10305 if t.is_self { " (you)" } else { "" },
10306 t.body.trim()
10307 ));
10308 }
10309 }
10310 text
10311 }
10312
10313 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10320 let mut turns = Vec::new();
10321 for j in &self.state.judgements {
10322 if j.ranking.is_empty() {
10323 continue;
10324 }
10325 let reasons = j
10326 .reasons
10327 .iter()
10328 .map(|(k, v)| format!("- {k}: {v}"))
10329 .collect::<Vec<_>>()
10330 .join("\n");
10331 turns.push(Turn {
10332 who: format!("Judge {} (opening ranking)", j.judge),
10333 is_self: j.judge == self_idx + 1,
10334 body: format!(
10335 "Ranked {}{}{reasons}",
10336 j.ranking.iter().collect::<String>(),
10337 if reasons.is_empty() {
10338 ""
10339 } else {
10340 ", because:\n"
10341 }
10342 ),
10343 });
10344 }
10345 for t in self
10346 .state
10347 .deliberation
10348 .iter()
10349 .flat_map(|r| r.turns.iter())
10350 .chain(current)
10351 {
10352 turns.push(Turn {
10353 who: format!("Judge {}", t.judge),
10354 is_self: t.judge == self_idx + 1,
10355 body: t.body.clone(),
10356 });
10357 }
10358 turns
10359 }
10360}
10361
10362fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10364 agent::has_session(spec.kind, seat, sessions)
10365}
10366
10367fn next_untried_in_roster<'a>(
10388 roster: &'a [AgentSpec],
10389 start: usize,
10390 tried: &BTreeSet<String>,
10391) -> Option<&'a AgentSpec> {
10392 roster
10393 .get(start + 1..)?
10394 .iter()
10395 .find(|s| !tried.contains(&s.id))
10396}
10397
10398fn pick_successor<'a>(
10419 roster: &'a [AgentSpec],
10420 start: usize,
10421 tried: &BTreeSet<String>,
10422 carried: Option<&BTreeSet<String>>,
10423 others: &BTreeSet<String>,
10424) -> Option<&'a AgentSpec> {
10425 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10426 roster
10427 .get(start + 1..)
10428 .and_then(|tail| {
10429 tail.iter()
10430 .filter(free)
10431 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10432 })
10433 .or_else(|| {
10434 carried
10435 .is_some()
10436 .then(|| roster.iter().find(free))
10437 .flatten()
10438 })
10439 .or_else(|| match carried {
10440 Some(c) => next_for_seat(roster, start, tried, c),
10441 None => next_untried_in_roster(roster, start, tried),
10442 })
10443}
10444
10445fn next_for_seat<'a>(
10455 roster: &'a [AgentSpec],
10456 start: usize,
10457 round_tried: &BTreeSet<String>,
10458 carried_failed: &BTreeSet<String>,
10459) -> Option<&'a AgentSpec> {
10460 roster
10461 .get(start + 1..)?
10462 .iter()
10463 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10464 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10465}
10466
10467fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10473 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10474 return spec;
10475 };
10476 let ok = |id: &str| !h.failed.contains(id);
10477 if let Some(last) = h.last_ok.as_deref()
10478 && ok(last)
10479 && let Some(s) = roster.iter().find(|s| s.id == last)
10480 {
10481 return s.clone();
10482 }
10483 if ok(&spec.id) {
10484 return spec;
10485 }
10486 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10487 roster
10488 .get(from + 1..)
10489 .into_iter()
10490 .flatten()
10491 .chain(roster.iter())
10492 .find(|s| ok(&s.id))
10493 .cloned()
10494 .unwrap_or(spec)
10495}
10496
10497pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10501 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10502}
10503
10504const TIMED_OUT: &str = "timed out";
10507
10508impl FailClass {
10509 fn of(out: &AgentOutcome) -> Option<Self> {
10511 match out {
10512 AgentOutcome::Ok(_) => None,
10513 AgentOutcome::Quota(_) => Some(Self::Quota),
10514 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10515 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10516 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10517 AgentOutcome::ResourceBlocked(_) => None,
10520 }
10521 }
10522
10523 fn stem_word(&self) -> &'static str {
10525 match self {
10526 Self::Quota => "quota",
10527 _ => "handover",
10528 }
10529 }
10530}
10531
10532fn failure_signature(msg: &str) -> String {
10536 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10537 if let Some(code) = line
10540 .strip_prefix("process exited with no usable output (exit code ")
10541 .and_then(|s| s.strip_suffix(')'))
10542 .and_then(|s| s.parse::<i32>().ok())
10543 {
10544 line = format!("exited with some({code}) and no usable output");
10545 } else if matches!(
10546 line.as_str(),
10547 "process exited with no usable output (no exit code reported)"
10548 | "process exited with no usable output (exit code unavailable)"
10549 ) {
10550 line = "exited with none and no usable output".to_owned();
10551 }
10552 let mut out = Vec::new();
10553 for word in line.split_whitespace() {
10554 if word.contains('/') || word.contains('\\') {
10555 out.push("<path>".to_owned());
10556 continue;
10557 }
10558 let mut w = String::new();
10559 let mut in_digits = false;
10560 for c in word.chars() {
10561 if c.is_ascii_digit() {
10562 if !in_digits {
10563 w.push('#');
10564 }
10565 in_digits = true;
10566 } else {
10567 in_digits = false;
10568 w.push(c);
10569 }
10570 }
10571 out.push(w);
10572 }
10573 out.join(" ").chars().take(120).collect()
10574}
10575
10576fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10583 match cur {
10584 FailClass::Quota | FailClass::Timeout => true,
10585 FailClass::Other(_) => prev != Some(cur),
10586 }
10587}
10588
10589fn fail_reason(out: &AgentOutcome) -> String {
10591 match out {
10592 AgentOutcome::Ok(_) => String::new(),
10593 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10594 AgentOutcome::Dropped(o) => format!(
10595 "the CLI dropped the stream ({})",
10596 o.dropped
10597 .as_ref()
10598 .map(|d| d.why.as_str())
10599 .unwrap_or("it ended without delivering its answer")
10600 ),
10601 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10602 e.lines().next().unwrap_or("").chars().take(160).collect()
10603 }
10604 }
10605}
10606
10607pub(crate) fn record_handover(
10610 state: &mut RunState,
10611 node: &str,
10612 seat: &str,
10613 from: &str,
10614 to: &str,
10615 class: &FailClass,
10616 reason: &str,
10617) {
10618 let message = if *class == FailClass::Quota {
10619 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10620 } else {
10621 format!("{seat}: handed over {from} -> {to} ({reason})")
10622 };
10623 state.event(node, message);
10624 state.handovers.push(Handover {
10625 at: Timestamp::now(),
10626 node: node.to_owned(),
10627 seat: seat.to_owned(),
10628 from: from.to_owned(),
10629 to: to.to_owned(),
10630 reason: reason.to_owned(),
10631 });
10632}
10633
10634fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10648 commands.iter().any(|c| c.exit_code.is_none())
10649}
10650
10651fn verified_noop_claim(
10664 usable: bool,
10665 commands: &[agent::CommandEvidence],
10666 text: &str,
10667) -> Option<String> {
10668 (usable && !has_unconfirmed_command(commands))
10669 .then(|| verdict::verified_noop(text))
10670 .flatten()
10671}
10672
10673fn short(commit: &str) -> String {
10674 commit.chars().take(7).collect()
10675}
10676
10677fn make_executable(path: &Path) -> Result<()> {
10678 #[cfg(unix)]
10679 {
10680 use std::os::unix::fs::PermissionsExt as _;
10681 let mut perms = std::fs::metadata(path)?.permissions();
10682 perms.set_mode(0o755);
10683 std::fs::set_permissions(path, perms)?;
10684 }
10685 #[cfg(not(unix))]
10686 {
10687 let _ = path;
10688 }
10689 Ok(())
10690}
10691
10692struct WaveCtx<'a> {
10699 run: &'a str,
10702 node: &'a str,
10704 prompts: &'a Prompts,
10705 cache: Option<&'a Path>,
10707 round: Option<usize>,
10710 carry_seats: bool,
10714 pauses: Option<(&'a Pause, &'a Pause)>,
10717}
10718
10719fn repository_wait(
10721 state: &RunState,
10722 pauses: Option<(&Pause, &Pause)>,
10723) -> crate::repo_lock::WaitPolicy {
10724 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10725 let inherited = crate::repo_lock::WaitPolicy::current();
10726 crate::repo_lock::WaitPolicy::new(
10727 Duration::from_secs(state.config.graph.timeout_resource_wait),
10728 move || {
10729 inherited.cancelled()
10730 || pauses
10731 .as_ref()
10732 .is_some_and(|(a, b)| a.parked() || b.parked())
10733 },
10734 )
10735}
10736
10737fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10738 if error.is::<crate::repo_lock::ResourceBlocked>() {
10739 state.resource_block = Some(crate::run::ResourceBlock {
10740 cause: crate::run::ResourceBlockCause::Other,
10741 node: "repository".into(),
10742 kind: if error.to_string().contains("disk pressure") {
10743 "disk pressure"
10744 } else if error.to_string().contains("disk admission") {
10745 "disk admission"
10746 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10747 "incremental cache admission"
10748 } else {
10749 "repository coordination"
10750 }
10751 .into(),
10752 detail: error.to_string(),
10753 at: Timestamp::now(),
10754 });
10755 state.event("resource", error.to_string());
10756 }
10757}
10758
10759async fn coordinate_operation(
10760 state: &mut RunState,
10761 cwd: &Path,
10762 wait: &crate::repo_lock::WaitPolicy,
10763) -> Result<Option<crate::repo_lock::SeatLease>> {
10764 let result =
10765 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10766 .await;
10767 if let Err(e) = &result {
10768 record_repository_block(state, e);
10769 }
10770 result
10771}
10772
10773async fn run_one(
10775 job: SeatJob,
10776 sem: Arc<Semaphore>,
10777 ctx: &WaveCtx<'_>,
10778 state: &mut RunState,
10779 attempt: usize,
10780) -> (SeatState, AgentOutcome) {
10781 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10782 .await
10783 .pop()
10784 .expect("one job in, one result out");
10785 (seat, out)
10786}
10787
10788async fn wave(
10794 jobs: Vec<SeatJob>,
10795 sem: Arc<Semaphore>,
10796 ctx: &WaveCtx<'_>,
10797 state: &mut RunState,
10798 attempt: usize,
10799) -> Vec<(usize, SeatState, AgentOutcome)> {
10800 let WaveCtx {
10801 run,
10802 node,
10803 prompts,
10804 cache,
10805 round,
10806 carry_seats: _,
10807 pauses,
10808 } = *ctx;
10809 for job in &jobs {
10810 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10811 }
10812 if let Err(e) = state.save() {
10813 tracing::warn!("could not persist in-progress seats: {e:#}");
10818 }
10819 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10835 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10839 let cache_guard = if let Some(cache_dir) = cache {
10840 if jobs_had_a_writer {
10841 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10842 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10843 match acquire_cache_lease(
10844 state,
10845 cache_dir,
10846 &owner,
10847 resource_limit,
10848 node,
10849 node,
10850 &cancel,
10851 )
10852 .await
10853 {
10854 Ok(Some(g)) => Some(g),
10855 Ok(None) => {
10858 if let Some(block) = &mut state.resource_block {
10859 block.cause = crate::run::ResourceBlockCause::Cancelled {
10860 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10861 };
10862 }
10863 return block_wave(
10864 jobs,
10865 state,
10866 node,
10867 attempt,
10868 "parked while waiting for the shared build cache",
10869 );
10870 }
10871 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10874 Err(e) => {
10875 let why = format!("{e:#}");
10879 state.event(
10880 "resource",
10881 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10882 );
10883 return block_wave(jobs, state, node, attempt, &why);
10884 }
10885 }
10886 } else {
10887 None
10888 }
10889 } else {
10890 None
10891 };
10892 state.seats_spawning(node, attempt);
10895 if let Err(e) = state.save() {
10896 tracing::warn!("could not persist the start of a wave: {e:#}");
10897 }
10898 let slot_limit = resource_limit
10902 + jobs
10903 .iter()
10904 .map(|j| j.timeout)
10905 .max()
10906 .unwrap_or_default()
10907 .saturating_mul(jobs.len() as u32);
10908 let mut set = tokio::task::JoinSet::new();
10909 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10912 let overlay = prompts.overlay(node);
10913 let repository_wait = repository_wait(state, pauses);
10914 for (i, mut job) in jobs.into_iter().enumerate() {
10915 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10916 if cache.is_some() {
10917 job.prompt.push('\n');
10918 job.prompt
10919 .push_str(&prompt::build_cache_note(node, job.allow_write));
10920 }
10921 let sem = Arc::clone(&sem);
10922 let run = run.to_owned();
10923 let node = node.to_owned();
10924 let attachments = if node == "implement" {
10927 state.attachments.clone()
10928 } else {
10929 Vec::new()
10930 };
10931 let cache = cache
10942 .filter(|_| job.allow_write && cache_guard.is_some())
10943 .map(Path::to_path_buf);
10944 let spawned_tx = spawned_tx.clone();
10945 let repository_wait = repository_wait.clone();
10946 let coordination = state.config.coordination.clone();
10947 let fetched_base = state.fetched_base.clone();
10948 set.spawn(async move {
10949 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10953 Ok(p) => p,
10954 Err(_) => {
10955 return (
10956 i,
10957 job.seat,
10958 AgentOutcome::ResourceBlocked(format!(
10959 "no parallel slot became free within {}s",
10960 slot_limit.as_secs()
10961 )),
10962 false,
10963 );
10964 }
10965 };
10966 spawned_tx.send(job.seat.key.clone()).ok();
10967 let mut seat = job.seat;
10968 let out = repository_wait
10969 .scope(agent::invoke(
10970 &job.spec,
10971 &mut seat,
10972 &Invocation {
10973 coordination: coordination.as_ref(),
10974 fetched_base: fetched_base.as_ref(),
10975 cwd: &job.cwd,
10976 prompt: &job.prompt,
10977 timeout: job.timeout,
10978 allow_write: job.allow_write,
10979 sessions: job.sessions,
10980 artifacts: &job.artifacts,
10981 stem: &job.stem,
10982 run: &run,
10983 node: &node,
10984 cache_dir: cache.as_deref(),
10985 attachments: &attachments,
10986 writable: &[],
10987 },
10988 ))
10989 .await;
10990 let interrupted = out
10991 .as_ref()
10992 .err()
10993 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10994 let out = match out {
10995 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10996 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10997 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
11005 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
11006 Ok(o) => AgentOutcome::Failed(format!(
11007 "process exited with no usable output ({})",
11008 crate::notices::exit_status(o.exit_code)
11009 )),
11010 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
11011 AgentOutcome::ResourceBlocked(e.to_string())
11012 }
11013 Err(e) => AgentOutcome::Failed(e.to_string()),
11014 };
11015 (i, seat, out, interrupted)
11016 });
11017 }
11018 drop(spawned_tx);
11019 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
11020 loop {
11021 let joined = tokio::select! {
11022 Some(key) = spawned_rx.recv() => {
11023 state.seat_spawned(&key);
11025 if let Err(e) = state.save() {
11026 tracing::warn!("could not persist a seat's start: {e:#}");
11027 }
11028 continue;
11029 }
11030 joined = set.join_next() => joined,
11031 };
11032 let Some(joined) = joined else {
11033 break;
11034 };
11035 let (i, seat, out, interrupted) = match joined {
11036 Ok(v) => v,
11037 Err(e) => {
11041 tracing::error!("agent task panicked: {e}");
11042 continue;
11043 }
11044 };
11045 state.seat_finished(&seat.key);
11046 if interrupted
11047 && node == "implement"
11048 && let Some(label) = state
11049 .candidates
11050 .iter()
11051 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11052 .map(|candidate| candidate.label)
11053 {
11054 state.candidate_preservation.insert(
11055 label,
11056 "interrupted implementation awaiting persistence".into(),
11057 );
11058 state.interrupted_implementations.insert(label);
11059 }
11060 if let AgentOutcome::ResourceBlocked(why) = &out
11061 && state.resource_block.is_none()
11062 {
11063 state.resource_block = Some(crate::run::ResourceBlock {
11064 cause: crate::run::ResourceBlockCause::Other,
11065 node: node.to_owned(),
11066 kind: if why.contains("disk pressure") {
11067 "disk pressure"
11068 } else if why.contains("disk admission") {
11069 "disk admission"
11070 } else if why.contains(crate::incremental::BLOCKED) {
11071 "incremental cache admission"
11072 } else if why.contains("repository coordination") {
11073 "repository coordination"
11074 } else {
11075 "parallel slot"
11076 }
11077 .to_owned(),
11078 detail: why.clone(),
11079 at: Timestamp::now(),
11080 });
11081 let activity = if interrupted {
11082 "agent execution interrupted"
11083 } else {
11084 "no agent was started"
11085 };
11086 state.event("resource", format!("{node}: {activity} - {why}"));
11087 }
11088 record_jobs(state, node, round, &seat.key, &out);
11089 if let Err(e) = state.save() {
11090 tracing::warn!("could not persist a seat's completion: {e:#}");
11091 }
11092 if collected.len() <= i {
11093 collected.resize_with(i + 1, || None);
11094 }
11095 collected[i] = Some((i, seat, out));
11096 }
11097 if state
11103 .active
11104 .values()
11105 .any(|a| a.node == node && a.attempt == attempt)
11106 {
11107 state
11108 .active
11109 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11110 if let Err(e) = state.save() {
11111 tracing::warn!("could not persist the end of a wave: {e:#}");
11112 }
11113 }
11114 if let Some(cache_dir) = cache
11121 && jobs_had_a_writer
11122 {
11123 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11124 }
11125 if let Some(guard) = cache_guard {
11126 guard.release();
11127 }
11128 collected.into_iter().flatten().collect()
11129}
11130
11131fn record_jobs(
11142 state: &mut RunState,
11143 node: &str,
11144 round: Option<usize>,
11145 seat: &str,
11146 out: &AgentOutcome,
11147) {
11148 let commands: &[agent::CommandEvidence] = match out {
11149 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11150 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11151 };
11152 let checked_at = Timestamp::now();
11153 for c in commands {
11154 state.jobs.push(JobRecord {
11155 node: node.to_owned(),
11156 round,
11157 seat: seat.to_owned(),
11158 id: c.id.clone(),
11159 description: c.description.clone(),
11160 checked_at,
11161 status: match c.exit_code {
11162 Some(0) => JobStatus::Completed,
11163 Some(_) => JobStatus::Failed,
11164 None => JobStatus::Unknown,
11165 },
11166 exit_code: c.exit_code,
11167 result_summary: c.result_summary.clone(),
11168 source: c.source.clone(),
11169 });
11170 }
11171}
11172
11173fn round_is_clean(
11194 blocking: usize,
11195 e2e_ok: bool,
11196 answered: usize,
11197 expected: usize,
11198 quota_missing: usize,
11199 policy: IncompleteReviewPolicy,
11200) -> bool {
11201 if blocking != 0 || !e2e_ok {
11202 return false;
11203 }
11204 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11205 return true;
11206 }
11207 answered > 0 && expected - answered <= quota_missing
11208}
11209
11210pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11234 if max_rounds == 0
11235 || reviews.last().is_some_and(|r| {
11236 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11237 })
11238 {
11239 return Some(RunStatus::Gating);
11240 }
11241 let last = reviews.last()?;
11242 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11243 if reviews.len() < max_rounds && !stagnant {
11244 return None;
11245 }
11246 if last.incomplete() && last.blocking == 0 {
11247 return Some(RunStatus::Blocked);
11248 }
11249 if last.e2e_status() == E2eStatus::ResourceBlocked
11250 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11251 {
11252 return None;
11253 }
11254 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11255 RunStatus::Gating
11256 } else {
11257 RunStatus::Blocked
11258 })
11259}
11260
11261pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11276 if nudged {
11277 (full / 4).max(Duration::from_secs(120)).min(full)
11278 } else {
11279 full
11280 }
11281}
11282
11283#[allow(clippy::too_many_arguments)]
11312async fn ask_json_wave<T>(
11313 jobs: Vec<SeatJob>,
11314 sem: Arc<Semaphore>,
11315 retries: usize,
11316 roster: &[AgentSpec],
11317 ctx: &WaveCtx<'_>,
11318 losses: &mut Vec<QuotaLoss>,
11319 state: &mut RunState,
11320 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11321) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11322where
11323 T: serde::de::DeserializeOwned + Send + 'static,
11324{
11325 ask_wave_with(
11326 jobs,
11327 sem,
11328 retries,
11329 roster,
11330 ctx,
11331 losses,
11332 state,
11333 &|text: &str| {
11334 let v = verdict::extract_json::<T>(text)?;
11335 validate(&v)?;
11336 Ok(v)
11337 },
11338 )
11339 .await
11340}
11341
11342#[allow(clippy::too_many_arguments)]
11351async fn ask_wave_with<T>(
11352 jobs: Vec<SeatJob>,
11353 sem: Arc<Semaphore>,
11354 retries: usize,
11355 roster: &[AgentSpec],
11356 ctx: &WaveCtx<'_>,
11357 losses: &mut Vec<QuotaLoss>,
11358 state: &mut RunState,
11359 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11360) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11361where
11362 T: Send + 'static,
11363{
11364 let n = jobs.len();
11365 let originals: Vec<SeatJob> = jobs;
11366 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11367 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11368 let mut nudges: Vec<usize> = vec![0; n];
11375 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11379 let mut tried: Vec<BTreeSet<String>> = specs
11380 .iter()
11381 .map(|s| BTreeSet::from([s.id.clone()]))
11382 .collect();
11383 let starts: Vec<usize> = specs
11384 .iter()
11385 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11386 .collect();
11387 let carry = ctx.carry_seats && !roster.is_empty();
11388 let carried: Vec<BTreeSet<String>> = originals
11391 .iter()
11392 .map(|j| {
11393 state
11394 .seat_history
11395 .get(&j.seat.key)
11396 .filter(|_| carry)
11397 .map(|h| h.failed.clone())
11398 .unwrap_or_default()
11399 })
11400 .collect();
11401 let mut prev: Vec<Option<FailClass>> = originals
11402 .iter()
11403 .map(|j| {
11404 state
11405 .seat_history
11406 .get(&j.seat.key)
11407 .filter(|_| carry)
11408 .and_then(|h| h.last_fail.clone())
11409 })
11410 .collect();
11411 let next_agent =
11412 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11413 let others: BTreeSet<String> = specs
11414 .iter()
11415 .enumerate()
11416 .filter(|(j, _)| *j != i)
11417 .map(|(_, s)| s.id.clone())
11418 .collect();
11419 pick_successor(
11420 roster,
11421 starts[i],
11422 tried,
11423 carry.then(|| &carried[i]),
11424 &others,
11425 )
11426 .cloned()
11427 };
11428 let mut fresh: Vec<Option<String>> = vec![None; n];
11429 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11430 let mut pending: Vec<usize> = (0..n).collect();
11431
11432 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11437 for round in 0..max_rounds {
11438 if pending.is_empty() {
11439 break;
11440 }
11441 let mut batch = Vec::with_capacity(pending.len());
11442 let mut renudged: Vec<&str> = Vec::new();
11443 for &i in &pending {
11444 let src = &originals[i];
11445 let full: &str = match &src.handover {
11450 Some(h) if specs[i].id != src.spec.id => h,
11451 _ => &src.prompt,
11452 };
11453 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11456 (
11457 full.to_owned(),
11458 src.timeout,
11459 format!("{}-{word}-{}", src.stem, specs[i].id),
11460 )
11461 } else if nudges[i] == 0 {
11462 (full.to_owned(), src.timeout, src.stem.clone())
11463 } else {
11464 renudged.push(src.seat.key.as_str());
11465 let why = done[i]
11466 .as_ref()
11467 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11468 .unwrap_or_else(|| "no parsable answer".to_owned());
11469 let nudge = prompt::nudge(&why);
11470 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11471 let prompt = if nudged {
11472 nudge
11473 } else {
11474 format!("{full}\n\n---\n\n{nudge}")
11475 };
11476 (
11477 prompt,
11478 retry_budget(src.timeout, nudged),
11479 format!("{}-retry{}", src.stem, nudges[i]),
11480 )
11481 };
11482 batch.push(SeatJob {
11483 spec: specs[i].clone(),
11484 seat: seats[i].clone(),
11485 cwd: src.cwd.clone(),
11486 prompt,
11487 timeout,
11488 allow_write: src.allow_write,
11489 sessions: src.sessions,
11490 artifacts: src.artifacts.clone(),
11491 stem,
11492 handover: None,
11493 });
11494 }
11495
11496 if !renudged.is_empty() {
11497 state.event(
11498 ctx.node,
11499 format!("retry {round}: re-asking {}", renudged.join(", ")),
11500 );
11501 }
11502 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11503 let mut still = Vec::new();
11504 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11505 seats[i] = seat;
11506 if let AgentOutcome::ResourceBlocked(why) = &out {
11509 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11510 continue;
11511 }
11512 let class = FailClass::of(&out);
11513 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11516 && nudges[i] < retries
11517 && !roster.is_empty();
11518 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11519 let next = should_hand_over(prev[i].as_ref(), &cur)
11520 .then(|| next_agent(i, &tried[i], &specs))
11521 .flatten();
11522 if carry {
11523 let h = state
11524 .seat_history
11525 .entry(originals[i].seat.key.clone())
11526 .or_default();
11527 h.failed.insert(specs[i].id.clone());
11528 h.last_fail = Some(cur.clone());
11529 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11530 h.last_ok = None;
11531 }
11532 if let Err(e) = state.save() {
11535 tracing::warn!("could not persist a seat's failure history: {e:#}");
11536 }
11537 }
11538 if let Some(next) = next {
11539 record_handover(
11540 state,
11541 ctx.node,
11542 &originals[i].seat.key,
11543 &specs[i].id,
11544 &next.id,
11545 &cur,
11546 &fail_reason(&out),
11547 );
11548 tried[i].insert(next.id.clone());
11549 prev[i] = Some(cur.clone());
11550 seats[i] =
11551 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11552 specs[i] = next;
11553 fresh[i] = Some(cur.stem_word().to_owned());
11554 nudges[i] = 0;
11555 done[i] = Some(Err(anyhow::anyhow!(
11556 "handed over after: {}",
11557 fail_reason(&out)
11558 )));
11559 still.push(i);
11560 continue;
11561 }
11562 }
11563 if carry
11564 && !nudge_first
11565 && let Some(cur) = class.clone()
11566 {
11567 let h = state
11569 .seat_history
11570 .entry(originals[i].seat.key.clone())
11571 .or_default();
11572 h.failed.insert(specs[i].id.clone());
11573 h.last_fail = Some(cur);
11574 }
11575 let parsed = match out {
11576 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11577 AgentOutcome::Quota(o) => {
11578 last_quota[i] = o.quota.clone();
11579 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11580 }
11581 AgentOutcome::Dropped(o) => {
11585 let why = o
11586 .dropped
11587 .as_ref()
11588 .map(|d| d.why.as_str())
11589 .unwrap_or("the CLI ended the stream without delivering its answer");
11590 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11591 }
11592 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11593 Err(anyhow::anyhow!(e))
11594 }
11595 };
11596 let quota = class == Some(FailClass::Quota);
11597 let failed = parsed.is_err();
11598 if !failed && matches!(ctx.node, "judge" | "vote") {
11599 state
11602 .quota
11603 .retain(|loss| loss.seat != originals[i].seat.key);
11604 }
11605 done[i] = Some(parsed);
11606 if carry && !failed {
11607 let h = state
11608 .seat_history
11609 .entry(originals[i].seat.key.clone())
11610 .or_default();
11611 h.failed.remove(&specs[i].id);
11612 h.last_ok = Some(specs[i].id.clone());
11613 h.last_fail = None;
11614 }
11615 let agent_failure = class.is_some()
11623 && !nudge_first
11624 && !roster.is_empty()
11625 && next_agent(i, &tried[i], &specs).is_some();
11626 if failed && !quota && !agent_failure && nudges[i] < retries {
11627 nudges[i] += 1;
11628 still.push(i);
11629 }
11630 }
11631 pending = still;
11632 }
11633
11634 for (i, q) in last_quota.into_iter().enumerate() {
11637 if let Some(quota) = q {
11638 losses.push(QuotaLoss::from_report(
11639 originals[i].seat.key.clone(),
11640 ctx.node,
11641 Some("a),
11642 ));
11643 }
11644 }
11645
11646 if state.status != RunStatus::Stalled
11649 && ctx.node != "deliberate"
11650 && n > 0
11651 && losses.len() == n
11652 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11653 {
11654 block_machine_outage(state, ctx.node, losses);
11655 state.quota.append(losses);
11656 }
11657
11658 seats
11659 .into_iter()
11660 .zip(done)
11661 .zip(nudges)
11662 .map(|((seat, res), attempts)| {
11663 (
11664 seat,
11665 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11666 attempts,
11667 )
11668 })
11669 .collect()
11670}
11671
11672pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11675 let mut auth: Vec<_> = losses
11676 .iter()
11677 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11678 .map(|loss| loss.agent.clone())
11679 .collect();
11680 auth.sort();
11681 auth.dedup();
11682 let (kind, detail) = if auth.is_empty() {
11683 (
11684 "agent usage limit",
11685 "All seats reached their agent usage limits; resume after usage is available"
11686 .to_owned(),
11687 )
11688 } else {
11689 (
11690 "agent authentication",
11691 format!(
11692 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11693 auth.join(", ")
11694 ),
11695 )
11696 };
11697 state.resource_block = Some(crate::run::ResourceBlock {
11698 node: node.to_owned(),
11699 kind: kind.to_owned(),
11700 cause: crate::run::ResourceBlockCause::Other,
11701 detail: detail.clone(),
11702 at: Timestamp::now(),
11703 });
11704 state.event(node, detail);
11705}
11706
11707fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11709 results
11710 .iter()
11711 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11712}
11713
11714fn block_wave(
11718 jobs: Vec<SeatJob>,
11719 state: &mut RunState,
11720 node: &str,
11721 attempt: usize,
11722 why: &str,
11723) -> Vec<(usize, SeatState, AgentOutcome)> {
11724 state.resource_wait = None;
11725 state
11726 .active
11727 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11728 if let Err(e) = state.save() {
11729 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11730 }
11731 jobs.into_iter()
11732 .enumerate()
11733 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11734 .collect()
11735}
11736
11737async fn acquire_cache_lease(
11750 state: &mut RunState,
11751 cache_dir: &Path,
11752 owner: &crate::cache::Owner,
11753 budget: Duration,
11754 node: &str,
11755 context: &str,
11756 cancel: &(dyn Fn() -> bool + Sync),
11757) -> Result<Option<crate::cache::Guard>> {
11758 let home = crate::run::home();
11759 let started = Instant::now();
11760 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11761 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11762 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11763 Err(e) => {
11764 state.event(
11765 "verify",
11766 format!("{context}: could not check the shared build cache: {e:#}"),
11767 );
11768 if let Err(e2) = state.save() {
11769 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11770 }
11771 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11772 }
11773 };
11774 state.event(
11775 "verify",
11776 format!(
11777 "{context}: waiting for the shared build cache at {} ({})",
11778 cache_dir.display(),
11779 busy.describe()
11780 ),
11781 );
11782 state.resource_wait = Some(crate::run::ResourceWait {
11785 node: context.to_owned(),
11786 kind: "shared build cache".to_owned(),
11787 holder: busy.describe(),
11788 since: Timestamp::now(),
11789 limit_secs: budget.as_secs(),
11790 });
11791 if let Err(e) = state.save() {
11792 tracing::warn!("could not persist a cache wait: {e:#}");
11793 }
11794 let remaining = budget.saturating_sub(started.elapsed());
11795 let waited = crate::cache::wait_for_unless(
11796 &home,
11797 cache_dir,
11798 owner,
11799 remaining,
11800 Duration::from_secs(5),
11801 cancel,
11802 )
11803 .await;
11804 state.resource_wait = None;
11805 let cause = match &waited {
11806 Ok(Some(_)) => None,
11807 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11808 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11809 Some(crate::run::ResourceBlockCause::Timeout {
11810 waited_secs: started.elapsed().as_secs(),
11811 limit_secs: budget.as_secs(),
11812 })
11813 }
11814 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11815 };
11816 if let Some(cause) = cause {
11817 let detail = match &waited {
11818 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11819 Err(error) => format!("{error:#}"),
11820 };
11821 state.resource_block = Some(crate::run::ResourceBlock {
11822 cause,
11823 node: node.to_owned(),
11824 kind: "shared build cache".into(),
11825 detail,
11826 at: Timestamp::now(),
11827 });
11828 state.event(
11829 "resource",
11830 state.resource_block.as_ref().unwrap().description(),
11831 );
11832 }
11833 if let Err(error) = state.save() {
11834 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11835 }
11836 waited
11837}
11838
11839#[derive(Debug)]
11844struct CacheCheckError(String);
11845
11846impl std::fmt::Display for CacheCheckError {
11847 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11848 f.write_str(&self.0)
11849 }
11850}
11851
11852impl std::error::Error for CacheCheckError {}
11853
11854fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11856 state
11857 .resource_block
11858 .get_or_insert_with(|| crate::run::ResourceBlock {
11859 cause: crate::run::ResourceBlockCause::Other,
11860 node: node.to_owned(),
11861 kind: "shared build cache".into(),
11862 detail: format!("{error:#}"),
11863 at: Timestamp::now(),
11864 });
11865 if let Err(error) = state.save() {
11866 tracing::warn!("could not persist cache resource block: {error:#}");
11867 }
11868}
11869
11870#[allow(clippy::too_many_arguments)]
11891async fn with_cache_lease<'s, F, Fut>(
11892 state: &'s mut RunState,
11893 cache_dir: Option<&Path>,
11894 node: &str,
11895 seat: &str,
11896 worktree: &Path,
11897 head: &str,
11898 budget: Duration,
11899 context: &str,
11900 wait: &crate::repo_lock::WaitPolicy,
11901 body: F,
11902) -> Result<(Vec<CommandOutcome>, bool)>
11903where
11904 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11905 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11906{
11907 let started = Instant::now();
11910 let home = crate::run::home();
11911 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11912 let guard = if let Some(cache_dir) = cache_dir {
11913 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11914 match acquire_cache_lease(
11915 state,
11916 cache_dir,
11917 &owner,
11918 resource_limit,
11919 node,
11920 context,
11921 &|| wait.cancelled(),
11922 )
11923 .await
11924 .and_then(|g| {
11925 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11926 }) {
11927 Ok(g) => Some(g),
11928 Err(e) => {
11929 record_cache_error(state, node, &e);
11930 return Ok((
11931 vec![CommandOutcome {
11932 command: "(waiting for the shared build cache)".into(),
11933 code: None,
11934 output_tail: e.to_string(),
11935 duration_ms: started.elapsed().as_millis() as u64,
11936 resource_blocked: true,
11937 }],
11938 false,
11939 ));
11940 }
11941 }
11942 } else {
11943 None
11944 };
11945 let _coordination = coordinate_operation(state, worktree, wait).await?;
11946 if let Some(cache_dir) = cache_dir {
11947 let identity = crate::cache::Identity::new(worktree, head);
11948 if let Err(e) = crate::cache::ensure_fresh(
11949 &home,
11950 cache_dir,
11951 &identity,
11952 &state.config.verify.cargo_manifests(),
11953 ) {
11954 record_cache_error(state, node, &e);
11955 state.event(
11956 "verify",
11957 format!(
11958 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11959 worktree.display(),
11960 short(head)
11961 ),
11962 );
11963 if let Some(guard) = guard {
11964 guard.release();
11965 }
11966 return Ok((
11967 vec![CommandOutcome {
11968 command: "(confirming the shared build cache is fresh)".into(),
11969 code: None,
11970 output_tail: e.to_string(),
11971 duration_ms: started.elapsed().as_millis() as u64,
11972 resource_blocked: true,
11973 }],
11974 false,
11975 ));
11976 }
11977 }
11978 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11979 if !timed_out_pids.is_empty() {
11980 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11981 }
11982 drop(_coordination);
11985 if let Some(guard) = guard {
11986 guard.release();
11987 }
11988 Ok((outcomes, retried))
11989}
11990
11991async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
12003 wait_for_pids_with(
12004 pids,
12005 crate::proc::pid_alive,
12006 LEASE_RELEASE_POLL,
12007 LEASE_RELEASE_MAX_WAIT,
12008 )
12009 .await;
12010}
12011
12012async fn wait_for_pids_with<F: Fn(u32) -> bool>(
12018 pids: &[u32],
12019 alive: F,
12020 poll: Duration,
12021 max_wait: Duration,
12022) {
12023 let deadline = Instant::now() + max_wait;
12024 loop {
12025 if pids.iter().all(|&pid| !alive(pid)) {
12026 return;
12027 }
12028 if Instant::now() >= deadline {
12029 return;
12030 }
12031 tokio::time::sleep(poll).await;
12032 }
12033}
12034
12035fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12042 outcomes.iter().any(|o| o.resource_blocked)
12043}
12044
12045pub(crate) enum GateFix {
12047 Retry,
12049 Stop,
12052 Defer,
12055}
12056
12057pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12065 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12066 red.peek().is_some()
12067 && red.all(|o| {
12068 !o.resource_blocked
12069 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12070 && !o.output_tail.trim().is_empty()
12071 })
12072}
12073
12074fn e2e_outcome_label(o: &CommandOutcome) -> String {
12078 if o.ok() {
12079 return "pass".to_owned();
12080 }
12081 let reason = if o.build_failed() {
12082 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12083 } else {
12084 format!("FAIL ({:?})", o.code)
12085 };
12086 format!(
12087 "{reason}\n{}",
12088 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12089 )
12090}
12091
12092fn rerunnable_failure(outcomes: &[CommandOutcome]) -> bool {
12099 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12100 red.peek().is_some()
12101 && !verify_inconclusive(outcomes)
12102 && red.all(|o| {
12103 !o.build_failed() && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12104 })
12105}
12106
12107const FAILED_TWICE: &str = "[this command failed twice on the same commit; the output below is \
12110 the second attempt's]";
12111
12112async fn run_e2e_with_retry(
12132 state: &mut RunState,
12133 shell: &[String],
12134 commands: &[String],
12135 worktree: &Path,
12136 timeout: Duration,
12137 context: &str,
12138 round: Option<usize>,
12139) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12140 let (mut e2e, mut timed_out_pids) = run_commands(
12141 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12142 )
12143 .await;
12144 for o in &e2e {
12145 state.event(
12146 "verify",
12147 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12148 );
12149 }
12150 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12154 if verify_retried {
12155 state.event(
12156 "verify",
12157 format!(
12158 "{context}: verify could not build/link, not a test result — retrying once \
12159 before concluding"
12160 ),
12161 );
12162 let retried = run_commands(
12163 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12164 )
12165 .await;
12166 e2e = retried.0;
12167 timed_out_pids.extend(retried.1);
12170 for o in &e2e {
12171 state.event(
12172 "verify",
12173 format!(
12174 "{context}: retry `{}` -> {}",
12175 o.command,
12176 e2e_outcome_label(o)
12177 ),
12178 );
12179 }
12180 }
12181 if state.config.graph.rerun_failed_e2e && rerunnable_failure(&e2e) {
12182 let failed: Vec<usize> = (0..e2e.len()).filter(|&i| !e2e[i].ok()).collect();
12183 let rerun_commands: Vec<String> = failed.iter().map(|&i| e2e[i].command.clone()).collect();
12184 let head = git::rev_parse(worktree, "HEAD").await.unwrap_or_default();
12185 state.event(
12186 "verify",
12187 format!(
12188 "{context}: {} failed with a test result — running it once more on the same \
12189 commit {} before deciding",
12190 rerun_commands
12191 .iter()
12192 .map(|c| format!("`{c}`"))
12193 .collect::<Vec<_>>()
12194 .join(", "),
12195 short(&head)
12196 ),
12197 );
12198 let (second, pids) = run_commands(
12199 state,
12200 "verify",
12201 "e2e",
12202 1 + usize::from(verify_retried),
12203 shell,
12204 &rerun_commands,
12205 worktree,
12206 timeout,
12207 )
12208 .await;
12209 timed_out_pids.extend(pids);
12210 for o in &second {
12211 state.event(
12212 "verify",
12213 format!(
12214 "{context}: rerun `{}` -> {}",
12215 o.command,
12216 e2e_outcome_label(o)
12217 ),
12218 );
12219 }
12220 let mut reproduced = false;
12221 for (slot, &i) in failed.iter().enumerate() {
12222 let Some(again) = second.get(slot) else {
12223 break;
12224 };
12225 let first = std::mem::replace(&mut e2e[i], again.clone());
12226 if again.ok() {
12227 let signature = crate::flake::signature(&first.output_tail, first.code);
12228 let flake = crate::run::VerifyFlake {
12229 round,
12230 context: context.to_owned(),
12231 head: head.clone(),
12232 command: first.command.clone(),
12233 first_failure: summarize_verify(&first.output_tail, EVENT_OUTPUT_TAIL),
12234 fingerprint: crate::flake::fingerprint(&state.repo, &signature),
12235 at: Timestamp::now(),
12236 task: None,
12237 filed: false,
12238 filing_error: None,
12239 };
12240 crate::flake::record(state, flake, &signature.label, &crate::queue::Queue::open());
12241 } else if !again.resource_blocked {
12242 reproduced = true;
12243 e2e[i].output_tail = format!("{FAILED_TWICE}\n{}", again.output_tail);
12244 }
12245 }
12246 if reproduced {
12247 state.event(
12248 "verify",
12249 format!("{context}: the failure reproduced on the second run — it stands"),
12250 );
12251 }
12252 }
12253 (e2e, verify_retried, timed_out_pids)
12254}
12255
12256#[allow(clippy::too_many_arguments)]
12272async fn run_commands(
12273 state: &mut RunState,
12274 node: &str,
12275 task: &str,
12276 attempt: usize,
12277 shell: &[String],
12278 commands: &[String],
12279 cwd: &Path,
12280 timeout: Duration,
12281) -> (Vec<CommandOutcome>, Vec<u32>) {
12282 if commands.is_empty() {
12283 return (Vec::new(), Vec::new());
12288 }
12289 let mut out = Vec::new();
12290 let mut timed_out_pids = Vec::new();
12291 let total = commands.len();
12292 for (idx, command) in commands.iter().enumerate() {
12293 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12294 if let Err(e) = state.save() {
12295 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12296 }
12297 let started = Instant::now();
12298 let mut cmd = tokio::process::Command::new(&shell[0]);
12299 cmd.quiet();
12300 for key in crate::worker::INVOCATION_ENV {
12304 cmd.env_remove(key);
12305 }
12306 cmd.args(&shell[1..])
12307 .arg(command)
12308 .current_dir(cwd)
12309 .stdin(std::process::Stdio::null())
12310 .stdout(std::process::Stdio::piped())
12311 .stderr(std::process::Stdio::piped())
12312 .kill_on_drop(true);
12313 let mut outputs = Vec::new();
12314 let protected = (|| -> Result<()> {
12315 crate::incremental::configure(&mut cmd, cwd)?;
12316 let mut paths = vec![cwd.to_owned()];
12317 if let Some(cache) = state.config.cache_dir() {
12318 paths.push(cache);
12319 }
12320 outputs.extend(crate::retention::OutputLease::admit(
12321 &crate::run::home(),
12322 &paths,
12323 state.config.disk.min_free_bytes,
12324 )?);
12325 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12326 outputs.push(guard);
12327 }
12328 if let Some(cache) = state.config.cache_dir() {
12329 outputs.push(crate::retention::OutputLease::cache(
12330 &crate::run::home(),
12331 &cache,
12332 false,
12333 )?);
12334 }
12335 Ok(())
12336 })();
12337 if let Err(e) = protected {
12338 let detail = format!("execution admission unavailable: {e:#}");
12339 state.resource_block = Some(crate::run::ResourceBlock {
12340 cause: crate::run::ResourceBlockCause::Other,
12341 node: node.to_owned(),
12342 kind: if detail.contains("disk admission") {
12343 "disk admission"
12344 } else {
12345 "output ownership"
12346 }
12347 .to_owned(),
12348 detail: detail.clone(),
12349 at: Timestamp::now(),
12350 });
12351 out.push(CommandOutcome {
12352 command: command.clone(),
12353 code: None,
12354 output_tail: detail,
12355 duration_ms: 0,
12356 resource_blocked: true,
12357 });
12358 break;
12359 }
12360 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12361 let (code, body) = match spawned {
12362 Ok(mut running) => {
12363 let pid = running.child().id();
12364 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12365 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12366 let mut paths = vec![cwd.to_owned()];
12367 if let Some(cache) = state.config.cache_dir() {
12368 paths.push(cache);
12369 }
12370 let mut pressure = None;
12371 let status = tokio::select! {
12372 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12373 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12374 pressure = Some(reason); None
12375 }
12376 };
12377 let (code, failure) = match status {
12378 Some(Ok(Ok(status))) => (
12379 status.code(),
12380 status
12381 .code()
12382 .is_none()
12383 .then(|| format!("command terminated: {status}")),
12384 ),
12385 Some(Ok(Err(error))) => {
12386 (None, Some(format!("failed waiting for command: {error}")))
12387 }
12388 Some(Err(_)) => {
12389 if !running.coordinated()
12393 && let Some(pid) = pid
12394 {
12395 timed_out_pids.push(pid);
12396 }
12397 (
12398 None,
12399 Some(format!("timed out after {}s", timeout.as_secs())),
12400 )
12401 }
12402 None => (None, None),
12403 };
12404 running.finish().await;
12405 if let Some(reason) = pressure {
12406 state.resource_block = Some(crate::run::ResourceBlock {
12407 cause: crate::run::ResourceBlockCause::Other,
12408 node: node.to_owned(),
12409 kind: "disk pressure".to_owned(),
12410 detail: reason.clone(),
12411 at: Timestamp::now(),
12412 });
12413 out.push(CommandOutcome {
12414 command: command.clone(),
12415 code: None,
12416 output_tail: reason,
12417 duration_ms: started.elapsed().as_millis() as u64,
12418 resource_blocked: true,
12419 });
12420 break;
12421 }
12422 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12423 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12424 if let Some(failure) = failure {
12425 body.push('\n');
12426 body.push_str(&failure);
12427 }
12428 (code, body)
12429 }
12430 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12431 };
12432 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12433 if !body.is_empty() {
12434 match crate::run::write_verification_artifact(state, task, &body) {
12438 Ok(name) if summary != body => {
12439 summary = format!("[full output: artifacts/{name}]\n{summary}");
12440 }
12441 Ok(_) => {}
12442 Err(e) => {
12443 let warning = format!("could not save the full {task} output: {e:#}");
12444 tracing::warn!("{warning}");
12445 state.event(node, warning);
12446 if summary != body {
12447 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12448 }
12449 }
12450 }
12451 }
12452 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12453 out.push(CommandOutcome {
12454 command: command.clone(),
12455 code,
12456 output_tail: summary,
12457 duration_ms: started.elapsed().as_millis() as u64,
12458 resource_blocked,
12459 });
12460 if resource_blocked {
12461 state.resource_block = Some(crate::run::ResourceBlock {
12462 cause: crate::run::ResourceBlockCause::Other,
12463 node: node.to_owned(),
12464 kind: "incremental cache admission".into(),
12465 detail: body,
12466 at: Timestamp::now(),
12467 });
12468 break;
12469 }
12470 }
12471 state.task_finished(task);
12472 if let Err(e) = state.save() {
12473 tracing::warn!("could not persist the end of {task}: {e:#}");
12474 }
12475 (out, timed_out_pids)
12476}
12477
12478fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12490 let repo = repo.display();
12491 match style {
12492 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12493 MergeStyle::Squash => {
12494 let subject = message
12497 .lines()
12498 .next()
12499 .unwrap_or(branch)
12500 .replace(['\\', '"', '$', '`'], "");
12501 format!(
12502 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12503 )
12504 }
12505 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12506 }
12507}
12508
12509const PR_TITLE_MAX: usize = 240;
12520
12521fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12532 let line = queue::first_line(text)?;
12533 let chars: Vec<char> = line.chars().collect();
12534 let (kept, mark) = if chars.len() <= max {
12535 (line.to_owned(), "")
12536 } else if let Some(end) = sentence_end(&chars, max) {
12537 (chars[..end].iter().collect::<String>(), "")
12538 } else {
12539 let room = max.saturating_sub(3);
12540 let cut = if chars[room].is_whitespace() {
12543 room
12544 } else {
12545 chars[..room]
12546 .iter()
12547 .rposition(|c| c.is_whitespace())
12548 .unwrap_or(room)
12549 };
12550 let head: String = chars[..cut].iter().collect();
12551 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12552 (head.to_owned(), "...")
12553 };
12554 if kept.is_empty() {
12555 return None;
12556 }
12557 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12558 return None;
12559 }
12560 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12561 return None;
12562 }
12563 Some(format!("{kept}{mark}"))
12564}
12565
12566fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12570 const MIN: usize = 20;
12571 for i in MIN..max.min(chars.len()) {
12572 if !matches!(chars[i], '.' | '!' | '?') {
12573 continue;
12574 }
12575 let Some(&next) = chars.get(i + 1) else {
12576 continue;
12577 };
12578 if !next.is_whitespace() {
12579 continue;
12580 }
12581 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12582 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12583 continue;
12584 }
12585 let word: String = chars[..i]
12586 .iter()
12587 .rev()
12588 .take_while(|c| !c.is_whitespace())
12589 .collect::<Vec<_>>()
12590 .into_iter()
12591 .rev()
12592 .collect();
12593 let word = word.to_ascii_lowercase();
12594 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12595 continue;
12596 }
12597 let end = chars[..i]
12598 .iter()
12599 .rposition(|c| !c.is_whitespace())
12600 .map_or(i, |p| p + 1);
12601 return Some(end);
12602 }
12603 None
12604}
12605
12606struct PrMessage {
12611 title: String,
12612 body: String,
12613}
12614
12615impl PrMessage {
12616 fn commit_message(&self) -> String {
12620 format!("{}\n\n{}", self.title, self.body)
12621 }
12622}
12623
12624fn title_marker(line: &str) -> Option<&str> {
12626 let line = line.trim();
12627 let head = line.get(..6)?;
12628 head.eq_ignore_ascii_case("title:")
12629 .then(|| line[6..].trim())
12630}
12631
12632fn summary_title(summary: &str) -> Option<String> {
12637 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12638 let raw = title_marker(first)?;
12639 if raw.is_empty() {
12640 return None;
12641 }
12642 let title = queue::title_from(raw, PR_TITLE_MAX);
12643 let lower = title.to_ascii_lowercase();
12644 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12645 return None;
12646 }
12647 Some(title)
12648}
12649
12650const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12652
12653pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12655
12656pub(crate) fn is_review_run(state: &RunState) -> bool {
12660 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12661}
12662
12663fn review_title(state: &RunState) -> Option<String> {
12668 english_subject(
12669 state.reviewed_commits.as_ref()?.first()?,
12670 state.config.graph.github_english(),
12671 )
12672}
12673
12674fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12677 let raw = raw.trim();
12678 if raw.is_empty() {
12679 return None;
12680 }
12681 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12682 return None;
12683 }
12684 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12685 return None;
12686 }
12687 let title = queue::title_from(raw, PR_TITLE_MAX);
12688 let lower = title.to_ascii_lowercase();
12689 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12690 return None;
12691 }
12692 Some(title)
12693}
12694
12695#[derive(Debug, Clone, PartialEq, Eq)]
12698struct BranchFacts {
12699 commits: Vec<(String, String)>,
12701 stat: String,
12703}
12704
12705const STAT_FILE_LINES: usize = 25;
12707const COMMITS_MAX_CHARS: usize = 20_000;
12709
12710async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12713 let commits = git::commit_log(repo, base, branch).await.ok()?;
12714 if commits.is_empty() {
12715 return None;
12716 }
12717 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12718 let lines: Vec<&str> = stat.lines().collect();
12719 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12720 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12721 let more = format!(" ... {omitted} more file(s)");
12722 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12723 kept.push(&more);
12724 kept.push(lines[lines.len() - 1]);
12725 kept.join("\n")
12726 } else {
12727 lines.join("\n")
12728 };
12729 Some(BranchFacts { commits, stat })
12730}
12731
12732fn markdown_safe(text: &str) -> String {
12735 text.replace("</details>", "</details>")
12736 .replace("\x60\x60\x60", "~~~")
12737}
12738
12739fn neutral_title(state: &RunState, winner: char) -> String {
12740 format!(
12741 "chore: land candidate {} of run {}",
12742 winner.to_ascii_uppercase(),
12743 state.id
12744 )
12745}
12746
12747pub(crate) async fn repair_landing_gate(
12750 state: &mut RunState,
12751 winner: &Candidate,
12752 outcomes: &[CommandOutcome],
12753) -> Result<GateFix> {
12754 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12755 return Ok(GateFix::Stop);
12756 }
12757 let mut runner = Runner {
12758 roles: state.config.resolve_roles()?,
12759 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12760 state: state.clone(),
12761 pause: Pause::new(),
12762 interrupt: Pause::new(),
12763 };
12764 let result = runner.gate_fix_round(winner, outcomes).await;
12765 *state = runner.state;
12766 result
12767}
12768
12769pub(crate) async fn verify_current_head(
12772 state: &mut RunState,
12773 winner: &Candidate,
12774 wait: &crate::repo_lock::WaitPolicy,
12775) -> Result<bool> {
12776 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12777 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12779 state
12780 .verification_history
12781 .push(crate::run::VerificationEvidence {
12782 superseded_by: Some(head.clone()),
12783 archived_at: Some(Timestamp::now()),
12784 fixes_spent: Some(state.gate_fixes.len()),
12785 gate_head: None,
12786 gate_ran: false,
12787 gate: Vec::new(),
12788 e2e: state.head_e2e.take(),
12789 });
12790 }
12791 if let Some(e) = &state.head_e2e
12792 && e.head == head
12793 {
12794 if e.passed {
12795 return Ok(true);
12796 }
12797 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12798 state.status = RunStatus::Blocked;
12799 state.save()?;
12800 return Ok(false);
12801 }
12802 } else if state.reviews.last().is_some_and(|r| {
12803 r.verified_head.as_deref() == Some(head.as_str())
12804 && (r.e2e_status() == E2eStatus::Passed
12805 || (r.e2e_status() == E2eStatus::NotConfigured
12806 && state.config.verify.e2e.is_empty()))
12807 }) || state
12808 .gate_repair_e2e
12809 .as_ref()
12810 .is_some_and(|e| e.head == head && e.passed)
12811 {
12812 return Ok(true);
12813 }
12814 if let Some(e) = &state.gate_repair_e2e
12815 && e.head == head
12816 && !e.passed
12817 && !e.outcomes.is_empty()
12818 && !verify_inconclusive(&e.outcomes)
12819 {
12820 state.status = RunStatus::Blocked;
12821 state.save()?;
12822 return Ok(false);
12823 }
12824 state.head_e2e = Some(crate::run::GateRepairE2e {
12825 head: head.clone(),
12826 passed: false,
12827 outcomes: Vec::new(),
12828 });
12829 state.save()?;
12830 let commands = state.config.verify.e2e.clone();
12831 let shell = state.config.shell();
12832 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12833 let cache_dir = state.config.cache_dir();
12834 let outcomes = if commands.is_empty() {
12835 Vec::new()
12836 } else {
12837 let (outcomes, _) = with_cache_lease(
12838 state,
12839 cache_dir.as_deref(),
12840 "e2e",
12841 "e2e",
12842 &winner.worktree,
12843 &head,
12844 timeout,
12845 "current head e2e",
12846 wait,
12847 |state, budget| async move {
12848 run_e2e_with_retry(
12849 state,
12850 &shell,
12851 &commands,
12852 &winner.worktree,
12853 budget,
12854 "current head e2e",
12855 None,
12856 )
12857 .await
12858 },
12859 )
12860 .await?;
12861 outcomes
12862 };
12863 let unresolved = verify_inconclusive(&outcomes);
12864 let passed = !unresolved
12865 && outcomes.iter().all(CommandOutcome::ok)
12866 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12867 if !outcomes.is_empty() && !unresolved {
12868 state.record_base_progress(&head);
12869 }
12870 state.head_e2e = Some(crate::run::GateRepairE2e {
12871 head: head.clone(),
12872 passed,
12873 outcomes,
12874 });
12875 if !passed && !unresolved {
12876 state.status = RunStatus::Blocked;
12877 state.event(
12878 "e2e",
12879 format!("verify.e2e failed on head {head}; nothing published"),
12880 );
12881 }
12882 state.save()?;
12883 Ok(passed)
12884}
12885
12886pub(crate) async fn check_gate_repair_e2e(
12889 state: &mut RunState,
12890 winner: &Candidate,
12891 wait: &crate::repo_lock::WaitPolicy,
12892) -> Result<GateFix> {
12893 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12894 return Ok(GateFix::Retry);
12895 }
12896 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12897 if state
12898 .gate_repair_e2e
12899 .as_ref()
12900 .is_some_and(|r| r.head == head && r.passed)
12901 {
12902 return Ok(GateFix::Retry);
12903 }
12904 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12905 head: head.clone(),
12906 passed: false,
12907 outcomes: Vec::new(),
12908 });
12909 state.save()?;
12910 let commands = state.config.verify.e2e.clone();
12911 let outcomes = if commands.is_empty() {
12912 Vec::new()
12913 } else {
12914 let shell = state.config.shell();
12915 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12916 let cache_dir = state.config.cache_dir();
12917 let (outcomes, _) = with_cache_lease(
12918 state,
12919 cache_dir.as_deref(),
12920 "e2e",
12921 "e2e",
12922 &winner.worktree,
12923 &head,
12924 timeout,
12925 "gate repair e2e",
12926 wait,
12927 |state, budget| async move {
12928 run_e2e_with_retry(
12929 state,
12930 &shell,
12931 &commands,
12932 &winner.worktree,
12933 budget,
12934 "gate repair e2e",
12935 None,
12936 )
12937 .await
12938 },
12939 )
12940 .await?;
12941 outcomes
12942 };
12943 let deferred = verify_inconclusive(&outcomes);
12944 let passed = outcomes.iter().all(CommandOutcome::ok)
12945 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12946 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12947 head,
12948 passed,
12949 outcomes,
12950 });
12951 state.save()?;
12952 if deferred {
12953 Ok(GateFix::Defer)
12954 } else if !passed {
12955 state.event(
12956 "gate",
12957 "verify.e2e failed after gate repair; nothing published",
12958 );
12959 Ok(GateFix::Stop)
12960 } else {
12961 Ok(GateFix::Retry)
12962 }
12963}
12964
12965pub(crate) async fn run_landing_gate(
12967 state: &mut RunState,
12968 winner: &Candidate,
12969 wait: &crate::repo_lock::WaitPolicy,
12970) -> Result<Vec<CommandOutcome>> {
12971 let shell = state.config.shell();
12972 let gate_commands = state.config.verify.gate.clone();
12973 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12974 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12975 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12976 Vec::new()
12977 } else {
12978 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12979 let cache_dir = state.config.cache_dir();
12980 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12981 let (outcomes, _) = with_cache_lease(
12982 state,
12983 cache_dir.as_deref(),
12984 "gate",
12985 "gate",
12986 &winner.worktree,
12987 &head,
12988 timeout,
12989 "final gate",
12990 wait,
12991 |state, budget| {
12992 let shell = shell.clone();
12993 let gate_commands = gate_commands.clone();
12994 let winner = winner.clone();
12995 async move {
12996 Runner::run_pre_gate(state, &winner).await;
12997 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12999 let (outcomes, timed_out_pids) = run_commands(
13000 state,
13001 "gate",
13002 "gate",
13003 0,
13004 &shell,
13005 &gate_commands,
13006 &winner.worktree,
13007 budget,
13008 )
13009 .await;
13010 (outcomes, false, timed_out_pids)
13011 }
13012 },
13013 )
13014 .await?;
13015 outcomes
13016 };
13017 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
13018 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
13019 let mut outcomes = outcomes;
13020 outcomes.push(CommandOutcome {
13021 command: "gate head stability".into(), code: None,
13022 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
13023 duration_ms: 0, resource_blocked: false,
13024 });
13025 return Ok(outcomes);
13026 }
13027 if outcomes.is_empty() {
13028 state.event("gate", "no gate commands configured; not tested");
13033 }
13034 for o in &outcomes {
13035 state.event(
13036 "gate",
13037 format!(
13038 "`{}` -> {}",
13039 o.command,
13040 if o.ok() {
13041 "pass".to_owned()
13042 } else {
13043 format!(
13044 "FAIL ({:?})\n{}",
13045 o.code,
13046 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
13047 )
13048 }
13049 ),
13050 );
13051 }
13052 Ok(outcomes)
13053}
13054
13055pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
13061 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
13062 ""
13063 } else {
13064 pr_title
13065 }
13066}
13067
13068pub fn landing_subject_source(state: &RunState) -> String {
13072 if is_review_run(state) {
13073 let winner = state.candidates.first().map_or('A', |c| c.label);
13074 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
13075 }
13076 state.instruction.clone()
13077}
13078
13079fn summary_without_title(summary: &str) -> String {
13082 let mut lines = summary.trim().lines().peekable();
13083 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
13084 lines.next();
13085 }
13086 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
13087}
13088
13089#[cfg(test)]
13101fn pr_message(state: &RunState, winner: char) -> PrMessage {
13102 pr_message_with(state, winner, None)
13103}
13104
13105#[cfg(test)]
13108fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
13109 let raw = pr_message_raw(
13110 state,
13111 winner,
13112 facts,
13113 crate::pr_outcome::FinalHead::Unreported,
13114 ChangeNote::NotAsked,
13115 );
13116 let id = crate::scrub::Identity::current();
13117 PrMessage {
13118 title: crate::scrub::scrub(&raw.title, &id),
13119 body: crate::scrub::scrub(&raw.body, &id),
13120 }
13121}
13122
13123#[derive(Debug, Clone, Copy)]
13125enum ChangeNote<'a> {
13126 NotAsked,
13128 Summary(&'a crate::run::ChangeSummary),
13130 Unavailable(&'a str),
13132}
13133
13134fn pr_message_raw(
13135 state: &RunState,
13136 winner: char,
13137 facts: Option<&BranchFacts>,
13138 head: crate::pr_outcome::FinalHead<'_>,
13139 change: ChangeNote<'_>,
13140) -> PrMessage {
13141 if !matches!(change, ChangeNote::NotAsked) {
13142 let (title, mut body) = match change {
13143 ChangeNote::Summary(c) => (
13144 c.title.clone(),
13145 format!("## Summary\n\n{}\n\n", c.body.trim()),
13146 ),
13147 ChangeNote::Unavailable(why) => (
13148 neutral_title(state, winner),
13149 format!(
13150 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13151 ),
13152 ),
13153 ChangeNote::NotAsked => unreachable!(),
13154 };
13155 body.push_str(&public_pr_evidence(state, head));
13156 body.push_str(&format!(
13157 "\n---\nmagi:run/{} magi:candidate-{}\n",
13158 state.id,
13159 winner.to_ascii_lowercase()
13160 ));
13161 return PrMessage { title, body };
13162 }
13163 pr_preview(state, winner, facts, head)
13164}
13165
13166fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13169 let mut body = crate::pr_outcome::review_rounds(state);
13170 body.push_str(&crate::pr_outcome::final_verification(state, head));
13171 let open = state.last_round_findings();
13172 if !open.is_empty() {
13173 body.push_str("## Open review findings\n\n");
13174 for f in open {
13175 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13176 }
13177 body.push('\n');
13178 }
13179 if let Some(c) = state.change_summary.as_ref()
13180 && head == crate::pr_outcome::FinalHead::At(&c.head)
13181 {
13182 body.push_str(&format!(
13183 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13184 c.base, c.head
13185 ));
13186 }
13187 body
13188}
13189
13190fn pr_preview(
13192 state: &RunState,
13193 winner: char,
13194 facts: Option<&BranchFacts>,
13195 head: crate::pr_outcome::FinalHead<'_>,
13196) -> PrMessage {
13197 let summary = state
13198 .candidates
13199 .iter()
13200 .find(|c| c.label == winner)
13201 .map(|c| c.summary.as_str())
13202 .unwrap_or_default();
13203 let review = is_review_run(state);
13206 let title = if review {
13207 facts
13208 .and_then(|f| {
13209 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13210 })
13211 .or_else(|| review_title(state))
13212 .or_else(|| {
13213 state
13214 .candidates
13215 .iter()
13216 .find(|c| c.label == winner)
13217 .filter(|c| !c.branch.starts_with("magi/"))
13218 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13219 })
13220 .unwrap_or_else(|| neutral_title(state, winner))
13221 } else {
13222 summary_title(summary).unwrap_or_else(|| {
13223 english_title(
13224 &state.instruction,
13225 PR_TITLE_MAX,
13226 state.config.graph.github_english(),
13227 )
13228 .unwrap_or_else(|| neutral_title(state, winner))
13229 })
13230 };
13231
13232 let mut body = String::new();
13233 let what = summary_without_title(summary);
13234 if !what.is_empty() {
13235 body.push_str("## Summary\n\n");
13236 body.push_str(&what);
13237 body.push_str("\n\n");
13238 }
13239
13240 body.push_str(&crate::pr_outcome::review_rounds(state));
13242 body.push_str(&crate::pr_outcome::final_verification(state, head));
13243 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13244
13245 let open = state.open_findings();
13246 if !open.is_empty() {
13247 body.push_str("## Open review findings\n\n");
13248 for f in &open {
13249 body.push_str(&format!(
13250 "- `{}` [{:?}] {}\n",
13251 f.id, f.severity, "Context retained in local review records."
13252 ));
13253 }
13254 body.push('\n');
13255 }
13256
13257 if let Some(fix) = fix
13258 && !fix.rejected.is_empty()
13259 {
13260 body.push_str("## Declined by the fixer\n\n");
13261 for r in &fix.rejected {
13262 body.push_str(&format!(
13263 "- `{}`: {}\n",
13264 r.id, "Reason retained in local review records."
13265 ));
13266 }
13267 body.push('\n');
13268 }
13269
13270 if review {
13271 body.push_str("## Commits under review\n\n");
13273 if let Some(facts) = facts {
13274 let mut left = COMMITS_MAX_CHARS;
13275 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13276 let mut entry = format!("- {}\n", markdown_safe(subject));
13277 for l in markdown_safe(text).lines() {
13278 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13279 }
13280 if left == 0 {
13281 body.push_str(&format!(
13282 "- ... {} more commit(s)\n",
13283 facts.commits.len() - i
13284 ));
13285 break;
13286 }
13287 if entry.len() > left {
13288 let mut end = left;
13291 while !entry.is_char_boundary(end) {
13292 end -= 1;
13293 }
13294 entry.truncate(end);
13295 entry.push_str("\n ... (truncated)\n");
13296 left = 0;
13297 } else {
13298 left -= entry.len();
13299 }
13300 body.push_str(&entry);
13301 }
13302 if !facts.stat.trim().is_empty() {
13303 body.push_str(&format!(
13304 "\n## Diff stat\n\n```\n{}\n```\n",
13305 markdown_safe(facts.stat.trim())
13306 ));
13307 }
13308 } else {
13309 match &state.reviewed_commits {
13310 Some(subjects) => {
13311 for s in subjects {
13312 body.push_str(&format!("- {}\n", s.trim()));
13313 }
13314 }
13315 None => {
13316 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13319 body.push_str(rest.trim());
13320 body.push('\n');
13321 }
13322 }
13323 }
13324 } else {
13325 let task = state.instruction.trim();
13326 let task = if task.is_empty() {
13327 "(empty task)"
13328 } else {
13329 task
13330 };
13331 body.push_str(&format!(
13332 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13333 task.replace("</details>", "</details>")
13334 ));
13335 }
13336
13337 body.push_str(&format!(
13338 "\n---\nmagi:run/{} magi:candidate-{}\n",
13339 state.id,
13340 winner.to_ascii_lowercase()
13341 ));
13342
13343 PrMessage { title, body }
13344}
13345
13346fn seeded_instruction(state: &RunState) -> String {
13349 let mut inputs = Vec::new();
13350 if let Some(start) = &state.sources.start_from {
13351 inputs.push(format!(
13352 "Start from `{}` at `{}` (including ancestry).",
13353 start.token, start.sha
13354 ));
13355 }
13356 for pick in &state.sources.pick_commits {
13357 inputs.push(format!(
13358 "Pick `{}` at `{}` (single commit).",
13359 pick.token, pick.sha
13360 ));
13361 }
13362 if inputs.is_empty() {
13363 state.instruction.clone()
13364 } else {
13365 format!(
13366 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13367 state.instruction,
13368 inputs.join("\n")
13369 )
13370 }
13371}
13372
13373async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13377 let base = &state.base_branch;
13378 let mut against = base.clone();
13379 if mode == MergeMode::Pr {
13380 let remote = &state.config.merge.remote;
13384 let tracking = format!("{remote}/{base}");
13385 let fetched = git::fetch(repo, remote, base).await;
13386 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13387 against = tracking;
13388 } else {
13389 return false;
13390 }
13391 }
13392 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13393}
13394
13395fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13398 let mut detail = format!(
13399 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13400 nothing to open a pull request for"
13401 );
13402 detail.push_str(
13403 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13404 );
13405 detail
13406}
13407
13408#[derive(Debug, PartialEq, Eq)]
13411enum PrPlan {
13412 Create,
13413 Adopt { url: String, title: String },
13414 Stop(String),
13415}
13416
13417fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13420 match found {
13421 Ok(land::OpenPr::None) => PrPlan::Create,
13422 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13423 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13424 "several open pull requests exist for this branch, not picking one: {}",
13425 urls.join(" ")
13426 )),
13427 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13428 }
13429}
13430
13431pub(crate) async fn gh_pr_create(
13433 cwd: &Path,
13434 base: &str,
13435 head: &str,
13436 title: &str,
13437 body: &str,
13438) -> Result<String> {
13439 let out = tokio::process::Command::new("gh")
13440 .args([
13441 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13442 ])
13443 .current_dir(cwd)
13444 .quiet()
13445 .stdin(std::process::Stdio::null())
13446 .output()
13447 .await
13448 .context("spawn gh")?;
13449 if out.status.success() {
13450 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13451 } else {
13452 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13453 }
13454}
13455
13456async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13458 match std::fs::symlink_metadata(path) {
13459 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13460 git::worktree_remove_clean(repo, path).await?;
13463 return Ok(());
13464 }
13465 Err(e) => return Err(e.into()),
13466 Ok(_) => {}
13467 }
13468 if !git::is_clean(path).await? {
13469 bail!(
13470 "worktree has uncommitted work; refusing removal: {}",
13471 path.display()
13472 );
13473 }
13474 if !git::worktree_remove_clean(repo, path).await? {
13475 bail!("worktree removal refused: {}", path.display());
13476 }
13477 Ok(())
13478}
13479
13480async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13483 if !crate::retention::read_seat(path) || !path.exists() {
13484 return remove_preserved_worktree(&state.repo, path).await;
13485 }
13486 let owned = std::fs::symlink_metadata(path)?.is_dir()
13487 && path.parent().map(Path::canonicalize).transpose()?
13488 == Some(state.worktree_root().canonicalize()?)
13489 && crate::retention::worktrees(state)
13490 .await?
13491 .contains(&path.canonicalize()?);
13492 let common = crate::repo_lock::common_dir(&state.repo).await?;
13493 if !owned
13494 || common.is_none()
13495 || crate::repo_lock::common_dir(path).await? != common
13496 || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13497 .await?
13498 .ok()
13499 || state.candidates.iter().any(|c| c.worktree == path)
13500 {
13501 return remove_preserved_worktree(&state.repo, path).await;
13502 }
13503 let changes = git::status_porcelain(path).await?;
13504 if !git::worktree_remove(&state.repo, path).await? {
13505 bail!(
13506 "read-only seat worktree removal refused: {}",
13507 path.display()
13508 );
13509 }
13510 if !changes.is_empty() {
13511 let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13512 let paths: String = paths.chars().take(1024).collect();
13513 state.event(
13514 "fold",
13515 format!(
13516 "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13517 path.file_name().unwrap_or_default().to_string_lossy(),
13518 changes.lines().count()
13519 ),
13520 );
13521 }
13522 Ok(())
13523}
13524
13525pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13534 fold_run_with_policy(state, drop_winner, home, false).await
13535}
13536
13537pub async fn fold_run_overriding_recovery_retention(
13539 state: &mut RunState,
13540 drop_winner: bool,
13541 home: &Path,
13542) -> Result<Vec<String>> {
13543 fold_run_with_policy(state, drop_winner, home, true).await
13544}
13545
13546async fn fold_run_with_policy(
13547 state: &mut RunState,
13548 drop_winner: bool,
13549 home: &Path,
13550 override_recovery_retention: bool,
13551) -> Result<Vec<String>> {
13552 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state)
13553 && !override_recovery_retention
13554 {
13555 bail!("refusing fold: {reason}");
13556 }
13557 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13558 bail!("candidate preservation incomplete; refusing cleanup");
13559 }
13560 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13564 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13565 if override_recovery_retention {
13567 *state = RunState::load_under(&state.id, home)?;
13568 anyhow::ensure!(
13569 state.status.done() && !state.status.resumable(),
13570 "recovery retention override requires a finished, non-resumable run"
13571 );
13572 let claimed = crate::daemon::is_working_on_other(home, &state.id, Timestamp::now())?;
13573 anyhow::ensure!(
13574 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
13575 "run is live or its ownership is unknown"
13576 );
13577 anyhow::ensure!(
13578 state.candidate_preservation.is_empty() && state.interrupted_implementations.is_empty(),
13579 "candidate preservation incomplete; refusing cleanup"
13580 );
13581 }
13582 let retention_reason = crate::pr_recovery::source_retention_reason(home, state);
13583 if let Some(reason) = &retention_reason
13584 && !override_recovery_retention
13585 {
13586 bail!("refusing fold: {reason}");
13587 }
13588 let _driver = crate::driver::maintenance(home, &state.id)?;
13589 let mut output_guards = Vec::new();
13590 for entry in std::fs::read_dir(state.worktree_root())
13591 .into_iter()
13592 .flatten()
13593 .flatten()
13594 {
13595 if entry.path().join(".git").is_file() {
13596 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13597 }
13598 }
13599 let repo = state.repo.clone();
13600 let wait = repository_wait(state, None);
13603 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13604 Ok(lease) => lease,
13605 Err(e) => {
13606 state.save_under(home)?;
13607 return Err(e);
13608 }
13609 };
13610 let _source_outputs = if retention_reason.is_some()
13613 || (state.status == RunStatus::Merged && state.merge.as_ref().is_some_and(|merge| merge.ok))
13614 {
13615 verify_merged_fold_sources(state, drop_winner, home).await?
13616 } else {
13617 Vec::new()
13618 };
13619 if let Some(reason) = retention_reason {
13620 state.event(
13621 "fold",
13622 format!(
13623 "operator CLI --override-recovery-retention authorized deletion for run {}: bypassed {reason}",
13624 state.id
13625 ),
13626 );
13627 state.save_under(home)?;
13628 }
13629 let root = state.worktree_root();
13630 let winner = state.tally.as_ref().map(|t| t.winner);
13631 let mut removed = Vec::new();
13632
13633 for i in 0..state.candidates.len() {
13634 let c = state.candidates[i].clone();
13635 let is_winner = Some(c.label) == winner;
13636 if is_winner && !drop_winner {
13637 continue;
13638 }
13639 let existed = c.worktree.exists();
13640 remove_preserved_worktree(&repo, &c.worktree).await?;
13643 if existed {
13644 removed.push(c.worktree.to_string_lossy().into_owned());
13645 }
13646 let handed_over = state.released_branches.contains(&c.branch);
13649 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13650 if !git::branch_delete(&repo, &c.branch).await? {
13651 bail!("candidate branch removal refused");
13652 }
13653 removed.push(c.branch.clone());
13654 }
13655 state.candidates[i].folded = true;
13656 }
13657
13658 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13659 let path = name.path();
13660 let keep = !drop_winner
13661 && winner.is_some_and(|w| {
13662 path.file_name()
13663 .is_some_and(|n| n == format!("cand-{w}").as_str())
13664 });
13665 if keep {
13666 continue;
13667 }
13668 remove_read_seat_worktree(state, &path).await?;
13669 removed.push(path.to_string_lossy().into_owned());
13670 }
13671
13672 remove_if_empty(&root);
13681
13682 if state.enabled_worktree_config && drop_winner {
13683 git::release_worktree_config(&repo).await.ok();
13687 state.enabled_worktree_config = false;
13688 }
13689 state.save_under(home)?;
13690 Ok(removed)
13691}
13692
13693async fn verify_merged_fold_sources(
13694 state: &RunState,
13695 drop_winner: bool,
13696 home: &Path,
13697) -> Result<Vec<std::fs::File>> {
13698 let winner = state.tally.as_ref().map(|t| t.winner);
13699 let mut heads = BTreeSet::new();
13700 for candidate in &state.candidates {
13701 if Some(candidate.label) == winner && !drop_winner {
13702 continue;
13703 }
13704 anyhow::ensure!(
13705 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13706 "candidate recovery source is outside the inventoried run root: {}",
13707 candidate.worktree.display()
13708 );
13709 match std::fs::symlink_metadata(&candidate.worktree) {
13710 Ok(_) => {}
13711 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13712 Err(error) => return Err(error.into()),
13713 }
13714 if !state.released_branches.contains(&candidate.branch)
13715 && git::branch_exists(&state.repo, &candidate.branch).await?
13716 {
13717 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13718 }
13719 }
13720 let mut cargo_guards = Vec::new();
13721 let entries = match std::fs::read_dir(state.worktree_root()) {
13722 Ok(entries) => Some(entries),
13723 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13724 Err(error) => return Err(error.into()),
13725 };
13726 for entry in entries.into_iter().flatten() {
13727 let path = entry?.path();
13728 let metadata = std::fs::symlink_metadata(&path)?;
13729 anyhow::ensure!(
13730 metadata.is_dir() && !metadata.file_type().is_symlink(),
13731 "non-directory or symlink recovery source retained: {}",
13732 path.display()
13733 );
13734 if !drop_winner
13735 && winner.is_some_and(|w| {
13736 path.file_name()
13737 .is_some_and(|name| name == format!("cand-{w}").as_str())
13738 })
13739 {
13740 continue;
13741 }
13742 anyhow::ensure!(
13743 path.join(".git").is_file(),
13744 "unregistered recovery source retained: {}",
13745 path.display()
13746 );
13747 anyhow::ensure!(
13748 git::git(
13749 &path,
13750 &[
13751 "status",
13752 "--porcelain",
13753 "--untracked-files=all",
13754 "--ignored",
13755 "--",
13756 ".",
13757 ":(exclude)target"
13758 ]
13759 )
13760 .await?
13761 .is_empty(),
13762 "source changes or ignored recovery files retained: {}",
13763 path.display()
13764 );
13765 match std::fs::symlink_metadata(path.join("target")) {
13766 Ok(_) => {
13767 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13768 }
13769 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13770 Err(error) => return Err(error.into()),
13771 }
13772 for checkpoint in [
13773 "rebase-merge",
13774 "rebase-apply",
13775 "sequencer",
13776 "CHERRY_PICK_HEAD",
13777 "MERGE_HEAD",
13778 "REBASE_HEAD",
13779 "REVERT_HEAD",
13780 "BISECT_LOG",
13781 "AUTO_MERGE",
13782 "MERGE_MSG",
13783 ] {
13784 let location = git::git(
13785 &path,
13786 &[
13787 "rev-parse",
13788 "--path-format=absolute",
13789 "--git-path",
13790 checkpoint,
13791 ],
13792 )
13793 .await?;
13794 anyhow::ensure!(
13795 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13796 if e.kind() == std::io::ErrorKind::NotFound),
13797 "Git recovery checkpoint retained: {checkpoint}"
13798 );
13799 }
13800 heads.insert(git::rev_parse(&path, "HEAD").await?);
13801 }
13802 crate::fold_evidence::preserve(state, home, &heads).await?;
13803
13804 Ok(cargo_guards)
13805}
13806
13807fn remove_if_empty(dir: &Path) {
13818 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13819 std::fs::remove_dir(dir).ok();
13820 }
13821}
13822
13823pub fn worst_open(state: &RunState) -> Option<Severity> {
13825 state
13826 .reviews
13827 .last()?
13828 .reviews
13829 .iter()
13830 .flat_map(|r| r.findings.iter())
13831 .map(|f| f.severity)
13832 .max()
13833}
13834
13835async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13839 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13840 git::acquire_worktree_config(repo).await?;
13841 state.enabled_worktree_config = true;
13842 }
13843 Ok(())
13844}
13845
13846#[cfg(test)]
13847mod tests {
13848 #[tokio::test]
13849 async fn launcher_base_fetch_requires_remote_success() {
13850 let tmp = tempfile::tempdir().unwrap();
13851 let repo = tmp.path().join("repo");
13852 let remote = tmp.path().join("remote");
13853 std::fs::create_dir(&repo).unwrap();
13854 std::fs::create_dir(&remote).unwrap();
13855 git::git(&remote, &["init", "--bare", "-b", "main"])
13856 .await
13857 .unwrap();
13858 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13859 git::git(
13860 &repo,
13861 &[
13862 "-c",
13863 "user.name=Fixture",
13864 "-c",
13865 "user.email=fixture@example.invalid",
13866 "commit",
13867 "--allow-empty",
13868 "-m",
13869 "fixture",
13870 ],
13871 )
13872 .await
13873 .unwrap();
13874 git::git(
13875 &repo,
13876 &["remote", "add", "origin", remote.to_str().unwrap()],
13877 )
13878 .await
13879 .unwrap();
13880 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13881 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13882 assert_eq!(
13883 resolve_base(&repo, "main", "origin", None)
13884 .await
13885 .unwrap()
13886 .commit,
13887 head
13888 );
13889 #[cfg(unix)]
13890 {
13891 let policy: crate::config::Coordination = toml::from_str(
13892 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13893 ).unwrap();
13894 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13895 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13896 let held = crate::repo_lock::SeatLease::operation_with(
13897 &repo,
13898 Some(&policy),
13899 &crate::repo_lock::WaitPolicy::default(),
13900 )
13901 .await
13902 .unwrap();
13903 assert!(held.is_some());
13904 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13905 tokio::pin!(fetch);
13906 assert!(
13907 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13908 .await
13909 .is_err()
13910 );
13911 assert!(
13912 !repo.join(".git/FETCH_HEAD").exists(),
13913 "no fetch before admission"
13914 );
13915 drop(held);
13916 let fetched = fetch.await.unwrap();
13917 assert_eq!(fetched.commit, head);
13918 assert_eq!(fetched.remote, "origin");
13919 assert_eq!(fetched.branch, "main");
13920 }
13921 git::git(
13923 &repo,
13924 &[
13925 "remote",
13926 "set-url",
13927 "origin",
13928 tmp.path().join("missing").to_str().unwrap(),
13929 ],
13930 )
13931 .await
13932 .unwrap();
13933 let error = resolve_base(&repo, "main", "origin", None)
13934 .await
13935 .unwrap_err()
13936 .to_string();
13937 assert!(error.contains("cannot read origin/main"));
13938 assert!(error.contains("refusing to branch off"));
13939 }
13940
13941 #[test]
13942 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13943 let leaked = vec!["chore(deps): update a crate".to_owned()];
13944 let own = "fix(daemon): apply a chosen action";
13945 assert!(should_retitle("", own, &leaked));
13946 assert!(should_retitle(
13947 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13948 own,
13949 &leaked
13950 ));
13951 assert!(should_retitle(
13952 "chore: land candidate A of run 1",
13953 own,
13954 &leaked
13955 ));
13956 assert!(should_retitle(
13957 "magi: candidate A (uncommitted work)",
13958 own,
13959 &leaked
13960 ));
13961 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13962 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13963 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13964 }
13965
13966 #[test]
13967 fn pr_merge_plan_creates_adopts_or_stops() {
13968 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13969 assert_eq!(
13970 pr_merge_plan(Ok(land::OpenPr::One {
13971 url: "u".into(),
13972 title: "t".into()
13973 })),
13974 PrPlan::Adopt {
13975 url: "u".into(),
13976 title: "t".into()
13977 }
13978 );
13979 let PrPlan::Stop(many) =
13980 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13981 else {
13982 panic!("many must stop");
13983 };
13984 assert!(many.contains('a') && many.contains('b'));
13985 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13986 panic!("a failed lookup must stop");
13987 };
13988 assert!(err.contains("bad token"));
13989 }
13990
13991 use super::*;
13992 use crate::run::GateStatus;
13993 use std::collections::BTreeMap;
13994 use std::time::Duration;
13995
13996 fn conductor() -> AgentSpec {
13997 AgentSpec {
13998 id: "conductor".to_owned(),
13999 kind: crate::config::AgentKind::Command,
14000 model: None,
14001 command: vec!["true".to_owned()],
14002 extra_args: Vec::new(),
14003 env: BTreeMap::new(),
14004 prompt_delivery: None,
14005 max_turn_secs: None,
14006 }
14007 }
14008
14009 fn spec(id: &str) -> AgentSpec {
14010 AgentSpec {
14011 id: id.to_owned(),
14012 kind: crate::config::AgentKind::Command,
14013 model: None,
14014 command: vec!["true".to_owned()],
14015 extra_args: Vec::new(),
14016 env: BTreeMap::new(),
14017 prompt_delivery: None,
14018 max_turn_secs: None,
14019 }
14020 }
14021
14022 fn ids(xs: &[&str]) -> BTreeSet<String> {
14023 xs.iter().map(|s| (*s).to_owned()).collect()
14024 }
14025
14026 #[test]
14027 fn pick_successor_skips_an_agent_another_seat_holds() {
14028 let roster = [spec("a"), spec("b"), spec("c")];
14030 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14031 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14032 }
14033
14034 #[test]
14035 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
14036 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
14038 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14039 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14040 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
14041 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
14042 }
14043
14044 #[test]
14045 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
14046 let roster = [spec("a"), spec("b")];
14047 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14048 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14049 let carried = ids(&["b"]);
14050 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
14051 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14052 }
14053
14054 #[test]
14055 fn pick_successor_returns_none_without_any_untried_successor() {
14056 let roster = [spec("a"), spec("b")];
14057 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
14058 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
14059 let carried = ids(&["a"]);
14060 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
14061 }
14062
14063 #[test]
14064 fn pick_successor_rescue_avoids_another_seats_occupant() {
14065 let roster = [spec("a"), spec("b"), spec("c")];
14067 let carried = ids(&["b", "c"]);
14068 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
14069 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14070 }
14071
14072 #[test]
14073 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
14074 let roster = [spec("a"), spec("b"), spec("c")];
14075 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
14076 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14077 }
14078
14079 #[test]
14080 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
14081 let roster = [spec("a"), spec("b"), spec("c")];
14082 let failed = ids(&["a", "b", "c"]);
14083 let mut tried = ids(&["b"]);
14085 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
14086 assert_eq!(first.id, "a");
14087 tried.insert(first.id.clone());
14088 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
14089 assert_eq!(second.id, "c");
14090 tried.insert(second.id.clone());
14091 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
14092 }
14093
14094 #[test]
14095 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
14096 let roster = [spec("a"), spec("b")];
14097 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
14098 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14099 }
14100
14101 #[test]
14102 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
14103 let roster = [spec("a"), spec("b"), spec("c")];
14104 let h = SeatHistory {
14105 failed: ids(&["a"]),
14106 last_ok: Some("b".to_owned()),
14107 last_fail: None,
14108 };
14109 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
14110 let gone = SeatHistory {
14112 last_ok: Some("zzz".to_owned()),
14113 ..h.clone()
14114 };
14115 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
14116 let failed = SeatHistory {
14117 failed: ids(&["a", "b"]),
14118 last_ok: Some("b".to_owned()),
14119 last_fail: None,
14120 };
14121 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
14122 }
14123
14124 #[test]
14125 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
14126 let first = SeatState::new("review-1", "alpha", 7);
14127 let next = handover_seat("review-1", "gamma", 7);
14128 assert_ne!(first.claude_session, next.claude_session);
14129 }
14130
14131 #[test]
14132 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
14133 let mut state = state_with_summary("x", "y");
14134 let a = handover_seat("review-1", "beta", state.next_seat_seed());
14135 let b = handover_seat("review-1", "beta", state.next_seat_seed());
14136 assert_ne!(a.claude_session, b.claude_session);
14137 }
14138
14139 #[test]
14140 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
14141 let roster = [spec("a"), spec("b")];
14142 let h = SeatHistory {
14143 failed: ids(&["a", "b"]),
14144 ..SeatHistory::default()
14145 };
14146 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
14147 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
14148 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
14149 }
14150
14151 #[test]
14158 fn failure_signature_ignores_numbers_and_paths() {
14159 assert_eq!(
14160 failure_signature("exited with Some(2) and no usable output"),
14161 failure_signature("exited with Some(137) and no usable output")
14162 );
14163 assert_eq!(
14164 failure_signature("process exited with no usable output (exit code 2)"),
14165 failure_signature("process exited with no usable output (exit code 137)")
14166 );
14167 for (old, new) in [
14168 (
14169 "exited with Some(2) and no usable output",
14170 "process exited with no usable output (exit code 2)",
14171 ),
14172 (
14173 "exited with None and no usable output",
14174 "process exited with no usable output (exit code unavailable)",
14175 ),
14176 ] {
14177 let previous = FailClass::Other(failure_signature(old));
14178 let current = FailClass::Other(failure_signature(new));
14179 assert_eq!(previous, current, "resumption retains the failure class");
14180 assert!(!should_hand_over(Some(&previous), ¤t));
14181 }
14182 assert_eq!(
14183 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
14184 failure_signature("cannot open /var/x.txt: denied")
14185 );
14186 assert_ne!(failure_signature("boom"), failure_signature("bang"));
14187 }
14188
14189 #[test]
14190 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
14191 let other = FailClass::Other("x".into());
14192 assert!(should_hand_over(None, &FailClass::Quota));
14193 assert!(should_hand_over(Some(&other), &FailClass::Quota));
14194 assert!(should_hand_over(
14195 Some(&FailClass::Timeout),
14196 &FailClass::Timeout
14197 ));
14198 assert!(should_hand_over(None, &other));
14199 assert!(!should_hand_over(Some(&other), &other));
14200 assert!(should_hand_over(
14201 Some(&other),
14202 &FailClass::Other("y".into())
14203 ));
14204 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14206 assert!(should_hand_over(Some(&FailClass::Quota), &other));
14207 }
14208
14209 #[test]
14210 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14211 let a = SeatState::new("judge-1", "alpha", 7);
14212 let b = handover_seat("judge-1", "beta", 7);
14213 assert_ne!(a.claude_session, b.claude_session);
14214 assert_eq!(b.turns, 0);
14215 }
14216
14217 #[test]
14218 fn a_timeout_is_classified_apart_from_other_failures() {
14219 assert_eq!(
14220 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14221 Some(FailClass::Timeout)
14222 );
14223 assert!(matches!(
14224 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14225 Some(FailClass::Other(_))
14226 ));
14227 }
14228
14229 #[test]
14230 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14231 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14232 let tried = BTreeSet::from(["beta".to_owned()]);
14233 let next = next_untried_in_roster(&roster, 1, &tried);
14236 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14237 }
14238
14239 #[test]
14240 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14241 let roster = vec![spec("alpha"), spec("beta")];
14242 let tried = BTreeSet::from(["beta".to_owned()]);
14243 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14247 }
14248
14249 #[test]
14250 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14251 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14252 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14253 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14257 }
14258
14259 #[test]
14260 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14261 let roster = vec![spec("a"), spec("a"), spec("b")];
14262 let tried = BTreeSet::from(["a".to_owned()]);
14263 let next = next_untried_in_roster(&roster, 0, &tried);
14264 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14265 }
14266
14267 #[test]
14268 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14269 let roster = vec![spec("a"), spec("b")];
14270 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14271 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14272 }
14273
14274 #[test]
14275 fn remove_if_empty_only_ever_takes_a_bare_directory() {
14276 let dir = tempfile::tempdir().unwrap();
14277 let bay = dir.path().join("ffff");
14278
14279 remove_if_empty(&bay);
14281 assert!(!bay.exists());
14282
14283 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14286 remove_if_empty(&bay);
14287 assert!(bay.exists(), "non-empty directory must survive");
14288
14289 std::fs::remove_dir(bay.join("cand-A")).unwrap();
14291 remove_if_empty(&bay);
14292 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14293 }
14294
14295 #[test]
14304 fn a_full_panel_that_found_nothing_is_clean() {
14305 assert!(round_is_clean(
14306 0,
14307 true,
14308 2,
14309 2,
14310 0,
14311 IncompleteReviewPolicy::Block
14312 ));
14313 }
14314
14315 #[test]
14316 fn a_missing_seat_is_never_clean_under_the_default_policy() {
14317 assert!(!round_is_clean(
14318 0,
14319 true,
14320 1,
14321 2,
14322 0,
14323 IncompleteReviewPolicy::Block
14324 ));
14325 }
14326
14327 #[test]
14328 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14329 assert!(!round_is_clean(
14330 1,
14331 true,
14332 1,
14333 2,
14334 0,
14335 IncompleteReviewPolicy::Warn
14336 ));
14337 }
14338
14339 #[test]
14340 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14341 assert!(round_is_clean(
14342 0,
14343 true,
14344 1,
14345 2,
14346 0,
14347 IncompleteReviewPolicy::Warn
14348 ));
14349 }
14350
14351 #[test]
14352 fn a_full_panel_with_an_open_finding_is_not_clean() {
14353 assert!(!round_is_clean(
14354 1,
14355 true,
14356 2,
14357 2,
14358 0,
14359 IncompleteReviewPolicy::Block
14360 ));
14361 }
14362
14363 #[test]
14364 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14365 assert!(!round_is_clean(
14366 0,
14367 false,
14368 2,
14369 2,
14370 0,
14371 IncompleteReviewPolicy::Block
14372 ));
14373 }
14374
14375 #[test]
14382 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14383 assert!(round_is_clean(
14386 0,
14387 true,
14388 1,
14389 2,
14390 1,
14391 IncompleteReviewPolicy::Block
14392 ));
14393 }
14394
14395 #[test]
14396 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14397 assert!(!round_is_clean(
14400 0,
14401 true,
14402 1,
14403 2,
14404 0,
14405 IncompleteReviewPolicy::Block
14406 ));
14407 }
14408
14409 #[test]
14410 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14411 assert!(!round_is_clean(
14412 1,
14413 true,
14414 1,
14415 2,
14416 1,
14417 IncompleteReviewPolicy::Block
14418 ));
14419 assert!(!round_is_clean(
14420 0,
14421 false,
14422 1,
14423 2,
14424 1,
14425 IncompleteReviewPolicy::Block
14426 ));
14427 }
14428
14429 #[test]
14430 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14431 assert!(!round_is_clean(
14435 0,
14436 true,
14437 0,
14438 2,
14439 2,
14440 IncompleteReviewPolicy::Block
14441 ));
14442 }
14443
14444 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14445 CommandOutcome {
14446 command: "test".to_owned(),
14447 code,
14448 output_tail: String::new(),
14449 duration_ms: 0,
14450 resource_blocked,
14451 }
14452 }
14453
14454 #[cfg(unix)]
14455 #[tokio::test]
14456 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14457 ask_test_home();
14458 let temp = tempfile::tempdir().unwrap();
14459 let mut runner = runner_at(RunStatus::Gating);
14460 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14461 let commands = vec![
14462 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14463 "touch must-not-run".to_owned(),
14464 ];
14465 let (outcomes, retried, pids) = run_e2e_with_retry(
14466 &mut runner.state,
14467 &shell,
14468 &commands,
14469 temp.path(),
14470 Duration::from_secs(5),
14471 "fixture",
14472 None,
14473 )
14474 .await;
14475 assert_eq!(outcomes.len(), 1);
14476 assert!(outcomes[0].resource_blocked);
14477 assert!(!outcomes[0].build_failed());
14478 assert!(!retried);
14479 assert!(pids.is_empty());
14480 assert!(!temp.path().join("must-not-run").exists());
14481 assert_eq!(
14482 runner.state.resource_block.as_ref().unwrap().kind,
14483 "incremental cache admission"
14484 );
14485 }
14486
14487 #[test]
14488 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14489 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14490 assert!(
14491 !verify_inconclusive(&[outcome(Some(1), false)]),
14492 "an ordinary failure is still evidence about the patch"
14493 );
14494 assert!(verify_inconclusive(&[outcome(None, true)]));
14495 assert!(
14496 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14497 "one inconclusive outcome taints the whole batch"
14498 );
14499 assert!(!verify_inconclusive(&[]));
14500 }
14501
14502 #[tokio::test]
14503 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14504 let calls = std::sync::atomic::AtomicUsize::new(0);
14508 let started = Instant::now();
14509 wait_for_pids_with(
14510 &[123],
14511 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14512 Duration::from_millis(5),
14513 Duration::from_secs(5),
14514 )
14515 .await;
14516 assert!(
14517 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14518 "must keep checking rather than deciding on the first answer"
14519 );
14520 assert!(
14521 started.elapsed() < Duration::from_secs(1),
14522 "must return the moment it is confirmed dead, not wait out the ceiling"
14523 );
14524 }
14525
14526 #[tokio::test]
14527 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14528 let started = Instant::now();
14529 wait_for_pids_with(
14530 &[123],
14531 |_| true, Duration::from_millis(5),
14533 Duration::from_millis(30),
14534 )
14535 .await;
14536 let elapsed = started.elapsed();
14537 assert!(
14538 elapsed >= Duration::from_millis(30),
14539 "must not give up before its own ceiling: {elapsed:?}"
14540 );
14541 assert!(
14542 elapsed < Duration::from_secs(1),
14543 "must not wait past its own ceiling either: {elapsed:?}"
14544 );
14545 }
14546
14547 #[tokio::test]
14548 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14549 let started = Instant::now();
14550 wait_for_pids_with(
14551 &[],
14552 |_| true,
14553 Duration::from_secs(5),
14554 Duration::from_secs(5),
14555 )
14556 .await;
14557 assert!(
14558 started.elapsed() < Duration::from_millis(200),
14559 "an empty pid list has nothing to confirm"
14560 );
14561 }
14562
14563 fn review_round(
14569 clean: bool,
14570 blocking: usize,
14571 answered: usize,
14572 expected: usize,
14573 progressed: bool,
14574 e2e_ok: bool,
14575 ) -> ReviewRound {
14576 ReviewRound {
14577 round: 1,
14578 head: "h".to_owned(),
14579 verified_head: None,
14580 verified_at: None,
14581 reviews: Vec::new(),
14582 e2e: vec![CommandOutcome {
14583 command: "test".to_owned(),
14584 code: Some(if e2e_ok { 0 } else { 1 }),
14585 output_tail: String::new(),
14586 duration_ms: 0,
14587 resource_blocked: false,
14588 }],
14589 verify_retried: false,
14590 e2e_deferred: false,
14591 e2e_defer_reason: None,
14592 fix: None,
14593 blocking,
14594 answered,
14595 expected,
14596 clean,
14597 progressed,
14598 vote_split: false,
14599 reconsideration: Vec::new(),
14600 verdict: None,
14601 }
14602 }
14603
14604 #[test]
14605 fn review_conclusion_is_none_when_nothing_has_run() {
14606 assert_eq!(review_conclusion(&[], 3), None);
14607 }
14608
14609 #[test]
14610 fn review_conclusion_is_none_while_rounds_remain() {
14611 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14612 assert_eq!(review_conclusion(&rounds, 3), None);
14613 }
14614
14615 #[test]
14616 fn review_conclusion_is_gating_once_a_round_is_clean() {
14617 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14618 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14619 }
14620
14621 #[test]
14622 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14623 let rounds = vec![
14624 review_round(false, 1, 2, 2, true, true),
14625 review_round(false, 1, 2, 2, true, true),
14626 ];
14627 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14628 }
14629
14630 #[test]
14631 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14632 let rounds = vec![
14633 review_round(false, 1, 2, 2, true, true),
14634 review_round(false, 1, 2, 2, true, false),
14635 ];
14636 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14637 }
14638
14639 #[test]
14640 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14641 let mut blocked = review_round(false, 1, 2, 2, true, false);
14648 blocked.e2e[0].resource_blocked = true;
14649 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14650 assert_eq!(review_conclusion(&rounds, 2), None);
14651 }
14652
14653 #[test]
14654 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14655 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14657 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14658 }
14659
14660 #[test]
14661 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14662 let rounds = vec![
14663 review_round(false, 1, 2, 2, false, true),
14664 review_round(false, 1, 2, 2, false, true),
14665 ];
14666 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14667 }
14668
14669 fn secs(n: u64) -> Duration {
14670 Duration::from_secs(n)
14671 }
14672
14673 #[tokio::test]
14676 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14677 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14678 if std::env::var_os(CHILD).is_none() {
14679 let output = std::process::Command::new(std::env::current_exe().unwrap())
14683 .args([
14684 "--exact",
14685 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14686 "--nocapture",
14687 ])
14688 .env(CHILD, "1")
14689 .env_remove("MAGI_WORKER_HOME")
14690 .env_remove("MAGI_WORKER_ID")
14691 .output()
14692 .unwrap();
14693 assert!(
14694 output.status.success()
14695 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14696 "{}\n{}",
14697 String::from_utf8_lossy(&output.stdout),
14698 String::from_utf8_lossy(&output.stderr)
14699 );
14700 return;
14701 }
14702 ask_test_home();
14703 let tmp = tempfile::tempdir().unwrap();
14704 let repo = tmp.path().join("repo");
14705 std::fs::create_dir_all(&repo).unwrap();
14706 init_repo(&repo);
14707 let g = |args: &[&str]| {
14708 let out = std::process::Command::new("git")
14709 .args(args)
14710 .current_dir(&repo)
14711 .quiet()
14712 .output()
14713 .unwrap();
14714 assert!(
14715 out.status.success(),
14716 "{}",
14717 String::from_utf8_lossy(&out.stderr)
14718 );
14719 };
14720 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14721 g(&["checkout", "-b", "historical"]);
14722 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14723 g(&["add", "."]);
14724 g(&["commit", "-m", "old alternative"]);
14725 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14726 g(&["checkout", "-b", "magi/task-selected", &base]);
14727 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14728 g(&["add", "."]);
14729 g(&["commit", "-m", "measurement infrastructure"]);
14730 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14731 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14732 g(&["add", "."]);
14733 g(&["commit", "-m", "dependent implementation"]);
14734 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14735 g(&["checkout", "main"]);
14736 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14737 g(&["add", "."]);
14738 g(&["commit", "-m", "unrelated base change"]);
14739 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14740 let origin = tmp.path().join("origin.git");
14741 g(&[
14742 "clone",
14743 "--bare",
14744 repo.to_str().unwrap(),
14745 origin.to_str().unwrap(),
14746 ]);
14747 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14748 g(&["fetch", "origin"]);
14749 let selection = crate::pr_recovery::Selection {
14750 url: "https://example.test/project/pull/1".into(),
14751 branch: "magi/task-selected".into(),
14752 head: head.clone(),
14753 source_head: None,
14754 source_run: None,
14755 source_proof: None,
14756 disposition: Some(crate::pr_recovery::Disposition::Review),
14757 instruction: "preserve measured behavior".into(),
14758 approved_at: jiff::Timestamp::now(),
14759 };
14760 let mut template = runner_at(RunStatus::Reviewing);
14761 template.state.repo = repo.clone();
14762 template.state.base_commit = current.clone();
14763 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14764 template.state.origin = Some(crate::run::Origin::queue("selected"));
14765 template.state.pr_head = Some(selection.branch.clone());
14766 template.state.pr_recovery = Some(selection.clone());
14767 let mut runner =
14768 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14769 .await
14770 .unwrap();
14771 let wt = runner.state.winner().unwrap().worktree.clone();
14772 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14773 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14774 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14775 assert!(runner.state.seeds.is_empty());
14776 assert_eq!(runner.state.instruction, "preserve measured behavior");
14777 runner.sync_to_base().await.unwrap();
14778 assert_eq!(
14779 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14780 "prerequisite"
14781 );
14782 assert_eq!(
14783 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14784 "measured implementation"
14785 );
14786 assert_eq!(
14787 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14788 "current main change"
14789 );
14790 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14791 assert_eq!(
14792 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14793 head
14794 );
14795
14796 let mut partial = runner_at(RunStatus::Reviewing);
14798 partial.state.repo = repo.clone();
14799 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14800 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14801 run: Some("exit 1".into()),
14802 ..Default::default()
14803 }];
14804 let selection = runner.state.pr_recovery.clone().unwrap();
14805 partial.state.pr_recovery = Some(selection.clone());
14806 partial.state.pr_head = Some(selection.branch.clone());
14807 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14808 let id = partial.state.id.clone();
14809 let error = match Runner::open_selected(
14810 &repo,
14811 partial.state,
14812 partial.roles,
14813 1,
14814 runner.state.base_commit.clone(),
14815 selection,
14816 )
14817 .await
14818 {
14819 Ok(_) => panic!("setup must fail"),
14820 Err(e) => e,
14821 };
14822 assert_eq!(
14823 error
14824 .downcast_ref::<crate::run::StartupFailure>()
14825 .unwrap()
14826 .run,
14827 id
14828 );
14829 let failed = RunState::load(&id).unwrap();
14830 assert_eq!(failed.status, RunStatus::Failed);
14831 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14832 assert!(Runner::resume(&id).is_err());
14833 runner.state.status = RunStatus::Ready;
14834 runner.state.driver_exited = true;
14835 runner.state.save().unwrap();
14836 let mut retry = runner_at(RunStatus::Reviewing);
14837 retry.state.repo = repo.clone();
14838 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14839 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14840 reselected.approved_at = jiff::Timestamp::now();
14841 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14842 retry.state.pr_head = Some(reselected.branch.clone());
14843 retry.state.pr_recovery = Some(reselected.clone());
14844 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14845 .await
14846 .unwrap();
14847 let restarted = Runner::open_selected(
14848 &repo,
14849 retry.state,
14850 retry.roles,
14851 1,
14852 runner.state.base_commit.clone(),
14853 reselected,
14854 )
14855 .await
14856 .unwrap();
14857 assert_eq!(
14858 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14859 .await
14860 .unwrap(),
14861 head
14862 );
14863 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14864 assert_eq!(
14865 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14866 head
14867 );
14868 }
14869
14870 #[tokio::test]
14871 #[cfg(unix)]
14872 async fn fallback_and_interrupted_restoration_require_coordination() {
14873 use std::os::fd::AsRawFd as _;
14874 ask_test_home();
14875 let tmp = tempfile::tempdir().unwrap();
14876 let mut runner = registered_runner(tmp.path()).await;
14877 runner.state.config.graph.timeout_resource_wait = 1;
14878 let winner = runner.state.candidates[0].clone();
14879 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14880 let leftover = winner.worktree.join("leftover");
14881 std::fs::write(&leftover, "untrusted edits").unwrap();
14882 let next = tmp.path().join("next-fixer");
14883 let mut failed = spec("first");
14884 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14885 let mut successor = spec("second");
14886 successor.command = vec![
14887 "sh".into(),
14888 "-c".into(),
14889 format!("touch '{}'; echo done", next.display()),
14890 ];
14891 let attempts = vec![
14892 (failed.clone(), "first".into()),
14893 (successor.clone(), "second".into()),
14894 ];
14895 let lock_path = runner.state.repo.join(".git/updater.lock");
14896 let external = std::fs::File::create(&lock_path).unwrap();
14897 assert_eq!(
14898 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14899 0
14900 );
14901 let artifacts = tmp.path().join("artifacts");
14902 let build = |spec, seat| SeatJob {
14903 spec,
14904 seat,
14905 cwd: winner.worktree.clone(),
14906 prompt: "fixture".into(),
14907 timeout: Duration::from_secs(10),
14908 allow_write: true,
14909 sessions: false,
14910 artifacts: artifacts.clone(),
14911 stem: "fix".into(),
14912 handover: None,
14913 };
14914 let (_, _, out) = runner
14915 .ask_fixer_among(
14916 attempts.clone(),
14917 "graft",
14918 None,
14919 Some((&winner.worktree, &head)),
14920 build,
14921 )
14922 .await;
14923 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14924 assert!(leftover.exists(), "contention must prevent clean");
14925 assert!(!next.exists());
14926 assert!(runner.state.handovers.is_empty());
14927 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14929 from_run: "earlier".into(),
14930 generation: 1,
14931 mode: crate::recovery::Mode::VerifyRepair,
14932 reasons: Vec::new(),
14933 reviewed_head: None,
14934 rounds: Vec::new(),
14935 open_findings: Vec::new(),
14936 evidence: None,
14937 criteria_inherited: true,
14938 repair: crate::recovery::Repair::Started {
14939 before: head.clone(),
14940 },
14941 });
14942 assert!(
14943 runner
14944 .recovery_repair(&winner)
14945 .await
14946 .unwrap_err()
14947 .is::<crate::repo_lock::ResourceBlocked>()
14948 );
14949 assert!(matches!(
14950 runner.state.recovery.as_ref().unwrap().repair,
14951 crate::recovery::Repair::Started { .. }
14952 ));
14953 assert!(leftover.exists());
14954 drop(external);
14955 runner.recovery_repair(&winner).await.unwrap();
14956 assert!(!leftover.exists());
14957 assert!(matches!(
14958 runner.state.recovery.as_ref().unwrap().repair,
14959 crate::recovery::Repair::Done { .. }
14960 ));
14961 std::fs::remove_file(&lock_path).unwrap();
14963 std::fs::create_dir(&lock_path).unwrap();
14964 std::fs::write(&leftover, "untrusted edits").unwrap();
14965 let (_, _, out) = runner
14966 .ask_fixer_among(
14967 attempts,
14968 "graft",
14969 None,
14970 Some((&winner.worktree, &head)),
14971 build,
14972 )
14973 .await;
14974 assert!(matches!(out, AgentOutcome::Failed(_)));
14975 assert!(leftover.exists());
14976 assert!(!next.exists());
14977 assert_eq!(
14978 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14979 head
14980 );
14981 }
14982
14983 #[cfg(unix)]
14984 async fn registered_runner(root: &Path) -> Runner {
14985 repository_runner(root, crate::config::test_coordination()).await
14986 }
14987
14988 async fn repository_runner(
14989 root: &Path,
14990 coordination: Option<crate::config::Coordination>,
14991 ) -> Runner {
14992 ask_test_home();
14993 let repo = root.join("repo");
14994 std::fs::create_dir_all(&repo).unwrap();
14995 init_repo(&repo);
14996 if coordination.is_some() {
14997 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14998 }
14999 let mut runner = runner_at(RunStatus::Prep);
15000 runner.state.repo = repo.clone();
15001 runner.state.config.coordination = coordination;
15002 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15003 runner.state.config.graph.worktree_root = Some(root.join("wt"));
15004 runner.state.config.graph.advise = false;
15005 runner.roles.implementers = vec![spec("implement")];
15006 runner.roles.judges = vec![spec("judge")];
15007 runner.prep().await.unwrap();
15008 runner
15009 }
15010
15011 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
15012
15013 #[tokio::test]
15014 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
15015 let tmp = tempfile::tempdir().unwrap();
15016 let mut runner = repository_runner(tmp.path(), None).await;
15017 let prompts = runner.state.config.prompts.clone();
15018 let run = runner.state.id.clone();
15019 let ctx = WaveCtx {
15020 carry_seats: false,
15021 pauses: None,
15022 run: &run,
15023 node: "advise",
15024 prompts: &prompts,
15025 cache: None,
15026 round: None,
15027 };
15028 let jobs: Vec<_> = ["alpha", "beta"]
15029 .into_iter()
15030 .map(|id| SeatJob {
15031 spec: scripted(id, AUTH_OUTAGE),
15032 seat: SeatState::new(id, id, 7),
15033 prompt: "propose".into(),
15034 cwd: runner.state.candidates[0].worktree.clone(),
15035 timeout: Duration::from_secs(10),
15036 allow_write: false,
15037 sessions: false,
15038 artifacts: agent::artifacts_dir(&runner.state.dir()),
15039 stem: id.into(),
15040 handover: None,
15041 })
15042 .collect();
15043 let mut losses = Vec::new();
15044 let results = ask_wave_with(
15045 jobs.clone(),
15046 runner.sem.clone(),
15047 2,
15048 &[],
15049 &ctx,
15050 &mut losses,
15051 &mut runner.state,
15052 &|_: &str| Ok(()),
15053 )
15054 .await;
15055 assert!(
15056 results
15057 .iter()
15058 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
15059 );
15060 assert_eq!(runner.state.quota.len(), 2);
15061 assert!(
15062 runner
15063 .state
15064 .quota
15065 .iter()
15066 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
15067 );
15068 assert!(!runner.state.advise_attempted);
15069 runner.stop_for_resource().unwrap();
15070 assert_eq!(runner.state.status, RunStatus::Blocked);
15071 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
15072 assert_eq!(
15073 runner.state.resource_block.as_ref().unwrap().kind,
15074 "agent authentication"
15075 );
15076 runner.state.resource_block = None;
15079 runner.state.quota.clear();
15080 let mut mixed = jobs;
15081 mixed[1].spec.command = vec!["false".into()];
15082 let results = ask_wave_with(
15083 mixed,
15084 runner.sem.clone(),
15085 0,
15086 &[],
15087 &ctx,
15088 &mut losses,
15089 &mut runner.state,
15090 &|_: &str| Ok(()),
15091 )
15092 .await;
15093 assert!(results.iter().all(|(_, res, _)| res.is_err()));
15094 assert!(runner.state.resource_block.is_none());
15095 assert_eq!(losses.len(), 1);
15096 }
15097
15098 #[tokio::test]
15099 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
15100 let tmp = tempfile::tempdir().unwrap();
15101 let mut runner = repository_runner(tmp.path(), None).await;
15102 let implementer = scripted("implement", AUTH_OUTAGE);
15103 runner.roles.implementers = vec![implementer.clone()];
15104 runner.state.config.agents = vec![implementer];
15105 runner.implement().await.unwrap();
15106 assert_eq!(runner.state.status, RunStatus::Blocked);
15107 assert_eq!(runner.state.quota.len(), 1);
15108 assert_eq!(
15109 runner.state.quota[0].cause,
15110 agent::QuotaCause::Authentication
15111 );
15112 let candidate = &runner.state.candidates[0];
15113 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
15114 assert_eq!(runner.state.seats["impl-A"].turns, 1);
15115 }
15116
15117 #[tokio::test]
15118 #[cfg(unix)]
15119 async fn parked_implementer_persists_completed_work() {
15120 for registered in [false, true] {
15121 let tmp = tempfile::tempdir().unwrap();
15122 let mut runner = registered_runner(tmp.path()).await;
15123 if !registered {
15124 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
15125 }
15126 let marker = tmp.path().join("started");
15127 let release = tmp.path().join("release");
15128 let mut implementer = spec("implement");
15129 implementer.command = vec![
15130 "sh".into(),
15131 "-c".into(),
15132 format!(
15133 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
15134 marker.display(),
15135 release.display()
15136 ),
15137 ];
15138 runner.roles.implementers = vec![implementer.clone()];
15139 runner.state.config.agents = vec![implementer];
15140 let pause = runner.pause.clone();
15141 let (result, ()) = tokio::join!(runner.implement(), async {
15142 tokio::time::timeout(Duration::from_secs(10), async {
15143 while !marker.exists() {
15144 tokio::time::sleep(Duration::from_millis(10)).await;
15145 }
15146 })
15147 .await
15148 .unwrap();
15149 pause.park();
15150 std::fs::write(&release, "go").unwrap();
15151 });
15152 result.unwrap();
15153 let candidate = &runner.state.candidates[0];
15154 assert!(candidate.commits > 0, "{candidate:?}");
15155 assert!(candidate.summary.contains("saved-result"));
15156 assert_eq!(candidate.files, 1);
15157 let before = candidate.commits;
15158 runner.implement().await.unwrap();
15159 assert_eq!(runner.state.candidates[0].commits, before);
15160 }
15161 }
15162
15163 #[tokio::test]
15164 #[cfg(unix)]
15165 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
15166 let tmp = tempfile::tempdir().unwrap();
15167 let mut runner = registered_runner(tmp.path()).await;
15168 let winner = runner.state.candidates[0].clone();
15169 let cache = tmp.path().join("cache");
15170 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
15171 runner.state.config.graph.timeout_resource_wait = 0;
15172 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
15173 let crate::cache::AcquireOutcome::Acquired(held) =
15174 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
15175 else {
15176 panic!("cache must be free")
15177 };
15178 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15179 from_run: "earlier".into(),
15180 generation: 1,
15181 mode: crate::recovery::Mode::VerifyRepair,
15182 reasons: Vec::new(),
15183 reviewed_head: None,
15184 rounds: Vec::new(),
15185 open_findings: Vec::new(),
15186 evidence: None,
15187 criteria_inherited: true,
15188 repair: crate::recovery::Repair::NotStarted,
15189 });
15190 runner.recovery_repair(&winner).await.unwrap();
15191 assert!(matches!(
15192 runner.state.recovery.as_ref().unwrap().repair,
15193 crate::recovery::Repair::WaitingForRestore { .. }
15194 ));
15195 drop(held);
15196 runner.state.resource_block = None;
15197 runner.state.config.graph.timeout_resource_wait = 1;
15198 runner.recovery_repair(&winner).await.unwrap();
15199 assert!(matches!(
15200 runner.state.recovery.as_ref().unwrap().repair,
15201 crate::recovery::Repair::Done { .. }
15202 ));
15203 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15204 }
15205
15206 #[tokio::test]
15207 #[cfg(unix)]
15208 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15209 use std::os::fd::AsRawFd as _;
15210 for graft in [false, true] {
15211 let tmp = tempfile::tempdir().unwrap();
15212 let mut runner = registered_runner(tmp.path()).await;
15213 runner.state.config.graph.timeout_resource_wait = 1;
15214 let winner = runner.state.candidates[0].clone();
15215 let marker = tmp.path().join("fixer-started");
15216 let mut fixer = spec("fixer");
15217 fixer.command = vec![
15218 "sh".into(),
15219 "-c".into(),
15220 format!("echo started >> '{}'; echo done", marker.display()),
15221 ];
15222 runner.roles.fixer = Some(vec![fixer]);
15223 if graft {
15224 let mut loser = winner.clone();
15225 loser.label = 'B';
15226 loser.index = 1;
15227 runner.state.candidates.push(loser);
15228 runner.state.config.graph.graft_runner_up = true;
15229 runner.state.tally = Some(Tally {
15230 first_choice: BTreeMap::new(),
15231 borda: BTreeMap::new(),
15232 winner: winner.label,
15233 rankings: 1,
15234 unanimous_initial: true,
15235 deliberated: false,
15236 changed_votes: 0,
15237 unanimous_final: true,
15238 tie_break: None,
15239 judges: 1,
15240 present: 1,
15241 quorum: 1,
15242 met_quorum: true,
15243 uncontested: None,
15244 });
15245 runner.state.judgements.push(crate::run::Judgement {
15246 judge: 1,
15247 seat: "judge-1".into(),
15248 agent: "judge".into(),
15249 ranking: vec![winner.label, 'B'],
15250 reasons: BTreeMap::new(),
15251 confidence: None,
15252 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15253 order: vec![0, 1],
15254 failed: None,
15255 duration_ms: 0,
15256 });
15257 } else {
15258 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15259 from_run: "earlier".into(),
15260 generation: 1,
15261 mode: crate::recovery::Mode::VerifyRepair,
15262 reasons: Vec::new(),
15263 reviewed_head: None,
15264 rounds: Vec::new(),
15265 open_findings: Vec::new(),
15266 evidence: None,
15267 criteria_inherited: true,
15268 repair: crate::recovery::Repair::NotStarted,
15269 });
15270 }
15271 let reader = crate::repo_lock::SeatLease::for_checkout(
15272 &winner.worktree,
15273 false,
15274 crate::config::test_coordination().as_ref(),
15275 )
15276 .await
15277 .unwrap();
15278 let pause = runner.pause.clone();
15279 let (result, ()) = tokio::join!(
15280 async {
15281 if graft {
15282 runner.graft().await
15283 } else {
15284 runner.recovery_repair(&winner).await
15285 }
15286 },
15287 async {
15288 tokio::time::sleep(Duration::from_millis(100)).await;
15289 pause.park();
15290 }
15291 );
15292 result.unwrap();
15293 assert!(!marker.exists(), "a blocked seat must not spawn");
15294 runner.state = RunState::load(&runner.state.id).unwrap();
15296 if graft {
15297 assert_eq!(
15298 runner.state.graft.as_ref().unwrap().outcome,
15299 GraftOutcome::WaitingForRestore
15300 );
15301 } else {
15302 assert!(matches!(
15303 runner.state.recovery.as_ref().unwrap().repair,
15304 crate::recovery::Repair::WaitingForRestore { .. }
15305 ));
15306 }
15307 drop(reader);
15308 runner.pause = Pause::new();
15309 runner.state.resource_block = None;
15310 let external =
15312 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15313 assert_eq!(
15314 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15315 0
15316 );
15317 let leftover = winner.worktree.join("leftover");
15318 std::fs::write(&leftover, "earlier fallback edits").unwrap();
15319 let result = if graft {
15320 runner.graft().await
15321 } else {
15322 runner.recovery_repair(&winner).await
15323 };
15324 assert!(
15325 result
15326 .unwrap_err()
15327 .is::<crate::repo_lock::ResourceBlocked>()
15328 );
15329 assert!(leftover.exists());
15330 assert!(!marker.exists());
15331 drop(external);
15332 runner.state.resource_block = None;
15333 let result = if graft {
15334 runner.graft().await
15335 } else {
15336 runner.recovery_repair(&winner).await
15337 };
15338 result.unwrap();
15339 assert!(!leftover.exists(), "restore must precede retry");
15340 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15341 if graft {
15342 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15343 } else {
15344 assert!(matches!(
15345 runner.state.recovery.as_ref().unwrap().repair,
15346 crate::recovery::Repair::Done { .. }
15347 ));
15348 }
15349 }
15350 }
15351
15352 #[tokio::test]
15353 #[cfg(unix)]
15354 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15355 use std::os::fd::AsRawFd as _;
15356 let tmp = tempfile::tempdir().unwrap();
15357 let mut runner = registered_runner(tmp.path()).await;
15358 runner.state.config.graph.timeout_resource_wait = 1;
15359 let winner = runner.state.candidates[0].clone();
15360 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15361 runner.state.tally = Some(Tally {
15362 first_choice: BTreeMap::new(),
15363 borda: BTreeMap::new(),
15364 winner: winner.label,
15365 rankings: 0,
15366 unanimous_initial: false,
15367 deliberated: false,
15368 changed_votes: 0,
15369 unanimous_final: false,
15370 tie_break: None,
15371 judges: 0,
15372 present: 0,
15373 quorum: 0,
15374 met_quorum: true,
15375 uncontested: Some("fixture".into()),
15376 });
15377 runner.state.graft = Some(GraftRecord {
15378 pre: pre.clone(),
15379 post: None,
15380 lines: None,
15381 limit: None,
15382 agent: None,
15383 outcome: GraftOutcome::Running,
15384 requested: Vec::new(),
15385 adoption: None,
15386 adoption_error: None,
15387 });
15388 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15389 git::commit_all(&winner.worktree, "unfinished graft")
15390 .await
15391 .unwrap();
15392 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15393 let leftover = winner.worktree.join("leftover");
15394 std::fs::write(&leftover, "untrusted edits").unwrap();
15395 let path = runner.state.repo.join(".git/updater.lock");
15396 let external = std::fs::File::create(&path).unwrap();
15397 assert_eq!(
15398 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15399 0
15400 );
15401 assert!(
15402 runner
15403 .graft()
15404 .await
15405 .unwrap_err()
15406 .is::<crate::repo_lock::ResourceBlocked>()
15407 );
15408 assert!(matches!(
15409 runner.state.graft.as_ref().unwrap().outcome,
15410 GraftOutcome::Running
15411 ));
15412 assert_eq!(
15413 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15414 unfinished
15415 );
15416 assert!(leftover.exists());
15417 drop(external);
15418 std::fs::remove_file(&path).unwrap();
15419 std::fs::create_dir(&path).unwrap();
15420 assert!(runner.graft().await.is_err());
15421 assert!(matches!(
15422 runner.state.graft.as_ref().unwrap().outcome,
15423 GraftOutcome::Running
15424 ));
15425 assert_eq!(
15426 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15427 unfinished
15428 );
15429 assert!(leftover.exists());
15430 std::fs::remove_dir(&path).unwrap();
15431 runner.graft().await.unwrap();
15432 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15433 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15434 assert!(!leftover.exists());
15435 }
15436
15437 #[tokio::test]
15438 #[cfg(unix)]
15439 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15440 use std::os::fd::AsRawFd as _;
15441 let tmp = tempfile::tempdir().unwrap();
15442 let mut runner = registered_runner(tmp.path()).await;
15443 runner.state.prep_complete = Some(false);
15444 runner.state.config.graph.timeout_resource_wait = 1;
15445 runner.state.save().unwrap();
15446 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15447 assert_eq!(
15448 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15449 0
15450 );
15451 runner.execute().await.unwrap();
15452 let disk = RunState::load(&runner.state.id).unwrap();
15453 assert!(disk.driver_exited);
15454 assert_eq!(disk.driver_pid, None);
15455 assert_eq!(runner.state.status, RunStatus::Blocked);
15456 assert_eq!(
15457 runner.state.resource_block.as_ref().unwrap().kind,
15458 "repository coordination"
15459 );
15460 assert!(runner.state.prep_failure.is_none());
15461 assert!(runner.state.seats.is_empty());
15462 runner.pause.park();
15463 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15466 .await
15467 .unwrap()
15468 .unwrap();
15469 assert!(runner.state.parked);
15470 assert!(runner.state.prep_failure.is_none());
15471 let disk = RunState::load(&runner.state.id).unwrap();
15472 assert!(disk.driver_exited);
15473 assert_eq!(disk.driver_pid, None);
15474 }
15475
15476 #[tokio::test]
15477 #[cfg(unix)]
15478 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15479 use std::os::fd::AsRawFd as _;
15480 let tmp = tempfile::tempdir().unwrap();
15481 let mut runner = registered_runner(tmp.path()).await;
15482 runner.state.prep_complete = Some(false);
15483 runner.state.config.graph.timeout_resource_wait = 30;
15484 runner.state.save().unwrap();
15485 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15486 assert_eq!(
15487 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15488 0
15489 );
15490 assert!(
15491 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15492 .await
15493 .is_err()
15494 );
15495 let disk = RunState::load(&runner.state.id).unwrap();
15496 assert!(disk.driver_exited);
15497 assert_eq!(disk.driver_pid, None);
15498 assert!(disk.status.resumable());
15499 assert!(runner.state.driver_exited);
15500 assert_eq!(runner.state.driver_pid, None);
15501 runner.state.save().unwrap();
15502 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15503 }
15504
15505 #[tokio::test]
15506 #[cfg(unix)]
15507 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15508 let tmp = tempfile::tempdir().unwrap();
15509 let mut runner = registered_runner(tmp.path()).await;
15510 let winner = runner.state.candidates[0].clone();
15511 let cache = tmp.path().join("cache");
15512 let started = tmp.path().join("pre-gate-started");
15513 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15514 runner.state.config.verify.gate =
15515 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15516 let home = crate::run::home();
15517 let owner =
15518 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15519 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15520 crate::cache::AcquireOutcome::Acquired(g) => g,
15521 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15522 };
15523 let mut gate = Box::pin(runner.run_gate(&winner));
15524 assert!(
15525 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15526 .await
15527 .is_err()
15528 );
15529 assert!(!started.exists(), "pre-gate must wait for the cache too");
15530 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15534 let editor = crate::repo_lock::SeatLease::checkout_with(
15535 &winner.worktree,
15536 true,
15537 crate::config::test_coordination().as_ref(),
15538 &wait,
15539 )
15540 .await
15541 .unwrap();
15542 drop(editor);
15543 guard.release();
15544 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15545 .await
15546 .unwrap()
15547 .unwrap();
15548 drop(gate);
15549 assert!(outcomes.iter().all(CommandOutcome::ok));
15550 assert!(started.exists());
15551 }
15552
15553 #[tokio::test]
15554 #[cfg(unix)]
15555 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15556 use std::os::fd::AsRawFd as _;
15557
15558 ask_test_home();
15559 let tmp = tempfile::tempdir().unwrap();
15560 let repo = tmp.path().join("repo");
15561 std::fs::create_dir_all(&repo).unwrap();
15562 init_repo(&repo);
15563 let common = repo.join(".git");
15564 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15565 let mut runner = runner_at(RunStatus::Prep);
15566 runner.state.repo = repo.clone();
15567 runner.state.config.coordination = crate::config::test_coordination();
15568 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15569 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15570 runner.state.config.graph.advise = false;
15571 runner.roles.implementers = vec![spec("implement")];
15572 runner.roles.judges = vec![spec("judge")];
15573 runner.prep().await.unwrap();
15574 let winner = runner.state.candidates[0].clone();
15575 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15578 let completed = tmp.path().join("seat-completed");
15579 runner.roles.implementers[0].command = vec![
15580 "sh".into(),
15581 "-c".into(),
15582 format!("touch '{}'; echo completed", completed.display()),
15583 ];
15584 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15585 assert_eq!(
15586 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15587 0
15588 );
15589 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15590 let mut implement = Box::pin(runner.implement());
15591 tokio::select! {
15592 result = &mut implement => panic!("rescue must wait: {result:?}"),
15593 result = tokio::time::timeout(Duration::from_secs(10), async {
15594 while !completed.exists() {
15595 tokio::time::sleep(Duration::from_millis(20)).await;
15596 }
15597 }) => result.unwrap(),
15598 }
15599 assert!(
15600 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15601 .await
15602 .is_err()
15603 );
15604 assert_eq!(
15605 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15606 before
15607 );
15608 assert_eq!(
15609 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15610 0
15611 );
15612 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15613 .await
15614 .unwrap()
15615 .unwrap();
15616 drop(implement);
15617 assert_ne!(
15618 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15619 before
15620 );
15621 assert!(git::is_clean(&winner.worktree).await.unwrap());
15622
15623 let started = tmp.path().join("pre-gate-started");
15624 let proceed = tmp.path().join("pre-gate-proceed");
15625 let checked = tmp.path().join("gate-ran");
15626 runner.state.config.verify.pre_gate = vec![format!(
15627 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15628 started.display(),
15629 proceed.display()
15630 )];
15631 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15632 assert_eq!(
15633 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15634 0
15635 );
15636 let mut gate = Box::pin(runner.run_gate(&winner));
15637 assert!(
15638 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15639 .await
15640 .is_err()
15641 );
15642 assert!(!started.exists());
15643 assert!(!checked.exists());
15644 assert!(git::is_clean(&winner.worktree).await.unwrap());
15645 assert_eq!(
15646 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15647 0
15648 );
15649 tokio::select! {
15650 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15651 result = tokio::time::timeout(Duration::from_secs(10), async {
15652 while !started.exists() {
15653 tokio::time::sleep(Duration::from_millis(20)).await;
15654 }
15655 }) => result.unwrap(),
15656 }
15657 assert_eq!(
15658 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15659 -1,
15660 "mechanical edits must hold the real lock"
15661 );
15662 std::fs::write(proceed, "ready").unwrap();
15663 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15664 .await
15665 .unwrap()
15666 .unwrap();
15667 drop(gate);
15668 assert!(outcomes.iter().all(CommandOutcome::ok));
15669 assert!(checked.exists());
15670 assert!(runner.state.pre_gate_commit.is_some());
15671 assert!(git::is_clean(&winner.worktree).await.unwrap());
15672 assert_eq!(
15673 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15674 0,
15675 "gate completion must release coordination"
15676 );
15677 drop(outsider);
15678 std::fs::remove_file(common.join("updater.lock")).unwrap();
15679 std::fs::create_dir(common.join("updater.lock")).unwrap();
15680 std::fs::remove_file(&started).unwrap();
15681 std::fs::remove_file(&checked).unwrap();
15682 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15683 let error = runner.run_gate(&winner).await.unwrap_err();
15684 assert!(
15685 error
15686 .to_string()
15687 .contains("acquire mandatory repository lock")
15688 );
15689 assert!(
15690 !started.exists(),
15691 "failed acquisition must prevent pre-gate"
15692 );
15693 assert!(!checked.exists(), "failed acquisition must prevent gate");
15694 assert!(git::is_clean(&winner.worktree).await.unwrap());
15695 assert_eq!(
15696 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15697 head
15698 );
15699 }
15700
15701 #[tokio::test]
15702 #[cfg(unix)]
15703 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15704 use std::os::fd::AsRawFd as _;
15705 use std::time::Duration;
15706
15707 ask_test_home();
15708 let tmp = tempfile::tempdir().unwrap();
15709 let repo = tmp.path().join("repo");
15710 std::fs::create_dir_all(&repo).unwrap();
15711 init_repo(&repo);
15712 let common = repo.join(".git");
15713 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15714 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15715 assert_eq!(
15716 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15717 0
15718 );
15719 let mut runner = runner_at(RunStatus::Prep);
15720 runner.state.repo = repo.clone();
15721 runner.state.config.coordination = crate::config::test_coordination();
15722 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15723 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15724 runner.state.config.graph.advise = true;
15725 runner.state.config.graph.advisors = 1;
15726 runner.roles.implementers = vec![spec("implement")];
15727 runner.roles.judges = vec![spec("judge")];
15728 let root = runner.state.worktree_root();
15729 let started = tmp.path().join("setup-started");
15730 let proceed = tmp.path().join("setup-proceed");
15731 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15732 run: Some(format!(
15733 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15734 started.display(),
15735 proceed.display()
15736 )),
15737 ..Default::default()
15738 }];
15739 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15740 let config = git::git(&repo, &["config", "--local", "--list"])
15741 .await
15742 .unwrap();
15743 let mut prep = Box::pin(runner.prep());
15744 assert!(
15745 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15746 .await
15747 .is_err()
15748 );
15749 assert!(
15750 !root.exists(),
15751 "contention must prevent even checkout creation"
15752 );
15753 assert!(!started.exists(), "contention must prevent setup");
15754 assert_eq!(
15755 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15756 branches
15757 );
15758 assert_eq!(
15759 git::git(&repo, &["config", "--local", "--list"])
15760 .await
15761 .unwrap(),
15762 config
15763 );
15764 drop(lock);
15765 tokio::select! {
15766 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15767 result = tokio::time::timeout(Duration::from_secs(10), async {
15768 while !started.exists() {
15769 tokio::time::sleep(Duration::from_millis(20)).await;
15770 }
15771 }) => result.unwrap(),
15772 }
15773 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15774 assert_eq!(
15775 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15776 -1
15777 );
15778 assert!(
15779 tokio::time::timeout(
15780 Duration::from_millis(100),
15781 crate::repo_lock::SeatLease::for_checkout(
15782 &repo,
15783 true,
15784 crate::config::test_coordination().as_ref()
15785 )
15786 )
15787 .await
15788 .is_err(),
15789 "preparation must also exclude local editors"
15790 );
15791 std::fs::write(proceed, "ready").unwrap();
15792 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15793 .await
15794 .unwrap()
15795 .unwrap();
15796 drop(prep);
15797 assert_eq!(runner.state.prep_complete, Some(true));
15798 assert!(root.join("judge-1").exists());
15799 assert!(root.join("advisor-1").exists());
15800 assert_eq!(
15801 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15802 0
15803 );
15804 }
15805
15806 #[tokio::test]
15807 #[cfg(unix)]
15808 async fn preparation_lock_failure_prevents_repository_changes() {
15809 ask_test_home();
15810 let tmp = tempfile::tempdir().unwrap();
15811 let repo = tmp.path().join("repo");
15812 std::fs::create_dir_all(&repo).unwrap();
15813 init_repo(&repo);
15814 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15815 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15818 let mut runner = runner_at(RunStatus::Prep);
15819 runner.state.repo = repo.clone();
15820 runner.state.config.coordination = crate::config::test_coordination();
15821 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15822 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15823 runner.roles.implementers = vec![spec("implement")];
15824 let error = runner.prep().await.unwrap_err();
15825 assert!(
15826 error
15827 .to_string()
15828 .contains("acquire mandatory repository lock")
15829 );
15830 assert!(!runner.state.worktree_root().exists());
15831 assert!(runner.state.candidates.is_empty());
15832 assert!(!repo.join(".git/config.worktree").exists());
15833 assert_eq!(
15834 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15835 "* main"
15836 );
15837 }
15838
15839 #[tokio::test]
15840 async fn preparation_records_creation_intent_before_git_can_fail() {
15841 ask_test_home();
15842 let tmp = tempfile::tempdir().unwrap();
15843 let repo = tmp.path().join("repo");
15844 std::fs::create_dir_all(&repo).unwrap();
15845 init_repo(&repo);
15846 let mut runner = runner_at(RunStatus::Prep);
15847 runner.state.repo = repo.clone();
15848 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15849 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15850 runner.state.config.blind.commit_msg_hook = false;
15851 runner.roles.implementers = vec![spec("one")];
15852 let label = blind::assign_labels(1, runner.state.seed)[0];
15853 let branch = runner.state.branch_for(label);
15854 git::git(&repo, &["branch", &branch]).await.unwrap();
15857 assert!(runner.prep().await.is_err());
15858 let saved = RunState::load(&runner.state.id).unwrap();
15859 assert!(saved.candidates.is_empty());
15860 let event = saved
15861 .events
15862 .iter()
15863 .find(|e| e.node == "prep-intent")
15864 .unwrap();
15865 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15866 assert_eq!(intent["branch"], branch);
15867 assert_eq!(intent["start"], runner.state.base_commit);
15868 assert_eq!(
15869 intent["worktree"],
15870 runner
15871 .state
15872 .worktree_root()
15873 .join(format!("cand-{label}"))
15874 .to_string_lossy()
15875 .as_ref()
15876 );
15877 assert_eq!(intent["picks"], serde_json::json!([]));
15878 }
15879
15880 #[tokio::test]
15881 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15882 ask_test_home();
15883 let tmp = tempfile::tempdir().unwrap();
15884 let repo = tmp.path().join("repo");
15885 std::fs::create_dir_all(&repo).unwrap();
15886 init_repo(&repo);
15887 let mut runner = runner_at(RunStatus::Prep);
15888 runner.state.repo = repo.clone();
15889 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15890 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15891 runner.state.sources.pick_commits.push(refs::Input {
15892 token: "missing".into(),
15893 sha: "0".repeat(40),
15894 provenance: refs::Provenance::ExplicitCli,
15895 });
15896 assert!(runner.prep().await.is_err());
15897 assert!(runner.state.candidates.is_empty());
15898 assert!(!runner.state.worktree_root().exists());
15899 }
15900
15901 #[tokio::test]
15902 async fn prose_and_legacy_inference_never_import_into_candidates() {
15903 ask_test_home();
15904 let tmp = tempfile::tempdir().unwrap();
15905 let repo = tmp.path().join("repo");
15906 std::fs::create_dir_all(&repo).unwrap();
15907 init_repo(&repo);
15908 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15909 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15910 .await
15911 .unwrap();
15912 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15913 git::git(&repo, &["add", "."]).await.unwrap();
15914 git::git(&repo, &["commit", "-m", "historical implementation"])
15915 .await
15916 .unwrap();
15917 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15918 git::git(&repo, &["checkout", "main"]).await.unwrap();
15919 for instruction in [
15920 format!("Example: magi/history/A at {source}"),
15921 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15922 format!("Never import magi/history/A or {source}"),
15923 format!("Conversation answer: I mentioned {source} only as background"),
15924 ] {
15925 let mut runner = runner_at(RunStatus::Prep);
15926 runner.state.repo = repo.clone();
15927 runner.state.base_commit = base.clone();
15928 runner.state.instruction = instruction.clone();
15929 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15930 runner.state.config.blind.commit_msg_hook = false;
15931 runner.state.seeds = vec![refs::Seed {
15932 token: "magi/history/A".into(),
15933 sha: source.clone(),
15934 kind: refs::SeedKind::Unmerged,
15935 branch: true,
15936 detail: String::new(),
15937 }];
15938 runner.roles.implementers = vec![spec("one")];
15939 assert_eq!(seeded_instruction(&runner.state), instruction);
15940 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15942 legacy.as_object_mut().unwrap().remove("sources");
15943 runner.state = serde_json::from_value(legacy).unwrap();
15944 assert_eq!(runner.state.sources, refs::Sources::default());
15945 runner.state.save().unwrap();
15946 runner.state = RunState::load(&runner.state.id).unwrap();
15947 runner.prep().await.unwrap();
15948 for candidate in &runner.state.candidates {
15949 assert_eq!(
15950 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15951 base
15952 );
15953 assert!(!candidate.worktree.join("unwanted").exists());
15954 }
15955 }
15956 }
15957
15958 #[tokio::test]
15959 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15960 ask_test_home();
15961 let tmp = tempfile::tempdir().unwrap();
15962 let repo = tmp.path().join("repo");
15963 std::fs::create_dir_all(&repo).unwrap();
15964 init_repo(&repo);
15965 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15966 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15967 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15968 git::git(&repo, &["add", "."]).await.unwrap();
15969 git::git(&repo, &["commit", "-m", "required seed"])
15970 .await
15971 .unwrap();
15972 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15973 git::git(&repo, &["checkout", "main"]).await.unwrap();
15974 let mut runner = runner_at(RunStatus::Prep);
15975 runner.state.repo = repo.clone();
15976 runner.state.base_commit = base.clone();
15977 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15978 runner.state.config.graph.advise = false;
15979 runner.state.config.blind.commit_msg_hook = false;
15980 runner.state.sources =
15981 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15982 .await
15983 .unwrap();
15984 runner.roles.implementers = vec![spec("one"), spec("two")];
15985 runner.roles.judges = vec![spec("judge")];
15986 let label = blind::assign_labels(2, runner.state.seed)[0];
15987 let branch = runner.state.branch_for(label);
15988 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15989 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15990 .await
15991 .unwrap();
15992 runner.state.candidates.push(Candidate {
15993 index: 0,
15994 label,
15995 agent: "one".into(),
15996 branch,
15997 worktree,
15998 summary: String::new(),
15999 stat: String::new(),
16000 files: 0,
16001 commits: 0,
16002 empty: false,
16003 failed: None,
16004 verified_noop: None,
16005 duration_ms: 0,
16006 folded: false,
16007 });
16008 runner.state.save().unwrap();
16009 runner.state = RunState::load(&runner.state.id).unwrap();
16011 assert_eq!(runner.state.prep_complete, Some(false));
16012 runner.prep().await.unwrap();
16013 assert_eq!(runner.state.prep_complete, Some(true));
16014 assert_eq!(runner.state.candidates.len(), 2);
16015 for candidate in &runner.state.candidates {
16016 assert_eq!(
16017 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
16018 git::tree_of(&repo, &seed).await.unwrap()
16019 );
16020 assert_eq!(
16021 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
16022 "required commit"
16023 );
16024 }
16025 assert!(runner.state.worktree_root().join("judge-1").exists());
16026 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
16027 runner.prep().await.unwrap();
16028 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
16029
16030 runner.state.prep_complete = Some(false);
16031 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
16032 run: Some(format!(
16033 "test -f '{}'",
16034 tmp.path().join("dependency-ready").display()
16035 )),
16036 ..Default::default()
16037 }];
16038 assert!(runner.prep().await.is_err());
16039 assert!(runner.state.prep_failure.is_some());
16040 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
16041 runner.state = RunState::load(&runner.state.id).unwrap();
16042 assert!(
16043 runner.prep().await.is_err(),
16044 "correcting the cause alone must not auto-retry"
16045 );
16046 let failed_at = runner.state.events.last().unwrap().at;
16047 assert!(
16048 runner.retry_preparation(failed_at).is_err(),
16049 "old approvals cannot unlock the hold"
16050 );
16051 let mut task = crate::queue::Task::new(
16052 "retry".into(),
16053 "retry setup".into(),
16054 repo.clone(),
16055 crate::queue::Source::Human,
16056 );
16057 task.start(runner.state.id.clone());
16058 task.hold_machine(Some("setup failed".into()));
16059 task.release_operator("cli", None);
16060 runner
16061 .retry_preparation(task.resumption.as_ref().unwrap().at)
16062 .unwrap();
16063 runner.prep().await.unwrap();
16064 assert!(runner.state.prep_failure.is_none());
16065 assert_eq!(runner.state.prep_complete, Some(true));
16066 assert_eq!(runner.state.candidates.len(), 2);
16067 assert!(runner.state.worktree_root().join("judge-1").exists());
16068 for candidate in &runner.state.candidates {
16069 assert_eq!(
16070 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
16071 git::tree_of(&repo, &seed).await.unwrap()
16072 );
16073 }
16074 }
16075
16076 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
16077 let mut heads = Vec::new();
16078 for candidate in candidates {
16079 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
16080 }
16081 heads
16082 }
16083
16084 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
16085 ask_test_home();
16086 let tmp = tempfile::tempdir().unwrap();
16087 let repo = tmp.path().join("repo");
16088 std::fs::create_dir(&repo).unwrap();
16089 init_repo(&repo);
16090 let mut runner = runner_at(RunStatus::Reviewing);
16091 runner.state.repo = repo.clone();
16092 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16093 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16094 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16095 let branch = format!("candidate-{label}");
16096 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
16097 git::worktree_add_branch(&repo, &worktree, &branch, "main")
16098 .await
16099 .unwrap();
16100 if label == 'B' {
16101 std::fs::write(worktree.join("change"), "winner").unwrap();
16102 git::rescue_commit(&worktree, "candidate change")
16103 .await
16104 .unwrap();
16105 }
16106 runner.state.candidates.push(Candidate {
16107 index,
16108 label,
16109 agent: "alpha".into(),
16110 branch,
16111 worktree,
16112 summary: "completed reply".into(),
16113 stat: String::new(),
16114 files: usize::from(label == 'B'),
16115 commits: usize::from(label == 'B'),
16116 empty: label == 'A',
16117 failed: None,
16118 verified_noop: None,
16119 duration_ms: 1,
16120 folded: false,
16121 });
16122 }
16123 runner.state.judge_skipped = true;
16124 runner.state.tally = Some(Tally {
16125 first_choice: BTreeMap::new(),
16126 borda: BTreeMap::new(),
16127 winner: 'B',
16128 rankings: 0,
16129 unanimous_initial: false,
16130 deliberated: false,
16131 changed_votes: 0,
16132 unanimous_final: false,
16133 tie_break: None,
16134 judges: 0,
16135 present: 0,
16136 quorum: 0,
16137 met_quorum: true,
16138 uncontested: Some("only B produced a change".into()),
16139 });
16140 runner.state.save().unwrap();
16141 (tmp, runner)
16142 }
16143
16144 #[tokio::test]
16145 async fn fold_losers_persists_discard_before_later_teardown_failure() {
16146 let (_tmp, mut runner) = preservation_resume_fixture().await;
16147 runner.state.candidates[0].folded = true;
16148 runner.state.config.graph.advise = true;
16149 runner.state.config.graph.advisors = 2;
16150 let repo = runner.state.repo.clone();
16151 let root = runner.state.worktree_root();
16152 let discarded = root.join("advisor-1");
16153 git::worktree_add_detached(&repo, &discarded, "main")
16154 .await
16155 .unwrap();
16156 std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
16157 let protected = root.join("advisor-2");
16158 git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
16159 .await
16160 .unwrap();
16161 std::fs::write(protected.join("keep"), "uncommitted").unwrap();
16162 runner.state.save().unwrap();
16163
16164 assert!(runner.fold_losers().await.is_err());
16165 assert!(!discarded.exists());
16166 assert!(protected.join("keep").exists());
16167 let saved = RunState::load(&runner.state.id).unwrap();
16168 assert!(saved.events.iter().any(|event| {
16169 event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
16170 }));
16171 }
16172
16173 #[tokio::test]
16174 async fn read_seat_teardown_discards_only_owned_detached_seats() {
16175 let (tmp, mut runner) = preservation_resume_fixture().await;
16176 let repo = runner.state.repo.clone();
16177 let root = runner.state.worktree_root();
16178 for name in ["advisor-3", "judge-1", "review-2"] {
16179 let path = root.join(name);
16180 git::worktree_add_detached(&repo, &path, "main")
16181 .await
16182 .unwrap();
16183 std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
16184 std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
16185 remove_read_seat_worktree(&mut runner.state, &path)
16186 .await
16187 .unwrap();
16188 assert!(!path.exists());
16189 }
16190 let outside = tmp.path().join("advisor-1");
16191 let unknown = root.join("unknown");
16192 let attached = root.join("advisor-4");
16193 git::worktree_add_detached(&repo, &outside, "main")
16194 .await
16195 .unwrap();
16196 git::worktree_add_detached(&repo, &unknown, "main")
16197 .await
16198 .unwrap();
16199 git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16200 .await
16201 .unwrap();
16202 let foreign = tmp.path().join("foreign");
16203 std::fs::create_dir(&foreign).unwrap();
16204 init_repo(&foreign);
16205 let wrong_repo = root.join("judge-2");
16206 git::worktree_add_detached(&foreign, &wrong_repo, "main")
16207 .await
16208 .unwrap();
16209 let candidate = runner.state.candidates[0].worktree.clone();
16210 for path in [outside, unknown, attached, wrong_repo, candidate] {
16211 std::fs::write(path.join("uncommitted"), "keep").unwrap();
16212 assert!(
16213 remove_read_seat_worktree(&mut runner.state, &path)
16214 .await
16215 .is_err()
16216 );
16217 assert_eq!(
16218 std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16219 "keep"
16220 );
16221 }
16222 }
16223
16224 #[tokio::test]
16225 #[cfg(unix)]
16226 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16227 for (solo, failed_checkpoint) in
16228 [(true, false), (false, false), (true, true), (false, true)]
16229 {
16230 let (_tmp, mut runner) = preservation_resume_fixture().await;
16231 if solo {
16232 runner.state.candidates.retain(|c| c.label == 'A');
16233 }
16234 runner.state.candidates[0].empty = false;
16235 runner.state.candidates[0].summary.clear();
16236 runner.state.tally = None;
16237 runner.state.judge_skipped = false;
16238 runner.state.status = RunStatus::Implementing;
16239 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16240 let mut implementer = spec("alpha");
16241 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16242 runner.roles.implementers = vec![implementer];
16243 let patch = runner.state.dir().join("artifacts/cand-A.patch");
16244 if failed_checkpoint {
16245 std::fs::create_dir_all(&patch).unwrap();
16246 }
16247 runner.state.save().unwrap();
16248 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16249 drop(pressure);
16250 runner.state = RunState::load(&runner.state.id).unwrap();
16251 assert_eq!(runner.state.status, RunStatus::Blocked);
16252 assert!(runner.state.interrupted_implementations.contains(&'A'));
16253 assert_eq!(runner.state.candidates[0].commits, 0);
16254 assert!(!runner.state.candidates[0].empty);
16255 assert!(runner.state.candidates[0].failed.is_none());
16256 if failed_checkpoint {
16257 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16258 std::fs::remove_dir(patch).unwrap();
16259 }
16260 runner.state.resource_block = None;
16261 runner.state.config.disk.min_free_bytes = 0;
16262 runner.roles.implementers[0].command = vec![
16263 "sh".into(),
16264 "-c".into(),
16265 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16266 ];
16267 runner.implement().await.unwrap();
16268 assert!(runner.state.interrupted_implementations.is_empty());
16269 assert!(runner.state.candidate_preservation.is_empty());
16270 assert_eq!(runner.state.candidates[0].commits, 1);
16271 assert!(!runner.state.candidates[0].empty);
16272 assert!(
16273 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16274 .unwrap()
16275 .contains("retried empty interrupted seat")
16276 );
16277 if !solo {
16278 assert_eq!(runner.state.candidates[1].commits, 1);
16279 }
16280 }
16281 }
16282
16283 #[tokio::test]
16284 #[cfg(unix)]
16285 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16286 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16287 let (_tmp, mut runner) = preservation_resume_fixture().await;
16288 runner.state.candidates.retain(|c| c.label == 'A');
16289 runner.state.candidates[0].empty = false;
16290 runner.state.candidates[0].summary.clear();
16291 runner.state.tally = None;
16292 runner.state.judge_skipped = false;
16293 runner.state.status = RunStatus::Implementing;
16294 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16297 let wt = runner.state.candidates[0].worktree.clone();
16298 std::fs::write(
16299 wt.join("interrupted-source"),
16300 "retain implementation work\n",
16301 )
16302 .unwrap();
16303 if committed {
16304 git::rescue_commit(&wt, "implementation checkpoint")
16305 .await
16306 .unwrap();
16307 }
16308 let lock = git::git(
16309 &wt,
16310 &[
16311 "rev-parse",
16312 "--path-format=absolute",
16313 "--git-path",
16314 "index.lock",
16315 ],
16316 )
16317 .await
16318 .unwrap();
16319 if block_rescue {
16320 std::fs::write(&lock, "fixture-owned lock").unwrap();
16321 }
16322 let mut implementer = spec("alpha");
16323 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16324 runner.roles.implementers = vec![implementer];
16325 runner.state.save().unwrap();
16326 let result = runner.implement().await;
16327 drop(pressure);
16328 assert_eq!(result.is_err(), block_rescue);
16329 assert_eq!(
16330 runner.state.resource_block.as_ref().unwrap().kind,
16331 "disk pressure"
16332 );
16333 assert!(
16334 runner
16335 .state
16336 .events
16337 .iter()
16338 .any(|e| e.message.contains("agent execution interrupted"))
16339 );
16340 assert!(
16341 runner
16342 .state
16343 .dir()
16344 .join("artifacts/candidate-A-recovery/staged.patch")
16345 .exists()
16346 );
16347 if block_rescue {
16348 let saved = RunState::load(&runner.state.id).unwrap();
16349 assert!(saved.candidate_preservation.contains_key(&'A'));
16350 assert!(!saved.candidates[0].empty);
16351 assert_eq!(
16352 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16353 "retain implementation work\n"
16354 );
16355 std::fs::remove_file(lock).unwrap();
16356 runner.state = saved;
16357 } else {
16358 assert_eq!(runner.state.status, RunStatus::Blocked);
16359 assert_eq!(runner.state.candidates[0].commits, 1);
16360 }
16361 runner.roles.implementers.clear();
16364 runner.state.config.disk.min_free_bytes = 0;
16365 runner.state.resource_block = None;
16366 runner.implement().await.unwrap();
16367 assert!(runner.state.candidate_preservation.is_empty());
16368 assert_eq!(runner.state.candidates[0].commits, 1);
16369 assert!(!runner.state.candidates[0].empty);
16370 assert!(git::is_clean(&wt).await.unwrap());
16371 assert!(
16372 crate::run::read_artifact(&runner.state, "cand-A.patch")
16373 .unwrap()
16374 .contains("retain implementation work")
16375 );
16376 }
16377 }
16378
16379 #[tokio::test]
16380 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16381 let (_tmp, mut runner) = preservation_resume_fixture().await;
16382 runner.implement().await.unwrap();
16384 assert!(runner.state.candidate_preservation.is_empty());
16385 assert!(
16386 !runner
16387 .state
16388 .dir()
16389 .join("artifacts/candidate-A-recovery")
16390 .exists()
16391 );
16392 runner.fold_losers().await.unwrap();
16393 assert!(!runner.state.candidates[0].worktree.exists());
16394 for schema in [14, crate::run::SCHEMA] {
16395 runner.state.schema = schema;
16396 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16397 runner.state.save().unwrap();
16398 runner.state = RunState::load(&runner.state.id).unwrap();
16399 runner.implement().await.unwrap();
16400 assert!(runner.state.candidate_preservation.is_empty());
16401 assert!(runner.state.candidates[0].folded);
16402 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16403 }
16404 }
16405
16406 #[tokio::test]
16407 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16408 let (_tmp, mut runner) = preservation_resume_fixture().await;
16409 runner.state.candidates.retain(|c| c.label == 'A');
16410 runner.state.tally = None;
16411 runner.state.judge_skipped = false;
16412 let candidate = &mut runner.state.candidates[0];
16413 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16414 candidate.failed = Some("deployment validation unavailable".into());
16415 std::fs::write(
16416 candidate.worktree.join("repair"),
16417 "future launcher policy\n",
16418 )
16419 .unwrap();
16420 runner
16421 .state
16422 .candidate_preservation
16423 .insert('A', "awaiting persistence".into());
16424 runner.persist_candidates().await.unwrap();
16425 let candidate = &runner.state.candidates[0];
16426 assert!(!candidate.empty);
16427 assert_eq!(candidate.commits, 1);
16428 assert_eq!(candidate.files, 1);
16429 assert!(candidate.failed.is_none());
16430 assert!(candidate.summary.contains("EPERM"));
16431 assert!(
16432 crate::run::read_artifact(&runner.state, "cand-A.patch")
16433 .unwrap()
16434 .contains("future launcher policy")
16435 );
16436 assert!(runner.state.candidate_preservation.is_empty());
16437 }
16438
16439 #[tokio::test]
16440 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16441 let (_tmp, mut runner) = preservation_resume_fixture().await;
16442 let wt = runner.state.candidates[0].worktree.clone();
16443 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16444 runner.state.schema = 14;
16445 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16446 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16447 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16448 for _ in 0..2 {
16449 assert!(runner.implement().await.is_err());
16450 assert_eq!(runner.state.status, RunStatus::Blocked);
16451 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16452 assert!(
16453 runner.state.candidates[0].empty,
16454 "old verdict must remain unusable"
16455 );
16456 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16457 assert!(runner.state.judge_skipped);
16458 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16459 assert!(
16460 crate::run::read_artifact(&runner.state, "cand-A.patch")
16461 .unwrap()
16462 .contains("owned edits")
16463 );
16464 assert!(wt.exists());
16465 assert_eq!(
16466 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16467 .await
16468 .unwrap(),
16469 1
16470 );
16471 runner.state = RunState::load(&runner.state.id).unwrap();
16472 }
16473 }
16474
16475 #[tokio::test]
16476 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16477 let (_tmp, mut runner) = preservation_resume_fixture().await;
16478 let wt = runner.state.candidates[0].worktree.clone();
16479 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16480 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16481 let index_lock = PathBuf::from(
16482 git::git(
16483 &wt,
16484 &[
16485 "rev-parse",
16486 "--path-format=absolute",
16487 "--git-path",
16488 "index.lock",
16489 ],
16490 )
16491 .await
16492 .unwrap(),
16493 );
16494 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16495 runner.state.schema = 14;
16496 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16497 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16498 assert!(runner.implement().await.is_err());
16499 assert!(runner.after_implement().is_err());
16500 assert!(runner.fold_losers().await.is_err());
16501 assert_eq!(
16502 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16503 "failed seat edits"
16504 );
16505 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16506 runner.state = RunState::load(&runner.state.id).unwrap();
16507 std::fs::remove_file(index_lock).unwrap();
16508 assert!(runner.implement().await.is_err());
16510 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16511 assert!(runner.state.candidates[0].empty);
16512 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16513 assert!(
16514 crate::run::read_artifact(&runner.state, "cand-A.patch")
16515 .unwrap()
16516 .contains("failed seat edits")
16517 );
16518 assert!(wt.exists());
16519 }
16520
16521 #[tokio::test]
16522 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16523 let (_tmp, mut runner) = preservation_resume_fixture().await;
16524 let repo = runner.state.repo.clone();
16525 let common = git::git(
16526 &repo,
16527 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16528 )
16529 .await
16530 .unwrap();
16531 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16532 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16533 assert!(runner.fold_losers().await.is_err());
16534 assert!(!runner.state.candidates[0].worktree.exists());
16535 assert!(!runner.state.candidates[0].folded);
16536 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16537 runner.state = RunState::load(&runner.state.id).unwrap();
16538 std::fs::remove_file(branch_lock).unwrap();
16539 let judge = runner.state.worktree_root().join("judge-1");
16540 git::worktree_add_detached(&repo, &judge, "main")
16541 .await
16542 .unwrap();
16543 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16544 git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16545 .await
16546 .unwrap();
16547 runner.roles.judges = vec![spec("judge")];
16548 assert!(runner.fold_losers().await.is_err());
16549 runner.state = RunState::load(&runner.state.id).unwrap();
16550 assert!(
16551 runner.state.candidates[0].folded,
16552 "candidate progress saved before judge failure"
16553 );
16554 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16555 assert!(judge.join("uncommitted").exists());
16556 git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16557 .await
16558 .unwrap();
16559 runner.fold_losers().await.unwrap();
16560 assert!(!judge.exists());
16561 runner.fold_losers().await.unwrap();
16562 }
16563
16564 #[tokio::test]
16565 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16566 let (_tmp, mut runner) = preservation_resume_fixture().await;
16567 runner.state.candidates.retain(|c| c.label == 'B');
16568 runner.state.tally = None;
16569 runner.state.judge_skipped = false;
16570 runner.state.prep_complete = Some(true);
16571 runner.state.config.graph.advise = false;
16572 runner.state.config.graph.selected_implementer = Some("alpha".into());
16573 let wt = runner.state.candidates[0].worktree.clone();
16574 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16575 runner
16576 .state
16577 .candidate_preservation
16578 .insert('B', "awaiting persistence".into());
16579 let index_lock = git::git(
16580 &wt,
16581 &[
16582 "rev-parse",
16583 "--path-format=absolute",
16584 "--git-path",
16585 "index.lock",
16586 ],
16587 )
16588 .await
16589 .unwrap();
16590 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16591 runner.state.save().unwrap();
16592 assert!(runner.execute_graph().await.is_err());
16593 assert_eq!(runner.state.status, RunStatus::Blocked);
16594 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16595 std::fs::remove_file(index_lock).unwrap();
16596 for _ in 0..2 {
16599 runner.state = RunState::load(&runner.state.id).unwrap();
16600 runner.pause.park();
16601 runner.execute_graph().await.unwrap();
16602 assert_eq!(runner.state.status, RunStatus::Implementing);
16603 assert!(runner.state.candidate_preservation.is_empty());
16604 assert_eq!(runner.state.candidates[0].commits, 2);
16605 assert!(runner.state.tally.is_none());
16606 }
16607 runner.implement().await.unwrap();
16608 assert_eq!(runner.state.status, RunStatus::Judging);
16609 runner.state.status = RunStatus::Blocked;
16611 runner.state.candidates[0].failed = Some("implementation failed".into());
16612 runner.state.save().unwrap();
16613 let error = runner.execute_graph().await.unwrap_err();
16614 assert!(
16615 error
16616 .to_string()
16617 .contains("selected implementation is held")
16618 );
16619 }
16620
16621 #[tokio::test]
16622 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16623 let (_tmp, mut runner) = preservation_resume_fixture().await;
16624 let repo = runner.state.repo.clone();
16625 let candidate = runner.state.candidates[0].clone();
16626 let registered = git::worktree_holding(&repo, &candidate.branch)
16629 .await
16630 .unwrap()
16631 .unwrap();
16632 assert_eq!(
16633 registered.canonicalize().unwrap(),
16634 candidate.worktree.canonicalize().unwrap()
16635 );
16636 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16637 assert_eq!(
16638 git::worktree_holding(&repo, &candidate.branch)
16639 .await
16640 .unwrap(),
16641 Some(registered)
16642 );
16643 assert!(
16644 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16645 "the stale registration initially keeps the branch checked out"
16646 );
16647 runner.fold_losers().await.unwrap();
16648 assert!(runner.state.candidates[0].folded);
16649 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16650 assert_eq!(
16651 git::worktree_holding(&repo, &candidate.branch)
16652 .await
16653 .unwrap(),
16654 None
16655 );
16656 runner.state = RunState::load(&runner.state.id).unwrap();
16657 runner.fold_losers().await.unwrap();
16658 }
16659
16660 #[tokio::test]
16661 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16662 let (_tmp, mut runner) = preservation_resume_fixture().await;
16663 let repo = runner.state.repo.clone();
16664 let candidate = runner.state.candidates[0].clone();
16665 let winner = runner.state.candidates[1].clone();
16666 let registered = git::worktree_holding(&repo, &candidate.branch)
16667 .await
16668 .unwrap()
16669 .unwrap();
16670 assert_eq!(
16671 registered.canonicalize().unwrap(),
16672 candidate.worktree.canonicalize().unwrap()
16673 );
16674 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16675 assert_eq!(
16676 git::worktree_holding(&repo, &candidate.branch)
16677 .await
16678 .unwrap(),
16679 Some(registered)
16680 );
16681 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16682
16683 for _ in 0..2 {
16684 fold_run(&mut runner.state, false, &crate::run::home())
16685 .await
16686 .unwrap();
16687 runner.state = RunState::load(&runner.state.id).unwrap();
16688 assert!(runner.state.candidates[0].folded);
16689 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16690 assert_eq!(
16691 git::worktree_holding(&repo, &candidate.branch)
16692 .await
16693 .unwrap(),
16694 None
16695 );
16696 assert!(!runner.state.candidates[1].folded);
16697 assert!(winner.worktree.exists());
16698 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16699 }
16700 }
16701 #[tokio::test]
16702 async fn fold_recovery_override_archives_sources_and_keeps_safety_checks() {
16703 for sidecar in [false, true] {
16704 let tmp = tempfile::tempdir().unwrap();
16705 let home = tmp.path().join("home");
16706 let repo = tmp.path().join("repo");
16707 std::fs::create_dir(&repo).unwrap();
16708 init_repo(&repo);
16709 let mut state = runner_at(RunStatus::Superseded).state;
16710 state.repo = repo.clone();
16711 state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16712 state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16713 state.driver_exited = true;
16714 state.reserve_under(&home).unwrap();
16715 git::git(
16716 &repo,
16717 &["update-ref", "refs/remotes/origin/main", &state.base_commit],
16718 )
16719 .await
16720 .unwrap();
16721 let wt = state.worktree_root().join("cand-A");
16722 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16723 .await
16724 .unwrap();
16725 std::fs::write(wt.join("independent"), "retained implementation\n").unwrap();
16726 git::commit_all(&wt, "fixture: independent source")
16727 .await
16728 .unwrap();
16729 let head = git::rev_parse(&wt, "HEAD").await.unwrap();
16730 state.candidates.push(Candidate {
16731 index: 0,
16732 label: 'A',
16733 agent: "fixture".into(),
16734 branch: "candidate".into(),
16735 worktree: wt.clone(),
16736 summary: String::new(),
16737 stat: String::new(),
16738 files: 0,
16739 commits: 0,
16740 empty: false,
16741 failed: None,
16742 verified_noop: None,
16743 duration_ms: 0,
16744 folded: false,
16745 });
16746 let mut task = crate::queue::Task::new(
16747 "task".into(),
16748 "task".into(),
16749 repo.clone(),
16750 crate::queue::Source::Human,
16751 );
16752 task.runs.push(state.id.clone());
16753 crate::queue::Queue::at(home.join("queue"))
16754 .put(&mut task)
16755 .unwrap();
16756 if sidecar {
16757 crate::pr_recovery::retain_sources(&task, &home).unwrap();
16758 }
16759 state.save_under(&home).unwrap();
16760 let reason = crate::pr_recovery::source_retention_reason(&home, &state).unwrap();
16761 assert!(
16762 fold_run(&mut state, true, &home)
16763 .await
16764 .unwrap_err()
16765 .to_string()
16766 .contains(&reason)
16767 );
16768 let mut disk = state.config.disk.clone();
16769 disk.fold_grace_secs = 0;
16770 assert_eq!(
16771 crate::clean::fold_due(
16772 &home.join("runs"),
16773 &home,
16774 &state.worktree_root(),
16775 &disk,
16776 Timestamp::now() + std::time::Duration::from_secs(10),
16777 )
16778 .await
16779 .unwrap(),
16780 (0, 0)
16781 );
16782 assert!(wt.exists());
16783 state.status = RunStatus::Blocked;
16784 state.save_under(&home).unwrap();
16785 assert!(
16786 fold_run_overriding_recovery_retention(&mut state, true, &home)
16787 .await
16788 .is_err()
16789 );
16790 state.status = RunStatus::Superseded;
16791 state.driver_exited = false;
16792 state.driver_claimed = None;
16793 state.save_under(&home).unwrap();
16794 assert!(
16795 fold_run_overriding_recovery_retention(&mut state, true, &home)
16796 .await
16797 .is_err()
16798 );
16799 state.driver_pid = Some(std::process::id());
16800 state.driver_started_at = crate::proc::process_started_at(std::process::id());
16801 state.save_under(&home).unwrap();
16802 assert!(
16803 fold_run_overriding_recovery_retention(&mut state, true, &home)
16804 .await
16805 .is_err()
16806 );
16807 state.driver_exited = true;
16808 state.driver_pid = None;
16809 state.driver_started_at = None;
16810 state.save_under(&home).unwrap();
16811 let lease = crate::driver::maintenance(&home, &state.id).unwrap();
16812 assert!(
16813 fold_run_overriding_recovery_retention(&mut state, true, &home)
16814 .await
16815 .is_err()
16816 );
16817 drop(lease);
16818 #[cfg(unix)]
16819 {
16820 let output = crate::retention::OutputLease::take(&wt, false).unwrap();
16821 assert!(
16822 fold_run_overriding_recovery_retention(&mut state, true, &home)
16823 .await
16824 .is_err()
16825 );
16826 drop(output);
16827 }
16828 let owner = crate::pr_head::owner_lock(&home, &state.id).unwrap();
16829 assert!(
16830 fold_run_overriding_recovery_retention(&mut state, true, &home)
16831 .await
16832 .is_err()
16833 );
16834 drop(owner);
16835 let checkpoint = git::git(
16836 &wt,
16837 &[
16838 "rev-parse",
16839 "--path-format=absolute",
16840 "--git-path",
16841 "CHERRY_PICK_HEAD",
16842 ],
16843 )
16844 .await
16845 .unwrap();
16846 std::fs::write(&checkpoint, &head).unwrap();
16847 assert!(
16848 fold_run_overriding_recovery_retention(&mut state, true, &home)
16849 .await
16850 .is_err()
16851 );
16852 std::fs::remove_file(checkpoint).unwrap();
16853 std::fs::write(wt.join("dirty"), "keep").unwrap();
16854 assert!(
16855 fold_run_overriding_recovery_retention(&mut state, true, &home)
16856 .await
16857 .is_err()
16858 );
16859 assert!(wt.join("dirty").exists());
16860 std::fs::remove_file(wt.join("dirty")).unwrap();
16861 let evidence = home
16862 .join("runs")
16863 .join(&state.id)
16864 .join("artifacts/fold-evidence");
16865 std::fs::create_dir_all(evidence.parent().unwrap()).unwrap();
16866 std::fs::write(&evidence, "blocked archive").unwrap();
16867 assert!(
16868 fold_run_overriding_recovery_retention(&mut state, true, &home)
16869 .await
16870 .is_err()
16871 );
16872 assert!(wt.exists());
16873 assert!(
16874 !RunState::load_under(&state.id, &home)
16875 .unwrap()
16876 .events
16877 .iter()
16878 .any(|event| event.message.contains("--override-recovery-retention"))
16879 );
16880 std::fs::remove_file(&evidence).unwrap();
16881 fold_run_overriding_recovery_retention(&mut state, true, &home)
16882 .await
16883 .unwrap();
16884 assert!(!wt.exists());
16885 assert!(!git::branch_exists(&repo, "candidate").await.unwrap());
16886 assert!(
16887 git::worktree_holding(&repo, "candidate")
16888 .await
16889 .unwrap()
16890 .is_none()
16891 );
16892 assert!(
16893 std::fs::read_to_string(evidence.join(format!("{head}.patch")))
16894 .unwrap()
16895 .contains("retained implementation")
16896 );
16897 assert!(evidence.join(format!("{head}.index")).is_file());
16898 let saved = RunState::load_under(&state.id, &home).unwrap();
16899 assert_eq!(saved.status, RunStatus::Superseded);
16900 assert!(saved.candidates[0].folded);
16901 assert!(saved.events.iter().any(|event| {
16902 event.message.contains("--override-recovery-retention")
16903 && event.message.contains(&state.id)
16904 && event.message.contains(&reason)
16905 }));
16906 assert_eq!(
16907 crate::pr_recovery::explicit_sources_retained(&home, &state.id),
16908 sidecar
16909 );
16910 }
16911 }
16912
16913 #[tokio::test]
16914 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16915 ask_test_home();
16916 let tmp = tempfile::tempdir().unwrap();
16917 let repo = tmp.path().join("repo");
16918 std::fs::create_dir(&repo).unwrap();
16919 init_repo(&repo);
16920 let mut runner = runner_at(RunStatus::Implementing);
16921 runner.state.repo = repo.clone();
16922 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16923 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16924 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16925 let branch = format!("candidate-{label}");
16926 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16927 git::worktree_add_branch(&repo, &wt, &branch, "main")
16928 .await
16929 .unwrap();
16930 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16931 git::git(&wt, &["add", "README.md"]).await.unwrap();
16932 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16933 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16934 runner.state.candidates.push(Candidate {
16935 index,
16936 label,
16937 agent: "alpha".to_owned(),
16938 branch,
16939 worktree: wt,
16940 summary: "completed implementation".to_owned(),
16941 stat: String::new(),
16942 files: 0,
16943 commits: 0,
16944 empty: false,
16945 failed: None,
16946 verified_noop: None,
16947 duration_ms: 42,
16948 folded: false,
16949 });
16950 runner
16951 .state
16952 .candidate_preservation
16953 .insert(label, "awaiting persistence".to_owned());
16954 }
16955 let wt = runner.state.candidates[0].worktree.clone();
16956 let index_lock = git::git(
16957 &wt,
16958 &[
16959 "rev-parse",
16960 "--path-format=absolute",
16961 "--git-path",
16962 "index.lock",
16963 ],
16964 )
16965 .await
16966 .unwrap();
16967 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16968 runner.state.save().unwrap();
16969 assert!(runner.persist_candidates().await.is_err());
16970 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16971 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16972 assert_eq!(
16973 runner.state.candidates[1].commits, 1,
16974 "other results are still collected"
16975 );
16976 assert!(
16977 !runner.state.candidates[0].empty,
16978 "failed rescue is not no change"
16979 );
16980 assert_eq!(
16981 runner.state.candidates[0].summary,
16982 "completed implementation"
16983 );
16984 assert!(runner.after_implement().is_err());
16985 let home = tmp.path().join("fold-home");
16986 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16987 assert!(wt.join("new-file").exists());
16988 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16989 assert!(recovery.join("staged.patch").exists());
16990 assert!(recovery.join("unstaged.patch").exists());
16991 assert_eq!(
16992 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16993 "untracked edits\n"
16994 );
16995 runner.state.begin_pass();
16996 runner.state.end_pass_parked(ParkCause::Upgrade {
16997 from: "1.0".into(),
16998 to: Some("1.1".into()),
16999 });
17000 runner.state.save().unwrap();
17001 runner.state = RunState::load(&runner.state.id).unwrap();
17002 assert!(matches!(
17003 runner.state.passes.last().unwrap().end,
17004 Some(crate::run::PassEnd::Parked {
17005 cause: ParkCause::Upgrade { .. }
17006 })
17007 ));
17008 std::fs::remove_file(index_lock).unwrap();
17009 runner.implement().await.unwrap();
17011 assert!(runner.state.candidate_preservation.is_empty());
17012 assert!(
17013 runner
17014 .state
17015 .candidates
17016 .iter()
17017 .all(|c| c.commits == 1 && !c.empty)
17018 );
17019 assert!(git::is_clean(&wt).await.unwrap());
17020 assert!(
17021 crate::run::read_artifact(&runner.state, "cand-A.patch")
17022 .unwrap()
17023 .contains("later tracked edits")
17024 );
17025 }
17026
17027 #[tokio::test]
17028 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
17029 ask_test_home();
17030 let tmp = tempfile::tempdir().unwrap();
17031 let repo = tmp.path().join("repo");
17032 std::fs::create_dir(&repo).unwrap();
17033 init_repo(&repo);
17034 let wt = tmp.path().join("candidate");
17035 git::worktree_add_branch(&repo, &wt, "candidate", "main")
17036 .await
17037 .unwrap();
17038 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
17039 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
17040 assert!(wt.join("uncommitted").exists());
17041 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
17042 std::fs::remove_file(wt.join("uncommitted")).unwrap();
17043 let mut runner = runner_at(RunStatus::Implementing);
17044 runner.state.repo = repo.clone();
17045 runner.state.base_commit = "missing-base".to_owned();
17046 runner.state.candidates.push(Candidate {
17047 index: 0,
17048 label: 'A',
17049 agent: "alpha".into(),
17050 branch: "candidate".into(),
17051 worktree: wt.clone(),
17052 summary: "completed".into(),
17053 stat: String::new(),
17054 files: 0,
17055 commits: 0,
17056 empty: false,
17057 failed: None,
17058 verified_noop: None,
17059 duration_ms: 1,
17060 folded: false,
17061 });
17062 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
17063 assert!(
17064 fold_run(&mut runner.state, true, &tmp.path().join("home"))
17065 .await
17066 .is_err()
17067 );
17068 assert!(!runner.state.candidates[0].folded);
17069 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
17070 std::fs::remove_file(wt.join("uncommitted")).unwrap();
17071 runner
17072 .state
17073 .candidate_preservation
17074 .insert('A', "pending".into());
17075 assert!(runner.persist_candidates().await.is_err());
17076 assert!(!runner.state.candidates[0].empty);
17077 assert!(!runner.state.candidates[0].folded);
17078 assert!(wt.exists());
17079 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
17080 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
17081 std::fs::create_dir_all(&patch_path).unwrap();
17082 assert!(
17083 runner.persist_candidates().await.is_err(),
17084 "artifact failure stays pending"
17085 );
17086 assert!(runner.state.candidate_preservation.contains_key(&'A'));
17087 assert!(!runner.state.candidates[0].empty);
17088 std::fs::remove_dir(patch_path).unwrap();
17089 runner.persist_candidates().await.unwrap();
17090 assert!(
17091 runner.state.candidates[0].empty,
17092 "only successful inspection confirms no change"
17093 );
17094 }
17095
17096 fn init_repo(dir: &Path) {
17097 let run = |args: &[&str]| {
17098 let out = std::process::Command::new("git")
17099 .args(args)
17100 .current_dir(dir)
17101 .quiet()
17102 .output()
17103 .expect("spawn git");
17104 assert!(
17105 out.status.success(),
17106 "git {args:?} failed: {}",
17107 String::from_utf8_lossy(&out.stderr)
17108 );
17109 };
17110 run(&["init", "-b", "main"]);
17111 run(&["config", "user.name", "magi test"]);
17112 run(&["config", "user.email", "magi@example.com"]);
17113 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
17114 run(&["add", "-A"]);
17115 run(&["commit", "-m", "init"]);
17116 }
17117
17118 fn ask_test_home() {
17126 crate::run::pin_test_home();
17127 }
17128
17129 #[test]
17130 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
17131 ask_test_home();
17132 let mut runner = runner_at(RunStatus::Prep);
17133 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17134 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
17137 let error = guard
17138 .finish(
17139 &mut runner.state,
17140 Err(
17141 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
17142 .into(),
17143 ),
17144 )
17145 .unwrap_err();
17146 assert_eq!(
17147 error.downcast_ref::<std::io::Error>().unwrap().kind(),
17148 std::io::ErrorKind::PermissionDenied
17149 );
17150 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
17151 let message = format!("{error:#}");
17152 assert!(message.contains("original refusal"));
17153 assert!(!message.contains("could not save terminal"));
17154 assert_eq!(
17155 error.chain().count(),
17156 2,
17157 "only startup attribution wraps the original error"
17158 );
17159 let saved = RunState::load(&runner.state.id).unwrap();
17160 assert_eq!(saved.status, RunStatus::Blocked);
17161 assert!(!saved.startup.unwrap().complete);
17162 }
17163
17164 #[test]
17165 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
17166 ask_test_home();
17167 let mut runner = runner_at(RunStatus::Prep);
17168 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17169 let id = runner.state.id.clone();
17170 drop(guard);
17171 let saved = RunState::load(&id).unwrap();
17172 assert_eq!(saved.status, RunStatus::Blocked);
17173 assert!(!saved.startup.as_ref().unwrap().complete);
17174 assert!(
17175 saved
17176 .startup
17177 .unwrap()
17178 .failure
17179 .unwrap()
17180 .contains("interrupted")
17181 );
17182 assert!(Runner::resume(&id).is_err());
17183 }
17184
17185 #[test]
17186 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
17187 ask_test_home();
17188 let mut runner = runner_at(RunStatus::Prep);
17189 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17190 let error = guard
17191 .finish(
17192 &mut runner.state,
17193 Err(
17194 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
17195 .into(),
17196 ),
17197 )
17198 .unwrap_err();
17199 assert_eq!(
17200 error.downcast_ref::<std::io::Error>().unwrap().kind(),
17201 std::io::ErrorKind::PermissionDenied
17202 );
17203 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
17204 let mut task = crate::queue::Task::new(
17205 "review".into(),
17206 "criteria".into(),
17207 runner.state.repo.clone(),
17208 crate::queue::Source::Human,
17209 );
17210 task.record_startup_failure(failure.clone());
17211 task.record_startup_failure(failure.clone());
17212 assert_eq!(task.startup_failures.len(), 1);
17213 assert!(task.runs.is_empty());
17214 assert_eq!(task.attempts, 0);
17215 }
17216
17217 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
17218 ask_test_home();
17219 let tmp = tempfile::tempdir().unwrap();
17220 let repo = tmp.path().join("repo");
17221 std::fs::create_dir(&repo).unwrap();
17222 init_repo(&repo);
17223 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17224 let origin = tmp.path().join("origin.git");
17225 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
17226 .await
17227 .unwrap();
17228 git::git(
17229 &repo,
17230 &["remote", "add", "origin", origin.to_str().unwrap()],
17231 )
17232 .await
17233 .unwrap();
17234 git::git(&repo, &["checkout", "-b", "fix/retained"])
17235 .await
17236 .unwrap();
17237 std::fs::write(repo.join("retained"), "retained change").unwrap();
17238 git::git(&repo, &["add", "."]).await.unwrap();
17239 git::git(&repo, &["commit", "-m", "retained implementation"])
17240 .await
17241 .unwrap();
17242 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
17243 git::git(&repo, &["checkout", "main"]).await.unwrap();
17244 std::fs::write(repo.join("base-change"), "new base").unwrap();
17245 git::git(&repo, &["add", "."]).await.unwrap();
17246 git::git(&repo, &["commit", "-m", "base moved"])
17247 .await
17248 .unwrap();
17249 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17250 git::git(&repo, &["checkout", "fix/retained"])
17251 .await
17252 .unwrap();
17253 let mut runner = runner_at(RunStatus::Voting);
17254 runner.state.repo = repo.clone();
17255 runner.state.base_commit = base;
17256 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17257 runner.state.config.disk.min_free_bytes = 0;
17258 runner.state.config.graph.review_rounds = 1;
17259 runner.state.config.verify.e2e = vec!["true".into()];
17260 runner.state.config.verify.gate = vec!["true".into()];
17261 runner.state.candidates = vec![Candidate {
17262 index: 0,
17263 label: 'A',
17264 agent: "fixture".into(),
17265 branch: "fix/retained".into(),
17266 worktree: repo.clone(),
17267 summary: String::new(),
17268 stat: String::new(),
17269 files: 1,
17270 commits: 1,
17271 empty: false,
17272 failed: None,
17273 verified_noop: None,
17274 duration_ms: 0,
17275 folded: false,
17276 }];
17277 runner.state.tally = Some(Tally {
17278 first_choice: BTreeMap::new(),
17279 borda: BTreeMap::new(),
17280 winner: 'A',
17281 rankings: 0,
17282 unanimous_initial: true,
17283 deliberated: false,
17284 changed_votes: 0,
17285 unanimous_final: true,
17286 tie_break: None,
17287 judges: 0,
17288 present: 0,
17289 quorum: 0,
17290 met_quorum: true,
17291 uncontested: Some("review-only".into()),
17292 });
17293 let mut task = crate::queue::Task::new(
17294 "fixture".into(),
17295 "fixture".into(),
17296 repo,
17297 crate::queue::Source::Human,
17298 );
17299 task.link_run(&runner.state.id);
17300 crate::queue::Queue::open().put(&mut task).unwrap();
17301 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
17302 runner.state.base_sync = Some(BaseSync {
17303 tip: String::new(),
17304 behind: 1,
17305 attempts: BASE_SYNC_ROUNDS,
17306 conflict: None,
17307 already_in: None,
17308 });
17309 runner.sync_to_base().await.unwrap();
17310 assert_eq!(runner.state.status, RunStatus::Blocked);
17311 (tmp, runner, head, task.id)
17312 }
17313
17314 #[tokio::test]
17315 async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
17316 let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
17317 runner.state.status = RunStatus::Voting;
17318 runner.state.base_sync = None;
17319 runner.state.base_sync_recovery = Default::default();
17320 let scratch = runner.state.dir().join("base-sync");
17321 std::fs::create_dir_all(&scratch).unwrap();
17322 std::fs::write(scratch.join("obstruction"), "keep").unwrap();
17323 runner.sync_to_base().await.unwrap();
17324 assert!(runner.state.resource_block.is_some());
17325 assert!(!runner.state.base_sync_blocked());
17326 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
17327 assert_eq!(runner.state.base_episode_used(), 0);
17328 assert!(runner.state.base_sync_recovery.operations.is_empty());
17329 assert!(runner.state.rebase_fixes.is_empty());
17330 assert_eq!(
17331 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17332 head
17333 );
17334 std::fs::remove_dir_all(&scratch).unwrap();
17335 runner.state.resource_block = None;
17336 runner.sync_to_base().await.unwrap();
17337 assert!(!runner.state.base_sync_blocked());
17338 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17339 assert_eq!(runner.state.base_episode_used(), 1);
17340 assert!(runner.state.base_sync_recovery.operations[0].finished);
17341 assert!(runner.state.rebase_fixes.is_empty());
17342 assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
17343 assert!(git::is_clean(&runner.state.repo).await.unwrap());
17344 }
17345
17346 #[cfg(unix)]
17347 #[tokio::test]
17348 async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
17349 for routed in [false, true] {
17350 let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
17351 let home = crate::run::home();
17352 let queue = crate::queue::Queue::open();
17353 let task = queue.get(&owner).unwrap();
17354 runner.state.task_instruction = Some(task.instruction.clone());
17355 runner.state.driver_exited = true;
17356 runner.state.save().unwrap();
17357 let questions = crate::ask::Questions::at(home.join("questions"));
17358 let mut q = crate::ask::Question::new(
17359 runner.state.id.clone(),
17360 "base-recovery".into(),
17361 "owner".into(),
17362 "Recover bounded base movement?".into(),
17363 String::new(),
17364 vec!["recover".into()],
17365 );
17366 q.actions.insert(
17367 "recover".into(),
17368 crate::ask::ChoiceAction::RecoverBase {
17369 run: runner.state.id.clone(),
17370 stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
17371 episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
17372 },
17373 );
17374 q.routed_task = Some(owner.clone());
17375 q.answer(crate::ask::Answer::Choice("recover".into()))
17376 .unwrap();
17377 questions.put(&mut q).unwrap();
17378 crate::base_recovery_control::recover(
17379 &queue,
17380 &home,
17381 crate::base_recovery_control::Request {
17382 run: runner.state.id.clone(),
17383 head: head.clone(),
17384 actor: "decisions".into(),
17385 request: q.id.clone(),
17386 decision: Some((owner.clone(), q.answered_at.unwrap())),
17387 expected_stop: Some((
17388 runner.state.base_sync_recovery.stop.clone().unwrap(),
17389 runner.state.base_sync_recovery.episodes.last().unwrap().id,
17390 )),
17391 },
17392 )
17393 .await
17394 .unwrap();
17395 if routed {
17396 questions
17397 .update(&q.id, |q| {
17398 q.routed_task = Some("another-owner".into());
17399 Ok(())
17400 })
17401 .unwrap();
17402 } else {
17403 let mut task = queue.get(&owner).unwrap();
17404 task.hold_manual(Some("new owner decision".into()));
17405 task.status = crate::queue::TaskStatus::Queued;
17407 queue.put(&mut task).unwrap();
17408 }
17409 runner.state = RunState::load(&runner.state.id).unwrap();
17410 runner.sync_to_base().await.unwrap();
17411 let approval = runner
17412 .state
17413 .base_sync_recovery
17414 .authorizations
17415 .last()
17416 .unwrap();
17417 assert!(approval.consumed);
17418 assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17419 "routing"
17420 } else {
17421 "owner decision"
17422 }));
17423 assert_eq!(
17424 runner.state.base_sync.as_ref().unwrap().attempts,
17425 BASE_SYNC_ROUNDS
17426 );
17427 assert_eq!(
17428 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17429 head
17430 );
17431 }
17432 }
17433
17434 #[cfg(unix)]
17435 #[tokio::test]
17436 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17437 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17438 if std::env::var_os(CHILD).is_none() {
17439 let home = tempfile::tempdir().unwrap();
17440 let output = std::process::Command::new(std::env::current_exe().unwrap())
17441 .args([
17442 "--exact",
17443 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17444 "--nocapture",
17445 ])
17446 .env(CHILD, "1")
17447 .env("MAGI_HOME", home.path())
17448 .env("MAGI_WORKER_HOME", home.path())
17449 .env("MAGI_WORKER_ID", "abcdef")
17450 .output()
17451 .unwrap();
17452 assert!(
17453 output.status.success()
17454 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17455 "{}\n{}",
17456 String::from_utf8_lossy(&output.stdout),
17457 String::from_utf8_lossy(&output.stderr)
17458 );
17459 return;
17460 }
17461 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17462 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17463 let home = crate::run::home();
17464 let queue = crate::queue::Queue::open();
17465 let mut owner = queue.get(&task).unwrap();
17466 owner.release_operator("test-release", None);
17467 owner.start(runner.state.id.clone());
17468 queue.put(&mut owner).unwrap();
17469 runner.state.driver_exited = true;
17470 runner.state.task_instruction = Some(owner.instruction.clone());
17471 runner.state.save().unwrap();
17472 let _claim = queue.claim(&task).unwrap();
17473 let reserve = |id: &str| {
17474 let directory = home.join("workers/jobs").join(id);
17475 std::fs::create_dir_all(&directory).unwrap();
17476 let job = crate::worker::Job {
17477 id: id.into(),
17478 sequence: 1,
17479 kind: "task".into(),
17480 key: format!("task:{task}"),
17481 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17482 generation: "fixture-generation".into(),
17483 version: None,
17484 status: crate::worker::Status::Running,
17485 result: None,
17486 pid: Some(std::process::id()),
17487 created_at: 0,
17488 finished_at: None,
17489 };
17490 std::fs::write(
17491 directory.join("job.json"),
17492 serde_json::to_vec(&job).unwrap(),
17493 )
17494 .unwrap();
17495 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17496 };
17497 let _own = reserve("abcdef");
17498 let other = reserve("fedcba");
17499 assert!(
17500 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17501 .await
17502 .is_err()
17503 );
17504 assert!(
17505 RunState::load(&runner.state.id)
17506 .unwrap()
17507 .base_sync_recovery
17508 .authorizations
17509 .is_empty()
17510 );
17511 drop(other);
17512 assert!(crate::daemon::is_working_on(
17513 &home,
17514 &runner.state.id,
17515 Timestamp::now()
17516 ));
17517 assert!(
17518 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17519 );
17520 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17521 .await
17522 .unwrap();
17523 assert!(
17524 RunState::load(&runner.state.id)
17525 .unwrap()
17526 .base_recovery_pending()
17527 );
17528 }
17529
17530 #[cfg(unix)]
17531 #[tokio::test]
17532 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17533 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17534 let home = crate::run::home();
17535 runner.state.driver_pid = Some(std::process::id());
17536 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17537 runner.state.driver_exited = false;
17538 runner.state.begin_pass();
17539 runner.state.save().unwrap();
17540 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17541 driver.record(&runner.state).unwrap();
17542 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17543 drop(driver);
17545 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17546 assert_eq!(
17547 runner.state.liveness_under(&home, false),
17548 crate::run::Liveness::Dead
17549 );
17550 let queue = crate::queue::Queue::open();
17551 let _claim = queue.claim(&task).unwrap();
17552 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17553 .await
17554 .unwrap();
17555 let disk = RunState::load(&runner.state.id).unwrap();
17556 assert!(!disk.driver_exited);
17557 assert!(disk.base_recovery_pending());
17558 }
17559
17560 #[cfg(unix)]
17561 #[tokio::test]
17562 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17563 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17564 if std::env::var_os(CHILD).is_none() {
17565 use std::os::unix::fs::PermissionsExt;
17566 let tmp = tempfile::tempdir().unwrap();
17567 let gh = tmp.path().join("gh");
17568 std::fs::write(&gh, r#"#!/bin/sh
17569case "$1 $2" in
17570 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17571 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17572 *) exit 2 ;;
17573esac
17574"#).unwrap();
17575 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17576 let mut paths = vec![tmp.path().to_path_buf()];
17577 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17578 let output = std::process::Command::new(std::env::current_exe().unwrap())
17579 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17580 .env(CHILD, "1")
17581 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17582 .env("PATH", std::env::join_paths(paths).unwrap())
17583 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17584 .output().unwrap();
17585 assert!(
17586 output.status.success()
17587 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17588 "{}\n{}",
17589 String::from_utf8_lossy(&output.stdout),
17590 String::from_utf8_lossy(&output.stderr)
17591 );
17592 return;
17593 }
17594 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17595 let repo = runner.state.repo.clone();
17596 git::git(&repo, &["checkout", "main"]).await.unwrap();
17597 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17598 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17599 git::git(&repo, &["checkout", "fix/retained"])
17600 .await
17601 .unwrap();
17602 runner.state.pr = Some(crate::run::PrRecord {
17603 url: "https://example.test/project/pull/1".into(),
17604 number: 1,
17605 state: "open".into(),
17606 checks: "unknown".into(),
17607 round: 0,
17608 rounds: 1,
17609 red_at_merge: Vec::new(),
17610 });
17611 runner.sync_to_base().await.unwrap();
17612 assert_eq!(runner.state.status, RunStatus::Blocked);
17613 assert_eq!(
17614 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17615 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17616 );
17617 assert!(runner.state.base_sync_closure_pending());
17618 assert!(crate::handover::can_resume(&runner.state));
17619 let queue = crate::queue::Queue::open();
17620 assert!(!crate::base_sync_recovery::held_task(
17621 &queue,
17622 &queue.get(&task).unwrap()
17623 ));
17624 assert!(runner.state.authorize_base_sync(&task).is_err());
17625 runner.state.base_sync_recovery.stop = None;
17627 runner.state.status = RunStatus::Voting;
17628 runner.state.gate_ran = true;
17629 runner.state.config.graph.review_rounds = 0;
17630 runner.state.save().unwrap();
17631 assert!(crate::handover::can_resume(&runner.state));
17632 runner.state = RunState::load(&runner.state.id).unwrap();
17633 runner.execute_graph().await.unwrap();
17634 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17635 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17636 assert_eq!(
17637 runner.state.base_sync.as_ref().unwrap().attempts,
17638 BASE_SYNC_ROUNDS
17639 );
17640 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17641 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17642 assert!(runner.state.reviews.is_empty());
17643 assert!(runner.state.gate.is_empty());
17644 }
17645
17646 #[cfg(unix)]
17647 #[tokio::test]
17648 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17649 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17650 runner.state.driver_exited = true;
17651 runner.state.save().unwrap();
17652 let home = crate::run::home();
17653 let queue = crate::queue::Queue::open();
17654 let _claim = queue.claim(&task).unwrap();
17655 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17656 assert!(
17657 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17658 .await
17659 .is_err()
17660 );
17661 drop(lease);
17662 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17663 assert!(
17664 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17665 .await
17666 .is_err()
17667 );
17668 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17669 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17670 .await
17671 .unwrap();
17672 let disk = RunState::load(&runner.state.id).unwrap();
17673 assert!(disk.base_recovery_pending());
17674 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17675 assert!(
17676 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17677 .await
17678 .is_err()
17679 );
17680 assert_eq!(
17681 RunState::load(&runner.state.id)
17682 .unwrap()
17683 .base_sync_recovery
17684 .authorizations
17685 .len(),
17686 1
17687 );
17688 let mut owner = queue.get(&task).unwrap();
17689 owner.runs.push("20260101-000000-ffff".into());
17690 queue.put(&mut owner).unwrap();
17691 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17692 .await
17693 .unwrap_err();
17694 assert!(error.to_string().contains("run task ownership changed"));
17695 }
17696
17697 #[cfg(unix)]
17698 #[tokio::test]
17699 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17700 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17701 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17702 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17703 legacy["status"] = serde_json::json!("voting");
17704 runner.state = serde_json::from_value(legacy).unwrap();
17705 runner.state.save().unwrap();
17706 for _ in 0..3 {
17707 runner.execute_graph().await.unwrap();
17708 assert_eq!(runner.state.status, RunStatus::Blocked);
17709 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17710 assert!(!crate::handover::can_resume(&runner.state));
17711 assert_eq!(
17712 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17713 head
17714 );
17715 assert!(runner.state.reviews.is_empty());
17716 assert!(!runner.state.gate_ran);
17717 }
17718 let notice = crate::notices::run_ended(&runner.state).unwrap();
17719 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17720 runner.state.begin_pass();
17721 runner.state.status = RunStatus::Voting;
17722 runner.deliberate().await.unwrap();
17723 runner.vote().await.unwrap();
17724 runner.finish_after_tally().await.unwrap();
17725 assert_eq!(runner.state.status, RunStatus::Blocked);
17726 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17727 assert_eq!(
17728 crate::notices::run_ended(&runner.state).unwrap().since,
17729 notice.since
17730 );
17731 }
17732
17733 #[cfg(unix)]
17734 #[tokio::test]
17735 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17736 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17737 for detail in [
17738 "fix/retained conflicts with origin/main and did not rebase: conflict",
17739 "rebased fix/retained locally but origin refused the push",
17740 "unknown failure",
17741 ] {
17742 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17743 runner.state.record_base_sync_stop(&head);
17744 assert!(runner.state.authorize_base_sync(&task).is_err());
17745 runner.state.status = RunStatus::Voting;
17746 runner.execute_graph().await.unwrap();
17747 assert_eq!(runner.state.status, RunStatus::Blocked);
17748 assert!(!crate::handover::can_resume(&runner.state));
17749 assert_eq!(
17750 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17751 head
17752 );
17753 }
17754 }
17755
17756 #[cfg(unix)]
17757 #[tokio::test]
17758 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17759 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17760 runner.state.authorize_base_sync(&task).unwrap();
17761 assert!(runner.state.consume_base_recovery(&head));
17762 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17763 runner.state.reserve_base_operation(&head, &tip);
17764 runner.state.save().unwrap();
17765 runner.state = RunState::load(&runner.state.id).unwrap();
17766 assert!(!runner.state.consume_base_recovery(&head));
17767 runner.sync_to_base().await.unwrap();
17768 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17769 assert_eq!(runner.state.base_episode_used(), 1);
17770 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17771 assert!(
17772 runner
17773 .state
17774 .base_sync_recovery
17775 .operations
17776 .last()
17777 .unwrap()
17778 .finished
17779 );
17780 assert!(!runner.state.base_sync_blocked());
17781 runner.sync_to_base().await.unwrap();
17782 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17783 }
17784
17785 #[cfg(unix)]
17786 #[tokio::test]
17787 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17788 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17789 let original = runner
17791 .state
17792 .base_sync
17793 .as_ref()
17794 .unwrap()
17795 .conflict
17796 .clone()
17797 .unwrap();
17798 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17799 original
17800 .replace("synchronization(s)", "rebase(s)")
17801 .replace("synchronizing", "rebasing"),
17802 );
17803 runner.state.base_sync_recovery = Default::default();
17804 runner.state.record_base_sync_stop(&head);
17805 assert_eq!(
17806 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17807 crate::base_sync_recovery::StopKind::Exhausted
17808 );
17809 runner.state.authorize_base_sync(&task).unwrap();
17810 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17811 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17812 .as_object_mut()
17813 .unwrap();
17814 auth.remove("interval");
17815 auth.remove("scope");
17816 runner.state = serde_json::from_value(legacy).unwrap();
17817 assert!(runner.state.consume_base_recovery(&head));
17818 assert_eq!(runner.state.base_episode_used(), 3);
17819 runner.sync_to_base().await.unwrap();
17820 assert_eq!(runner.state.base_episode_used(), 4);
17821 runner.state.resource_block = Some(crate::run::ResourceBlock {
17822 node: "gate".into(),
17823 kind: "cache".into(),
17824 detail: "legacy wait".into(),
17825 at: Timestamp::now(),
17826 cause: crate::run::ResourceBlockCause::Unrecorded,
17827 });
17828 runner.state.save().unwrap();
17829 assert!(
17830 runner
17831 .state
17832 .base_sync_recovery
17833 .episodes
17834 .last()
17835 .unwrap()
17836 .boundary
17837 .is_none()
17838 );
17839 runner.state.resource_block = None;
17840 advance_base_fixture(&runner).await;
17841 runner.sync_to_base().await.unwrap();
17842 assert!(runner.state.base_sync_blocked());
17843 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17844 }
17845
17846 #[cfg(unix)]
17847 async fn advance_base_fixture(runner: &Runner) {
17848 let repo = &runner.state.repo;
17849 for args in [
17850 vec!["checkout", "main"],
17851 vec!["commit", "--allow-empty", "-m", "base advanced"],
17852 vec!["push", "origin", "main"],
17853 vec!["checkout", "fix/retained"],
17854 ] {
17855 git::git(repo, &args).await.unwrap();
17856 }
17857 }
17858
17859 #[cfg(unix)]
17860 fn fresh_base_interval_fixture(runner: &mut Runner) {
17861 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17862 runner.state.base_sync.as_mut().unwrap().conflict = None;
17863 runner.state.base_sync_recovery = Default::default();
17864 runner.state.status = RunStatus::Voting;
17865 }
17866
17867 #[cfg(unix)]
17868 #[tokio::test]
17869 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17870 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17871 fresh_base_interval_fixture(&mut runner);
17872 runner.state.begin_pass();
17873 runner.sync_to_base().await.unwrap();
17874 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17875 runner.state.resource_block = Some(crate::run::ResourceBlock {
17876 node: "gate".into(),
17877 kind: "build cache".into(),
17878 detail: "busy".into(),
17879 at: Timestamp::now(),
17880 cause: crate::run::ResourceBlockCause::Timeout {
17881 waited_secs: 1,
17882 limit_secs: 1,
17883 },
17884 });
17885 runner.state.save().unwrap();
17886 let queue = crate::queue::Queue::open();
17887 let mut owner = queue.get(&task).unwrap();
17888 runner.state.task_instruction = Some(owner.instruction.clone());
17889 owner.release_operator("fixture-owner", None);
17890 queue.put(&mut owner).unwrap();
17891 let approval = owner.resumption.clone().unwrap();
17892 runner.state.resource_block = None;
17893 runner.state.save().unwrap();
17894 runner.state = RunState::load(&runner.state.id).unwrap();
17895 advance_base_fixture(&runner).await;
17896 runner.sync_to_base().await.unwrap();
17897 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17898 assert_eq!(runner.state.base_episode_used(), 1);
17899 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17900 assert!(
17901 runner.state.base_sync_recovery.episodes[0]
17902 .boundary
17903 .as_ref()
17904 .unwrap()
17905 .consumed
17906 );
17907 assert!(
17908 !runner
17909 .state
17910 .renew_base_episode(&task, &approval, &head, "base")
17911 );
17912 for _ in 0..3 {
17913 owner.release_operator("fixture-owner", None);
17914 queue.put(&mut owner).unwrap();
17915 runner.state = RunState::load(&runner.state.id).unwrap();
17916 runner.state.begin_pass();
17917 runner.sync_to_base().await.unwrap();
17918 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17919 assert_eq!(runner.state.base_episode_used(), 1);
17920 }
17921 runner.state.resource_block = Some(crate::run::ResourceBlock {
17922 node: "gate".into(),
17923 kind: "disk pressure".into(),
17924 detail: "free space required".into(),
17925 at: Timestamp::now(),
17926 cause: crate::run::ResourceBlockCause::Other,
17927 });
17928 runner.state.end_pass_parked(ParkCause::Interrupt {
17929 reason: "disk pressure".into(),
17930 });
17931 runner.state.save().unwrap();
17932 owner.release_operator("fixture-owner-after-disk", None);
17933 queue.put(&mut owner).unwrap();
17934 runner.state.resource_block = None;
17935 advance_base_fixture(&runner).await;
17936 runner.sync_to_base().await.unwrap();
17937 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17938 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17939 assert_eq!(runner.state.base_episode_used(), 1);
17940 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17941 }
17942
17943 #[cfg(unix)]
17944 #[tokio::test]
17945 async fn base_recovery_continuous_advancement_stops_at_four() {
17946 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17947 fresh_base_interval_fixture(&mut runner);
17948 for spent in 1..=BASE_SYNC_ROUNDS {
17949 runner.sync_to_base().await.unwrap();
17950 assert!(!runner.state.base_sync_blocked());
17951 assert_eq!(runner.state.base_episode_used(), spent);
17952 advance_base_fixture(&runner).await;
17953 }
17954 runner.sync_to_base().await.unwrap();
17955 assert_eq!(
17956 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17957 crate::base_sync_recovery::StopKind::Exhausted
17958 );
17959 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17960 assert!(stop.unverified_sync);
17961 let action = runner.state.base_sync_next_action();
17962 assert!(action.contains("never reached its gate"));
17963 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17964 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17965 runner.state = RunState::load(&runner.state.id).unwrap();
17966 runner.sync_to_base().await.unwrap();
17967 assert_eq!(runner.state.base_episode_used(), 4);
17968 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17969 }
17970
17971 #[cfg(unix)]
17972 #[tokio::test]
17973 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17974 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17975 runner.state.ensure_base_episode();
17976 runner.state.record_base_progress(&head);
17978 runner.state.base_sync_recovery.stop = None;
17979 runner.state.record_base_sync_stop(&head);
17980 assert!(
17981 !runner
17982 .state
17983 .base_sync_recovery
17984 .stop
17985 .as_ref()
17986 .unwrap()
17987 .unverified_sync
17988 );
17989
17990 runner.state.finish_base_operation(Some(&head));
17991 runner.state.gate_ran = true;
17993 runner.state.gate_head = Some("older-head".into());
17994 runner.state.base_sync_recovery.stop = None;
17995 runner.state.record_base_sync_stop(&head);
17996 runner.state.save().unwrap();
17997 runner.state = RunState::load(&runner.state.id).unwrap();
17998 assert!(
17999 runner
18000 .state
18001 .base_sync_recovery
18002 .stop
18003 .as_ref()
18004 .unwrap()
18005 .unverified_sync
18006 );
18007
18008 runner.state.gate_head = Some(head.clone());
18009 runner.state.base_sync_recovery.stop = None;
18010 runner.state.record_base_sync_stop(&head);
18011 assert!(
18012 !runner
18013 .state
18014 .base_sync_recovery
18015 .stop
18016 .as_ref()
18017 .unwrap()
18018 .unverified_sync
18019 );
18020 assert!(
18021 !runner
18022 .state
18023 .base_sync_next_action()
18024 .contains("never reached its gate")
18025 );
18026
18027 let mut legacy = serde_json::to_value(&runner.state).unwrap();
18029 legacy["base_sync_recovery"]["stop"]
18030 .as_object_mut()
18031 .unwrap()
18032 .remove("unverified_sync");
18033 legacy["base_sync_recovery"]["episodes"][0]
18034 .as_object_mut()
18035 .unwrap()
18036 .remove("last_sync_head");
18037 runner.state = serde_json::from_value(legacy).unwrap();
18038 assert!(
18039 !runner
18040 .state
18041 .base_sync_recovery
18042 .stop
18043 .as_ref()
18044 .unwrap()
18045 .unverified_sync
18046 );
18047 assert!(
18048 runner.state.base_sync_recovery.episodes[0]
18049 .last_sync_head
18050 .is_none()
18051 );
18052 }
18053
18054 #[cfg(unix)]
18055 #[tokio::test]
18056 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
18057 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18058 runner.state.authorize_base_sync(&task).unwrap();
18059 assert!(
18060 runner
18061 .state
18062 .base_sync_next_action()
18063 .contains("Approval recorded")
18064 );
18065 assert!(
18066 runner
18067 .state
18068 .base_sync_next_action()
18069 .contains("4/4 in episode 0, 4 lifetime")
18070 );
18071 assert!(runner.state.consume_base_recovery(&head));
18072 assert!(
18073 runner
18074 .state
18075 .base_sync_next_action()
18076 .contains("0/4 in episode 1, 4 lifetime")
18077 );
18078 }
18079
18080 #[cfg(unix)]
18081 #[tokio::test]
18082 async fn base_recovery_changed_head_invalidates_without_spending() {
18083 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
18084 runner.state.authorize_base_sync(&task).unwrap();
18085 git::git(
18086 &runner.state.repo,
18087 &["commit", "--allow-empty", "-m", "external head change"],
18088 )
18089 .await
18090 .unwrap();
18091 runner.sync_to_base().await.unwrap();
18092 let a = runner
18093 .state
18094 .base_sync_recovery
18095 .authorizations
18096 .last()
18097 .unwrap();
18098 assert!(a.consumed);
18099 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
18100 runner.state = RunState::load(&runner.state.id).unwrap();
18101 assert!(
18102 runner
18103 .state
18104 .base_sync_next_action()
18105 .contains("Approval invalidated")
18106 );
18107 assert!(!runner.state.base_recovery_pending());
18108 assert!(runner.state.base_sync_blocked());
18109 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18110 }
18111
18112 #[cfg(unix)]
18113 #[tokio::test]
18114 async fn base_recovery_stale_fetched_base_invalidates_approval() {
18115 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18116 runner.state.driver_exited = true;
18117 runner.state.save().unwrap();
18118 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18119 .await
18120 .unwrap();
18121 runner.state = RunState::load(&runner.state.id).unwrap();
18122 advance_base_fixture(&runner).await;
18123 runner.sync_to_base().await.unwrap();
18124 let a = runner
18125 .state
18126 .base_sync_recovery
18127 .authorizations
18128 .last()
18129 .unwrap();
18130 assert!(
18131 a.consumed
18132 && a.invalidated
18133 .as_deref()
18134 .unwrap()
18135 .contains("fetched base changed")
18136 );
18137 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18138 assert_eq!(
18139 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18140 head
18141 );
18142 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18143 .await
18144 .unwrap();
18145 runner.state = RunState::load(&runner.state.id).unwrap();
18146 runner.sync_to_base().await.unwrap();
18147 assert!(!runner.state.base_sync_blocked());
18148 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18149 }
18150
18151 #[cfg(unix)]
18152 #[tokio::test]
18153 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
18154 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18155 runner.state.authorize_base_sync(&task).unwrap();
18156 assert!(runner.state.consume_base_recovery(&head));
18157 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18158 runner.state.reserve_base_operation(&head, &tip);
18159 runner.state.save().unwrap();
18160 let scratch = runner.state.dir().join("base-sync");
18161 assert_eq!(
18162 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18163 .await
18164 .unwrap(),
18165 crate::rebase::Rebased::Applied
18166 );
18167 runner.state = RunState::load(&runner.state.id).unwrap();
18168 runner.sync_to_base().await.unwrap();
18169 assert!(
18170 runner
18171 .state
18172 .base_sync_recovery
18173 .operations
18174 .last()
18175 .unwrap()
18176 .finished
18177 );
18178 assert_eq!(runner.state.base_episode_used(), 1);
18179 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18180 assert!(!runner.state.base_sync_blocked());
18181 }
18182
18183 #[cfg(unix)]
18184 #[tokio::test]
18185 async fn base_recovery_crash_preserves_untracked_files() {
18186 for ignored in [false, true] {
18187 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18188 runner.state.authorize_base_sync(&task).unwrap();
18189 assert!(runner.state.consume_base_recovery(&head));
18190 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18191 runner.state.reserve_base_operation(&head, &tip);
18192 runner.state.save().unwrap();
18193 let scratch = runner.state.dir().join("base-sync");
18194 assert_eq!(
18195 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18196 .await
18197 .unwrap(),
18198 crate::rebase::Rebased::Applied
18199 );
18200 let repo = &runner.state.repo;
18201 let result = git::rev_parse(repo, "HEAD").await.unwrap();
18202 assert_ne!(result, head);
18203 if ignored {
18204 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
18205 .await
18206 .unwrap();
18207 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
18208 }
18209 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
18210 assert!(
18211 git::git_raw(repo, &["diff", "--quiet", &head])
18212 .await
18213 .unwrap()
18214 .ok()
18215 );
18216 runner.state = RunState::load(&runner.state.id).unwrap();
18218 runner.sync_to_base().await.unwrap();
18219 assert!(runner.state.base_sync_blocked());
18220 assert!(!runner.state.base_sync_recovery.operations[0].finished);
18221 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18222 assert_eq!(
18223 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18224 result
18225 );
18226 assert_eq!(
18227 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
18228 "retained work\n"
18229 );
18230 assert!(
18231 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
18232 .await
18233 .unwrap()
18234 .ok()
18235 );
18236 }
18237 }
18238
18239 #[cfg(unix)]
18240 #[tokio::test]
18241 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
18242 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18243 runner.state.authorize_base_sync(&task).unwrap();
18244 assert!(runner.state.consume_base_recovery(&head));
18245 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18246 runner.state.reserve_base_operation(&head, &tip);
18247 runner.state.save().unwrap();
18248 let scratch = runner.state.dir().join("base-sync");
18249 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18250 .await
18251 .unwrap();
18252 git::sync_to_head(&runner.state.repo).await.unwrap();
18253 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
18254 git::git(&runner.state.repo, &["add", "foreign"])
18255 .await
18256 .unwrap();
18257 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
18258 .await
18259 .unwrap();
18260 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18261 runner.state = RunState::load(&runner.state.id).unwrap();
18262 runner.sync_to_base().await.unwrap();
18263 assert!(runner.state.base_sync_blocked());
18264 assert!(
18265 !runner
18266 .state
18267 .base_sync_recovery
18268 .operations
18269 .last()
18270 .unwrap()
18271 .finished
18272 );
18273 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18274 assert_eq!(
18275 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18276 foreign_head
18277 );
18278 }
18279
18280 #[cfg(unix)]
18281 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
18282 {
18283 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18284 fresh_base_interval_fixture(&mut runner);
18285 let repo = &runner.state.repo;
18286 git::git(repo, &["checkout", "main"]).await.unwrap();
18287 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
18288 git::git(repo, &["add", "retained"]).await.unwrap();
18289 git::git(repo, &["commit", "-m", "upstream retained change"])
18290 .await
18291 .unwrap();
18292 git::git(repo, &["push", "origin", "main"]).await.unwrap();
18293 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
18294 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
18295 let scratch = runner.state.dir().join("base-sync");
18296 assert!(matches!(
18297 git::rebase_start(repo, &scratch, "fix/retained", &tip)
18298 .await
18299 .unwrap(),
18300 git::RebaseStart::Conflicted(_)
18301 ));
18302 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
18303 agent: "fixture".into(),
18304 paths: vec!["retained".into()],
18305 from: Some(head.clone()),
18306 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
18307 index_outside: None,
18308 finished: false,
18309 error: None,
18310 launcher: None,
18311 infra: None,
18312 outage: None,
18313 });
18314 (tmp, runner, head, tip, scratch)
18315 }
18316
18317 #[cfg(unix)]
18318 #[tokio::test]
18319 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
18320 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18321 std::fs::write(
18324 scratch.join("retained"),
18325 "upstream and retained resolution\n",
18326 )
18327 .unwrap();
18328 runner.state.resource_block = Some(crate::run::ResourceBlock {
18329 node: "rebase".into(),
18330 kind: "repository coordination".into(),
18331 detail: "busy".into(),
18332 at: Timestamp::now(),
18333 cause: crate::run::ResourceBlockCause::Other,
18334 });
18335 runner.state.save().unwrap();
18336 let newer = git::git(
18339 &runner.state.repo,
18340 &[
18341 "commit-tree",
18342 "main^{tree}",
18343 "-p",
18344 "main",
18345 "-m",
18346 "base advanced during wait",
18347 ],
18348 )
18349 .await
18350 .unwrap();
18351 git::git(
18352 &runner.state.repo,
18353 &[
18354 "push",
18355 "origin",
18356 &format!("{}:refs/heads/main", newer.trim()),
18357 ],
18358 )
18359 .await
18360 .unwrap();
18361 runner.state = RunState::load(&runner.state.id).unwrap();
18362 runner.state.resource_block = None;
18363 runner.sync_to_base().await.unwrap();
18364 assert!(!runner.state.base_sync_blocked());
18365 let op = &runner.state.base_sync_recovery.operations[0];
18366 assert_eq!(op.head, head);
18367 assert_eq!(op.tip, tip);
18368 assert!(op.finished);
18369 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18370 assert_eq!(runner.state.rebase_fixes.len(), 1);
18371 let latest = git::rev_parse(&runner.state.repo, "origin/main")
18372 .await
18373 .unwrap();
18374 assert_ne!(latest, tip);
18375 runner.state = RunState::load(&runner.state.id).unwrap();
18376 runner.sync_to_base().await.unwrap();
18377 assert!(!runner.state.base_sync_blocked());
18378 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
18379 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
18380 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18381 }
18382
18383 #[cfg(unix)]
18384 #[tokio::test]
18385 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
18386 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18387 runner.state.reserve_base_operation(&head, &tip);
18388 runner.state.save().unwrap();
18389 std::fs::write(
18390 scratch.join("retained"),
18391 "upstream and retained resolution\n",
18392 )
18393 .unwrap();
18394 git::git(&scratch, &["add", "retained"]).await.unwrap();
18395 git::git(
18396 &scratch,
18397 &["-c", "core.editor=true", "rebase", "--continue"],
18398 )
18399 .await
18400 .unwrap();
18401 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18402 assert!(
18403 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18404 .await
18405 .unwrap()
18406 .is_empty()
18407 );
18408 assert_eq!(
18409 crate::rebase::finish(
18410 &mut runner.state,
18411 &scratch,
18412 "fix/retained",
18413 &head,
18414 &tip,
18415 &["retained".into()],
18416 ""
18417 )
18418 .await
18419 .unwrap(),
18420 crate::rebase::Rebased::Applied
18421 );
18422 assert!(!scratch.exists());
18423 runner.state = RunState::load(&runner.state.id).unwrap();
18425 let op = &runner.state.base_sync_recovery.operations[0];
18426 assert!(!op.finished);
18427 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18428 runner.sync_to_base().await.unwrap();
18429 assert!(!runner.state.base_sync_blocked());
18430 assert!(runner.state.base_sync_recovery.operations[0].finished);
18431 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18432 assert_eq!(runner.state.rebase_fixes.len(), 1);
18433 let winner = runner.state.winner().unwrap().clone();
18434 let wait = runner.repository_wait();
18435 assert!(
18436 verify_current_head(&mut runner.state, &winner, &wait)
18437 .await
18438 .unwrap()
18439 );
18440 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18441 }
18442
18443 #[cfg(unix)]
18444 #[tokio::test]
18445 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18446 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18447 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18448 runner.state.reserve_base_operation(&head, &tip);
18449 runner.state.save().unwrap();
18450 std::fs::write(
18451 scratch.join("retained"),
18452 "upstream and retained resolution\n",
18453 )
18454 .unwrap();
18455 git::git(&scratch, &["add", "retained"]).await.unwrap();
18456 git::git(
18457 &scratch,
18458 &["-c", "core.editor=true", "rebase", "--continue"],
18459 )
18460 .await
18461 .unwrap();
18462 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18463 assert!(!git::rebase_in_progress(&scratch).await);
18464 assert!(
18465 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18466 .await
18467 .unwrap()
18468 .is_empty()
18469 );
18470 match damage {
18471 Some("record") => {
18472 let path =
18473 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18474 .await
18475 .unwrap();
18476 std::fs::write(scratch.join(path), "").unwrap();
18477 }
18478 Some("foreign") => {
18479 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18480 git::git(&scratch, &["add", "foreign"]).await.unwrap();
18481 git::git(&scratch, &["commit", "-m", "foreign work"])
18482 .await
18483 .unwrap();
18484 }
18485 Some("dirty") => {
18486 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18487 }
18488 _ => {}
18489 }
18490 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18491 runner.state = RunState::load(&runner.state.id).unwrap();
18493 assert!(
18494 runner.state.base_sync_recovery.operations[0]
18495 .validated_head
18496 .is_none()
18497 );
18498 runner.sync_to_base().await.unwrap();
18499 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18500 assert_eq!(runner.state.base_episode_used(), 1);
18501 assert_eq!(runner.state.rebase_fixes.len(), 1);
18502 assert_eq!(
18503 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18504 retained_head
18505 );
18506 if damage.is_some() {
18507 assert!(runner.state.base_sync_blocked());
18508 assert!(scratch.exists());
18509 assert!(!runner.state.base_sync_recovery.operations[0].finished);
18510 assert!(
18511 runner.state.base_sync_recovery.operations[0]
18512 .validated_head
18513 .is_none()
18514 );
18515 } else {
18516 assert!(!runner.state.base_sync_blocked());
18517 assert!(!scratch.exists());
18518 assert!(runner.state.base_sync_recovery.operations[0].finished);
18519 assert_eq!(
18520 runner.state.base_sync_recovery.operations[0]
18521 .validated_head
18522 .as_deref(),
18523 Some(result.as_str())
18524 );
18525 let winner = runner.state.winner().unwrap().clone();
18526 let wait = runner.repository_wait();
18527 assert!(
18528 verify_current_head(&mut runner.state, &winner, &wait)
18529 .await
18530 .unwrap()
18531 );
18532 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18533 runner.state = RunState::load(&runner.state.id).unwrap();
18534 runner.sync_to_base().await.unwrap();
18535 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18536 }
18537 }
18538 }
18539
18540 #[cfg(unix)]
18541 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18542 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18543 runner.state.prep_complete = Some(true);
18545 runner.state.base_sync = None;
18546 runner.state.base_sync_recovery = Default::default();
18547 runner.state.config.graph.gate_fix_rounds = 1;
18548 let mut review = review_round(true, 0, 1, 1, false, true);
18549 review.head = head.clone();
18550 review.verified_head = Some(head.clone());
18551 runner.state.reviews = vec![review];
18552 runner.state.gate = vec![CommandOutcome {
18553 command: "old gate".into(),
18554 code: Some(1),
18555 output_tail: "old failure".into(),
18556 duration_ms: 0,
18557 resource_blocked: false,
18558 }];
18559 runner.state.gate_head = Some(head.clone());
18560 runner.state.gate_ran = true;
18561 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18562 agent: "fixture".into(),
18563 failed: runner.state.gate.clone(),
18564 notes: "spent".into(),
18565 committed: false,
18566 error: None,
18567 }];
18568 runner.state.land_approval = Some(crate::run::LandApproval {
18569 question: "old approval".into(),
18570 head: head.clone(),
18571 });
18572 runner.state.status = RunStatus::Blocked;
18573 (tmp, runner, head)
18574 }
18575
18576 #[cfg(unix)]
18577 #[tokio::test]
18578 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18579 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18580 let (tmp, mut runner, old) = resumed_gate_fixture().await;
18581 let phases = tmp.path().join("phases");
18582 runner.state.config.verify.e2e = vec![format!(
18583 "echo e2e >> '{}'; {}",
18584 phases.display(),
18585 if e2e_ok {
18586 "true"
18587 } else {
18588 "echo new-e2e-failure; false"
18589 }
18590 )];
18591 runner.state.config.verify.gate = vec![format!(
18592 "echo gate >> '{}'; {}",
18593 phases.display(),
18594 if gate_ok {
18595 "true"
18596 } else {
18597 "echo new-gate-failure; false"
18598 }
18599 )];
18600 runner.state.save().unwrap();
18601 runner.state = RunState::load(&runner.state.id).unwrap();
18602 runner.sync_to_base().await.unwrap();
18603 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18604 assert_ne!(head, old);
18605 runner.state = RunState::load(&runner.state.id).unwrap();
18607 runner.execute_graph().await.unwrap();
18608 assert_eq!(runner.state.gate_fixes.len(), 1);
18609 assert_eq!(runner.state.reviews.len(), 1);
18610 assert_eq!(runner.state.reviews[0].head, old);
18611 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18612 assert_eq!(runner.state.verification_history.len(), 1);
18613 let history = &runner.state.verification_history[0];
18614 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18615 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18616 assert!(history.archived_at.is_some());
18617 assert_eq!(history.fixes_spent, Some(1));
18618 assert_eq!(history.gate[0].output_tail, "old failure");
18619 let archive_events: Vec<_> = runner
18620 .state
18621 .events
18622 .iter()
18623 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18624 .collect();
18625 assert_eq!(archive_events.len(), 1);
18626 assert!(archive_events[0].message.contains(&short(&old)));
18627 assert!(archive_events[0].message.contains(&short(&head)));
18628 assert!(
18629 archive_events[0]
18630 .message
18631 .contains("1 gate-fix round(s) already spent stay spent")
18632 );
18633 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18634 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18635 if e2e_ok {
18636 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18637 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18638 } else {
18639 assert!(!runner.state.gate_ran);
18640 }
18641 assert_eq!(
18642 runner.state.status,
18643 if e2e_ok && gate_ok {
18644 RunStatus::Ready
18645 } else {
18646 RunStatus::Blocked
18647 }
18648 );
18649 if !e2e_ok || !gate_ok {
18650 let cause = runner.state.verification_blocking_cause().unwrap();
18651 assert!(cause.contains(&head));
18652 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18653 runner.state.event("land", "rebased successfully");
18655 let notice = crate::notices::run_ended(&runner.state).unwrap();
18656 assert!(!format!("{notice:?}").contains("rebased successfully"));
18657 assert!(runner.state.merge.is_none());
18658 }
18659 let before = std::fs::read_to_string(&phases).unwrap();
18660 runner.state.save().unwrap();
18661 runner.state = RunState::load(&runner.state.id).unwrap();
18662 runner.review_loop().await.unwrap();
18663 runner.gate().await.unwrap();
18664 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18665 assert_eq!(runner.state.verification_history.len(), 1);
18666 let history = &runner.state.verification_history[0];
18667 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18668 assert!(history.archived_at.is_some());
18669 assert_eq!(history.fixes_spent, Some(1));
18670 }
18671 }
18672
18673 #[cfg(unix)]
18674 #[tokio::test]
18675 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18676 let (tmp, mut runner, head) = resumed_gate_fixture().await;
18677 let marker = tmp.path().join("gate-ran");
18678 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18679 runner.review_loop().await.unwrap();
18680 runner.gate().await.unwrap();
18681 assert_eq!(runner.state.status, RunStatus::Blocked);
18682 assert!(!marker.exists());
18683 assert!(runner.state.verification_history.is_empty());
18684 runner.state.gate_head = None;
18685 runner.gate().await.unwrap();
18686 assert!(marker.exists());
18687 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18688 assert_eq!(runner.state.verification_history.len(), 1);
18689 assert_eq!(runner.state.gate_fixes.len(), 1);
18690 }
18691
18692 #[cfg(unix)]
18693 #[tokio::test]
18694 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18695 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18696 runner.sync_to_base().await.unwrap();
18697 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18698 runner.state.reconcile_verification_head(&head);
18699 for outcomes in [
18701 Vec::new(),
18702 vec![CommandOutcome {
18703 command: "true".into(),
18704 code: None,
18705 output_tail: "cache unavailable".into(),
18706 duration_ms: 0,
18707 resource_blocked: true,
18708 }],
18709 ] {
18710 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18711 head: head.clone(),
18712 passed: false,
18713 outcomes,
18714 });
18715 runner.state.gate_ran = false;
18716 runner.state.gate.clear();
18717 runner.state.save().unwrap();
18718 runner.state = RunState::load(&runner.state.id).unwrap();
18719 runner.merge().await.unwrap();
18720 assert!(runner.state.merge.is_none());
18721 runner.review_loop().await.unwrap();
18722 runner.gate().await.unwrap();
18723 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18724 assert!(runner.state.gate_allows_progress());
18725 assert_eq!(runner.state.gate_fixes.len(), 1);
18726 assert!(runner.state.merge.is_none());
18727 }
18728 git::git(
18730 &runner.state.repo,
18731 &["commit", "--allow-empty", "-m", "external update"],
18732 )
18733 .await
18734 .unwrap();
18735 runner.merge().await.unwrap();
18736 assert!(runner.state.merge.is_none());
18737 assert_eq!(runner.state.status, RunStatus::Blocked);
18738 }
18739
18740 #[cfg(unix)]
18741 #[tokio::test]
18742 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18743 for waiting_on_gate in [false, true] {
18744 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18745 runner.sync_to_base().await.unwrap();
18746 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18747 runner.state.reconcile_verification_head(&head);
18748 if waiting_on_gate {
18749 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18750 head: head.clone(),
18751 passed: true,
18752 outcomes: Vec::new(),
18753 });
18754 }
18755 let cache = tmp.path().join("cache");
18756 runner.state.config.verify.e2e =
18757 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18758 runner.state.config.verify.gate = vec!["true".into()];
18759 runner.state.config.graph.timeout_resource_wait = 1;
18760 let owner =
18761 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18762 let guard =
18763 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18764 crate::cache::AcquireOutcome::Acquired(g) => g,
18765 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18766 };
18767 runner.review_loop().await.unwrap();
18768 runner.gate().await.unwrap();
18769 assert!(!runner.state.gate_ran);
18770 assert_eq!(runner.state.gate_fixes.len(), 1);
18771 assert!(runner.state.resource_block.is_some());
18772 runner.merge().await.unwrap();
18773 assert!(runner.state.merge.is_none());
18774 drop(guard);
18775 runner.state = RunState::load(&runner.state.id).unwrap();
18776 runner.state.resource_block = None;
18778 runner.review_loop().await.unwrap();
18779 runner.gate().await.unwrap();
18780 assert!(runner.state.gate_allows_progress());
18781 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18782 assert_eq!(runner.state.gate_fixes.len(), 1);
18783 }
18784 }
18785
18786 #[cfg(unix)]
18787 #[tokio::test]
18788 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18789 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18790 runner.sync_to_base().await.unwrap();
18791 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18792 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18793 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18795 runner.review_loop().await.unwrap();
18796 runner.gate().await.unwrap();
18797 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18798 assert_ne!(before, after);
18799 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18800 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18801 assert!(!runner.state.gate_allows_progress());
18802 runner.merge().await.unwrap();
18803 assert!(runner.state.merge.is_none());
18804 assert!(
18805 runner
18806 .state
18807 .verification_history
18808 .iter()
18809 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18810 );
18811
18812 runner.state.reviews[0].clean = false;
18814 runner.state.reviews[0].answered = 0;
18815 runner.state.config.verify.e2e = vec!["true".into()];
18816 runner.review_loop().await.unwrap();
18817 runner.gate().await.unwrap();
18818 runner.merge().await.unwrap();
18819 assert_eq!(runner.state.status, RunStatus::Blocked);
18820 assert!(runner.state.merge.is_none());
18821 assert!(
18822 runner
18823 .state
18824 .verification_blocking_cause()
18825 .unwrap()
18826 .contains("review blocked")
18827 );
18828 }
18829
18830 #[cfg(unix)]
18831 #[tokio::test]
18832 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18833 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18834 runner.state.gate_head = None;
18835 runner.state.config.verify.gate =
18836 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18837 runner.review_loop().await.unwrap();
18838 runner.gate().await.unwrap();
18839 assert_ne!(
18840 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18841 old
18842 );
18843 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18844 assert!(!runner.state.gate_allows_progress());
18845 assert_eq!(runner.state.status, RunStatus::Blocked);
18846 runner.merge().await.unwrap();
18847 assert!(runner.state.merge.is_none());
18848 }
18849
18850 #[cfg(unix)]
18851 #[tokio::test]
18852 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18853 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18854 let mut old_review = review_round(true, 0, 1, 1, false, true);
18855 old_review.head = head.clone();
18856 old_review.verified_head = Some(head.clone());
18857 runner.state.reviews = vec![old_review];
18858 runner.state.gate = vec![CommandOutcome {
18859 command: "old gate".into(),
18860 code: None,
18861 output_tail: "timed out after 2400s".into(),
18862 duration_ms: 2_400_000,
18863 resource_blocked: false,
18864 }];
18865 runner.state.gate_ran = true;
18866 runner.state.gate_head = Some(head.clone());
18867 runner.state.land_approval = Some(crate::run::LandApproval {
18868 question: "old-head-approval".into(),
18869 head: head.clone(),
18870 });
18871 runner.state.authorize_base_sync(&task).unwrap();
18872 runner.sync_to_base().await.unwrap();
18873 assert!(!runner.state.base_sync_blocked());
18874 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18875 assert_ne!(head, fresh);
18876 assert_eq!(runner.state.reviews.len(), 1);
18877 assert!(!runner.state.gate_ran);
18878 let saved = &runner.state.base_sync_recovery.authorizations[0];
18879 assert_eq!(saved.reviews[0].head, head);
18880 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18881 assert!(saved.gate_ran);
18882 assert!(!saved.gate[0].ok());
18883 runner.review_loop().await.unwrap();
18884 runner.gate().await.unwrap();
18885 assert_eq!(runner.state.reviews[0].head, head);
18886 assert_eq!(
18887 runner.state.reviews[0].verified_head.as_deref(),
18888 Some(head.as_str())
18889 );
18890 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18891 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18892 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18893 assert!(runner.state.gate_ran);
18894 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18895 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18896 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18897 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18898 assert_eq!(
18899 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18900 "retained change"
18901 );
18902 }
18903
18904 #[cfg(unix)]
18905 #[tokio::test]
18906 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18907 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18908 let queue = crate::queue::Queue::open();
18909 let mut owner = queue.get(&task).unwrap();
18910 owner.release_operator("test-resume", None);
18911 queue.put(&mut owner).unwrap();
18912 let generation = owner.resumption.as_ref().unwrap().generation;
18913 runner.state.task_instruction = Some(owner.instruction.clone());
18914 runner.state.driver_exited = true;
18915 runner.state.save().unwrap();
18916 let home = crate::run::home();
18917 let _claim = queue.claim(&task).unwrap();
18918 owner.resumption.as_mut().unwrap().at =
18919 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18920 queue.put(&mut owner).unwrap();
18921 assert!(
18922 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18923 .await
18924 .is_err(),
18925 "an approval from before the stop grants nothing"
18926 );
18927 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18928 queue.put(&mut owner).unwrap();
18929 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18930 .await
18931 .unwrap();
18932 assert!(
18933 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18934 .await
18935 .is_err()
18936 );
18937 runner.state = RunState::load(&runner.state.id).unwrap();
18938 assert_eq!(
18939 runner.state.base_sync_recovery.authorizations[0].generation,
18940 Some(generation)
18941 );
18942 runner.sync_to_base().await.unwrap();
18943 let repo = runner.state.repo.clone();
18944 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18945 git::git(&repo, &["checkout", "main"]).await.unwrap();
18946 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18947 git::git(&repo, &["add", "."]).await.unwrap();
18948 git::git(&repo, &["commit", "-m", "move base again"])
18949 .await
18950 .unwrap();
18951 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18952 git::git(&repo, &["checkout", "fix/retained"])
18953 .await
18954 .unwrap();
18955 runner.sync_to_base().await.unwrap();
18956 assert!(!runner.state.base_sync_blocked());
18957 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18958 assert_eq!(runner.state.base_episode_used(), 2);
18959 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18960 assert_ne!(fresh, head);
18961 assert!(!runner.state.base_recovery_pending());
18962 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18964 queue.put(&mut owner).unwrap();
18965 assert!(
18966 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18967 .await
18968 .is_err()
18969 );
18970 runner.state = RunState::load(&runner.state.id).unwrap();
18971 runner.sync_to_base().await.unwrap();
18972 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18973 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18974 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18975 }
18976
18977 #[cfg(unix)]
18978 #[tokio::test]
18979 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18980 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18981 runner.state.base_sync = None;
18982 runner.state.status = RunStatus::Blocked;
18983 let mut second = runner.state.candidates[0].clone();
18984 second.label = 'B';
18985 runner.state.candidates.push(second);
18986 assert!(runner.state.votes.is_empty());
18987 assert!(runner.state.deliberation.is_empty());
18988 runner.deliberate().await.unwrap();
18989 runner.vote().await.unwrap();
18990 assert_eq!(runner.state.status, RunStatus::Blocked);
18991 assert!(runner.state.votes.is_empty());
18992 assert!(runner.state.deliberation.is_empty());
18993 }
18994
18995 fn runner_at(status: RunStatus) -> Runner {
18998 let mut state = RunState::new(
18999 PathBuf::from("/nonexistent/repo"),
19000 "main".to_owned(),
19001 "deadbeef".to_owned(),
19002 "task".to_owned(),
19003 Config::default(),
19004 );
19005 state.config.disk.min_free_bytes = 0;
19007 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
19010 state.id = format!(
19011 "20000101-000000-{:04x}",
19012 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
19013 );
19014 state.status = status;
19015 Runner {
19016 state,
19017 roles: ResolvedRoles {
19018 implementers: Vec::new(),
19019 judges: Vec::new(),
19020 reviewers: Vec::new(),
19021 fixer: None,
19022 conductor: conductor(),
19023 implementer_roster: Vec::new(),
19024 judge_roster: Vec::new(),
19025 reviewer_roster: Vec::new(),
19026 },
19027 sem: Arc::new(Semaphore::new(1)),
19028 pause: Pause::new(),
19029 interrupt: Pause::new(),
19030 }
19031 }
19032
19033 #[tokio::test]
19034 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
19035 crate::run::pin_test_home();
19036 let temp = tempfile::tempdir().unwrap();
19037 let root = temp.path().canonicalize().unwrap();
19038 let repo = root.join("repo");
19039 std::fs::create_dir(&repo).unwrap();
19040 init_repo(&repo);
19041 git::git(&repo, &["config", "core.autocrlf", "false"])
19042 .await
19043 .unwrap();
19044 let mut runner = runner_at(RunStatus::Blocked);
19045 runner.state.repo = repo.clone();
19046 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
19047 runner.state.config.agents = vec![conductor()];
19048 runner.state.config.graph.selected_implementer = Some("conductor".into());
19049 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
19050 let (shell, command, setup_output) = if cfg!(windows) {
19051 (
19052 vec!["cmd".into(), "/C".into()],
19053 "echo ready>seat-ready",
19054 "ready\r\n",
19055 )
19056 } else {
19057 (
19058 vec!["sh".into(), "-c".into()],
19059 "printf ready > seat-ready",
19060 "ready",
19061 )
19062 };
19063 runner.state.config.verify.shell = Some(shell);
19064 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
19065 run: Some(command.into()),
19066 ..Default::default()
19067 }];
19068 runner.state.save().unwrap();
19069 let path = runner.state.worktree_root().join("judge-1");
19070 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
19071 .await
19072 .unwrap();
19073 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
19074 git::git(&path, &["add", "."]).await.unwrap();
19075 let tree = git::git(&path, &["write-tree"]).await.unwrap();
19076 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
19077 .await
19078 .unwrap();
19079 git::git(&path, &["checkout", "--detach", &head])
19080 .await
19081 .unwrap();
19082 let evidence = crate::fold_evidence::snapshot(
19083 &runner.state,
19084 &crate::run::home().canonicalize().unwrap(),
19085 &head,
19086 64 * 1024 * 1024,
19087 )
19088 .await
19089 .unwrap();
19090 let mut contents = BTreeMap::new();
19091 contents.insert(
19092 repo.join("logs/HEAD"),
19093 format!("{head} {head} fixture\n").into_bytes(),
19094 );
19095 let record = serde_json::json!({
19096 "version":1, "run":runner.state.id, "source":path, "head":head,
19097 "evidence":evidence, "admin":{}, "admin_contents":contents,
19098 "history":{head.clone():evidence}, "phase":"evicted"
19099 });
19100 std::fs::write(
19101 runner.state.dir().join("artifacts/eviction-judge-1.json"),
19102 serde_json::to_vec(&record).unwrap(),
19103 )
19104 .unwrap();
19105 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
19106 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
19107 .await
19108 .unwrap();
19109 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
19110 assert!(
19111 !git::git_raw(&repo, &["cat-file", "-e", &head])
19112 .await
19113 .unwrap()
19114 .ok()
19115 );
19116 let mut resumed = Runner::resume(&runner.state.id).unwrap();
19117 let error = resumed.execute().await.unwrap_err();
19118 assert!(
19119 error
19120 .to_string()
19121 .contains("selected implementation is held"),
19122 "{error:#}"
19123 );
19124 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
19125 assert_eq!(
19126 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
19127 setup_output
19128 );
19129 assert_eq!(
19130 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
19131 "unrelated seat"
19132 );
19133 let error = resumed.execute().await.unwrap_err();
19134 assert!(
19135 error
19136 .to_string()
19137 .contains("selected implementation is held"),
19138 "{error:#}"
19139 );
19140 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
19141 git::git(&path, &["add", "."]).await.unwrap();
19142 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
19143 let next = git::git(
19144 &path,
19145 &[
19146 "commit-tree",
19147 &next_tree,
19148 "-p",
19149 &head,
19150 "-m",
19151 "next review checkpoint",
19152 ],
19153 )
19154 .await
19155 .unwrap();
19156 git::git(&path, &["checkout", "--detach", &next])
19157 .await
19158 .unwrap();
19159 resumed.state.reviews.push(
19160 serde_json::from_value(serde_json::json!({
19161 "round":1, "head":next, "reviews":[]
19162 }))
19163 .unwrap(),
19164 );
19165 resumed.state.save().unwrap();
19166 let error = resumed.execute().await.unwrap_err();
19167 assert!(
19168 error
19169 .to_string()
19170 .contains("selected implementation is held"),
19171 "{error:#}"
19172 );
19173 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
19174 }
19175
19176 #[tokio::test]
19177 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
19178 crate::run::pin_test_home();
19179 let mut runner = runner_at(RunStatus::Gating);
19180 runner.state.save().unwrap();
19181 let id = runner.state.id.clone();
19182 let passes = runner.state.passes.len();
19183 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19184 let mut claiming = Box::pin(runner.claim_driver());
19185 std::future::poll_fn(|cx| {
19186 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
19187 std::task::Poll::Ready(())
19188 })
19189 .await;
19190 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
19191 drop(owner);
19192 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
19193 .await
19194 .unwrap()
19195 .unwrap();
19196 assert_eq!(execution.runner.state.passes.len(), passes + 1);
19197 assert!(execution.runner.state.resource_block.is_none());
19198 assert_eq!(
19199 execution.runner.state.liveness(false),
19200 crate::run::Liveness::Live
19201 );
19202 drop(execution);
19203 assert!(RunState::load(&id).unwrap().driver_exited);
19204 }
19205
19206 #[tokio::test]
19207 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
19208 crate::run::pin_test_home();
19209 let mut runner = runner_at(RunStatus::Gating);
19210 runner.state.save().unwrap();
19211 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
19214 assert!(runner.claim_driver().await.is_err());
19215 assert!(runner.state.driver_exited);
19216 assert_eq!(runner.state.driver_pid, None);
19217 let disk = RunState::load(&runner.state.id).unwrap();
19218 assert!(disk.driver_exited);
19219 assert_eq!(disk.driver_pid, None);
19220 assert_eq!(disk.status, RunStatus::Gating);
19221 }
19222
19223 #[tokio::test]
19224 async fn an_early_graph_error_releases_its_driver() {
19225 crate::run::pin_test_home();
19226 let mut runner = runner_at(RunStatus::Blocked);
19227 runner.state.config.graph.selected_implementer = Some("selected".into());
19228 runner.state.save().unwrap();
19229 assert!(runner.execute().await.is_err());
19230 let disk = RunState::load(&runner.state.id).unwrap();
19231 assert!(disk.driver_exited);
19232 assert_eq!(disk.driver_pid, None);
19233 assert_eq!(disk.status, RunStatus::Blocked);
19234 }
19235
19236 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
19237 crate::run::MergeRepair {
19238 question: "repair-question".into(),
19239 head: "old-head".into(),
19240 pr: "https://github.com/o/r/pull/42".into(),
19241 reviews: vec![review_round(false, 1, 1, 1, false, true)],
19242 gate: vec![outcome(Some(0), false)],
19243 gate_head: Some("old-head".into()),
19244 gate_fixes: Vec::new(),
19245 pre_gate: Vec::new(),
19246 phase,
19247 failure: None,
19248 }
19249 }
19250
19251 #[tokio::test]
19252 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
19253 crate::run::pin_test_home();
19254 let mut runner = runner_at(RunStatus::Landing);
19255 runner.state.merge_repairs.push(merge_repair_request(
19256 crate::run::MergeRepairPhase::Requested,
19257 ));
19258 runner.state.save().unwrap();
19259 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
19260 assert!(
19261 runner
19262 .execute()
19263 .await
19264 .unwrap_err()
19265 .to_string()
19266 .contains("already running")
19267 );
19268 assert_eq!(
19269 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19270 crate::run::MergeRepairPhase::Requested
19271 );
19272 drop(claim);
19273 let mut disk = runner.state.clone();
19274 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
19275 disk.save().unwrap();
19276 assert!(
19277 runner
19278 .execute()
19279 .await
19280 .unwrap_err()
19281 .to_string()
19282 .contains("decision changed")
19283 );
19284 assert_eq!(
19285 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19286 crate::run::MergeRepairPhase::Complete
19287 );
19288 }
19289
19290 #[test]
19291 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
19292 crate::run::pin_test_home();
19293 let mut runner = runner_at(RunStatus::Landing);
19294 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
19295 runner.state.reviews = request.reviews.clone();
19296 runner.state.gate = request.gate.clone();
19297 runner.state.gate_ran = true;
19298 runner.state.land_approval = Some(crate::run::LandApproval {
19299 question: "old".into(),
19300 head: "old-head".into(),
19301 });
19302 runner.state.merge_repairs.push(request);
19303 runner.begin_merge_repair_review(0).unwrap();
19304 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
19305 assert!(!runner.state.gate_status().ok());
19306 assert!(runner.state.land_approval.is_none());
19307 let disk = RunState::load(&runner.state.id).unwrap();
19308 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
19309 assert!(disk.merge_repairs[0].gate[0].ok());
19310 assert_eq!(
19311 disk.merge_repairs[0].phase,
19312 crate::run::MergeRepairPhase::Reviewing
19313 );
19314 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
19315 assert!(disk.reviews.is_empty());
19316 }
19317
19318 #[tokio::test]
19319 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
19320 crate::run::pin_test_home();
19321 for phase in [
19322 crate::run::MergeRepairPhase::Fixing,
19323 crate::run::MergeRepairPhase::Requested,
19324 ] {
19325 let mut runner = runner_at(RunStatus::Landing);
19326 runner.state.config.graph.land_rounds = 0;
19327 runner.state.merge_repairs.push(merge_repair_request(phase));
19328 runner.state.save().unwrap();
19329 assert!(!runner.repair_merge_review().await.unwrap());
19330 assert_eq!(runner.state.status, RunStatus::Blocked);
19331 assert_eq!(
19332 runner.state.merge_repairs[0].phase,
19333 crate::run::MergeRepairPhase::Failed
19334 );
19335 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
19336 assert!(
19337 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
19338 "interrupted"
19339 } else {
19340 "budget exhausted"
19341 })
19342 );
19343 assert!(runner.state.merge.is_none());
19344 }
19345 }
19346
19347 #[test]
19351 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
19352 crate::run::pin_test_home();
19353 let mut runner = runner_at(RunStatus::Implementing);
19354 let interrupt = Pause::new();
19355 runner.watch_interrupt(interrupt.clone());
19356
19357 interrupt.park_because("task a1b2 asked to run first");
19358
19359 assert!(runner.park_here().expect("park_here"));
19360 assert!(runner.state.parked);
19361 let last = runner.state.events.last().expect("a park event");
19362 assert_eq!(last.node, "park");
19363 assert!(
19364 last.message.contains("task a1b2 asked to run first"),
19365 "expected the interrupt reason in {:?}",
19366 last.message
19367 );
19368 }
19369
19370 #[test]
19373 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
19374 crate::run::pin_test_home();
19375 let mut runner = runner_at(RunStatus::Implementing);
19376 runner.state.begin_pass();
19377 let shutdown = Pause::new();
19378 runner.on_pause(shutdown.clone());
19379 let upgrade = ParkCause::Upgrade {
19380 from: "0.121.0".to_owned(),
19381 to: Some("0.121.1".to_owned()),
19382 };
19383 shutdown.park_for(upgrade.clone());
19384 shutdown.park_for(ParkCause::Operator);
19385
19386 assert!(runner.park_here().expect("park_here"));
19387 assert_eq!(
19388 runner.state.passes[0].end,
19389 Some(crate::run::PassEnd::Parked { cause: upgrade })
19390 );
19391 let last = runner.state.events.last().expect("a park event");
19392 assert!(last.message.contains("0.121.0"), "{}", last.message);
19393 }
19394
19395 #[tokio::test]
19396 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
19397 crate::run::pin_test_home();
19398 let mut runner = runner_at(RunStatus::Reviewing);
19399 runner.state.save().unwrap();
19400 let mut execution = runner.claim_driver().await.expect("first execution");
19401 assert_eq!(execution.runner.state.passes.len(), 1);
19402 execution
19403 .driver
19404 .finish(&mut execution.runner.state)
19405 .await
19406 .unwrap();
19407 }
19408
19409 #[tokio::test]
19410 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19411 crate::run::pin_test_home();
19412 let mut runner = runner_at(RunStatus::Reviewing);
19413 runner.state.config.agents = vec![conductor()];
19414 runner.state.config.graph.timeout_resource_wait = 0;
19415 runner.state.save().unwrap();
19416 let id = runner.state.id.clone();
19417 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19418 assert!(runner.claim_driver().await.is_err());
19419 drop(owner);
19420 drop(runner);
19421 let mut resumed = Runner::resume(&id).unwrap();
19422 assert_eq!(resumed.state.driver_claimed, Some(false));
19423 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19424 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19425 let mut execution = resumed
19426 .claim_driver()
19427 .await
19428 .expect("retry initial claim after reload");
19429 assert_eq!(execution.runner.state.driver_claimed, Some(true));
19430 assert_eq!(execution.runner.state.passes.len(), 1);
19431 execution
19432 .driver
19433 .finish(&mut execution.runner.state)
19434 .await
19435 .unwrap();
19436 }
19437
19438 #[tokio::test]
19439 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19440 crate::run::pin_test_home();
19441 let mut runner = runner_at(RunStatus::Reviewing);
19442 runner.state.config.agents = vec![conductor()];
19443 runner.state.save().unwrap();
19444 let id = runner.state.id.clone();
19445 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19446 let mut resumed = Runner::resume(&id).unwrap();
19447 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19448 assert!(resumed.claim_driver().await.is_err());
19449 assert!(RunState::load(&id).unwrap().passes.is_empty());
19450 drop(lease);
19451 let mut execution = resumed.claim_driver().await.unwrap();
19452 execution
19453 .driver
19454 .finish(&mut execution.runner.state)
19455 .await
19456 .unwrap();
19457 }
19458
19459 #[tokio::test]
19460 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19461 crate::run::pin_test_home();
19462 let mut runner = runner_at(RunStatus::Reviewing);
19463 let state = &mut runner.state;
19464 state.config.agents = vec![conductor()];
19465 state.driver_claimed = None;
19466 state.driver_pid = Some(std::process::id());
19467 state.driver_started_at = None;
19468 state.begin_pass();
19469 state.save().unwrap();
19470 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19471 std::fs::write(
19472 state.dir().join("driver.json"),
19473 serde_json::to_vec(&serde_json::json!({
19474 "pid": state.driver_pid, "started_at": null,
19475 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19476 }))
19477 .unwrap(),
19478 )
19479 .unwrap();
19480 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19481 drop(lease);
19482 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19483 let mut resumed = Runner::resume(&state.id).unwrap();
19484 assert!(!resumed.state.driver_exited);
19485 let mut execution = resumed
19486 .claim_driver()
19487 .await
19488 .expect("lease proves previous execution ended");
19489 assert_eq!(execution.runner.state.passes.len(), 2);
19490 execution
19491 .driver
19492 .finish(&mut execution.runner.state)
19493 .await
19494 .unwrap();
19495 }
19496
19497 #[tokio::test]
19498 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19499 crate::run::pin_test_home();
19500 let mut runner = runner_at(RunStatus::Reviewing);
19501 let mut json = serde_json::to_value(&runner.state).unwrap();
19502 json.as_object_mut().unwrap().remove("driver_claimed");
19503 runner.state = serde_json::from_value(json).unwrap();
19504 assert_eq!(runner.state.driver_claimed, None);
19505 runner.state.save().unwrap();
19506 let mut execution = runner
19507 .claim_driver()
19508 .await
19509 .expect("the execution lease, not a missing marker, excludes a second driver");
19510 assert_eq!(execution.runner.state.passes.len(), 1);
19511 execution
19512 .driver
19513 .finish(&mut execution.runner.state)
19514 .await
19515 .unwrap();
19516 }
19517
19518 #[tokio::test]
19519 async fn driver_admission_refuses_a_proven_live_driver() {
19520 crate::run::pin_test_home();
19521 let mut runner = runner_at(RunStatus::Reviewing);
19522 let pid = std::process::id();
19523 runner.state.driver_pid = Some(pid);
19524 runner.state.driver_started_at = crate::proc::process_started_at(pid);
19525 assert!(runner.state.driver_started_at.is_some());
19526 runner.state.save().unwrap();
19527 let error = runner
19528 .claim_driver()
19529 .await
19530 .err()
19531 .expect("a live driver must refuse");
19532 assert!(error.to_string().contains("live driver"), "{error:#}");
19533 let saved = RunState::load(&runner.state.id).unwrap();
19534 assert_eq!(saved.driver_pid, Some(pid));
19535 assert_eq!(saved.status, RunStatus::Reviewing);
19536 assert!(saved.passes.is_empty());
19537 }
19538
19539 #[test]
19540 fn parking_resource_halt_persists_the_pass_once() {
19541 crate::run::pin_test_home();
19542 let mut runner = runner_at(RunStatus::Reviewing);
19543 runner.state.begin_pass();
19544 runner.state.resource_block = Some(crate::run::ResourceBlock {
19545 node: "fix".into(),
19546 kind: "shared build cache".into(),
19547 detail: "cancelled".into(),
19548 at: Timestamp::now(),
19549 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19550 });
19551 let cause = ParkCause::Upgrade {
19552 from: "old".into(),
19553 to: Some("new".into()),
19554 };
19555 runner.pause.park_for(cause.clone());
19556 runner.stop_for_resource().unwrap();
19559 assert!(runner.resource_halt().unwrap());
19560 assert!(runner.park_here().unwrap());
19561 let saved = RunState::load(&runner.state.id).unwrap();
19562 assert!(saved.parked);
19563 assert_eq!(saved.status, RunStatus::Reviewing);
19564 assert!(saved.passes[0].ended_at.is_some());
19565 assert_eq!(
19566 saved.passes[0].end,
19567 Some(crate::run::PassEnd::Parked {
19568 cause: cause.clone()
19569 })
19570 );
19571 assert_eq!(
19572 saved.resource_block.unwrap().cause,
19573 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19574 );
19575 assert_eq!(
19576 saved
19577 .events
19578 .iter()
19579 .filter(|event| event.node == "park")
19580 .count(),
19581 1
19582 );
19583 }
19584
19585 #[test]
19593 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19594 crate::run::pin_test_home();
19595 let mut runner = runner_at(RunStatus::Implementing);
19596 let shutdown = Pause::new();
19597 runner.on_pause(shutdown.clone());
19598 let interrupt = Pause::new();
19599 runner.watch_interrupt(interrupt.clone());
19600
19601 assert!(!runner.park_here().expect("park_here"));
19603 assert!(!runner.state.parked);
19604
19605 interrupt.park_because("test");
19607 assert!(!shutdown.parked());
19608 assert!(runner.park_here().expect("park_here"));
19609 }
19610
19611 #[tokio::test]
19625 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19626 crate::run::pin_test_home();
19627 let mut runner = runner_at(RunStatus::Implementing);
19628 let interrupt = Pause::new();
19629 runner.watch_interrupt(interrupt.clone());
19630
19631 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19632 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19633
19634 let node = async move {
19638 started_tx.send(()).expect("send started");
19639 finish_rx.await.expect("recv finish");
19640 "node finished"
19641 };
19642
19643 let interrupter = async move {
19644 started_rx.await.expect("recv started");
19645 interrupt.park_because("higher-priority task waiting");
19647 tokio::task::yield_now().await;
19651 finish_tx.send(()).expect("send finish");
19652 };
19653
19654 let (node_result, ()) = tokio::join!(node, interrupter);
19655 assert_eq!(
19656 node_result, "node finished",
19657 "the in-flight call ran to completion"
19658 );
19659
19660 assert!(runner.park_here().expect("park_here"));
19663 assert!(runner.state.parked);
19664 }
19665
19666 #[test]
19672 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19673 crate::run::pin_test_home();
19674 let mut runner = runner_at(RunStatus::Judging);
19675 runner.state.config.agents = vec![conductor()];
19679 runner.state.candidates = vec![Candidate {
19680 index: 0,
19681 label: 'A',
19682 agent: "alpha".to_owned(),
19683 branch: "magi/x/A".to_owned(),
19684 worktree: PathBuf::from("/nonexistent/worktree"),
19685 summary: "did the thing".to_owned(),
19686 stat: "1 file changed".to_owned(),
19687 files: 1,
19688 commits: 1,
19689 empty: false,
19690 failed: None,
19691 verified_noop: None,
19692 duration_ms: 1234,
19693 folded: false,
19694 }];
19695 let run_id = runner.state.id.clone();
19696
19697 let interrupt = Pause::new();
19698 runner.watch_interrupt(interrupt.clone());
19699 interrupt.park_because("task c3d4 asked to run first");
19700 assert!(runner.park_here().expect("park_here"));
19701
19702 let resumed = Runner::resume(&run_id).expect("resume");
19703 assert_eq!(resumed.state.candidates.len(), 1);
19704 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19705 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19706 assert_eq!(resumed.state.status, runner.state.status);
19707 assert!(
19708 resumed.state.parked,
19709 "still parked until `execute` actually walks the graph again"
19710 );
19711 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19712 }
19713
19714 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19716 let mut q = ask::Question::new(
19717 run.to_owned(),
19718 "implement".to_owned(),
19719 "impl-A".to_owned(),
19720 "Which storage backend should the cache use?".to_owned(),
19721 String::new(),
19722 vec!["SQLite".to_owned(), "Redis".to_owned()],
19723 );
19724 store.put(&mut q).unwrap();
19725 q
19726 }
19727
19728 #[test]
19729 fn a_failed_runs_open_question_is_abandoned() {
19730 ask_test_home();
19731 let store = ask::Questions::open();
19732 let mut runner = runner_at(RunStatus::Failed);
19733 let run = runner.state.id.clone();
19734 let q = ask_open_question(&store, &run);
19735
19736 runner.settle_questions();
19737
19738 let back = store.get(&q.id).unwrap();
19739 assert!(
19740 !back.status.open(),
19741 "the seat that asked died with the run; nobody is left to read an answer"
19742 );
19743 assert!(
19744 back.detail.contains(&run) && back.detail.contains("failed"),
19745 "the reason names what the run became, not just that it is gone: {}",
19746 back.detail
19747 );
19748 }
19749
19750 #[test]
19751 fn a_merged_runs_open_question_is_abandoned_too() {
19752 ask_test_home();
19753 let store = ask::Questions::open();
19754 for status in [RunStatus::Merged, RunStatus::Ready] {
19757 let mut runner = runner_at(status);
19758 let run = runner.state.id.clone();
19759 let q = ask_open_question(&store, &run);
19760
19761 runner.settle_questions();
19762
19763 let back = store.get(&q.id).unwrap();
19764 assert!(
19765 !back.status.open(),
19766 "{status:?} run's question must not outlive the run"
19767 );
19768 }
19769 }
19770
19771 #[test]
19772 fn a_still_resumable_runs_open_question_is_left_alone() {
19773 ask_test_home();
19774 let store = ask::Questions::open();
19775 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19781 let mut runner = runner_at(status);
19782 let run = runner.state.id.clone();
19783 let q = ask_open_question(&store, &run);
19784
19785 runner.settle_questions();
19786
19787 let back = store.get(&q.id).unwrap();
19788 assert!(
19789 back.status.open(),
19790 "{status:?} is still alive; the question must still be waiting"
19791 );
19792 }
19793 }
19794
19795 #[test]
19796 fn settle_questions_never_touches_an_already_answered_question() {
19797 ask_test_home();
19798 let store = ask::Questions::open();
19799 let mut runner = runner_at(RunStatus::Failed);
19800 let run = runner.state.id.clone();
19801 let mut q = ask_open_question(&store, &run);
19802 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19803 .unwrap();
19804 store.put(&mut q).unwrap();
19805
19806 runner.settle_questions();
19811 runner.settle_questions();
19812
19813 let back = store.get(&q.id).unwrap();
19814 assert_eq!(
19815 back.status,
19816 ask::QuestionStatus::Answered,
19817 "a real answer is a decision on record, never overwritten by a sweep"
19818 );
19819 }
19820
19821 #[tokio::test]
19832 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19833 crate::run::pin_test_home();
19834 let tmp = tempfile::tempdir().expect("tempdir");
19835 let repo = tmp.path().join("repo");
19836 std::fs::create_dir_all(&repo).unwrap();
19837 init_repo(&repo);
19838
19839 let mut config = Config::default();
19840 config.graph.worktree_root = Some(tmp.path().join("wt"));
19841
19842 let mut state = RunState::new(
19843 repo.clone(),
19844 "main".to_owned(),
19845 "deadbeef".to_owned(),
19846 "task".to_owned(),
19847 config,
19848 );
19849 let root = state.worktree_root();
19850 let wt_a = root.join("cand-A");
19851 let wt_b = root.join("cand-B");
19852 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19853 .await
19854 .expect("worktree A");
19855 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19856 .await
19857 .expect("worktree B");
19858
19859 state.candidates = vec![
19860 Candidate {
19861 index: 0,
19862 label: 'A',
19863 agent: "alpha".to_owned(),
19864 branch: "magi/x/A".to_owned(),
19865 worktree: wt_a.clone(),
19866 summary: String::new(),
19867 stat: String::new(),
19868 files: 0,
19869 commits: 0,
19870 empty: false,
19871 failed: None,
19872 verified_noop: None,
19873 duration_ms: 0,
19874 folded: false,
19875 },
19876 Candidate {
19877 index: 1,
19878 label: 'B',
19879 agent: "beta".to_owned(),
19880 branch: "magi/x/B".to_owned(),
19881 worktree: wt_b.clone(),
19882 summary: String::new(),
19883 stat: String::new(),
19884 files: 0,
19885 commits: 0,
19886 empty: false,
19887 failed: None,
19888 verified_noop: None,
19889 duration_ms: 0,
19890 folded: false,
19891 },
19892 ];
19893 state.tally = Some(Tally {
19894 first_choice: BTreeMap::from([('A', 1)]),
19895 borda: BTreeMap::new(),
19896 winner: 'A',
19897 rankings: 1,
19898 unanimous_initial: true,
19899 deliberated: false,
19900 changed_votes: 0,
19901 unanimous_final: true,
19902 tie_break: None,
19903 judges: 1,
19904 present: 1,
19905 quorum: 1,
19906 met_quorum: true,
19907 uncontested: None,
19908 });
19909 state.status = RunStatus::Ready;
19910
19911 fold_run(&mut state, false, &crate::run::home())
19912 .await
19913 .expect("fold_run");
19914
19915 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19916 assert!(
19917 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19918 "the unmerged winner's branch survives"
19919 );
19920 assert!(
19921 !state.candidates[0].folded,
19922 "the winner is not marked folded"
19923 );
19924
19925 assert!(!wt_b.exists(), "the loser's worktree is removed");
19926 assert!(
19927 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19928 "the loser's branch is removed"
19929 );
19930 assert!(state.candidates[1].folded, "the loser is marked folded");
19931 }
19932
19933 #[tokio::test]
19934 #[cfg(unix)]
19935 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19936 use std::os::fd::AsRawFd as _;
19937 use std::time::Duration;
19938
19939 let tmp = tempfile::tempdir().unwrap();
19940 let repo = tmp.path().join("repo");
19941 std::fs::create_dir_all(&repo).unwrap();
19942 init_repo(&repo);
19943 let common = repo.join(".git");
19944 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19945 let home = tmp.path().join("home");
19946 let mut config = Config::default();
19947 config.coordination = crate::config::test_coordination();
19948 config.graph.worktree_root = Some(tmp.path().join("wt"));
19949 config.graph.timeout_resource_wait = 1;
19950 let mut state = RunState::new(
19951 repo.clone(),
19952 "main".into(),
19953 "deadbeef".into(),
19954 "task".into(),
19955 config,
19956 );
19957 let root = state.worktree_root();
19958 let candidate = root.join("cand-A");
19959 let reviewer = root.join("review-1");
19960 let branch = state.branch_for('A');
19961 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19962 .await
19963 .unwrap();
19964 git::worktree_add_detached(&repo, &reviewer, "main")
19965 .await
19966 .unwrap();
19967 git::acquire_worktree_config(&repo).await.unwrap();
19968 state.enabled_worktree_config = true;
19969 state.status = RunStatus::Ready;
19970 state.candidates.push(Candidate {
19971 index: 0,
19972 label: 'A',
19973 agent: "implement".into(),
19974 branch: branch.clone(),
19975 worktree: candidate.clone(),
19976 summary: String::new(),
19977 stat: String::new(),
19978 files: 0,
19979 commits: 0,
19980 empty: false,
19981 failed: None,
19982 verified_noop: None,
19983 duration_ms: 0,
19984 folded: false,
19985 });
19986 state.save_under(&home).unwrap();
19987 let config_before = std::fs::read(common.join("config")).unwrap();
19988 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19989 .await
19990 .unwrap();
19991 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19994 assert_eq!(
19995 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19996 0
19997 );
19998 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19999 assert!(
20000 tokio::time::timeout(Duration::from_millis(200), &mut fold)
20001 .await
20002 .is_err()
20003 );
20004 assert!(candidate.exists());
20005 assert!(reviewer.exists());
20006 assert!(git::branch_exists(&repo, &branch).await.unwrap());
20007 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
20008 assert_eq!(
20009 git::git(&repo, &["worktree", "list", "--porcelain"])
20010 .await
20011 .unwrap(),
20012 worktrees_before
20013 );
20014 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
20015 .await
20016 .unwrap()
20017 .unwrap_err();
20018 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
20019 drop(fold);
20020 assert!(candidate.exists() && reviewer.exists());
20021 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
20022 assert_eq!(
20023 state.resource_block.as_ref().unwrap().kind,
20024 "repository coordination"
20025 );
20026 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
20027 drop(external);
20028 let mut fold = Box::pin(fold_run(&mut state, true, &home));
20029 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
20030 .await
20031 .unwrap()
20032 .unwrap();
20033 drop(fold);
20034 assert!(removed.contains(&branch));
20035 assert!(!root.exists());
20036 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
20037 assert!(state.candidates[0].folded);
20038 assert!(!state.enabled_worktree_config);
20039 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
20040 let enabled = git::git_raw(
20041 &repo,
20042 &["config", "--local", "--get", "extensions.worktreeConfig"],
20043 )
20044 .await
20045 .unwrap();
20046 assert!(!enabled.ok(), "the last config reference was released");
20047 }
20048
20049 #[tokio::test]
20052 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
20053 let tmp = tempfile::tempdir().expect("tempdir");
20054 let repo = tmp.path().join("repo");
20055 std::fs::create_dir_all(&repo).unwrap();
20056 init_repo(&repo);
20057 let home = tmp.path().join("home");
20058
20059 let mut config = Config::default();
20060 config.graph.worktree_root = Some(tmp.path().join("wt"));
20061 let mut state = RunState::new(
20062 repo.clone(),
20063 "main".to_owned(),
20064 "deadbeef".to_owned(),
20065 "task".to_owned(),
20066 config,
20067 );
20068 git::git(&repo, &["branch", "magi/x/A", "main"])
20070 .await
20071 .expect("branch");
20072 state.candidates = vec![Candidate {
20073 index: 0,
20074 label: 'A',
20075 agent: "alpha".to_owned(),
20076 branch: "magi/x/A".to_owned(),
20077 worktree: state.worktree_root().join("cand-A"),
20078 summary: String::new(),
20079 stat: String::new(),
20080 files: 0,
20081 commits: 0,
20082 empty: false,
20083 failed: None,
20084 verified_noop: None,
20085 duration_ms: 0,
20086 folded: true,
20087 }];
20088 state.released_to = Some("20260901-000000-new1".to_owned());
20089 state.released_branches = vec!["magi/x/A".to_owned()];
20090
20091 fold_run(&mut state, true, &home).await.expect("fold_run");
20092 state.status = RunStatus::Merged;
20093 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
20094 state.merge = Some(crate::run::MergeOutcome {
20095 mode: MergeMode::Pr,
20096 ok: true,
20097 detail: "confirmed merged".into(),
20098 empty: false,
20099 });
20100 fold_run(&mut state, true, &home)
20101 .await
20102 .expect("merged fold");
20103
20104 assert!(
20105 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
20106 "the handed-over branch survives a fold"
20107 );
20108 }
20109
20110 #[tokio::test]
20114 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
20115 let tmp = tempfile::tempdir().expect("tempdir");
20116 let repo = tmp.path().join("repo");
20117 std::fs::create_dir_all(&repo).unwrap();
20118 init_repo(&repo);
20119 let run = |args: &[&str]| {
20120 let out = std::process::Command::new("git")
20121 .quiet()
20122 .args(args)
20123 .current_dir(&repo)
20124 .output()
20125 .expect("spawn git");
20126 assert!(out.status.success(), "git {args:?}");
20127 };
20128 run(&["branch", "magi/x/A"]);
20129 run(&["checkout", "-q", "-b", "magi/x/B"]);
20130 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
20131 run(&["add", "-A"]);
20132 run(&["commit", "-q", "-m", "work"]);
20133 run(&["checkout", "-q", "main"]);
20134
20135 let probe = RunState::new(
20139 repo.clone(),
20140 "main".to_owned(),
20141 "deadbeef".to_owned(),
20142 "task".to_owned(),
20143 Config::default(),
20144 );
20145 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
20146 let bare = tmp.path().join("origin.git");
20147 let out = std::process::Command::new("git")
20148 .quiet()
20149 .args(["init", "-q", "--bare"])
20150 .arg(&bare)
20151 .output()
20152 .expect("spawn git");
20153 assert!(out.status.success(), "git init --bare");
20154 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
20155 run(&["push", "-q", "origin", "main"]);
20156
20157 let mut state = RunState::new(
20158 repo.clone(),
20159 "main".to_owned(),
20160 "deadbeef".to_owned(),
20161 "task".to_owned(),
20162 Config::default(),
20163 );
20164 state.seeds = vec![refs::Seed {
20165 token: "magi/27b2/A".to_owned(),
20166 kind: refs::SeedKind::Unresolved,
20167 sha: String::new(),
20168 branch: true,
20169 detail: "no branch or commit named magi/27b2/A".to_owned(),
20170 }];
20171
20172 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
20173 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
20174 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
20175 let detail = empty_candidate_detail(&state, "main");
20176 assert!(detail.starts_with("empty candidate"), "{detail}");
20177 assert!(!detail.contains("magi/27b2/A"), "{detail}");
20178 assert!(detail.contains("--start-from"), "{detail}");
20179 }
20180
20181 #[tokio::test]
20190 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
20191 let tmp = tempfile::tempdir().expect("tempdir");
20192 let repo = tmp.path().join("repo");
20193 std::fs::create_dir_all(&repo).unwrap();
20194 init_repo(&repo);
20195
20196 let mut config = Config::default();
20197 config.merge.mode = MergeMode::Local;
20198
20199 let mut state = RunState::new(
20200 repo.clone(),
20201 "main".to_owned(),
20202 "deadbeef".to_owned(),
20203 "task".to_owned(),
20204 config,
20205 );
20206 state.candidates = vec![Candidate {
20207 index: 0,
20208 label: 'A',
20209 agent: "alpha".to_owned(),
20210 branch: "does-not-exist".to_owned(),
20211 worktree: repo.clone(),
20212 summary: String::new(),
20213 stat: String::new(),
20214 files: 0,
20215 commits: 0,
20216 empty: false,
20217 failed: None,
20218 verified_noop: None,
20219 duration_ms: 0,
20220 folded: false,
20221 }];
20222 state.tally = Some(Tally {
20223 first_choice: BTreeMap::from([('A', 1)]),
20224 borda: BTreeMap::new(),
20225 winner: 'A',
20226 rankings: 1,
20227 unanimous_initial: true,
20228 deliberated: false,
20229 changed_votes: 0,
20230 unanimous_final: true,
20231 tie_break: None,
20232 judges: 0,
20233 present: 0,
20234 quorum: 0,
20235 met_quorum: true,
20236 uncontested: Some("only candidate A produced a change".to_owned()),
20237 });
20238 state.reviews = vec![ReviewRound {
20239 round: 1,
20240 head: "deadbeef".to_owned(),
20241 verified_head: None,
20242 verified_at: None,
20243 reviews: Vec::new(),
20244 e2e: Vec::new(),
20245 fix: None,
20246 blocking: 0,
20247 answered: 0,
20248 expected: 0,
20249 clean: true,
20250 verify_retried: false,
20251 e2e_deferred: false,
20252 e2e_defer_reason: None,
20253 progressed: false,
20254 vote_split: false,
20255 reconsideration: Vec::new(),
20256 verdict: None,
20257 }];
20258 state.gate = vec![CommandOutcome {
20259 command: "test".to_owned(),
20260 code: Some(0),
20261 output_tail: String::new(),
20262 duration_ms: 0,
20263 resource_blocked: false,
20264 }];
20265 state.gate_ran = true;
20266 state.status = RunStatus::Ready;
20271 state.merge = Some(MergeOutcome {
20272 mode: MergeMode::Local,
20273 ok: false,
20274 detail: "already concluded".to_owned(),
20275 empty: false,
20276 });
20277
20278 let mut runner = Runner {
20279 state,
20280 roles: ResolvedRoles {
20281 implementers: Vec::new(),
20282 judges: Vec::new(),
20283 reviewers: Vec::new(),
20284 fixer: None,
20285 conductor: conductor(),
20286 implementer_roster: Vec::new(),
20287 judge_roster: Vec::new(),
20288 reviewer_roster: Vec::new(),
20289 },
20290 sem: Arc::new(Semaphore::new(1)),
20291 pause: Pause::new(),
20292 interrupt: Pause::new(),
20293 };
20294
20295 runner.merge().await.expect("merge");
20296
20297 assert_eq!(
20298 runner.state.status,
20299 RunStatus::Ready,
20300 "a concluded run's status must not change on reentry"
20301 );
20302 assert_eq!(
20303 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20304 Some("already concluded"),
20305 "merge must not run again once the node already recorded an outcome"
20306 );
20307 }
20308
20309 #[tokio::test]
20318 async fn merge_refuses_a_gate_that_has_not_actually_run() {
20319 let tmp = tempfile::tempdir().expect("tempdir");
20320 let repo = tmp.path().join("repo");
20321 std::fs::create_dir_all(&repo).unwrap();
20322 init_repo(&repo);
20323
20324 let mut config = Config::default();
20325 config.merge.mode = MergeMode::Local;
20326
20327 let mut state = RunState::new(
20328 repo.clone(),
20329 "main".to_owned(),
20330 "deadbeef".to_owned(),
20331 "task".to_owned(),
20332 config,
20333 );
20334 state.candidates = vec![Candidate {
20335 index: 0,
20336 label: 'A',
20337 agent: "alpha".to_owned(),
20338 branch: "does-not-exist".to_owned(),
20339 worktree: repo.clone(),
20340 summary: String::new(),
20341 stat: String::new(),
20342 files: 0,
20343 commits: 0,
20344 empty: false,
20345 failed: None,
20346 verified_noop: None,
20347 duration_ms: 0,
20348 folded: false,
20349 }];
20350 state.tally = Some(Tally {
20351 first_choice: BTreeMap::from([('A', 1)]),
20352 borda: BTreeMap::new(),
20353 winner: 'A',
20354 rankings: 1,
20355 unanimous_initial: true,
20356 deliberated: false,
20357 changed_votes: 0,
20358 unanimous_final: true,
20359 tie_break: None,
20360 judges: 0,
20361 present: 0,
20362 quorum: 0,
20363 met_quorum: true,
20364 uncontested: Some("only candidate A produced a change".to_owned()),
20365 });
20366 state.reviews = vec![ReviewRound {
20367 round: 1,
20368 head: "deadbeef".to_owned(),
20369 verified_head: None,
20370 verified_at: None,
20371 reviews: Vec::new(),
20372 e2e: Vec::new(),
20373 fix: None,
20374 blocking: 0,
20375 answered: 0,
20376 expected: 0,
20377 clean: true,
20378 verify_retried: false,
20379 e2e_deferred: false,
20380 e2e_defer_reason: None,
20381 progressed: false,
20382 vote_split: false,
20383 reconsideration: Vec::new(),
20384 verdict: None,
20385 }];
20386 state.gate = Vec::new();
20388 state.gate_ran = false;
20389 state.status = RunStatus::Gating;
20390
20391 let mut runner = Runner {
20392 state,
20393 roles: ResolvedRoles {
20394 implementers: Vec::new(),
20395 judges: Vec::new(),
20396 reviewers: Vec::new(),
20397 fixer: None,
20398 conductor: conductor(),
20399 implementer_roster: Vec::new(),
20400 judge_roster: Vec::new(),
20401 reviewer_roster: Vec::new(),
20402 },
20403 sem: Arc::new(Semaphore::new(1)),
20404 pause: Pause::new(),
20405 interrupt: Pause::new(),
20406 };
20407
20408 runner.merge().await.expect("merge");
20409
20410 assert!(
20411 runner.state.merge.is_none(),
20412 "an empty gate must never be read as a passing one: {:?}",
20413 runner.state.merge
20414 );
20415 }
20416
20417 #[tokio::test]
20424 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20425 let tmp = tempfile::tempdir().expect("tempdir");
20426 let repo = tmp.path().join("repo");
20427 std::fs::create_dir_all(&repo).unwrap();
20428 init_repo(&repo);
20429
20430 let config = Config::default();
20432
20433 let mut state = RunState::new(
20434 repo.clone(),
20435 "main".to_owned(),
20436 "deadbeef".to_owned(),
20437 "task".to_owned(),
20438 config,
20439 );
20440 state.candidates = vec![Candidate {
20441 index: 0,
20442 label: 'A',
20443 agent: "alpha".to_owned(),
20444 branch: "does-not-exist".to_owned(),
20445 worktree: repo.clone(),
20446 summary: String::new(),
20447 stat: String::new(),
20448 files: 0,
20449 commits: 0,
20450 empty: false,
20451 failed: None,
20452 verified_noop: None,
20453 duration_ms: 0,
20454 folded: false,
20455 }];
20456 state.tally = Some(Tally {
20457 first_choice: BTreeMap::from([('A', 1)]),
20458 borda: BTreeMap::new(),
20459 winner: 'A',
20460 rankings: 1,
20461 unanimous_initial: true,
20462 deliberated: false,
20463 changed_votes: 0,
20464 unanimous_final: true,
20465 tie_break: None,
20466 judges: 0,
20467 present: 0,
20468 quorum: 0,
20469 met_quorum: true,
20470 uncontested: Some("only candidate A produced a change".to_owned()),
20471 });
20472 state.reviews = vec![ReviewRound {
20473 round: 1,
20474 head: "deadbeef".to_owned(),
20475 verified_head: None,
20476 verified_at: None,
20477 reviews: Vec::new(),
20478 e2e: Vec::new(),
20479 fix: None,
20480 blocking: 0,
20481 answered: 0,
20482 expected: 0,
20483 clean: true,
20484 verify_retried: false,
20485 e2e_deferred: false,
20486 e2e_defer_reason: None,
20487 progressed: false,
20488 vote_split: false,
20489 reconsideration: Vec::new(),
20490 verdict: None,
20491 }];
20492
20493 let mut runner = Runner {
20494 state,
20495 roles: ResolvedRoles {
20496 implementers: Vec::new(),
20497 judges: Vec::new(),
20498 reviewers: Vec::new(),
20499 fixer: None,
20500 conductor: conductor(),
20501 implementer_roster: Vec::new(),
20502 judge_roster: Vec::new(),
20503 reviewer_roster: Vec::new(),
20504 },
20505 sem: Arc::new(Semaphore::new(1)),
20506 pause: Pause::new(),
20507 interrupt: Pause::new(),
20508 };
20509
20510 runner.gate().await.expect("gate");
20511 assert!(
20512 runner.state.gate_ran,
20513 "zero configured commands is still a real attempt, not an unrun gate"
20514 );
20515 assert!(runner.state.gate.is_empty());
20516 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20517 assert_ne!(
20518 runner.state.status,
20519 RunStatus::Blocked,
20520 "a gate with nothing to check must not read as failed"
20521 );
20522
20523 runner.merge().await.expect("merge");
20524 assert_eq!(
20525 runner.state.status,
20526 RunStatus::Ready,
20527 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20528 );
20529 }
20530
20531 #[tokio::test]
20542 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20543 crate::run::pin_test_home();
20544 let home = crate::run::home();
20545
20546 let tmp = tempfile::tempdir().expect("tempdir");
20547 let repo = tmp.path().join("repo");
20548 std::fs::create_dir_all(&repo).unwrap();
20549 init_repo(&repo);
20550 let cache_dir = tmp.path().join("target");
20553
20554 let mut config = Config::default();
20555 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20556 config.graph.timeout_verify = Some(2);
20560 config.graph.timeout_resource_wait = 0;
20561
20562 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20563 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20564 .expect("no io error acquiring directly")
20565 {
20566 crate::cache::AcquireOutcome::Acquired(g) => g,
20567 crate::cache::AcquireOutcome::Busy(b) => {
20568 panic!("expected the direct acquire to win the lease first: {b:?}")
20569 }
20570 };
20571
20572 let mut state = RunState::new(
20573 repo.clone(),
20574 "main".to_owned(),
20575 "deadbeef".to_owned(),
20576 "task".to_owned(),
20577 config,
20578 );
20579 state.candidates = vec![Candidate {
20580 index: 0,
20581 label: 'A',
20582 agent: "alpha".to_owned(),
20583 branch: "does-not-exist".to_owned(),
20584 worktree: repo.clone(),
20585 summary: String::new(),
20586 stat: String::new(),
20587 files: 0,
20588 commits: 0,
20589 empty: false,
20590 failed: None,
20591 verified_noop: None,
20592 duration_ms: 0,
20593 folded: false,
20594 }];
20595 state.tally = Some(Tally {
20596 first_choice: BTreeMap::from([('A', 1)]),
20597 borda: BTreeMap::new(),
20598 winner: 'A',
20599 rankings: 1,
20600 unanimous_initial: true,
20601 deliberated: false,
20602 changed_votes: 0,
20603 unanimous_final: true,
20604 tie_break: None,
20605 judges: 0,
20606 present: 0,
20607 quorum: 0,
20608 met_quorum: true,
20609 uncontested: Some("only candidate A produced a change".to_owned()),
20610 });
20611 state.reviews = vec![ReviewRound {
20612 round: 1,
20613 head: "deadbeef".to_owned(),
20614 verified_head: None,
20615 verified_at: None,
20616 reviews: Vec::new(),
20617 e2e: Vec::new(),
20618 fix: None,
20619 blocking: 0,
20620 answered: 0,
20621 expected: 0,
20622 clean: true,
20623 verify_retried: false,
20624 e2e_deferred: false,
20625 e2e_defer_reason: None,
20626 progressed: false,
20627 vote_split: false,
20628 reconsideration: Vec::new(),
20629 verdict: None,
20630 }];
20631
20632 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20633 state.reviews[0].e2e = vec![CommandOutcome {
20634 command: state.config.verify.e2e[0].clone(),
20635 code: Some(0),
20636 output_tail: String::new(),
20637 duration_ms: 0,
20638 resource_blocked: false,
20639 }];
20640 let mut runner = Runner {
20641 state,
20642 roles: ResolvedRoles {
20643 implementers: Vec::new(),
20644 judges: Vec::new(),
20645 reviewers: Vec::new(),
20646 fixer: None,
20647 conductor: conductor(),
20648 implementer_roster: Vec::new(),
20649 judge_roster: Vec::new(),
20650 reviewer_roster: Vec::new(),
20651 },
20652 sem: Arc::new(Semaphore::new(1)),
20653 pause: Pause::new(),
20654 interrupt: Pause::new(),
20655 };
20656
20657 runner.gate().await.expect("gate");
20658 assert!(runner.state.resource_block.is_none());
20659 assert!(
20660 !runner
20661 .state
20662 .events
20663 .iter()
20664 .any(|event| { event.message.contains("waiting for the shared build cache") }),
20665 "a gate with nothing to run must never request the held cache lease"
20666 );
20667 assert!(
20668 runner.state.gate_ran,
20669 "zero commands is still a real, immediate attempt"
20670 );
20671 assert!(runner.state.gate.is_empty());
20672 assert_ne!(
20673 runner.state.status,
20674 RunStatus::Blocked,
20675 "must not read as resource-blocked on a lease it never asked for"
20676 );
20677 }
20678
20679 #[cfg(unix)]
20682 #[tokio::test]
20683 async fn verification_is_standalone_inside_a_live_worker() {
20684 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20685 if std::env::var_os(CHILD).is_none() {
20686 let home = tempfile::tempdir().unwrap();
20687 let generation = crate::worker::pin_current(home.path()).unwrap();
20688 let output = std::process::Command::new(std::env::current_exe().unwrap())
20689 .args([
20690 "--exact",
20691 "graph::tests::verification_is_standalone_inside_a_live_worker",
20692 "--nocapture",
20693 ])
20694 .env(CHILD, "1")
20695 .env("MAGI_HOME", home.path())
20696 .env("MAGI_WORKER_HOME", home.path())
20697 .env("MAGI_WORKER_ID", "abcdef")
20698 .env("MAGI_WORKER_GENERATION", &generation.id)
20699 .env("MAGI_WORKER_EXE", &generation.executable)
20700 .env("MAGI_RUN", "verification-parent")
20701 .env("MAGI_NODE", "gate")
20702 .env("MAGI_SEAT", "worker-fixture")
20703 .env("MAGI_PROCESS_OWNER", "fixture-owner")
20704 .env("MAGI_NO_AUTOUPDATE", "0")
20705 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20706 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20707 .env(
20708 "VERIFY_EXPECT_TOOLCHAIN",
20709 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20710 )
20711 .env(
20712 "VERIFY_EXPECT_CACHE",
20713 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20714 )
20715 .output()
20716 .unwrap();
20717 assert!(
20718 output.status.success()
20719 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20720 "{}\n{}",
20721 String::from_utf8_lossy(&output.stdout),
20722 String::from_utf8_lossy(&output.stderr)
20723 );
20724 return;
20725 }
20726 let home = crate::run::home();
20727 let directory = home.join("workers/jobs/abcdef");
20728 std::fs::create_dir_all(&directory).unwrap();
20729 let job = crate::worker::Job {
20730 id: "abcdef".into(),
20731 sequence: 1,
20732 kind: "task".into(),
20733 key: "task:verification".into(),
20734 payload: serde_json::json!({}),
20735 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20736 version: Some(env!("CARGO_PKG_VERSION").into()),
20737 status: crate::worker::Status::Running,
20738 result: None,
20739 pid: Some(std::process::id()),
20740 created_at: 0,
20741 finished_at: None,
20742 };
20743 std::fs::write(
20744 directory.join("job.json"),
20745 serde_json::to_vec(&job).unwrap(),
20746 )
20747 .unwrap();
20748 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20749 let assert_worker = || {
20750 assert_eq!(
20751 crate::worker::current_job(&home).unwrap().unwrap().id,
20752 job.id
20753 );
20754 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20755 assert_eq!(
20756 crate::worker::helper_executable(&home).unwrap(),
20757 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20758 );
20759 assert!(
20760 crate::updater::Checker::new(&crate::config::Update {
20761 mode: crate::config::UpdateMode::Notify,
20762 interval: None,
20763 auto_restart: false,
20764 })
20765 .is_none(),
20766 "the driver remains pinned, even while tests are standalone"
20767 );
20768 };
20769 assert_worker();
20770 let repo = home.join("repo");
20771 std::fs::create_dir_all(&repo).unwrap();
20772 init_repo(&repo);
20773 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20776 test -z "${MAGI_WORKER_GENERATION+x}" &&
20777 test -z "${MAGI_WORKER_HOME+x}" &&
20778 test -z "${MAGI_WORKER_EXE+x}" &&
20779 test "$MAGI_HOME/repo" -ef . &&
20780 test "$MAGI_RUN" = verification-parent &&
20781 test "$MAGI_NODE" = gate &&
20782 test "$MAGI_SEAT" = worker-fixture &&
20783 test -n "$MAGI_PROCESS_OWNER" &&
20784 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20785 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20786 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20787 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20788 assert!(
20790 !std::process::Command::new("sh")
20791 .args(["-c", probe])
20792 .current_dir(&repo)
20793 .status()
20794 .unwrap()
20795 .success()
20796 );
20797 let unrelated_home = tempfile::tempdir().unwrap();
20798 assert!(
20799 crate::worker::current_job(unrelated_home.path()).is_err(),
20800 "an inherited worker id cannot be resolved in a unit fixture's home"
20801 );
20802 let mut runner = runner_at(RunStatus::Gating);
20803 runner.state.repo = repo.clone();
20804 runner.state.config.disk.min_free_bytes = 0;
20805 runner.state.config.coordination = crate::config::test_coordination();
20806 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20807 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20808 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20809 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20810 let winner = Candidate {
20811 index: 0,
20812 label: 'A',
20813 agent: "fixture".into(),
20814 branch: "main".into(),
20815 worktree: repo.clone(),
20816 summary: String::new(),
20817 stat: String::new(),
20818 files: 0,
20819 commits: 0,
20820 empty: false,
20821 failed: None,
20822 verified_noop: None,
20823 duration_ms: 0,
20824 folded: false,
20825 };
20826 let (e2e, retried, pids) = run_e2e_with_retry(
20827 &mut runner.state,
20828 &["sh".into(), "-c".into()],
20829 &[format!("{probe} && printf initial")],
20830 &repo,
20831 Duration::from_secs(10),
20832 "initial",
20833 None,
20834 )
20835 .await;
20836 assert!(!retried && pids.is_empty());
20837 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20838 assert_eq!(e2e[0].output_tail, "initial");
20839 let gate = runner.run_gate(&winner).await.unwrap();
20840 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20841 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20844 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20845 passed: false,
20846 outcomes: Vec::new(),
20847 });
20848 runner.state.save().unwrap();
20849 let mut resumed = RunState::load(&runner.state.id).unwrap();
20850 let wait = crate::repo_lock::WaitPolicy::default();
20851 assert!(matches!(
20852 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20853 .await
20854 .unwrap(),
20855 GateFix::Retry
20856 ));
20857 let saved = RunState::load(&resumed.id).unwrap();
20858 let repair = saved.gate_repair_e2e.unwrap();
20859 assert!(repair.passed);
20860 assert_eq!(repair.outcomes.len(), 1);
20861 assert!(repair.outcomes[0].ok());
20862 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20863 .await
20864 .unwrap();
20865 assert_eq!(landing.len(), 1);
20866 assert!(landing[0].ok(), "{landing:?}");
20867 assert_eq!(
20868 std::fs::read_to_string(repo.join("phases")).unwrap(),
20869 "pre\ngate\ne2e\npre\ngate\n"
20870 );
20871 assert_worker();
20872 }
20873
20874 #[cfg(unix)]
20875 #[tokio::test]
20876 async fn verification_keeps_the_panic_and_the_full_output() {
20877 crate::run::pin_test_home();
20878 let tmp = tempfile::tempdir().unwrap();
20879 let mut config = Config::default();
20880 config.disk.min_free_bytes = 0;
20881 let mut state = RunState::new(
20882 tmp.path().to_owned(),
20883 "main".into(),
20884 "base".into(),
20885 "verification evidence".into(),
20886 config,
20887 );
20888 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20889 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20890 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20891 let mut names = Vec::new();
20892 for _ in 0..2 {
20893 let (outcomes, _) = run_commands(
20894 &mut state,
20895 "gate",
20896 "gate",
20897 0,
20898 &["sh".into(), "-c".into()],
20899 &[script.into()],
20900 tmp.path(),
20901 Duration::from_secs(60),
20902 )
20903 .await;
20904 let tail = &outcomes[0].output_tail;
20905 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20906 assert!(tail.contains("boom assertion"));
20907 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20908 let line = tail.lines().next().unwrap();
20909 names.push(line.to_owned());
20910 }
20911 assert_ne!(names[0], names[1], "the second run must not overwrite");
20912 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20913 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20914 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20915
20916 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20918 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20919 let (outcomes, _) = run_commands(
20920 &mut state,
20921 "gate",
20922 "gate",
20923 0,
20924 &["sh".into(), "-c".into()],
20925 &["echo failure-output; exit 7".into()],
20926 tmp.path(),
20927 Duration::from_secs(10),
20928 )
20929 .await;
20930 assert_eq!(outcomes[0].code, Some(7));
20931 assert!(state.events.iter().any(|e| {
20932 e.node == "gate" && e.message.contains("could not save the full gate output:")
20933 }));
20934 }
20935
20936 #[cfg(unix)]
20937 #[tokio::test]
20938 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20939 crate::run::pin_test_home();
20940 let tmp = tempfile::tempdir().unwrap();
20941 let mut config = Config::default();
20942 config.disk.min_free_bytes = 0;
20943 let mut state = RunState::new(
20944 tmp.path().to_owned(),
20945 "main".into(),
20946 "base".into(),
20947 "verification evidence".into(),
20948 config,
20949 );
20950 let (outcomes, pids) = run_commands(
20951 &mut state,
20952 "gate",
20953 "gate",
20954 1,
20955 &["sh".into(), "-c".into()],
20956 &[
20957 "printf before-timeout; exec sleep 30".into(),
20958 "kill -TERM $$".into(),
20959 "exit 7".into(),
20960 "printf success".into(),
20961 ],
20962 tmp.path(),
20963 Duration::from_secs(1),
20964 )
20965 .await;
20966 wait_for_timed_out_children_to_die(&pids).await;
20967 assert_eq!(
20968 pids.len(),
20969 1,
20970 "unleased timeout retains its death-check PID"
20971 );
20972 assert_eq!(outcomes.len(), 4);
20973 assert_eq!(outcomes[0].code, None);
20974 assert!(outcomes[0].output_tail.contains("before-timeout"));
20975 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20976 assert_eq!(outcomes[1].code, None);
20977 assert!(outcomes[1].output_tail.contains("command terminated:"));
20978 assert!(!outcomes[1].output_tail.contains("timed out"));
20979 assert_eq!(outcomes[2].code, Some(7));
20980 assert!(!outcomes[2].output_tail.contains("timed out"));
20981 assert_eq!(outcomes[3].code, Some(0));
20982 assert_eq!(outcomes[3].output_tail, "success");
20983 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20984 }
20985
20986 #[tokio::test]
20996 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20997 crate::run::pin_test_home();
20998
20999 let tmp = tempfile::tempdir().expect("tempdir");
21000 let repo = tmp.path().join("repo");
21001 std::fs::create_dir_all(&repo).unwrap();
21002 init_repo(&repo);
21003
21004 let mut config = Config::default();
21005 config.disk.min_free_bytes = 0;
21006 config.verify.gate = vec![
21007 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
21008 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
21009 .to_owned(),
21010 ];
21011
21012 let mut state = RunState::new(
21013 repo.clone(),
21014 "main".to_owned(),
21015 "deadbeef".to_owned(),
21016 "task".to_owned(),
21017 config,
21018 );
21019 let run_id = state.id.clone();
21020 state.candidates = vec![Candidate {
21021 index: 0,
21022 label: 'A',
21023 agent: "alpha".to_owned(),
21024 branch: "does-not-exist".to_owned(),
21025 worktree: repo.clone(),
21026 summary: String::new(),
21027 stat: String::new(),
21028 files: 0,
21029 commits: 0,
21030 empty: false,
21031 failed: None,
21032 verified_noop: None,
21033 duration_ms: 0,
21034 folded: false,
21035 }];
21036 state.tally = Some(Tally {
21037 first_choice: BTreeMap::from([('A', 1)]),
21038 borda: BTreeMap::new(),
21039 winner: 'A',
21040 rankings: 1,
21041 unanimous_initial: true,
21042 deliberated: false,
21043 changed_votes: 0,
21044 unanimous_final: true,
21045 tie_break: None,
21046 judges: 0,
21047 present: 0,
21048 quorum: 0,
21049 met_quorum: true,
21050 uncontested: Some("only candidate A produced a change".to_owned()),
21051 });
21052 state.reviews = vec![ReviewRound {
21053 round: 1,
21054 head: "deadbeef".to_owned(),
21055 verified_head: None,
21056 verified_at: None,
21057 reviews: Vec::new(),
21058 e2e: Vec::new(),
21059 fix: None,
21060 blocking: 0,
21061 answered: 0,
21062 expected: 0,
21063 clean: true,
21064 verify_retried: false,
21065 e2e_deferred: false,
21066 e2e_defer_reason: None,
21067 progressed: false,
21068 vote_split: false,
21069 reconsideration: Vec::new(),
21070 verdict: None,
21071 }];
21072
21073 let mut runner = Runner {
21074 state,
21075 roles: ResolvedRoles {
21076 implementers: Vec::new(),
21077 judges: Vec::new(),
21078 reviewers: Vec::new(),
21079 fixer: None,
21080 conductor: conductor(),
21081 implementer_roster: Vec::new(),
21082 judge_roster: Vec::new(),
21083 reviewer_roster: Vec::new(),
21084 },
21085 sem: Arc::new(Semaphore::new(1)),
21086 pause: Pause::new(),
21087 interrupt: Pause::new(),
21088 };
21089
21090 let started_marker = repo.join("started.marker");
21091 let release_marker = repo.join("release.marker");
21092 let poller = tokio::spawn(async move {
21093 for _ in 0..100 {
21098 if started_marker.exists()
21099 && let Ok(s) = crate::run::RunState::load(&run_id)
21100 && let Some(a) = s.active.get("gate")
21101 {
21102 std::fs::write(&release_marker, b"go").expect("release marker");
21103 return Some(a.clone());
21104 }
21105 tokio::time::sleep(Duration::from_millis(50)).await;
21106 }
21107 None
21108 });
21109
21110 runner.gate().await.expect("gate");
21111 let captured = poller.await.expect("poller task");
21112 let captured = captured.expect(
21113 "the poller never saw a `gate` task entry in run.json while the command was \
21114 still blocked on its own release marker",
21115 );
21116
21117 assert_eq!(captured.task.as_deref(), Some("gate"));
21118 assert_eq!(captured.node, "gate");
21119 assert_eq!(captured.index, Some(1));
21120 assert_eq!(captured.total, Some(1));
21121 assert!(
21122 captured
21123 .command
21124 .as_deref()
21125 .is_some_and(|c| c.contains("started.marker")),
21126 "{captured:?}"
21127 );
21128
21129 assert!(
21130 runner.state.active.is_empty(),
21131 "the entry must be cleared once the command actually finished: {:?}",
21132 runner.state.active
21133 );
21134 assert!(runner.state.gate_ran);
21135 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
21136 }
21137
21138 #[tokio::test]
21141 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
21142 use crate::verdict::{Finding, ReviewVote, Severity};
21143 crate::run::pin_test_home();
21144 let tmp = tempfile::tempdir().expect("tempdir");
21145 let repo = tmp.path().join("repo");
21146 std::fs::create_dir_all(&repo).unwrap();
21147 init_repo(&repo);
21148
21149 for (severity, vote, expect) in [
21150 (Severity::Major, ReviewVote::Reject, true),
21151 (Severity::Minor, ReviewVote::Reject, false),
21152 (Severity::Major, ReviewVote::Approve, false),
21153 ] {
21154 let mut round = review_round(false, 1, 1, 1, false, true);
21155 round.reviews = vec![ReviewRecord {
21156 reviewer: 1,
21157 agent: "alpha".to_owned(),
21158 summary: String::new(),
21159 findings: vec![Finding {
21160 id: "R1-1-1".to_owned(),
21161 severity,
21162 file: None,
21163 line: None,
21164 title: "t".to_owned(),
21165 detail: String::new(),
21166 }],
21167 vote: Some(vote),
21168 failed: None,
21169 duration_ms: 0,
21170 attempts: 0,
21171 }];
21172 let mut state = RunState::new(
21173 repo.clone(),
21174 "main".to_owned(),
21175 "deadbeef".to_owned(),
21176 "task".to_owned(),
21177 Config::default(),
21178 );
21179 state.reviews = vec![round];
21180 let mut runner = Runner {
21181 state,
21182 roles: ResolvedRoles {
21183 implementers: Vec::new(),
21184 judges: Vec::new(),
21185 reviewers: Vec::new(),
21186 fixer: None,
21187 conductor: conductor(),
21188 implementer_roster: Vec::new(),
21189 judge_roster: Vec::new(),
21190 reviewer_roster: Vec::new(),
21191 },
21192 sem: Arc::new(Semaphore::new(1)),
21193 pause: Pause::new(),
21194 interrupt: Pause::new(),
21195 };
21196 let shell = runner.state.config.shell();
21197 runner
21198 .stop_reviewing("round budget spent", &shell, &repo)
21199 .await
21200 .expect("stop_reviewing");
21201 assert_eq!(runner.state.status, RunStatus::Gating);
21202 assert_eq!(
21203 runner.state.contested_handoff.is_some(),
21204 expect,
21205 "{severity:?} + {vote:?}"
21206 );
21207 }
21208 }
21209
21210 #[tokio::test]
21223 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
21224 crate::run::pin_test_home();
21225 let home = crate::run::home();
21226
21227 let tmp = tempfile::tempdir().expect("tempdir");
21228 let repo = tmp.path().join("repo");
21229 std::fs::create_dir_all(&repo).unwrap();
21230 init_repo(&repo);
21231 let head = crate::git::rev_parse(&repo, "HEAD")
21232 .await
21233 .expect("rev-parse");
21234 let cache_dir = tmp.path().join("target");
21237
21238 let mut config = Config::default();
21239 config.verify.e2e = vec![format!(
21240 "CARGO_TARGET_DIR='{}' test -f README.md",
21241 cache_dir.display()
21242 )];
21243 config.graph.review_rounds = 1;
21244 config.graph.timeout_verify = Some(2);
21247 config.graph.timeout_resource_wait = 2;
21248
21249 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21250 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21251 .expect("no io error acquiring directly")
21252 {
21253 crate::cache::AcquireOutcome::Acquired(g) => g,
21254 crate::cache::AcquireOutcome::Busy(b) => {
21255 panic!("expected the direct acquire to win the lease first: {b:?}")
21256 }
21257 };
21258
21259 let mut state = RunState::new(
21260 repo.clone(),
21261 "main".to_owned(),
21262 head.clone(),
21263 "task".to_owned(),
21264 config,
21265 );
21266 state.candidates = vec![Candidate {
21267 index: 0,
21268 label: 'A',
21269 agent: "alpha".to_owned(),
21270 branch: "does-not-exist".to_owned(),
21271 worktree: repo.clone(),
21272 summary: String::new(),
21273 stat: String::new(),
21274 files: 0,
21275 commits: 0,
21276 empty: false,
21277 failed: None,
21278 verified_noop: None,
21279 duration_ms: 0,
21280 folded: false,
21281 }];
21282 state.tally = Some(Tally {
21283 first_choice: BTreeMap::from([('A', 1)]),
21284 borda: BTreeMap::new(),
21285 winner: 'A',
21286 rankings: 1,
21287 unanimous_initial: true,
21288 deliberated: false,
21289 changed_votes: 0,
21290 unanimous_final: true,
21291 tie_break: None,
21292 judges: 0,
21293 present: 0,
21294 quorum: 0,
21295 met_quorum: true,
21296 uncontested: Some("only candidate A produced a change".to_owned()),
21297 });
21298 state.reviews = vec![ReviewRound {
21302 round: 1,
21303 head: head.clone(),
21304 verified_head: None,
21305 verified_at: None,
21306 reviews: Vec::new(),
21307 e2e: Vec::new(),
21308 fix: None,
21309 blocking: 1,
21310 answered: 1,
21311 expected: 1,
21312 clean: false,
21313 verify_retried: false,
21314 e2e_deferred: true,
21315 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
21316 progressed: false,
21317 vote_split: false,
21318 reconsideration: Vec::new(),
21319 verdict: None,
21320 }];
21321
21322 let mut runner = Runner {
21323 state,
21324 roles: ResolvedRoles {
21325 implementers: Vec::new(),
21326 judges: Vec::new(),
21327 reviewers: Vec::new(),
21328 fixer: None,
21329 conductor: conductor(),
21330 implementer_roster: Vec::new(),
21331 judge_roster: Vec::new(),
21332 reviewer_roster: Vec::new(),
21333 },
21334 sem: Arc::new(Semaphore::new(1)),
21335 pause: Pause::new(),
21336 interrupt: Pause::new(),
21337 };
21338
21339 let shell = runner.state.config.shell();
21340 runner
21341 .stop_reviewing("round budget spent", &shell, &repo)
21342 .await
21343 .expect("stop_reviewing");
21344
21345 let last = runner.state.reviews.last().expect("round record");
21346 assert_eq!(
21347 last.e2e_status(),
21348 E2eStatus::ResourceBlocked,
21349 "the shared cache is still held; the attempt must read as blocked, not deferred or \
21350 failed: {last:?}"
21351 );
21352 assert_eq!(
21353 last.verified_head.as_deref(),
21354 Some(head.as_str()),
21355 "which commit this attempt targeted is known even though nothing finished checking \
21356 it"
21357 );
21358 let first_attempt_at = last
21359 .verified_at
21360 .expect("when this attempt ran is known too");
21361 assert_ne!(
21362 runner.state.status,
21363 RunStatus::Blocked,
21364 "contention is evidence about the machine, not the patch — it must not settle the \
21365 run as blocked: {:?}",
21366 runner.state.status
21367 );
21368 assert!(
21369 !runner
21370 .state
21371 .events
21372 .iter()
21373 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
21374 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
21375 runner.state.events
21376 );
21377
21378 runner
21383 .stop_reviewing("round budget spent", &shell, &repo)
21384 .await
21385 .expect("stop_reviewing retry");
21386 assert_eq!(
21387 runner.state.reviews.len(),
21388 1,
21389 "no new round was started: {:?}",
21390 runner.state.reviews
21391 );
21392 let last = runner.state.reviews.last().expect("round record");
21393 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
21394 assert!(
21395 last.verified_at.expect("still known") > first_attempt_at,
21396 "a second reentry must be a fresh attempt, not a stale copy of the first"
21397 );
21398 assert_ne!(runner.state.status, RunStatus::Blocked);
21399
21400 held.release();
21401 }
21402
21403 #[cfg(unix)]
21404 #[tokio::test]
21405 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
21406 ask_test_home();
21407 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21408 let tmp = tempfile::tempdir().unwrap();
21409 let repo = tmp.path().join("repo");
21410 std::fs::create_dir(&repo).unwrap();
21411 init_repo(&repo);
21412 let run_git = |args: &[&str]| {
21413 let out = std::process::Command::new("git")
21414 .args(args)
21415 .current_dir(&repo)
21416 .output()
21417 .unwrap();
21418 assert!(
21419 out.status.success(),
21420 "{args:?}: {}",
21421 String::from_utf8_lossy(&out.stderr)
21422 );
21423 };
21424 let origin = tmp.path().join("origin.git");
21425 run_git(&["init", "--bare", origin.to_str().unwrap()]);
21426 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21427 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21428 run_git(&["add", "spec.rs"]);
21429 run_git(&["commit", "-m", "initial spec"]);
21430 run_git(&["push", "origin", "main"]);
21431 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21432 run_git(&["checkout", "-b", "fix/review"]);
21433 std::fs::write(
21434 repo.join("check.rs"),
21435 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21436 )
21437 .unwrap();
21438 run_git(&["add", "check.rs"]);
21439 run_git(&["commit", "-m", "use spec"]);
21440 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21441 run_git(&["checkout", "main"]);
21442 std::fs::write(
21443 repo.join("spec.rs"),
21444 "struct AgentSpec { max_turn_secs: u64 }\n",
21445 )
21446 .unwrap();
21447 run_git(&["add", "spec.rs"]);
21448 run_git(&["commit", "-m", "extend spec"]);
21449 run_git(&["push", "origin", "main"]);
21450 run_git(&["checkout", "fix/review"]);
21451
21452 let mut runner = runner_at(RunStatus::Reviewing);
21453 runner.state.repo = repo.clone();
21454 runner.state.base_commit = base;
21455 runner.state.pr_head = Some("fix/review".into());
21456 runner.state.config.graph.review_rounds = max_rounds;
21457 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21458 runner.state.config.graph.sessions = false;
21459 runner.state.config.disk.min_free_bytes = 0;
21460 runner.state.config.verify.e2e =
21461 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21462 runner.state.candidates = vec![Candidate {
21463 index: 0,
21464 label: 'A',
21465 agent: "fixture".into(),
21466 branch: "fix/review".into(),
21467 worktree: repo.clone(),
21468 summary: String::new(),
21469 stat: String::new(),
21470 files: 1,
21471 commits: 1,
21472 empty: false,
21473 failed: None,
21474 verified_noop: None,
21475 duration_ms: 0,
21476 folded: false,
21477 }];
21478 runner.state.tally = Some(Tally {
21479 first_choice: BTreeMap::new(),
21480 borda: BTreeMap::new(),
21481 winner: 'A',
21482 rankings: 0,
21483 unanimous_initial: true,
21484 deliberated: false,
21485 changed_votes: 0,
21486 unanimous_final: true,
21487 tie_break: None,
21488 judges: 0,
21489 present: 0,
21490 quorum: 0,
21491 met_quorum: true,
21492 uncontested: Some("review-only".into()),
21493 });
21494 let mut last = review_round(false, 0, 1, 1, false, false);
21495 last.head = reviewed.clone();
21496 last.e2e_deferred = true;
21497 last.e2e[0].resource_blocked = true;
21498 last.e2e[0].output_tail = "waiting for the shared build cache".into();
21499 if max_rounds == 1 {
21500 last.e2e.clear();
21501 }
21502 runner.state.reviews = vec![last];
21503 let calls = tmp.path().join("fix-calls");
21504 let mut fixer = spec("repair");
21505 fixer
21506 .env
21507 .insert("FIX_CALLS".into(), calls.display().to_string());
21508 fixer.command = vec!["sh".into(), "-c".into(), r#"
21509 echo fix >> "$FIX_CALLS"
21510 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21511 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21512 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21513 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21514 "#.into()];
21515 let mut reviewer = spec("reviewer");
21516 reviewer.command = vec![
21517 "sh".into(),
21518 "-c".into(),
21519 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21520 .into(),
21521 ];
21522 runner.roles.fixer = Some(vec![fixer]);
21523 runner.roles.reviewers = vec![reviewer];
21524
21525 runner.sync_to_base().await.unwrap();
21526 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21527 assert_ne!(rebased, reviewed);
21528 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21529 if reentry {
21530 let shell = runner.state.config.shell();
21533 runner
21534 .stop_unresolved_review("deferred verification", &shell, &repo)
21535 .await
21536 .unwrap();
21537 assert!(runner.catchup_repairable());
21538 assert!(!calls.exists());
21539 runner.state = RunState::load(&runner.state.id).unwrap();
21540 assert!(runner.catchup_repairable());
21541 }
21542 runner.review_loop().await.unwrap();
21543 let first = &runner.state.reviews[0];
21544 assert_eq!(first.head, reviewed);
21545 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21546 assert!(
21547 first.verify_retried,
21548 "a reproduced build failure gets exactly one retry: {:?}",
21549 first.e2e
21550 );
21551 assert!(first.e2e[0].output_tail.contains("E0063"));
21552 assert!(!runner.catchup_repairable());
21553 if max_rounds == 2 {
21554 assert_eq!(
21555 runner.state.status,
21556 RunStatus::Gating,
21557 "{}",
21558 crate::report::run(&runner.state)
21559 );
21560 assert_eq!(runner.state.reviews.len(), 2);
21561 assert!(first.fix.as_ref().unwrap().committed);
21562 assert!(runner.state.reviews[1].clean);
21563 assert_eq!(
21564 runner
21565 .state
21566 .reviews
21567 .iter()
21568 .filter(|r| r.fix.is_some())
21569 .count(),
21570 1
21571 );
21572 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21573 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21574 } else {
21575 assert_eq!(runner.state.status, RunStatus::Blocked);
21576 assert!(first.fix.is_none());
21577 assert!(!calls.exists());
21578 let recovery = crate::recovery::assess(
21579 "prior",
21580 Some(&runner.state),
21581 &runner.landing_base(),
21582 crate::recovery::Relation::Diverged,
21583 );
21584 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21585 let reasons = recovery.reasons.join("\n");
21586 assert!(
21587 reasons.contains(&reviewed) && reasons.contains(&rebased),
21588 "{reasons}"
21589 );
21590 assert!(
21591 reasons.contains("base sync to")
21592 && reasons.contains("additional review is required"),
21593 "{reasons}"
21594 );
21595 }
21596 }
21597 }
21598
21599 #[tokio::test]
21611 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21612 crate::run::pin_test_home();
21613 let home = crate::run::home();
21614
21615 let tmp = tempfile::tempdir().expect("tempdir");
21616 let repo = tmp.path().join("repo");
21617 std::fs::create_dir_all(&repo).unwrap();
21618 init_repo(&repo);
21619 let head = crate::git::rev_parse(&repo, "HEAD")
21620 .await
21621 .expect("rev-parse");
21622 let cache_dir = tmp.path().join("target");
21623
21624 let mut config = Config::default();
21625 config.verify.e2e = vec![format!(
21626 "CARGO_TARGET_DIR='{}' test -f README.md",
21627 cache_dir.display()
21628 )];
21629 config.graph.review_rounds = 1;
21630 config.graph.timeout_verify = Some(2);
21631 config.graph.timeout_resource_wait = 2;
21632
21633 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21634 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21635 .expect("no io error acquiring directly")
21636 {
21637 crate::cache::AcquireOutcome::Acquired(g) => g,
21638 crate::cache::AcquireOutcome::Busy(b) => {
21639 panic!("expected the direct acquire to win the lease first: {b:?}")
21640 }
21641 };
21642
21643 let mut state = RunState::new(
21644 repo.clone(),
21645 "main".to_owned(),
21646 head.clone(),
21647 "task".to_owned(),
21648 config,
21649 );
21650 state.candidates = vec![Candidate {
21651 index: 0,
21652 label: 'A',
21653 agent: "alpha".to_owned(),
21654 branch: "does-not-exist".to_owned(),
21655 worktree: repo.clone(),
21656 summary: String::new(),
21657 stat: String::new(),
21658 files: 0,
21659 commits: 0,
21660 empty: false,
21661 failed: None,
21662 verified_noop: None,
21663 duration_ms: 0,
21664 folded: false,
21665 }];
21666 state.tally = Some(Tally {
21667 first_choice: BTreeMap::from([('A', 1)]),
21668 borda: BTreeMap::new(),
21669 winner: 'A',
21670 rankings: 1,
21671 unanimous_initial: true,
21672 deliberated: false,
21673 changed_votes: 0,
21674 unanimous_final: true,
21675 tie_break: None,
21676 judges: 0,
21677 present: 0,
21678 quorum: 0,
21679 met_quorum: true,
21680 uncontested: Some("only candidate A produced a change".to_owned()),
21681 });
21682 state.reviews = vec![ReviewRound {
21686 round: 1,
21687 head: head.clone(),
21688 verified_head: Some(head.clone()),
21689 verified_at: Some(jiff::Timestamp::now()),
21690 reviews: Vec::new(),
21691 e2e: vec![CommandOutcome {
21692 command: format!(
21693 "CARGO_TARGET_DIR='{}' test -f README.md",
21694 cache_dir.display()
21695 ),
21696 code: None,
21697 output_tail: "waiting for the shared build cache".to_owned(),
21698 duration_ms: 0,
21699 resource_blocked: true,
21700 }],
21701 fix: None,
21702 blocking: 1,
21703 answered: 1,
21704 expected: 1,
21705 clean: false,
21706 verify_retried: false,
21707 e2e_deferred: false,
21708 e2e_defer_reason: None,
21709 progressed: false,
21710 vote_split: false,
21711 reconsideration: Vec::new(),
21712 verdict: None,
21713 }];
21714
21715 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21716 let mut runner = Runner {
21717 state,
21718 roles: ResolvedRoles {
21719 implementers: Vec::new(),
21720 judges: Vec::new(),
21721 reviewers: Vec::new(),
21722 fixer: None,
21723 conductor: conductor(),
21724 implementer_roster: Vec::new(),
21725 judge_roster: Vec::new(),
21726 reviewer_roster: Vec::new(),
21727 },
21728 sem: Arc::new(Semaphore::new(1)),
21729 pause: Pause::new(),
21730 interrupt: Pause::new(),
21731 };
21732
21733 runner.review_loop().await.expect("review_loop");
21738
21739 assert_eq!(
21740 runner.state.reviews.len(),
21741 1,
21742 "no new round was started on top of the unresolved one: {:?}",
21743 runner.state.reviews
21744 );
21745 let last = &runner.state.reviews[0];
21746 assert_eq!(
21747 last.e2e_status(),
21748 E2eStatus::ResourceBlocked,
21749 "still contended: {last:?}"
21750 );
21751 assert!(
21752 last.verified_at.expect("still known") > first_attempt_at,
21753 "review_loop must have actually retried the check, not left it exactly as found"
21754 );
21755 assert_ne!(
21756 runner.state.status,
21757 RunStatus::Blocked,
21758 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21759 runner.state.status
21760 );
21761
21762 held.release();
21763 }
21764
21765 #[tokio::test]
21766 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21767 crate::run::pin_test_home();
21768 for status in [RunStatus::Blocked, RunStatus::Gating] {
21769 let mut runner = runner_at(status);
21770 runner.state.config.graph.land = true;
21771 runner.state.gate_ran = true;
21772 runner.state.merge = Some(MergeOutcome {
21773 mode: MergeMode::Pr,
21774 ok: false,
21775 detail: "the fixer produced no commit".into(),
21776 empty: false,
21777 });
21778 let url = "https://example.invalid/o/r/pull/1";
21779 runner.state.pr = Some(crate::run::PrRecord {
21780 url: url.into(),
21781 number: 1,
21782 state: "open".into(),
21783 checks: "red".into(),
21784 round: 1,
21785 rounds: 1,
21786 red_at_merge: Vec::new(),
21787 });
21788 runner.state.save().unwrap();
21789 runner.execute_graph().await.unwrap();
21793 assert_eq!(runner.state.status, RunStatus::Blocked);
21794 assert!(runner.state.land_sync_pending);
21795 assert!(runner.state.events.iter().any(|event|
21796 event.node == "land" && event.message == format!("watching {url}")));
21797 assert_eq!(
21798 runner.state.merge.as_ref().unwrap().detail,
21799 "the fixer produced no commit"
21800 );
21801 }
21802 }
21803
21804 #[tokio::test]
21805 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21806 crate::run::pin_test_home();
21807 let tmp = tempfile::tempdir().expect("tempdir");
21808 let repo = tmp.path().join("repo");
21809 std::fs::create_dir_all(&repo).unwrap();
21810 init_repo(&repo);
21811
21812 let mut config = Config::default();
21813 config.merge.mode = MergeMode::Pr;
21814 config.graph.land = true;
21815 config.graph.land_approval = false;
21816
21817 let mut state = RunState::new(
21818 repo.clone(),
21819 "main".to_owned(),
21820 "deadbeef".to_owned(),
21821 "task".to_owned(),
21822 config,
21823 );
21824 state.candidates = vec![Candidate {
21825 index: 0,
21826 label: 'A',
21827 agent: "alpha".to_owned(),
21828 branch: "does-not-exist".to_owned(),
21829 worktree: repo.clone(),
21830 summary: String::new(),
21831 stat: String::new(),
21832 files: 0,
21833 commits: 0,
21834 empty: false,
21835 failed: None,
21836 verified_noop: None,
21837 duration_ms: 0,
21838 folded: false,
21839 }];
21840 state.tally = Some(Tally {
21841 first_choice: BTreeMap::from([('A', 1)]),
21842 borda: BTreeMap::new(),
21843 winner: 'A',
21844 rankings: 1,
21845 unanimous_initial: true,
21846 deliberated: false,
21847 changed_votes: 0,
21848 unanimous_final: true,
21849 tie_break: None,
21850 judges: 0,
21851 present: 0,
21852 quorum: 0,
21853 met_quorum: true,
21854 uncontested: Some("only candidate A produced a change".to_owned()),
21855 });
21856 state.reviews = vec![ReviewRound {
21857 round: 1,
21858 head: "deadbeef".to_owned(),
21859 verified_head: None,
21860 verified_at: None,
21861 reviews: Vec::new(),
21862 e2e: Vec::new(),
21863 fix: None,
21864 blocking: 0,
21865 answered: 0,
21866 expected: 0,
21867 clean: true,
21868 verify_retried: false,
21869 e2e_deferred: false,
21870 e2e_defer_reason: None,
21871 progressed: false,
21872 vote_split: false,
21873 reconsideration: Vec::new(),
21874 verdict: None,
21875 }];
21876 state.gate = vec![CommandOutcome {
21877 command: "test".to_owned(),
21878 code: Some(0),
21879 output_tail: String::new(),
21880 duration_ms: 0,
21881 resource_blocked: false,
21882 }];
21883 state.gate_ran = true;
21884 state.status = RunStatus::Landing;
21888 state.merge = Some(MergeOutcome {
21889 mode: MergeMode::Pr,
21890 ok: true,
21891 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21892 empty: false,
21893 });
21894
21895 ask_test_home();
21899 state.save().expect("save landing run");
21902 let store = ask::Questions::open();
21903 let q = ask_open_question(&store, &state.id);
21904
21905 let mut runner = Runner {
21906 state,
21907 roles: ResolvedRoles {
21908 implementers: Vec::new(),
21909 judges: Vec::new(),
21910 reviewers: Vec::new(),
21911 fixer: None,
21912 conductor: conductor(),
21913 implementer_roster: Vec::new(),
21914 judge_roster: Vec::new(),
21915 reviewer_roster: Vec::new(),
21916 },
21917 sem: Arc::new(Semaphore::new(1)),
21918 pause: Pause::new(),
21919 interrupt: Pause::new(),
21920 };
21921
21922 runner.execute().await.expect("execute");
21927
21928 assert_eq!(
21929 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21930 Some("https://example.invalid/x/y/pull/1"),
21931 "reentry must not push again or open a second pull request over the \
21932 one `land` is already watching"
21933 );
21934 assert_ne!(
21935 runner.state.status,
21936 RunStatus::Landing,
21937 "land could not actually reach the fake pull request, so it must \
21938 have given up rather than left the run silently parked forever"
21939 );
21940 assert_eq!(runner.state.status, RunStatus::Blocked);
21944 assert!(
21945 store.get(&q.id).unwrap().status.open(),
21946 "Blocked is still alive; settle_questions must have been a no-op here"
21947 );
21948 }
21949
21950 fn state_with_round(round: ReviewRound) -> RunState {
21951 let mut s = RunState::new(
21952 PathBuf::from("/repo"),
21953 "main".to_owned(),
21954 "abc1234".to_owned(),
21955 "add retries".to_owned(),
21956 Config::default(),
21957 );
21958 s.reviews = vec![round];
21959 s
21960 }
21961
21962 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21963 crate::verdict::Finding {
21964 id: id.to_owned(),
21965 severity,
21966 file: None,
21967 line: None,
21968 title: title.to_owned(),
21969 detail: String::new(),
21970 }
21971 }
21972
21973 #[test]
21974 fn pr_body_names_open_findings_and_declined_ones() {
21975 let round = ReviewRound {
21976 round: 2,
21977 head: "deadbee".to_owned(),
21978 verified_head: None,
21979 verified_at: None,
21980 reviews: vec![ReviewRecord {
21981 attempts: 0,
21982 reviewer: 1,
21983 agent: "alpha".to_owned(),
21984 summary: String::new(),
21985 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21986 vote: None,
21987 failed: None,
21988 duration_ms: 0,
21989 }],
21990 e2e: vec![CommandOutcome {
21991 command: "cargo test".to_owned(),
21992 code: Some(0),
21993 output_tail: String::new(),
21994 duration_ms: 0,
21995 resource_blocked: false,
21996 }],
21997 verify_retried: false,
21998 e2e_deferred: false,
21999 e2e_defer_reason: None,
22000 fix: Some(FixRecord {
22001 agent: "alpha".to_owned(),
22002 addressed: Vec::new(),
22003 rejected: vec![crate::verdict::Rejection {
22004 id: "R1-1-1".to_owned(),
22005 why: "not reachable from any caller".to_owned(),
22006 }],
22007 notes: String::new(),
22008 committed: true,
22009 failed: None,
22010 duration_ms: 0,
22011 continuation: None,
22012 }),
22013 blocking: 0,
22014 answered: 1,
22015 expected: 1,
22016 clean: false,
22017 progressed: true,
22018 vote_split: false,
22019 reconsideration: Vec::new(),
22020 verdict: None,
22021 };
22022 let state = state_with_round(round);
22023 let body = pr_message(&state, 'A').body;
22024
22025 assert!(body.contains("add retries"), "the task must still be there");
22026 assert!(body.contains("R2-1-1"), "{body}");
22027 assert!(!body.contains("unused import"), "{body}");
22028 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
22029 assert!(
22030 !body.contains("not reachable from any caller"),
22031 "the reason it was declined: {body}"
22032 );
22033 }
22034
22035 #[test]
22036 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
22037 let round = ReviewRound {
22038 round: 1,
22039 head: "deadbee".to_owned(),
22040 verified_head: None,
22041 verified_at: None,
22042 reviews: vec![ReviewRecord {
22043 attempts: 0,
22044 reviewer: 1,
22045 agent: "alpha".to_owned(),
22046 summary: String::new(),
22047 findings: Vec::new(),
22048 vote: None,
22049 failed: None,
22050 duration_ms: 0,
22051 }],
22052 e2e: Vec::new(),
22053 verify_retried: false,
22054 e2e_deferred: false,
22055 e2e_defer_reason: None,
22056 fix: None,
22057 blocking: 0,
22058 answered: 1,
22059 expected: 1,
22060 clean: true,
22061 progressed: false,
22062 vote_split: false,
22063 reconsideration: Vec::new(),
22064 verdict: None,
22065 };
22066 let state = state_with_round(round);
22067 let body = pr_message(&state, 'A').body;
22068 assert!(!body.contains("Open review findings"), "{body}");
22069 assert!(!body.contains("Declined"), "{body}");
22070 }
22071
22072 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
22073 let mut state = RunState::new(
22074 PathBuf::from("/repo"),
22075 "main".to_owned(),
22076 "abc1234".to_owned(),
22077 instruction.to_owned(),
22078 Config::default(),
22079 );
22080 state.config.disk.min_free_bytes = 0;
22082 state.candidates.push(Candidate {
22083 index: 0,
22084 label: 'A',
22085 agent: "alpha".to_owned(),
22086 branch: "magi/x/A".to_owned(),
22087 worktree: PathBuf::from("/wt"),
22088 summary: summary.to_owned(),
22089 stat: String::new(),
22090 files: 1,
22091 commits: 1,
22092 empty: false,
22093 failed: None,
22094 verified_noop: None,
22095 folded: false,
22096 duration_ms: 0,
22097 });
22098 state
22099 }
22100
22101 fn review_state(subjects: &[&str]) -> RunState {
22102 let mut state = state_with_summary(
22103 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
22104 "",
22105 );
22106 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22107 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
22108 state
22109 }
22110
22111 #[tokio::test]
22114 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
22115 ask_test_home();
22116 let tmp = tempfile::tempdir().unwrap();
22117 let repo = tmp.path().join("repo");
22118 std::fs::create_dir_all(&repo).unwrap();
22119 init_repo(&repo);
22120 let origin = tmp.path().join("origin.git");
22121 let g = |dir: &Path, args: &[&str]| {
22122 let out = std::process::Command::new("git")
22123 .args(args)
22124 .current_dir(dir)
22125 .quiet()
22126 .output()
22127 .expect("spawn git");
22128 assert!(
22129 out.status.success(),
22130 "git {args:?}: {}",
22131 String::from_utf8_lossy(&out.stderr)
22132 );
22133 };
22134 g(
22135 tmp.path(),
22136 &[
22137 "clone",
22138 "--bare",
22139 "-q",
22140 repo.to_str().unwrap(),
22141 origin.to_str().unwrap(),
22142 ],
22143 );
22144 g(
22145 &repo,
22146 &["remote", "add", "origin", origin.to_str().unwrap()],
22147 );
22148 let c1 = git::rev_parse(&repo, "main").await.unwrap();
22149
22150 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
22152 g(&repo, &["add", "-A"]);
22153 g(
22154 &repo,
22155 &["commit", "-q", "-m", "chore(deps): update a crate"],
22156 );
22157 g(&repo, &["push", "-q", "origin", "main"]);
22158 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
22159 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22160 g(&repo, &["add", "-A"]);
22161 g(
22162 &repo,
22163 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
22164 );
22165 g(&repo, &["checkout", "-q", "main"]);
22166
22167 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
22168 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
22169 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
22171 assert_eq!(fallback, c1);
22172
22173 let mut state = review_state(&[
22174 "chore(deps): update a crate",
22175 "fix(daemon): apply a chosen action",
22176 ]);
22177 state.repo = repo.clone();
22178 state.base_branch = "main".to_owned();
22179 state.base_commit = c1;
22180 refresh_reviewed_commits(&mut state, "feat/own").await;
22181 assert_eq!(
22182 state.reviewed_commits,
22183 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
22184 );
22185 assert_eq!(
22186 review_title(&state).as_deref(),
22187 Some("fix(daemon): apply a chosen action")
22188 );
22189 assert_eq!(
22190 leaked_subjects(&state, "feat/own").await,
22191 Some(vec!["chore(deps): update a crate".to_owned()])
22192 );
22193 state.config.merge.remote = "nowhere".to_owned();
22196 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
22197 }
22198
22199 struct SummaryFixture {
22203 _tmp: tempfile::TempDir,
22204 repo: PathBuf,
22205 reply: PathBuf,
22206 runner: Runner,
22207 }
22208
22209 fn summary_fixture() -> SummaryFixture {
22210 ask_test_home();
22211 let tmp = tempfile::tempdir().unwrap();
22212 let repo = tmp.path().join("repo");
22213 std::fs::create_dir_all(&repo).unwrap();
22214 init_repo(&repo);
22215 let origin = tmp.path().join("origin.git");
22216 let g = |args: &[&str]| {
22217 let out = std::process::Command::new("git")
22218 .args(args)
22219 .current_dir(&repo)
22220 .quiet()
22221 .output()
22222 .expect("spawn git");
22223 assert!(
22224 out.status.success(),
22225 "git {args:?}: {}",
22226 String::from_utf8_lossy(&out.stderr)
22227 );
22228 };
22229 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
22230 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
22231 g(&["push", "-q", "origin", "main"]);
22232 g(&["checkout", "-q", "-b", "feat/own"]);
22233 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22234 g(&["add", "-A"]);
22235 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
22236 let reply = tmp.path().join("reply.txt");
22237 std::fs::write(
22238 &reply,
22239 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
22240 )
22241 .unwrap();
22242 let mut runner = runner_at(RunStatus::Gating);
22243 let mut state = review_state(&["feat(own): add the own file"]);
22244 state.repo = repo.clone();
22245 state.base_branch = "main".to_owned();
22246 state.candidates[0].branch = "feat/own".to_owned();
22247 state.candidates[0].worktree = repo.clone();
22248 let mut author = spec("alpha");
22249 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
22250 author
22251 .env
22252 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
22253 state.config.agents = vec![author];
22254 runner.state = state;
22255 SummaryFixture {
22256 _tmp: tmp,
22257 repo,
22258 reply,
22259 runner,
22260 }
22261 }
22262
22263 async fn posted(f: &mut SummaryFixture) -> PrMessage {
22264 let winner = f.runner.state.candidates[0].clone();
22265 f.runner
22266 .guarded_pr_message(&winner, None, true)
22267 .await
22268 .unwrap()
22269 .expect("an English description never parks")
22270 }
22271
22272 fn clean_round_with_minor_findings() -> ReviewRound {
22273 let mut round = passing_round(2, None);
22274 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
22275 round.reviews = vec![ReviewRecord {
22276 attempts: 0,
22277 reviewer: 1,
22278 agent: "alpha".into(),
22279 summary: "内部レビュー".into(),
22280 findings: vec![
22281 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
22282 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
22283 ],
22284 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
22285 failed: None,
22286 duration_ms: 0,
22287 }];
22288 round
22289 }
22290
22291 #[tokio::test]
22292 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
22293 for review_only in [false, true] {
22294 let mut f = summary_fixture();
22295 if !review_only {
22296 f.runner.state.candidates[0].agent = "alpha".into();
22297 }
22298 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
22299 assert!(f.runner.state.reviews[0].clean);
22300 assert!(f.runner.state.reviews[0].fix.is_none());
22301 assert!(f.runner.state.open_findings().is_empty());
22302 assert_eq!(
22303 crate::pr_outcome::required_finding_ids(&f.runner.state),
22304 ["R2-1-1", "R2-1-2"]
22305 );
22306 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22307 let evidence = crate::pr_outcome::summary_evidence(
22308 &f.runner.state,
22309 crate::pr_outcome::FinalHead::At(&head),
22310 );
22311 assert!(
22312 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
22313 );
22314 let omitted = posted(&mut f).await;
22316 assert!(omitted.body.contains("No description"));
22317 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
22318 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
22319 assert!(f.runner.state.change_summary.is_none());
22320 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
22321 let described = posted(&mut f).await;
22322 assert!(described.body.contains("improve documentation for callers"));
22323 assert!(described.body.contains("remaining edge case"));
22324 assert!(described.body.contains("## Open review findings"));
22325 assert!(!described.body.contains("未修正"));
22326 }
22327 }
22328
22329 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
22330 let mut f = summary_fixture();
22331 if !review_only {
22332 f.runner.state.candidates[0].agent = "alpha".into();
22333 }
22334 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22335 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
22336 fixed
22337 .fix
22338 .as_mut()
22339 .unwrap()
22340 .rejected
22341 .push(crate::verdict::Rejection {
22342 id: "R1-1-1".into(),
22343 why: "呼び出されないコードのため辞退".into(),
22344 });
22345 let mut clean = clean_round_with_minor_findings();
22346 clean.verified_head = Some(head.clone());
22347 f.runner.state.reviews = vec![fixed, clean];
22348 ok_gate(&mut f.runner.state, Some(&head));
22349 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
22350 f.runner.state.config.agents.push(scripted("beta", reply));
22351 f.runner.state.config.roles.language_judge =
22352 Some(crate::config::AgentChoice::One("beta".into()));
22353 f
22354 }
22355
22356 #[tokio::test]
22357 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
22358 for review_only in [false, true] {
22359 let mut f = rewrite_fixture_with_findings(review_only,
22360 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
22361 let m = posted(&mut f).await;
22362 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22363 let expected =
22364 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
22365 assert!(m.body.contains(&expected));
22366 assert!(m.body.contains("## Review rounds"));
22367 assert!(
22368 m.body
22369 .contains("Gate (`verify.gate`): passed on the final commit")
22370 );
22371 assert!(
22372 m.body
22373 .contains("Tests (`verify.e2e`): passed on the final commit")
22374 );
22375 let c = f.runner.state.change_summary.as_ref().unwrap();
22376 assert!(
22377 m.body
22378 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
22379 );
22380 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22381 assert!(crate::pr_outcome::covers_required_findings(
22382 &f.runner.state,
22383 &m.body
22384 ));
22385 assert!(m.body.contains("documentation remains incomplete"));
22386 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
22387 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
22388 assert!(f.runner.state.github_text.is_none());
22389 }
22390 }
22391
22392 #[tokio::test]
22393 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
22394 for review_only in [false, true] {
22395 for reply in [
22396 "TITLE: fix: public description\nA short English description.",
22397 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
22398 ] {
22399 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
22400 let winner = f.runner.state.candidates[0].clone();
22401 let posted = f
22402 .runner
22403 .guarded_pr_message(&winner, None, true)
22404 .await
22405 .unwrap();
22406 assert!(posted.is_none());
22407 assert!(f.runner.state.parked);
22408 let withheld = f.runner.state.github_text.as_ref().unwrap();
22409 assert!(
22410 withheld.rewrite.is_none(),
22411 "an incomplete rewrite must not be selectable"
22412 );
22413 let q = ask::Questions::open()
22414 .get(withheld.question.as_ref().unwrap())
22415 .unwrap();
22416 assert!(
22417 !q.choices
22418 .contains(&crate::github_text::USE_REWRITE.to_owned())
22419 );
22420 assert!(
22421 q.detail.contains("R1-1-1")
22422 && q.detail.contains("R2-1-1")
22423 && q.detail.contains("R2-1-2")
22424 );
22425 }
22426 }
22427 }
22428
22429 #[tokio::test]
22430 async fn japanese_reports_stay_local_in_both_public_run_modes() {
22431 for review_only in [false, true] {
22432 let mut f = summary_fixture();
22433 if !review_only {
22434 f.runner.state.candidates[0].agent = "alpha".into();
22435 }
22436 f.runner.state.instruction = "内部タスクの説明".into();
22437 f.runner.state.candidates[0].summary = "内部の候補報告".into();
22438 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22439 f.runner.state.reviews = vec![
22440 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22441 round_with_notes(2, Some("二回目の内部修正報告です。")),
22442 passing_round(3, Some(&head)),
22443 ];
22444 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22445 ok_gate(&mut f.runner.state, Some(&head));
22446 let captured = f.repo.join("public-prompt.txt");
22447 let author = &mut f.runner.state.config.agents[0];
22448 author.command = vec![
22449 "sh".into(),
22450 "-c".into(),
22451 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22452 ];
22453 author.env.insert(
22454 "PROMPT_FILE".into(),
22455 captured.to_string_lossy().into_owned(),
22456 );
22457 let m = posted(&mut f).await;
22458 assert!(m.body.contains("Adds `own.txt`"));
22459 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22460 assert!(m.body.contains("R1-1-1"));
22461 assert!(
22462 m.body
22463 .contains("Gate (`verify.gate`): passed on the final commit")
22464 );
22465 assert!(
22466 m.body
22467 .contains("Tests (`verify.e2e`): passed on the final commit")
22468 );
22469 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22470 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22471 let prompt = std::fs::read_to_string(captured).unwrap();
22472 assert!(prompt.contains(&format!(
22473 "git diff {} {head}",
22474 f.runner.state.change_summary.as_ref().unwrap().base
22475 )));
22476 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22477 assert_eq!(
22478 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22479 "二回目の内部修正報告です。"
22480 );
22481 }
22482 }
22483
22484 #[tokio::test]
22485 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22486 for review_only in [false, true] {
22487 let mut f = summary_fixture();
22488 if !review_only {
22489 f.runner.state.candidates[0].agent = "alpha".into();
22490 }
22491 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22492 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22493 std::fs::write(&f.reply, "no usable description").unwrap();
22494 let failed = posted(&mut f).await;
22495 assert!(
22496 failed
22497 .body
22498 .contains("No description of what this change does could be generated")
22499 );
22500 assert!(failed.body.is_ascii());
22501 assert!(f.runner.state.github_text.is_none());
22502 assert!(f.runner.state.change_summary.is_none());
22503 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22504 let evidence = crate::pr_outcome::summary_evidence(
22505 &f.runner.state,
22506 crate::pr_outcome::FinalHead::At(&head),
22507 );
22508 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22509 assert!(
22510 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22511 .exists()
22512 );
22513 f.runner.state =
22514 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22515 std::fs::write(
22516 &f.reply,
22517 "TITLE: feat(own): add the own file\nAdds the committed own file.",
22518 )
22519 .unwrap();
22520 let retried = posted(&mut f).await;
22521 assert!(retried.body.contains("Adds the committed own file."));
22522 assert_eq!(
22523 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22524 "日本語の修正報告"
22525 );
22526 }
22527 }
22528
22529 #[tokio::test]
22530 async fn same_head_verification_and_findings_invalidate_public_prose() {
22531 let mut f = summary_fixture();
22532 posted(&mut f).await;
22533 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22534 let original = f
22535 .runner
22536 .state
22537 .change_summary
22538 .as_ref()
22539 .unwrap()
22540 .evidence
22541 .clone();
22542 ok_gate(&mut f.runner.state, Some(&head));
22543 std::fs::write(
22544 &f.reply,
22545 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22546 )
22547 .unwrap();
22548 let verified = posted(&mut f).await;
22549 assert!(verified.body.contains("now verified"));
22550 assert_ne!(
22551 f.runner.state.change_summary.as_ref().unwrap().evidence,
22552 original
22553 );
22554 let mut round = round_with_notes(1, Some("内部報告"));
22555 round.reviews.push(ReviewRecord {
22556 attempts: 0,
22557 reviewer: 1,
22558 agent: "alpha".into(),
22559 summary: "内部レビュー".into(),
22560 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22561 vote: None,
22562 failed: None,
22563 duration_ms: 0,
22564 });
22565 round
22566 .fix
22567 .as_mut()
22568 .unwrap()
22569 .rejected
22570 .push(crate::verdict::Rejection {
22571 id: "R1-1-1".into(),
22572 why: "呼び出し元から到達できない".into(),
22573 });
22574 f.runner.state.reviews = vec![round];
22575 let omitted = posted(&mut f).await;
22577 assert!(omitted.body.contains("No description"));
22578 assert!(omitted.body.contains("R1-1-1"));
22579 assert!(!omitted.body.contains("到達"));
22580 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22581 let described = posted(&mut f).await;
22582 assert!(described.body.contains("unreachable from callers"));
22583 assert!(described.body.contains("R1-1-1"));
22584 assert!(!described.body.contains("到達"));
22585 let evidence = f
22586 .runner
22587 .state
22588 .change_summary
22589 .as_ref()
22590 .unwrap()
22591 .evidence
22592 .clone();
22593 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22594 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22595 assert!(posted(&mut f).await.body.contains("rationale updated"));
22596 assert_ne!(
22597 f.runner.state.change_summary.as_ref().unwrap().evidence,
22598 evidence
22599 );
22600 f.runner
22602 .state
22603 .change_summary
22604 .as_mut()
22605 .unwrap()
22606 .evidence
22607 .clear();
22608 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22609 assert!(posted(&mut f).await.body.contains("current specification"));
22610 }
22611
22612 #[tokio::test]
22613 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22614 let mut f = summary_fixture();
22615 let m = posted(&mut f).await;
22616 assert_eq!(m.title, "feat(own): add the own file");
22617 assert!(
22618 m.body
22619 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22620 "{}",
22621 m.body
22622 );
22623 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22624 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22625 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22626 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22627 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22628 let kept = f.runner.state.change_summary.as_ref().unwrap();
22629 assert_eq!(
22630 (kept.head.as_str(), kept.base.as_str()),
22631 (head.as_str(), base.as_str())
22632 );
22633 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22634 }
22635
22636 #[tokio::test]
22637 async fn review_only_summary_receives_the_configured_github_language() {
22638 let mut f = summary_fixture();
22639 f.runner.state.config.graph.github_language = "ja".into();
22640 let captured = f.repo.join("summary-prompt.txt");
22641 let author = &mut f.runner.state.config.agents[0];
22642 author.command = vec![
22643 "sh".into(),
22644 "-c".into(),
22645 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22646 ];
22647 author.env.insert(
22648 "PROMPT_FILE".into(),
22649 captured.to_string_lossy().into_owned(),
22650 );
22651 std::fs::write(
22652 &f.reply,
22653 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22654 )
22655 .unwrap();
22656 let m = posted(&mut f).await;
22657 let prompt = std::fs::read_to_string(captured).unwrap();
22658 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22659 assert!(!prompt.contains("in English"));
22660 assert_eq!(m.title, "feat(own): ファイルを追加");
22661 assert!(m.body.contains("own.txt を追加します。"));
22662 }
22663
22664 #[tokio::test]
22665 async fn review_fixes_come_after_the_whole_change_summary() {
22666 let mut f = summary_fixture();
22667 f.runner.state.reviews = vec![
22668 round_with_notes(1, Some("handled the PATH case")),
22669 round_with_notes(2, None),
22670 ];
22671 let m = posted(&mut f).await;
22672 let summary = m.body.find("## Summary").expect("summary");
22673 let fixes = m.body.find("## Review rounds").expect("review rounds");
22674 assert!(summary < fixes, "{}", m.body);
22675 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22676 assert!(m.body.contains("## Final verification"), "{}", m.body);
22677 }
22678
22679 #[tokio::test]
22680 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22681 let mut f = summary_fixture();
22682 let first = posted(&mut f).await;
22683 let first_head = f.runner.state.change_summary.clone().unwrap().head;
22684 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22686 let again = posted(&mut f).await;
22687 assert_eq!((again.title, again.body), (first.title, first.body));
22688
22689 let g = |args: &[&str]| {
22690 let out = std::process::Command::new("git")
22691 .args(args)
22692 .current_dir(&f.repo)
22693 .quiet()
22694 .output()
22695 .expect("spawn git");
22696 assert!(out.status.success(), "git {args:?}");
22697 };
22698 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22699 g(&["add", "-A"]);
22700 g(&["commit", "-q", "-m", "fix(own): handle more"]);
22701 let second = posted(&mut f).await;
22702 let kept = f.runner.state.change_summary.clone().unwrap();
22703 assert_ne!(kept.head, first_head);
22704 assert!(second.body.contains("Not read yet."), "{}", second.body);
22705 assert!(!second.body.contains("so the first summary is read"));
22706 assert_eq!(second.title, "feat(own): other");
22707 }
22708
22709 #[tokio::test]
22710 async fn an_unavailable_summary_is_stated_and_never_invented() {
22711 let mut f = summary_fixture();
22712 std::fs::write(&f.reply, "no title line here").unwrap();
22713 let m = posted(&mut f).await;
22714 assert!(
22715 m.body
22716 .contains("No description of what this change does could be generated"),
22717 "{}",
22718 m.body
22719 );
22720 assert!(m.body.contains("internal reports remain in local records"));
22721 assert!(f.runner.state.change_summary.is_none());
22722 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22723 let winner = f.runner.state.candidates[0].clone();
22725 let preview = f
22726 .runner
22727 .guarded_pr_message(&winner, None, false)
22728 .await
22729 .unwrap()
22730 .unwrap();
22731 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22732 }
22733
22734 #[test]
22735 fn pr_message_review_single_commit_uses_its_subject() {
22736 let state = review_state(&["feat(nats): per-role user"]);
22737 let m = pr_message(&state, 'A');
22738 assert_eq!(m.title, "feat(nats): per-role user");
22739 assert!(!m.body.contains("Review the work already"), "{}", m.body);
22740 assert!(m.body.contains("## Commits under review"), "{}", m.body);
22741 }
22742
22743 #[test]
22744 fn pr_message_review_multi_commit_takes_the_oldest() {
22745 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22746 let m = pr_message(&state, 'A');
22747 assert_eq!(m.title, "feat: the change");
22748 for s in ["feat: the change", "fix: typo", "fix: again"] {
22749 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22750 }
22751 }
22752
22753 #[test]
22754 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22755 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22756 let mut state = review_state(&[first, "fix: later fixup"]);
22757 state.candidates[0].branch = "機能/ログイン".to_owned();
22758 let m = pr_message(&state, 'A');
22759 assert!(
22760 m.title.starts_with("chore: land candidate A of run"),
22761 "{}",
22762 m.title
22763 );
22764 }
22765 }
22766
22767 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22768 BranchFacts {
22769 commits: commits
22770 .iter()
22771 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22772 .collect(),
22773 stat: stat.to_owned(),
22774 }
22775 }
22776
22777 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22778 let mut r = review_round(true, 0, 1, 1, true, true);
22779 r.round = round;
22780 r.fix = notes.map(|n| FixRecord {
22781 agent: "fixer".to_owned(),
22782 addressed: Vec::new(),
22783 rejected: Vec::new(),
22784 notes: n.to_owned(),
22785 committed: true,
22786 failed: None,
22787 duration_ms: 0,
22788 continuation: None,
22789 });
22790 r
22791 }
22792
22793 #[test]
22794 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22795 let state = review_state(&["ignored"]);
22796 let f = facts(
22797 &[
22798 (
22799 "fix(login): resolve PATH on macOS",
22800 "Login shells skip rc files.",
22801 ),
22802 ("fix: address review", ""),
22803 ],
22804 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22805 );
22806 let m = pr_message_with(&state, 'A', Some(&f));
22807 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22808 assert!(
22809 m.body
22810 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22811 "{}",
22812 m.body
22813 );
22814 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22815 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22816 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22817 for banned in [
22818 "Review the work already",
22819 "no task statement",
22820 "Original task",
22821 ] {
22822 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22823 }
22824 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22825 }
22826
22827 #[test]
22828 fn pr_message_review_truncates_a_huge_first_commit_body() {
22829 let state = review_state(&["ignored"]);
22830 let f = facts(
22831 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22832 "s",
22833 );
22834 let m = pr_message_with(&state, 'A', Some(&f));
22835 assert!(m.body.len() < 30_000, "{}", m.body.len());
22836 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22837 assert!(
22838 m.body.contains("- ... 1 more commit(s)"),
22839 "{}",
22840 m.body.len()
22841 );
22842 assert!(m.body.ends_with("magi:candidate-a\n"));
22843 }
22844
22845 #[test]
22846 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22847 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22848 assert_eq!(m.title, "feat: x");
22849 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22850 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22851 assert!(!m.body.contains("no task statement"), "{}", m.body);
22852 }
22853
22854 #[test]
22855 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22856 let mut state = review_state(&["日本語の件名"]);
22857 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22858 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22859 state.candidates[0].branch = "機能/ログイン".to_owned();
22860 assert!(
22861 pr_message(&state, 'A')
22862 .title
22863 .starts_with("chore: land candidate A")
22864 );
22865 }
22866
22867 #[test]
22868 fn pr_message_review_fixes_survive_a_clean_final_round() {
22869 let mut state = review_state(&["feat: x"]);
22870 state.reviews = vec![
22871 round_with_notes(1, Some("handled the PATH case")),
22872 round_with_notes(2, None),
22873 ];
22874 let body = pr_message(&state, 'A').body;
22875 assert!(
22876 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22877 "{body}"
22878 );
22879 assert!(body.contains("### Round 1"), "{body}");
22880
22881 state.reviews = vec![
22882 round_with_notes(1, Some("first fix")),
22883 round_with_notes(2, Some("")),
22884 round_with_notes(3, Some("second fix")),
22885 round_with_notes(4, None),
22886 ];
22887 let body = pr_message(&state, 'A').body;
22888 assert!(
22889 body.contains("### Round 1") && !body.contains("first fix"),
22890 "{body}"
22891 );
22892 assert!(
22893 body.contains("### Round 3") && !body.contains("second fix"),
22894 "{body}"
22895 );
22896 assert!(body.contains("### Round 2"), "{body}");
22897 }
22898
22899 const FINAL: &str = "abcdef1234567890";
22900
22901 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22902 let mut r = round_with_notes(round, None);
22903 r.verified_head = verified.map(str::to_owned);
22904 r
22905 }
22906
22907 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22908 state.gate = vec![CommandOutcome {
22909 command: "gate".to_owned(),
22910 code: Some(0),
22911 output_tail: String::new(),
22912 duration_ms: 0,
22913 resource_blocked: false,
22914 }];
22915 state.gate_ran = true;
22916 state.gate_head = head.map(str::to_owned);
22917 }
22918
22919 fn body_at(state: &RunState) -> String {
22920 pr_message_raw(
22921 state,
22922 'A',
22923 None,
22924 crate::pr_outcome::FinalHead::At(FINAL),
22925 ChangeNote::NotAsked,
22926 )
22927 .body
22928 }
22929
22930 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22931
22932 #[test]
22933 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22934 let mut state = review_state(&["feat: x"]);
22935 state.reviews = vec![
22936 round_with_notes(1, Some(SANDBOXED)),
22937 passing_round(2, Some(FINAL)),
22938 ];
22939 ok_gate(&mut state, Some(FINAL));
22940 let body = body_at(&state);
22941 assert!(!body.contains("Handled the PATH case."), "{body}");
22942 assert!(!body.contains("could not commit"), "{body}");
22943 assert!(!body.contains("could not run"), "{body}");
22944 assert!(
22945 body.contains("The fix is committed on the branch"),
22946 "{body}"
22947 );
22948 assert!(!body.contains("sandbox"), "{body}");
22949 assert!(
22950 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22951 "{body}"
22952 );
22953 assert!(
22954 body.contains("Gate (`verify.gate`): passed on the final commit"),
22955 "{body}"
22956 );
22957 assert!(
22959 state.reviews[0]
22960 .fix
22961 .as_ref()
22962 .unwrap()
22963 .notes
22964 .contains("could not commit")
22965 );
22966 }
22967
22968 #[test]
22969 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22970 let mut state = review_state(&["feat: x"]);
22971 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22972 r1.fix.as_mut().unwrap().committed = false;
22973 let mut r2 = passing_round(2, Some(FINAL));
22974 r2.e2e[0].code = Some(1);
22975 state.reviews = vec![r1, r2];
22976 let body = body_at(&state);
22977 assert!(!body.contains("could not commit"), "{body}");
22978 assert!(!body.contains("could not run"), "{body}");
22979 assert!(body.contains("This round produced no commit"), "{body}");
22980 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22981 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22982 assert!(!body.contains("passed"), "{body}");
22983 }
22984
22985 #[test]
22986 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22987 let mut state = review_state(&["feat: x"]);
22988 state.reviews = vec![
22989 round_with_notes(1, Some(SANDBOXED)),
22990 passing_round(2, Some("0123456789abcdef")),
22991 ];
22992 ok_gate(&mut state, Some("0123456789abcdef"));
22993 let body = body_at(&state);
22994 assert!(!body.contains("HTTP tests could not run"), "{body}");
22995 assert!(!body.contains("could not commit"), "{body}");
22997 assert!(
22998 body.contains("passed on an earlier commit (`0123456`)"),
22999 "{body}"
23000 );
23001 assert!(!body.contains("passed on the final commit"), "{body}");
23002
23003 ok_gate(&mut state, None);
23005 let body = body_at(&state);
23006 assert!(
23007 body.contains("passed on a commit that was not recorded"),
23008 "{body}"
23009 );
23010 }
23011
23012 #[test]
23013 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
23014 let state = state_with_summary(
23015 "add retries to the client",
23016 "TITLE: feat: retries\n\nDid it.",
23017 );
23018 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
23019 assert_eq!(m.title, "feat: retries");
23020 assert!(
23021 m.body.contains("<summary>Original task</summary>"),
23022 "{}",
23023 m.body
23024 );
23025 assert!(!m.body.contains("Commits under review"), "{}", m.body);
23026 assert!(
23027 m.body
23028 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
23029 "{}",
23030 m.body
23031 );
23032 }
23033
23034 #[test]
23035 fn pr_message_review_bounds_a_long_english_subject() {
23036 let long = format!("feat: {}", "word ".repeat(100));
23037 let m = pr_message(&review_state(&[&long]), 'A');
23038 assert!(m.title.starts_with("feat: word"), "{}", m.title);
23039 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23040 }
23041
23042 #[test]
23043 fn pr_message_implementation_run_is_unchanged_by_review_support() {
23044 let state = state_with_summary("add retries\n\ndetails", "- did some things");
23045 let m = pr_message(&state, 'A');
23046 assert_eq!(m.title, "add retries");
23047 assert!(m.body.contains("<summary>Original task</summary>"));
23048 assert!(!m.body.contains("Commits under review"));
23049 assert_eq!(landing_subject_source(&state), state.instruction);
23050 }
23051
23052 #[test]
23053 fn review_run_squash_subject_is_the_change_not_the_prompt() {
23054 let state = review_state(&["feat: the change", "fix: typo"]);
23055 let source = landing_subject_source(&state);
23056 assert_eq!(land::merge_subject("", &source), "feat: the change");
23057 assert_eq!(
23058 land::merge_subject("magi: candidate A (uncommitted work)", &source),
23059 "feat: the change"
23060 );
23061 assert_eq!(
23063 land::merge_subject("feat: renamed by hand", &source),
23064 "feat: renamed by hand"
23065 );
23066 let blank = review_state(&["日本語"]);
23067 assert!(
23068 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
23069 );
23070 }
23071
23072 #[test]
23073 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
23074 let state = review_state(&["feat: the change"]);
23075 let source = landing_subject_source(&state);
23076 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
23077 assert_eq!(
23078 land::merge_subject(landing_title(&state, old), &source),
23079 "feat: the change"
23080 );
23081 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
23082 let task = state_with_summary("add retries", "");
23083 assert_eq!(landing_title(&task, old), old);
23084 }
23085
23086 #[test]
23087 fn pr_message_describes_the_change_not_the_task() {
23088 let state = state_with_summary(
23089 "今回やってほしいこと: results projector を直す",
23090 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
23091 );
23092 let m = pr_message(&state, 'A');
23093 assert_eq!(m.title, "fix(web): batch the runs list reads");
23094 assert!(
23095 m.body.starts_with("## Summary\n\n- reads run.json once"),
23096 "{}",
23097 m.body
23098 );
23099 assert!(!m.body.contains("TITLE:"), "{}", m.body);
23100 let task_at = m.body.find("今回やってほしいこと").unwrap();
23101 let details_at = m.body.find("<details>").unwrap();
23102 assert!(
23103 details_at < task_at,
23104 "the task lives inside <details>: {}",
23105 m.body
23106 );
23107 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
23108 assert!(m.body.contains("magi:candidate-a"));
23109 }
23110
23111 #[test]
23112 fn pr_message_falls_back_to_the_task_without_a_title_line() {
23113 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
23114 let m = pr_message(&state, 'A');
23115 assert_eq!(m.title, "add retries");
23116 assert!(
23117 m.body.contains("## Summary\n\n- did some things"),
23118 "{}",
23119 m.body
23120 );
23121
23122 let none = RunState::new(
23123 PathBuf::from("/repo"),
23124 "main".to_owned(),
23125 "abc1234".to_owned(),
23126 "add retries".to_owned(),
23127 Config::default(),
23128 );
23129 let m = pr_message(&none, 'A');
23130 assert_eq!(m.title, "add retries");
23131 assert!(!m.body.contains("## Summary"), "{}", m.body);
23132 }
23133
23134 #[test]
23135 fn pr_message_refuses_the_candidate_commit_subject() {
23136 for bad in [
23137 "TITLE: magi: candidate A (uncommitted work)",
23138 "TITLE: chore: stuff (uncommitted work)",
23139 "TITLE: ",
23140 ] {
23141 let state = state_with_summary("add retries", bad);
23142 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
23143 }
23144 }
23145
23146 #[test]
23147 fn pr_message_bounds_a_very_long_task_and_title() {
23148 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
23149 let state = state_with_summary(&long, "- nothing");
23150 let m = pr_message(&state, 'A');
23151 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23152 assert!(!m.title.contains('\n'));
23153
23154 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
23155 let m = pr_message(&state, 'A');
23156 assert!(m.title.starts_with("feat: "));
23157 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23158 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
23159 }
23160
23161 fn long_title_of(instruction: &str) -> String {
23162 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
23163 }
23164
23165 #[test]
23166 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
23167 let first = "Make the landing path keep a readable title for long tasks";
23168 let line = format!(
23169 "{first}. {}",
23170 "And then keep going with more words ".repeat(20)
23171 );
23172 let t = long_title_of(&line);
23173 assert_eq!(t, first);
23174 assert!(!t.starts_with("chore: land"));
23175 }
23176
23177 #[test]
23178 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
23179 let line = "word ".repeat(200);
23180 let t = long_title_of(&line);
23181 assert!(t.ends_with("word..."), "{t}");
23182 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
23183 }
23184
23185 #[test]
23186 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
23187 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
23188 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
23189 }
23190 }
23191
23192 #[test]
23193 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
23194 let mut state = state_with_summary("再試行を追加する", "- nothing");
23195 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
23196 state.config.graph.github_language = "ja".into();
23197 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
23198 }
23199
23200 #[test]
23201 fn pr_message_title_limit_is_exact() {
23202 let at = "a".repeat(PR_TITLE_MAX);
23203 assert_eq!(long_title_of(&at), at);
23204 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
23205 assert!(over.ends_with("..."), "{over}");
23206 assert_eq!(over.chars().count(), PR_TITLE_MAX);
23207 }
23208
23209 #[test]
23210 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
23211 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
23212 let t = long_title_of(&line);
23213 assert!(t.ends_with("..."), "{t}");
23214 assert!(t.is_ascii(), "{t}");
23215 }
23216
23217 #[test]
23218 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
23219 let line = format!(
23220 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
23221 "plus more filler words ".repeat(20)
23222 );
23223 let t = long_title_of(&line);
23224 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
23225 }
23226
23227 #[test]
23228 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
23229 let line = format!("\n\n# {}", "title words ".repeat(40));
23230 let state = state_with_summary(&line, "- nothing");
23231 let m = pr_message(&state, 'A');
23232 assert!(m.title.starts_with("title words"), "{}", m.title);
23233 assert_eq!(
23234 land::merge_subject(&m.title, &landing_subject_source(&state)),
23235 m.title
23236 );
23237 assert_eq!(
23239 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
23240 "feat: renamed by hand"
23241 );
23242 }
23243
23244 #[tokio::test]
23245 async fn github_text_rewrite_is_bounded_and_falls_back() {
23246 crate::run::pin_test_home();
23247 for (reply, accepted) in [
23248 (
23249 "TITLE: fix: retries\nAdd retries for failed requests.",
23250 true,
23251 ),
23252 (
23253 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
23254 false,
23255 ),
23256 (
23257 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
23258 true,
23259 ),
23260 (
23261 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
23262 true,
23263 ),
23264 (
23265 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
23266 true,
23267 ),
23268 (
23269 "TITLE: fix: retries\n日本語の説明です。 token=secret",
23270 false,
23271 ),
23272 ("TITLE: fix: retries\n", false),
23273 ] {
23274 let dir = tempfile::tempdir().unwrap();
23275 let mut runner = runner_at(RunStatus::Gating);
23276 runner.state = state_with_summary(
23277 "add retries",
23278 "TITLE: fix: retries\n日本語の説明を書きます。",
23279 );
23280 runner.state.repo = dir.path().to_owned();
23281 runner.state.candidates[0].worktree = dir.path().to_owned();
23282 let mut author = spec("alpha");
23283 author.command = vec![
23284 "sh".into(),
23285 "-c".into(),
23286 "printf '%s' \"$REWRITE_REPLY\"".into(),
23287 ];
23288 author.env.insert("REWRITE_REPLY".into(), reply.into());
23289 runner.state.config.agents = vec![author];
23290 seed_public_summary(&mut runner);
23291 let winner = runner.state.candidates[0].clone();
23292 let first = runner
23293 .guarded_pr_message(&winner, None, true)
23294 .await
23295 .unwrap();
23296 let message = if accepted {
23297 first.expect("a passing rewrite never asks")
23298 } else {
23299 assert!(first.is_none());
23301 assert!(runner.state.parked);
23302 let store = ask::Questions::open();
23303 let asked: Vec<_> = store
23304 .list()
23305 .into_iter()
23306 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23307 .collect();
23308 assert_eq!(asked.len(), 1);
23309 assert!(!asked[0].detail.contains("secret"));
23310 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
23311 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
23312 assert_eq!(
23313 asked[0]
23314 .choices
23315 .contains(&crate::github_text::USE_REWRITE.to_owned()),
23316 rewrite.is_some()
23317 );
23318 store
23321 .update(&asked[0].id, |q| {
23322 q.abandon("test");
23323 Ok(())
23324 })
23325 .unwrap();
23326 let again = runner
23327 .guarded_pr_message(&winner, None, true)
23328 .await
23329 .unwrap()
23330 .expect("an abandoned question falls back");
23331 let n = store
23332 .list()
23333 .into_iter()
23334 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23335 .count();
23336 assert_eq!(n, 1, "asked once per rejected text");
23337 again
23338 };
23339 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
23340 assert!(!message.body.contains("secret"));
23341 assert!(!message.body.contains("prefix"));
23342 assert!(!message.title.contains("secret"));
23343 assert_eq!(
23344 message.body.contains("Add retries for failed requests."),
23345 accepted
23346 );
23347 assert_eq!(
23348 runner.state.seats["github-text-rewrite"].turns, 1,
23349 "only one rewrite invocation"
23350 );
23351 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
23352 if accepted {
23353 assert!(runner.state.github_text.is_none());
23354 assert!(!runner.state.parked);
23355 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
23356 assert!(message.body.contains("## Final verification"));
23357 assert!(message.body.contains("Evidence: final diff"));
23358 assert!(
23359 message
23360 .body
23361 .contains(&format!("magi:run/{}", runner.state.id))
23362 );
23363 }
23364 if !accepted {
23365 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23366 assert!(
23367 message
23368 .body
23369 .contains(&format!("magi:run/{}", runner.state.id))
23370 );
23371 }
23372 }
23373 }
23374
23375 fn scripted(id: &str, reply: &str) -> AgentSpec {
23376 let mut a = spec(id);
23377 a.command = vec![
23378 "sh".into(),
23379 "-c".into(),
23380 "printf '%s' \"$REWRITE_REPLY\"".into(),
23381 ];
23382 a.env.insert("REWRITE_REPLY".into(), reply.into());
23383 a
23384 }
23385
23386 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
23387 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
23388
23389 fn seed_public_summary(runner: &mut Runner) {
23391 let repo = &runner.state.repo;
23392 let g = |args: &[&str]| {
23393 let out = std::process::Command::new("git")
23394 .args(args)
23395 .current_dir(repo)
23396 .quiet()
23397 .output()
23398 .unwrap();
23399 assert!(out.status.success(), "git {args:?}");
23400 String::from_utf8_lossy(&out.stdout).trim().to_owned()
23401 };
23402 g(&["init", "-q", "-b", "main"]);
23403 g(&[
23404 "-c",
23405 "user.name=Test",
23406 "-c",
23407 "user.email=test@example.invalid",
23408 "commit",
23409 "-q",
23410 "--allow-empty",
23411 "-m",
23412 "feat: fixture",
23413 ]);
23414 let head = g(&["rev-parse", "HEAD"]);
23415 g(&["update-ref", "refs/remotes/origin/main", &head]);
23416 let evidence = crate::pr_outcome::summary_evidence(
23417 &runner.state,
23418 crate::pr_outcome::FinalHead::At(&head),
23419 );
23420 let generated = &runner.state.candidates[0].summary;
23421 runner.state.change_summary = Some(crate::run::ChangeSummary {
23422 base: head.clone(),
23423 head,
23424 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23425 title: summary_title(generated).unwrap(),
23426 body: summary_without_title(generated),
23427 });
23428 }
23429
23430 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23431 let mut runner = runner_at(RunStatus::Gating);
23432 runner.state = state_with_summary("add retries", JA_SUMMARY);
23433 if review_only {
23434 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23435 }
23436 runner.state.repo = dir.to_owned();
23437 runner.state.candidates[0].worktree = dir.to_owned();
23438 runner.state.config.agents = agents;
23439 seed_public_summary(&mut runner);
23440 runner
23441 }
23442
23443 fn gate_questions(runner: &Runner) -> usize {
23444 ask::Questions::open()
23445 .list()
23446 .into_iter()
23447 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23448 .count()
23449 }
23450
23451 #[tokio::test]
23452 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23453 crate::run::pin_test_home();
23454 let dir = tempfile::tempdir().unwrap();
23455 let mut runner = rewrite_runner(
23456 dir.path(),
23457 true,
23458 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23459 );
23460 runner.state.config.roles.language_judge =
23461 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23462 let winner = runner.state.candidates[0].clone();
23463 let msg = runner
23464 .guarded_pr_message(&winner, None, true)
23465 .await
23466 .unwrap()
23467 .expect("an English rewrite posts without asking");
23468 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23469 assert!(msg.body.contains("<details>kept</details>"));
23470 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23471 assert_eq!(gate_questions(&runner), 0);
23472 assert!(runner.state.github_text.is_none());
23473 assert!(!runner.state.seats.contains_key("impl-A"));
23474 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23475 }
23476
23477 #[tokio::test]
23478 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23479 crate::run::pin_test_home();
23480 let dir = tempfile::tempdir().unwrap();
23481 let mut runner = rewrite_runner(
23482 dir.path(),
23483 false,
23484 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23485 );
23486 runner.state.config.roles.language_judge =
23487 Some(crate::config::AgentChoice::One("beta".into()));
23488 runner.state.seats.insert(
23489 "impl-A".into(),
23490 SeatState::new("impl-A", "alpha", runner.state.seed),
23491 );
23492 let winner = runner.state.candidates[0].clone();
23493 let msg = runner
23494 .guarded_pr_message(&winner, None, true)
23495 .await
23496 .unwrap()
23497 .expect("rewritten");
23498 assert!(msg.body.contains("<details>kept</details>"));
23499 assert_eq!(runner.state.seats["impl-A"].turns, 0);
23500 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23501 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23502 }
23503
23504 #[tokio::test]
23505 async fn rewrite_reuses_a_resumable_implementer_session() {
23506 crate::run::pin_test_home();
23507 let dir = tempfile::tempdir().unwrap();
23508 let mut runner = rewrite_runner(
23509 dir.path(),
23510 false,
23511 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23512 );
23513 runner.state.config.roles.language_judge =
23514 Some(crate::config::AgentChoice::One("beta".into()));
23515 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23516 seat.turns = 1;
23517 runner.state.seats.insert("impl-A".into(), seat);
23518 let winner = runner.state.candidates[0].clone();
23519 runner
23520 .guarded_pr_message(&winner, None, true)
23521 .await
23522 .unwrap()
23523 .expect("rewritten");
23524 assert_eq!(runner.state.seats["impl-A"].turns, 2);
23525 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23526 }
23527
23528 #[tokio::test]
23529 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23530 crate::run::pin_test_home();
23533 let dir = tempfile::tempdir().unwrap();
23534 let mut exhausted = spec("alpha");
23535 exhausted.command = vec!["false".into()];
23536 let mut runner = rewrite_runner(
23537 dir.path(),
23538 false,
23539 vec![exhausted, scripted("beta", EN_REWRITE)],
23540 );
23541 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23543 "alpha".into(),
23544 "beta".into(),
23545 ]));
23546 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23547 seat.turns = 1;
23548 runner.state.seats.insert("impl-A".into(), seat);
23549 let winner = runner.state.candidates[0].clone();
23550 let msg = runner
23551 .guarded_pr_message(&winner, None, true)
23552 .await
23553 .unwrap()
23554 .expect("the role rewrites after the implementer fails");
23555 assert!(msg.body.contains("`src/net.rs`"));
23556 assert_eq!(
23557 runner.state.seats["impl-A"].turns, 2,
23558 "the implementer was tried once"
23559 );
23560 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23561 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23562 }
23563
23564 #[tokio::test]
23565 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23566 crate::run::pin_test_home();
23567 let dir = tempfile::tempdir().unwrap();
23568 let mut broken = spec("bad");
23569 broken.command = vec!["false".into()];
23570 let mut runner = rewrite_runner(
23571 dir.path(),
23572 true,
23573 vec![
23574 broken,
23575 scripted("noformat", "Add retries."),
23576 scripted("good", EN_REWRITE),
23577 ],
23578 );
23579 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23580 "bad".into(),
23581 "noformat".into(),
23582 "good".into(),
23583 ]));
23584 let winner = runner.state.candidates[0].clone();
23585 let msg = runner
23586 .guarded_pr_message(&winner, None, true)
23587 .await
23588 .unwrap()
23589 .expect("the third agent rewrites");
23590 assert!(msg.body.contains("`src/net.rs`"));
23591 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23592 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23593 }
23594
23595 #[tokio::test]
23596 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23597 crate::run::pin_test_home();
23598 for (agents, role, saved) in [
23600 (
23601 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23602 Some("beta"),
23603 true,
23604 ),
23605 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23606 (Vec::new(), None, false),
23607 ] {
23608 let dir = tempfile::tempdir().unwrap();
23609 let mut runner = rewrite_runner(dir.path(), true, agents);
23610 runner.state.config.roles.language_judge =
23611 role.map(|r| crate::config::AgentChoice::One(r.into()));
23612 let winner = runner.state.candidates[0].clone();
23613 let first = runner
23614 .guarded_pr_message(&winner, None, true)
23615 .await
23616 .unwrap();
23617 assert!(first.is_none(), "withheld text parks the run");
23618 assert!(runner.state.parked);
23619 assert_eq!(gate_questions(&runner), 1);
23620 let g = runner.state.github_text.as_ref().unwrap();
23621 assert_eq!(g.rewrite.is_some(), saved);
23622 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23623 }
23624 }
23625
23626 #[tokio::test]
23627 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23628 crate::run::pin_test_home();
23629 let dir = tempfile::tempdir().unwrap();
23630 let mut runner = runner_at(RunStatus::Gating);
23631 runner.state =
23632 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23633 runner.state.repo = dir.path().to_owned();
23634 runner.state.candidates[0].worktree = dir.path().to_owned();
23635 let mut author = spec("alpha");
23636 author.command = vec![
23637 "sh".into(),
23638 "-c".into(),
23639 "printf '%s' \"$REWRITE_REPLY\"".into(),
23640 ];
23641 author.env.insert(
23642 "REWRITE_REPLY".into(),
23643 format!(
23644 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23645 "English sentence. ".repeat(600)
23646 ),
23647 );
23648 runner.state.config.agents = vec![author];
23649 seed_public_summary(&mut runner);
23650 let winner = runner.state.candidates[0].clone();
23651 assert!(
23652 runner
23653 .guarded_pr_message(&winner, None, true)
23654 .await
23655 .unwrap()
23656 .is_none()
23657 );
23658 let record = runner.state.github_text.clone().unwrap();
23659 let rewrite = record.rewrite.clone().unwrap();
23660 assert_eq!(rewrite.failed, ["body-language"]);
23661 assert!(
23662 rewrite
23663 .body
23664 .ends_with(&format!("magi:run/{}", runner.state.id))
23665 );
23666 let store = ask::Questions::open();
23667 let q = store.get(record.question.as_ref().unwrap()).unwrap();
23668 assert!(
23669 q.choices
23670 .contains(&crate::github_text::USE_REWRITE.to_owned())
23671 );
23672 for label in [
23673 "ORIGINAL",
23674 "FALLBACK",
23675 "REWRITE",
23676 "Rules still failed: body-language",
23677 "truncated",
23678 ] {
23679 assert!(q.detail.contains(label), "{label}");
23680 }
23681 let artifact = std::fs::read_to_string(crate::run::artifact_path(
23682 &runner.state,
23683 &crate::github_text::artifact_name(&record.fingerprint),
23684 ))
23685 .unwrap();
23686 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23687 runner.state =
23689 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23690 store
23691 .update(&q.id, |q| {
23692 q.say("まだ日本語")?;
23693 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23694 })
23695 .unwrap();
23696 assert!(
23697 runner
23698 .guarded_pr_message(&winner, None, true)
23699 .await
23700 .unwrap()
23701 .is_none()
23702 );
23703 let retry = runner.state.github_text.as_ref().unwrap();
23704 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23705 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23706 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23707 assert!(
23708 q.posting_comparison
23709 .as_ref()
23710 .unwrap()
23711 .matches_detail(&q.detail)
23712 );
23713 store
23714 .update(&q.id, |q| {
23715 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23716 })
23717 .unwrap();
23718 let message = runner
23719 .guarded_pr_message(&winner, None, true)
23720 .await
23721 .unwrap()
23722 .unwrap();
23723 assert_eq!(message.title, rewrite.title);
23724 assert_eq!(message.body, rewrite.body);
23725 assert_eq!(
23726 runner.state.github_text.as_ref().unwrap().choice,
23727 Some(GithubTextChoice::Rewrite)
23728 );
23729 let again = runner
23730 .guarded_pr_message(&winner, None, true)
23731 .await
23732 .unwrap()
23733 .unwrap();
23734 assert_eq!(again.title, message.title);
23735 assert_eq!(again.body, message.body);
23736 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23737 assert!(
23738 !runner
23739 .state
23740 .events
23741 .iter()
23742 .any(|e| e.message.contains(&rewrite.title)
23743 || e.message.contains("English sentence."))
23744 );
23745 }
23746
23747 #[tokio::test]
23748 async fn historical_posting_reply_never_expires_into_fallback() {
23749 crate::run::pin_test_home();
23750 let dir = tempfile::tempdir().unwrap();
23751 let mut runner = runner_at(RunStatus::Gating);
23752 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23753 runner.state.repo = dir.path().to_owned();
23754 runner.state.candidates[0].worktree = dir.path().to_owned();
23755 seed_public_summary(&mut runner);
23756 let winner = runner.state.candidates[0].clone();
23757 assert!(
23758 runner
23759 .guarded_pr_message(&winner, None, true)
23760 .await
23761 .unwrap()
23762 .is_none()
23763 );
23764 let id = runner
23765 .state
23766 .github_text
23767 .as_ref()
23768 .unwrap()
23769 .question
23770 .clone()
23771 .unwrap();
23772 let store = ask::Questions::open();
23773 let mut q = store.get(&id).unwrap();
23774 q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23775 q.answer(ask::Answer::Choice(
23776 crate::github_text::POST_ORIGINAL.into(),
23777 ))
23778 .unwrap();
23779 q.answer_delivered = true;
23780 store.put(&mut q).unwrap();
23781 let source = std::fs::read(store.path_of(&id)).unwrap();
23782 let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23783 std::fs::create_dir_all(&migration).unwrap();
23784 std::fs::write(
23785 migration.join("state.json"),
23786 serde_json::to_vec(&serde_json::json!({
23787 "schema": 1, "sources": [id], "pending": [id]
23788 }))
23789 .unwrap(),
23790 )
23791 .unwrap();
23792 for _ in 0..2 {
23793 assert!(
23794 runner
23795 .guarded_pr_message(&winner, None, true)
23796 .await
23797 .unwrap()
23798 .is_none()
23799 );
23800 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23801 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23802 }
23803 let home = crate::run::home();
23804 crate::task_chat::sweep(
23805 &crate::queue::Queue::at(home.join("queue")),
23806 &crate::talk::Talks::at(home.join("talks")),
23807 &store,
23808 &home,
23809 );
23810 let outbox = crate::task_inquiry::Outbox::at(&home);
23811 outbox
23812 .update_decision(&id, |d| {
23813 d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23814 d.source_answer_at = q.answered_at;
23815 Ok(true)
23816 })
23817 .unwrap();
23818 outbox
23819 .settle(
23820 &id,
23821 "owner-chat",
23822 "Owner confirms no publication remains necessary",
23823 None,
23824 )
23825 .unwrap();
23826 assert!(
23827 runner
23828 .guarded_pr_message(&winner, None, true)
23829 .await
23830 .unwrap()
23831 .is_none()
23832 );
23833 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23834 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23835 }
23836
23837 #[tokio::test]
23838 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23839 crate::run::pin_test_home();
23840 let dir = tempfile::tempdir().unwrap();
23841 let mut runner = runner_at(RunStatus::Gating);
23842 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23843 runner.state.repo = dir.path().to_owned();
23844 runner.state.candidates[0].worktree = dir.path().to_owned();
23845 seed_public_summary(&mut runner);
23846 let winner = runner.state.candidates[0].clone();
23847 assert!(
23848 runner
23849 .guarded_pr_message(&winner, None, true)
23850 .await
23851 .unwrap()
23852 .is_none()
23853 );
23854 let record = runner.state.github_text.as_ref().unwrap();
23855 assert!(record.rewrite.is_none());
23856 let mut old = serde_json::to_value(record).unwrap();
23858 old.as_object_mut().unwrap().remove("rewrite");
23859 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23860 assert!(old.rewrite.is_none());
23861 let id = record.question.clone().unwrap();
23862 ask::Questions::open()
23863 .update(&id, |q| {
23864 q.choices.push(crate::github_text::USE_REWRITE.into());
23865 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23866 })
23867 .unwrap();
23868 let message = runner
23869 .guarded_pr_message(&winner, None, true)
23870 .await
23871 .unwrap()
23872 .unwrap();
23873 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23874 assert_eq!(
23875 runner.state.github_text.as_ref().unwrap().choice,
23876 Some(GithubTextChoice::Fallback)
23877 );
23878 }
23879
23880 #[tokio::test]
23881 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23882 crate::run::pin_test_home();
23883 let dir = tempfile::tempdir().unwrap();
23884 let mut runner = runner_at(RunStatus::Gating);
23885 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23886 runner.state.repo = dir.path().to_owned();
23887 runner.state.candidates[0].worktree = dir.path().to_owned();
23888 seed_public_summary(&mut runner);
23889 let winner = runner.state.candidates[0].clone();
23890 assert!(
23891 runner
23892 .guarded_pr_message(&winner, None, true)
23893 .await
23894 .unwrap()
23895 .is_none()
23896 );
23897 let store = ask::Questions::open();
23898 let find = |run: &str| {
23899 store
23900 .list()
23901 .into_iter()
23902 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23903 .unwrap()
23904 };
23905 let run = runner.state.id.clone();
23906 let q = find(&run);
23907 let fp = runner
23908 .state
23909 .github_text
23910 .as_ref()
23911 .unwrap()
23912 .fingerprint
23913 .clone();
23914 let name = crate::github_text::artifact_name(&fp);
23915 let path = crate::run::artifact_path(&runner.state, &name);
23916 assert!(q.detail.contains(&path.display().to_string()));
23917 assert!(q.detail.contains("title-language: title"));
23918 let saved = std::fs::read_to_string(&path).expect("artifact written");
23919 assert!(saved.contains("日本語のタイトル"));
23920 assert!(q.cwd.is_none());
23921 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23922 let say = |id: &str, text: &str, choice: &str| {
23924 store
23925 .update(id, |q| {
23926 q.thread.push(ask::Turn {
23927 who: ask::Who::Operator,
23928 body: text.to_owned(),
23929 at: jiff::Timestamp::now(),
23930 note: None,
23931 });
23932 q.answer(ask::Answer::Choice(choice.to_owned()))
23933 })
23934 .unwrap();
23935 };
23936 say(&q.id, "まだ日本語", "use my text");
23937 assert!(
23938 runner
23939 .guarded_pr_message(&winner, None, true)
23940 .await
23941 .unwrap()
23942 .is_none()
23943 );
23944 let second = find(&run);
23945 assert_ne!(second.id, q.id);
23946 assert!(second.detail.contains("did not pass"));
23947 assert!(second.detail.contains("日本語のタイトル"));
23950 assert!(!second.detail.contains("まだ日本語"));
23951 assert!(second.detail.contains(&path.display().to_string()));
23952 say(&second.id, "fix: retry failed requests", "use my text");
23953 let message = runner
23954 .guarded_pr_message(&winner, None, true)
23955 .await
23956 .unwrap()
23957 .expect("a vetted replacement is posted");
23958 assert_eq!(message.title, "fix: retry failed requests");
23959 assert_eq!(
23960 runner
23961 .state
23962 .github_text
23963 .as_ref()
23964 .unwrap()
23965 .chosen_title
23966 .as_deref(),
23967 Some("fix: retry failed requests")
23968 );
23969 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23970 let again = runner
23972 .guarded_pr_message(&winner, None, true)
23973 .await
23974 .unwrap()
23975 .unwrap();
23976 assert_eq!(again.title, message.title);
23977 assert_eq!(
23978 store
23979 .list()
23980 .into_iter()
23981 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23982 .count(),
23983 2
23984 );
23985 }
23986
23987 #[test]
23988 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23989 let mut state = state_with_summary(
23993 "add retries",
23994 "TITLE: fix(web): batch reads\n- reads run.json once",
23995 );
23996 state.config.graph.language = "ja".to_owned();
23997 let m = pr_message(&state, 'A');
23998 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23999 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
24000
24001 let task = "今回やってほしいこと: results projector を直す";
24004 let mut state = state_with_summary(task, "- no title line");
24005 state.config.graph.language = "ja".to_owned();
24006 let m = pr_message(&state, 'A');
24007 assert_eq!(
24008 m.title,
24009 format!("chore: land candidate A of run {}", state.id)
24010 );
24011 assert!(
24012 m.body.contains(&format!(
24013 "<summary>Original task</summary>\n\n{task}\n\n</details>"
24014 )),
24015 "{}",
24016 m.body
24017 );
24018 }
24019
24020 #[test]
24021 fn pr_message_scrubs_home_paths_and_addresses() {
24022 let state = state_with_summary(
24023 "fix it in /Users/someone/src/x",
24024 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
24025 );
24026 let m = pr_message(&state, 'A');
24027 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
24028 assert!(!m.body.contains(leak), "{}", m.body);
24029 }
24030 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
24031 }
24032
24033 #[test]
24034 fn pr_message_survives_a_task_that_closes_details() {
24035 let state = state_with_summary("a </details> b", "TITLE: fix: x");
24036 let m = pr_message(&state, 'A');
24037 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
24038 }
24039
24040 #[test]
24041 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
24042 let cmd = manual_merge_command(
24043 MergeStyle::Squash,
24044 Path::new("/repo"),
24045 "b",
24046 "fix: \"quoted\" $(x) `y`\n\nbody",
24047 );
24048 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
24049 }
24050
24051 #[test]
24052 fn manual_merge_command_matches_the_configured_style() {
24053 let repo = Path::new("/repo");
24054 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
24055
24056 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
24057 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
24058
24059 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
24060 assert_eq!(
24061 squash,
24062 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
24063 \"Merge magi run 0832 (candidate A)\""
24064 );
24065
24066 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
24067 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
24068 }
24069
24070 #[test]
24071 fn a_nudge_gets_a_quarter_of_the_budget() {
24072 assert_eq!(retry_budget(secs(1200), true), secs(300));
24074 assert_eq!(retry_budget(secs(3600), true), secs(900));
24075 }
24076
24077 #[test]
24078 fn a_resent_prompt_keeps_the_whole_budget() {
24079 assert_eq!(retry_budget(secs(1200), false), secs(1200));
24082 assert_eq!(retry_budget(secs(60), false), secs(60));
24083 }
24084
24085 #[test]
24086 fn the_floor_never_exceeds_the_original_budget() {
24087 assert_eq!(retry_budget(secs(60), true), secs(60));
24091 assert_eq!(retry_budget(secs(480), true), secs(120));
24092 assert_eq!(retry_budget(secs(0), true), secs(0));
24093 }
24094
24095 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
24096 agent::CommandEvidence {
24097 id: "item1".to_owned(),
24098 description: "cargo test".to_owned(),
24099 exit_code,
24100 result_summary: String::new(),
24101 source: "codex".to_owned(),
24102 }
24103 }
24104
24105 #[test]
24106 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
24107 assert!(!has_unconfirmed_command(&[]));
24111 }
24112
24113 #[test]
24114 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
24115 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
24119 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
24120 assert!(!has_unconfirmed_command(&[
24121 evidence(Some(0)),
24122 evidence(Some(101))
24123 ]));
24124 }
24125
24126 #[test]
24127 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
24128 assert!(has_unconfirmed_command(&[
24129 evidence(Some(0)),
24130 evidence(None)
24131 ]));
24132 }
24133
24134 #[test]
24135 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
24136 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24137 assert_eq!(
24138 verified_noop_claim(true, &[], text).as_deref(),
24139 Some("already fixed by b32cfc4, on main.")
24140 );
24141 }
24142
24143 #[test]
24144 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
24145 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24148 assert!(verified_noop_claim(false, &[], text).is_none());
24149 }
24150
24151 #[test]
24152 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
24153 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24154 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
24155 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
24157 }
24158
24159 #[test]
24160 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
24161 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
24162 }
24163
24164 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
24167 runner.state.candidates = shape
24168 .iter()
24169 .enumerate()
24170 .map(|(i, &(empty, verified))| Candidate {
24171 index: i,
24172 label: (b'A' + i as u8) as char,
24173 agent: "sonnet".to_owned(),
24174 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
24175 worktree: PathBuf::from(format!("/wt/{i}")),
24176 summary: String::new(),
24177 stat: String::new(),
24178 files: 0,
24179 commits: 0,
24180 empty,
24181 failed: None,
24182 verified_noop: verified.map(str::to_owned),
24183 duration_ms: 0,
24184 folded: false,
24185 })
24186 .collect();
24187 }
24188
24189 #[test]
24190 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
24191 ask_test_home();
24192 let mut runner = runner_at(RunStatus::Implementing);
24193 set_candidates(
24194 &mut runner,
24195 &[
24196 (true, Some("already on main at b32cfc4")),
24197 (true, Some("same fix, see the existing test")),
24198 ],
24199 );
24200
24201 runner
24202 .after_implement()
24203 .expect("a verified no-op is not an error");
24204
24205 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
24206 }
24207
24208 #[test]
24209 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
24210 ask_test_home();
24211 let mut runner = runner_at(RunStatus::Implementing);
24212 set_candidates(
24216 &mut runner,
24217 &[(true, Some("already on main at b32cfc4")), (true, None)],
24218 );
24219
24220 let err = runner
24221 .after_implement()
24222 .expect_err("an unverified empty candidate must still fail the run");
24223
24224 assert!(
24225 err.to_string().contains("no candidate produced a change"),
24226 "{err}"
24227 );
24228 assert_eq!(runner.state.status, RunStatus::Failed);
24229 }
24230
24231 #[test]
24232 fn after_implement_still_fails_an_ordinary_all_empty_run() {
24233 ask_test_home();
24234 let mut runner = runner_at(RunStatus::Implementing);
24235 set_candidates(&mut runner, &[(true, None), (true, None)]);
24236
24237 let err = runner
24238 .after_implement()
24239 .expect_err("no candidate declared anything; this is an ordinary failure");
24240
24241 assert!(
24242 err.to_string().contains("no candidate produced a change"),
24243 "{err}"
24244 );
24245 assert_eq!(runner.state.status, RunStatus::Failed);
24246 }
24247}
24248
24249#[derive(Debug)]
24252struct GraftRestoreFailed(String);
24253
24254impl std::fmt::Display for GraftRestoreFailed {
24255 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24256 write!(
24257 f,
24258 "could not restore the winner to its pre-graft commit: {}",
24259 self.0
24260 )
24261 }
24262}
24263
24264impl std::error::Error for GraftRestoreFailed {}
24265
24266#[cfg(test)]
24270mod resource_wait_tests {
24271 use super::*;
24272 use std::collections::BTreeMap;
24273
24274 fn sh(id: &str, script: &str) -> AgentSpec {
24275 AgentSpec {
24276 id: id.to_owned(),
24277 kind: crate::config::AgentKind::Command,
24278 model: None,
24279 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
24280 extra_args: Vec::new(),
24281 env: BTreeMap::new(),
24282 prompt_delivery: None,
24283 max_turn_secs: None,
24284 }
24285 }
24286
24287 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
24288 SeatJob {
24289 seat: SeatState::new("impl-A", &spec.id, 1),
24290 spec,
24291 cwd: dir.to_path_buf(),
24292 prompt: "do it".to_owned(),
24293 timeout,
24294 allow_write: true,
24295 sessions: false,
24296 artifacts: dir.join("art"),
24297 stem: "impl-A".to_owned(),
24298 handover: None,
24299 }
24300 }
24301
24302 fn state(resource_wait_secs: u64) -> RunState {
24303 let mut config = Config::default();
24304 config.graph.timeout_resource_wait = resource_wait_secs;
24305 RunState::new(
24306 PathBuf::from("/repo"),
24307 "main".to_owned(),
24308 "abc1234".to_owned(),
24309 "task".to_owned(),
24310 config,
24311 )
24312 }
24313
24314 fn hold(cache: &Path) -> crate::cache::Guard {
24315 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
24316 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
24317 crate::cache::AcquireOutcome::Acquired(g) => g,
24318 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
24319 }
24320 }
24321
24322 async fn run(
24323 jobs: Vec<SeatJob>,
24324 sem: Arc<Semaphore>,
24325 cache: Option<&Path>,
24326 st: &mut RunState,
24327 ) -> Vec<(usize, SeatState, AgentOutcome)> {
24328 let prompts = Prompts::default();
24329 let ctx = WaveCtx {
24330 run: "r",
24331 node: "implement",
24332 prompts: &prompts,
24333 cache,
24334 round: None,
24335 carry_seats: false,
24336 pauses: None,
24337 };
24338 wave(jobs, sem, &ctx, st, 0).await
24339 }
24340
24341 #[tokio::test]
24342 async fn verification_cache_wait_preserves_the_command_budget() {
24343 crate::run::pin_test_home();
24344 let tmp = tempfile::tempdir().unwrap();
24345 let cache = tmp.path().join("target");
24346 let guard = hold(&cache);
24347 let release = tokio::spawn(async move {
24348 tokio::time::sleep(Duration::from_millis(100)).await;
24349 guard.release();
24350 });
24351 let mut st = state(10);
24352 let budget = Duration::from_millis(50);
24353 let wait = repository_wait(&st, None);
24354 let (outcomes, _) = with_cache_lease(
24355 &mut st,
24356 Some(&cache),
24357 "gate",
24358 "gate",
24359 tmp.path(),
24360 "abc1234",
24361 budget,
24362 "test gate",
24363 &wait,
24364 |_, available| async move {
24365 assert_eq!(
24366 available, budget,
24367 "contention must not shorten verification"
24368 );
24369 (Vec::new(), false, Vec::new())
24370 },
24371 )
24372 .await
24373 .unwrap();
24374 release.await.unwrap();
24375 assert!(outcomes.is_empty(), "{outcomes:?}");
24376 assert!(
24377 st.events
24378 .iter()
24379 .any(|event| event.message.contains("waiting for"))
24380 );
24381 assert!(st.resource_wait.is_none());
24382 hold(&cache).release();
24384 }
24385
24386 #[tokio::test]
24387 async fn verification_cache_wait_expires_without_starting_commands() {
24388 crate::run::pin_test_home();
24389 let tmp = tempfile::tempdir().unwrap();
24390 let cache = tmp.path().join("target");
24391 let _held = hold(&cache);
24392 let mut st = state(0);
24393 let wait = repository_wait(&st, None);
24394 let (outcomes, _) = with_cache_lease(
24395 &mut st,
24396 Some(&cache),
24397 "gate",
24398 "gate",
24399 tmp.path(),
24400 "abc1234",
24401 Duration::from_secs(60),
24402 "test gate",
24403 &wait,
24404 |_, _| async { panic!("commands must not start without the cache lease") },
24405 )
24406 .await
24407 .unwrap();
24408 assert_eq!(outcomes.len(), 1);
24409 assert!(outcomes[0].resource_blocked);
24410 assert!(st.resource_wait.is_none());
24411 assert!(matches!(
24412 st.resource_block.unwrap().cause,
24413 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24414 ));
24415 }
24416
24417 #[tokio::test]
24418 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24419 crate::run::pin_test_home();
24420 let tmp = tempfile::tempdir().unwrap();
24421 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24422 let cache = tmp.path().join("cache");
24423 let mut state = state(21600);
24424 let wait = repository_wait(&state, None);
24425 let (outcomes, _) = with_cache_lease(
24426 &mut state,
24427 Some(&cache),
24428 "gate",
24429 "gate",
24430 tmp.path(),
24431 "head",
24432 Duration::from_secs(60),
24433 "gate",
24434 &wait,
24435 |_, _| async { panic!("freshness failure must not start verification") },
24436 )
24437 .await
24438 .unwrap();
24439 assert_eq!(outcomes.len(), 1);
24440 assert!(outcomes[0].resource_blocked);
24441 let block = state.resource_block.unwrap();
24442 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24443 assert!(!block.description().contains("timed out"));
24444 }
24445
24446 #[tokio::test]
24447 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24448 crate::run::pin_test_home();
24449 let tmp = tempfile::tempdir().unwrap();
24450 let cache = tmp.path().join("cache");
24451 let _held = hold(&cache);
24452 let mut state = state(21600);
24453 let pause = Pause::new();
24454 let interrupt = Pause::new();
24455 let cause = ParkCause::Operator;
24456 pause.park_for(cause.clone());
24457 let prompts = Prompts::default();
24458 let ctx = WaveCtx {
24459 run: "test",
24460 node: "fix",
24461 prompts: &prompts,
24462 cache: Some(&cache),
24463 round: None,
24464 carry_seats: false,
24465 pauses: Some((&pause, &interrupt)),
24466 };
24467 let result = wave(
24468 vec![job(
24469 sh("writer", "exit 99"),
24470 tmp.path(),
24471 Duration::from_secs(1),
24472 )],
24473 Arc::new(Semaphore::new(1)),
24474 &ctx,
24475 &mut state,
24476 0,
24477 )
24478 .await;
24479 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24480 assert_eq!(result[0].1.turns, 0);
24481 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24482 assert_eq!(
24483 state.resource_block.unwrap().cause,
24484 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24485 );
24486 assert!(state.active.is_empty());
24487 }
24488
24489 #[tokio::test]
24490 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24491 crate::run::pin_test_home();
24492 let tmp = tempfile::tempdir().unwrap();
24493 let cache = tmp.path().join("target-a");
24494 let guard = hold(&cache);
24495 tokio::spawn(async move {
24497 tokio::time::sleep(Duration::from_millis(1500)).await;
24498 guard.release();
24499 });
24500 let mut st = state(60);
24501 let results = run(
24502 vec![job(
24503 sh("alpha", "echo ok"),
24504 tmp.path(),
24505 Duration::from_secs(1),
24506 )],
24507 Arc::new(Semaphore::new(1)),
24508 Some(&cache),
24509 &mut st,
24510 )
24511 .await;
24512 assert!(
24513 matches!(results[0].2, AgentOutcome::Ok(_)),
24514 "the wait must not be charged to the agent"
24515 );
24516 assert!(st.resource_block.is_none());
24517 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24518 assert!(st.resource_wait.is_none() && st.active.is_empty());
24519 }
24520
24521 #[tokio::test]
24522 #[cfg(unix)]
24523 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24524 use std::os::fd::AsRawFd as _;
24525 crate::run::pin_test_home();
24526 let tmp = tempfile::tempdir().unwrap();
24527 crate::git::git(tmp.path(), &["init"]).await.unwrap();
24528 let common = tmp.path().join(".git");
24529 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24530 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24531 assert_eq!(
24532 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24533 0
24534 );
24535 let mut st = state(1);
24536 st.config.coordination = crate::config::test_coordination();
24537 let mut reader = job(
24538 sh("reader", "echo must-not-run"),
24539 tmp.path(),
24540 Duration::from_secs(30),
24541 );
24542 reader.allow_write = false;
24543 let results = run(
24544 vec![reader.clone()],
24545 Arc::new(Semaphore::new(1)),
24546 None,
24547 &mut st,
24548 )
24549 .await;
24550 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24551 assert_eq!(results[0].1.turns, 0);
24552 assert!(FailClass::of(&results[0].2).is_none());
24553 assert_eq!(
24554 st.resource_block.as_ref().unwrap().kind,
24555 "repository coordination"
24556 );
24557 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24558 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24559 drop(external);
24560 let editor = crate::repo_lock::SeatLease::for_checkout(
24561 tmp.path(),
24562 true,
24563 crate::config::test_coordination().as_ref(),
24564 )
24565 .await
24566 .unwrap();
24567 let pause = Pause::new();
24568 let interrupt = Pause::new();
24569 let prompts = Prompts::default();
24570 let ctx = WaveCtx {
24571 run: "r",
24572 node: "review",
24573 prompts: &prompts,
24574 cache: None,
24575 round: None,
24576 carry_seats: false,
24577 pauses: Some((&pause, &interrupt)),
24578 };
24579 let (results, ()) = tokio::join!(
24580 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24581 async {
24582 tokio::time::sleep(Duration::from_millis(50)).await;
24583 pause.park();
24584 }
24585 );
24586 assert!(
24587 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24588 );
24589 assert_eq!(results[0].1.turns, 0);
24590 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24591 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24592 drop(editor);
24593 }
24594
24595 #[tokio::test]
24596 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24597 crate::run::pin_test_home();
24598 let tmp = tempfile::tempdir().unwrap();
24599 let cache = tmp.path().join("target-b");
24600 let _held = hold(&cache);
24601 let mut st = state(1);
24602 let results = run(
24603 vec![job(
24604 sh("alpha", "echo ok"),
24605 tmp.path(),
24606 Duration::from_secs(1),
24607 )],
24608 Arc::new(Semaphore::new(1)),
24609 Some(&cache),
24610 &mut st,
24611 )
24612 .await;
24613 let (_, seat, out) = &results[0];
24614 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24615 assert!(wave_blocked(&results));
24616 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24617 assert_eq!(seat.turns, 0, "nothing was asked");
24618 assert!(
24619 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24620 "no agent was spawned"
24621 );
24622 assert!(FailClass::of(out).is_none(), "not a failure class");
24623 let block = st.resource_block.as_ref().expect("recorded for the resume");
24624 assert_eq!(block.kind, "shared build cache");
24625 assert!(matches!(
24626 block.cause,
24627 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24628 ));
24629 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24630 assert!(st.active.is_empty());
24631 }
24632
24633 #[tokio::test]
24634 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24635 crate::run::pin_test_home();
24636 let tmp = tempfile::tempdir().unwrap();
24637 let mut st = state(1);
24638 let results = run(
24639 vec![job(
24640 sh("alpha", "echo ok"),
24641 tmp.path(),
24642 Duration::from_secs(1),
24643 )],
24644 Arc::new(Semaphore::new(0)),
24645 None,
24646 &mut st,
24647 )
24648 .await;
24649 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24650 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24651 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24652 }
24653
24654 #[tokio::test]
24655 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24656 crate::run::pin_test_home();
24657 let tmp = tempfile::tempdir().unwrap();
24658 let mut st = state(60);
24659 let results = run(
24660 vec![job(
24661 sh("alpha", "sleep 30"),
24662 tmp.path(),
24663 Duration::from_secs(1),
24664 )],
24665 Arc::new(Semaphore::new(1)),
24666 None,
24667 &mut st,
24668 )
24669 .await;
24670 let out = &results[0].2;
24671 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24672 let class = FailClass::of(out).expect("a failure");
24673 assert_eq!(class, FailClass::Timeout);
24674 assert!(should_hand_over(None, &class), "existing handover policy");
24675 assert!(st.resource_block.is_none());
24676 }
24677}