1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false, None).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true, None).await
1357 }
1358
1359 pub(crate) async fn authorize_base_recovery_control(
1360 home: &Path,
1361 task: &str,
1362 control: &mut crate::base_recovery_control::Transaction,
1363 ) -> Result<()> {
1364 let id = control.request.run.clone();
1365 Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366 }
1367
1368 async fn authorize_base_recovery_request(
1369 home: &Path,
1370 id: &str,
1371 task: &str,
1372 resumption: bool,
1373 mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374 ) -> Result<()> {
1375 let _owner = crate::pr_head::owner_lock(home, id)?;
1376 let mut state = RunState::load_under(id, home)?;
1377 crate::pr_head::ensure_owner(&state)?;
1378 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379 anyhow::ensure!(
1380 queued.execution_runs().last().is_some_and(|run| run == id)
1381 && queued.status != crate::queue::TaskStatus::Done
1382 && !queued.fresh_start
1383 && queued.review_branch.is_none(),
1384 "run task ownership changed"
1385 );
1386 anyhow::ensure!(
1387 state
1388 .origin
1389 .as_ref()
1390 .and_then(|o| o.task.as_deref())
1391 .is_none_or(|owner| owner == task),
1392 "run belongs to another task"
1393 );
1394 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395 anyhow::ensure!(
1396 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397 "run is live or its ownership is unknown"
1398 );
1399 let _execution = crate::driver::maintenance(home, id)?;
1402 anyhow::ensure!(
1403 state.status.resumable()
1404 && state.released_to.is_none()
1405 && state.retired_to.is_none()
1406 && state.pr_taken_over_by.is_none(),
1407 "run ownership is final"
1408 );
1409 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410 anyhow::ensure!(
1411 !state.pr_publication_pending
1412 && !crate::handover::landing_checkpoint(&state)
1413 && state.merge_decision.is_none(),
1414 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415 );
1416 let winner = state.winner().context("run has no retained winner")?;
1417 anyhow::ensure!(
1418 git::is_clean(&winner.worktree).await?,
1419 "retained worktree has uncommitted changes"
1420 );
1421 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422 anyhow::ensure!(
1423 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424 "retained worktree does not match the winner branch"
1425 );
1426 if let Some(control) = control.as_ref() {
1427 anyhow::ensure!(
1428 head.starts_with(&control.request.head),
1429 "retained HEAD changed since the owner request"
1430 );
1431 anyhow::ensure!(
1432 state
1433 .base_sync_recovery
1434 .stop
1435 .as_ref()
1436 .is_none_or(|stop| stop.head == head),
1437 "retained HEAD differs from the recorded stop head; inspect it first"
1438 );
1439 }
1440 state.ensure_base_episode();
1441 state.record_base_sync_stop(&head);
1442 state.save_under(home)?;
1443 let fetched =
1444 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1445 anyhow::ensure!(
1446 fetched.ok(),
1447 "cannot authorize base recovery without a fetched base"
1448 );
1449 let approved_tip = git::rev_parse(
1450 &state.repo,
1451 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1452 )
1453 .await?;
1454 if let Some(control) = control.as_deref_mut() {
1455 let queue = crate::queue::Queue::at(home.join("queue"));
1456 control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1457 if let Some(prior) = state
1458 .base_sync_recovery
1459 .authorizations
1460 .iter()
1461 .find(|a| a.request.as_deref() == Some(&control.key()))
1462 {
1463 anyhow::ensure!(
1464 !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1465 "recovery request was consumed or invalidated; it cannot queue again"
1466 );
1467 return control.enqueue(&queue);
1468 }
1469 }
1470 if resumption {
1471 anyhow::ensure!(
1472 crate::daemon::compatible_run_instruction(&state, &queued),
1473 "resumption instruction does not match the retained run"
1474 );
1475 let approval = queued
1476 .resumption
1477 .as_ref()
1478 .context("no operator resumption")?;
1479 anyhow::ensure!(
1480 approval.instruction == queued.instruction,
1481 "resumption approval does not match the current task instruction"
1482 );
1483 state.authorize_base_sync_resumption(task, approval)?;
1484 } else {
1485 state.authorize_base_sync(task)?;
1486 }
1487 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1488 state
1489 .base_sync_recovery
1490 .authorizations
1491 .last_mut()
1492 .unwrap()
1493 .scope = Some(crate::base_sync_recovery::Scope {
1494 tip: approved_tip,
1495 instruction: queued.instruction.clone(),
1496 episode,
1497 });
1498 if let Some(control) = control.as_deref_mut() {
1499 let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1500 authorization.request = Some(control.key());
1501 authorization.generation = control.generation();
1502 authorization.decision = control.decision();
1503 authorization.question = control
1504 .request
1505 .decision
1506 .as_ref()
1507 .map(|_| control.request.request.clone());
1508 }
1509 state.event(
1510 "base-recovery",
1511 "owner authorized a bounded base synchronization interval; fresh verification required",
1512 );
1513 state.save_under(home)?;
1514 if let Some(control) = control {
1515 control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1516 }
1517 Ok(())
1518 }
1519
1520 pub fn resume(id: &str) -> Result<Self> {
1522 let state = RunState::load(id)?;
1523 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1524 if state.startup.as_ref().is_some_and(|s| !s.complete)
1525 || (state.status == RunStatus::Prep
1526 && state.instruction.trim().is_empty()
1527 && state.candidates.is_empty()
1528 && state.prep_complete == Some(false))
1529 {
1530 bail!(
1531 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1532 );
1533 }
1534 crate::pr_head::ensure_owner(&state)?;
1535 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1536 bail!(
1537 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1538 );
1539 }
1540 if let Some(to) = &state.released_to {
1541 bail!(
1542 "run {} cannot be resumed: its worktree was released to run {}",
1543 state.short(),
1544 crate::run::short_of(to)
1545 );
1546 }
1547 let roles = state.config.resolve_roles()?;
1548 if let Some(id) = &state.config.graph.selected_implementer
1549 && state.candidates.iter().any(|c| c.agent != *id)
1550 {
1551 bail!(
1552 "replacement implementer pin does not match the saved candidates; task must stay held"
1553 );
1554 }
1555 let max_parallel = state.config.graph.max_parallel.max(1);
1556 Ok(Self {
1557 state,
1558 roles,
1559 sem: Arc::new(Semaphore::new(max_parallel)),
1560 pause: Pause::new(),
1561 interrupt: Pause::new(),
1562 })
1563 }
1564
1565 pub async fn execute(&mut self) -> Result<()> {
1572 anyhow::ensure!(
1573 self.state.startup.as_ref().is_none_or(|s| s.complete),
1574 "startup incomplete; inspect retained evidence before explicit recovery"
1575 );
1576 if let Some(selection) = &self.state.pr_recovery
1577 && (selection.source_proof.is_some() || selection.source_run.is_some())
1578 {
1579 Box::pin(selection.verify_source(&self.state.repo)).await?;
1580 }
1581 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1584 || self.state.land_approval.is_some()
1585 || self.state.merge_decision.is_some()
1586 || !self.state.merge_repairs.is_empty()
1587 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1588 {
1589 let claim = FixClaim::acquire(&self.state.dir())?;
1590 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1591 let disk = RunState::load(&self.state.id)?;
1592 let checkpoint = |s: &RunState| {
1593 (
1594 s.merge_repairs.len(),
1595 s.merge_repairs
1596 .last()
1597 .map(|r| (r.question.clone(), r.phase)),
1598 s.land_approval
1599 .as_ref()
1600 .map(|a| (a.question.clone(), a.head.clone())),
1601 )
1602 };
1603 if checkpoint(&disk) != checkpoint(&self.state) {
1604 bail!(
1605 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1606 );
1607 }
1608 Some(claim)
1609 } else {
1610 None
1611 };
1612 let mut execution = self.claim_driver().await?;
1613 let runner = &mut *execution.runner;
1614 let wait = runner.repository_wait();
1615 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1618 let result = match result {
1619 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1620 record_repository_block(&mut runner.state, &e);
1621 runner.park_here()?;
1622 Ok(())
1623 }
1624 other => other,
1625 };
1626 execution.driver.finish(&mut runner.state).await?;
1627 let ended = if result.is_err() {
1628 Some({
1629 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1630 if let Err(error) = &result {
1631 notice.message = format!(
1632 "Run {} stopped with an error: {}",
1633 runner.state.id,
1634 error.to_string().lines().next().unwrap_or("unknown error")
1635 );
1636 }
1637 notice
1638 })
1639 } else {
1640 crate::notices::run_ended(&runner.state)
1641 };
1642 if let Some(notice) = ended {
1643 crate::notices::raise(notice);
1644 }
1645 result
1646 }
1647
1648 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1649 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1650 }
1651
1652 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1653 let wait = self.repository_wait();
1654 coordinate_operation(&mut self.state, cwd, &wait).await
1655 }
1656
1657 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1659 let owner = match crate::pr_head::owner_lock_wait(
1660 &crate::run::home(),
1661 &self.state.id,
1662 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1663 )
1664 .await
1665 {
1666 Ok(owner) => owner,
1667 Err(error) => {
1668 let detail = format!("worktree maintenance or ownership update: {error:#}");
1669 self.state.resource_block = Some(crate::run::ResourceBlock {
1673 cause: crate::run::ResourceBlockCause::Other,
1674 node: "start".into(),
1675 kind: "worktree maintenance".into(),
1676 detail: detail.clone(),
1677 at: Timestamp::now(),
1678 });
1679 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1680 }
1681 };
1682 let disk = RunState::load(&self.state.id)?;
1683 crate::pr_head::ensure_owner(&disk)?;
1684 let claimed =
1685 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1686 anyhow::ensure!(
1691 disk.liveness(claimed) != crate::run::Liveness::Live,
1692 "run already has a live driver; wait for its execution to end"
1693 );
1694 if let Some(to) = &disk.released_to {
1695 bail!(
1696 "run {} cannot continue: its worktree was released to run {}",
1697 self.state.short(),
1698 crate::run::short_of(to)
1699 );
1700 }
1701 anyhow::ensure!(
1702 serde_json::to_value(&disk.base_sync_recovery)?
1703 == serde_json::to_value(&self.state.base_sync_recovery)?,
1704 "base recovery changed since this run was loaded; reload the saved run"
1705 );
1706 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1707 let mut execution = DriverExecution {
1708 runner: self,
1709 driver,
1710 };
1711 let _owner = owner;
1714 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1715 .await?;
1716 let state = &mut execution.runner.state;
1717 state.parked = false;
1722 state.resource_wait = None;
1723 state.resource_block = None;
1724 state.clear_active();
1731 let pid = std::process::id();
1746 state.driver_pid = Some(pid);
1747 state.driver_started_at = crate::proc::process_started_at(pid);
1748 state.driver_exited = false;
1749 state.driver_claimed = Some(true);
1750 state.begin_pass();
1751 execution.driver.bind(state);
1752 state.save()?;
1753 execution.driver.record(state)?;
1754 Ok(execution)
1755 }
1756
1757 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1760 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1761 if let Err(error) = &result
1762 && !error.is::<crate::repo_lock::ResourceBlocked>()
1763 && self
1764 .state
1765 .repair_grants
1766 .last()
1767 .is_some_and(|g| g.conflict.is_none())
1768 {
1769 self.state.repair_grants.last_mut().unwrap().phase =
1770 crate::repair_budget::Phase::Complete;
1771 self.state.status = RunStatus::Blocked;
1772 self.state.save()?;
1773 bail!(
1774 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1775 self.state.passes.last().map_or(0, |p| p.index)
1776 );
1777 }
1778 result
1779 }
1780
1781 async fn execute_granted_repair(&mut self) -> Result<()> {
1784 use crate::repair_budget::{Kind, Phase};
1785 let index = self.state.repair_grants.len() - 1;
1786 let winner = self
1787 .state
1788 .winner()
1789 .cloned()
1790 .context("repair winner missing")?;
1791 loop {
1792 let grant = self.state.repair_grants[index].clone();
1793 match grant.phase {
1794 Phase::Pending => {
1795 if grant.spent >= grant.receipt.request.rounds {
1796 self.state.repair_grants[index].phase = Phase::Complete;
1797 self.state.status = RunStatus::Blocked;
1798 self.state.save()?;
1799 return Ok(());
1800 }
1801 self.state.repair_grants[index].spent += 1;
1804 self.state.repair_grants[index].phase = Phase::Fixing;
1805 self.state.save()?;
1806 match grant.receipt.request.kind {
1807 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1808 Kind::Gate => {
1809 let failed = self.state.gate.clone();
1810 let _ = self.gate_fix_round(&winner, &failed).await?;
1811 }
1812 Kind::Review => {
1813 let mut record = self
1814 .state
1815 .reviews
1816 .last()
1817 .cloned()
1818 .context("review failure missing")?;
1819 let round = record.round;
1820 let step = self.run_fix_round(&winner, &mut record).await?;
1821 self.state.reviews[round - 1] = record;
1822 if matches!(step, FixStep::Recorded { report_lost: true }) {
1823 self.state.repair_grants[index].phase = Phase::Complete;
1824 self.state.status = RunStatus::Blocked;
1825 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1826 self.state.save()?;
1827 return Ok(());
1828 }
1829 if matches!(step, FixStep::Resource) {
1830 self.stop_for_resource()?;
1831 return Ok(());
1832 }
1833 self.state.save()?;
1834 }
1835 }
1836 if self.resource_halt()? {
1837 return Ok(());
1838 }
1839 }
1840 Phase::Fixing => {
1841 if grant.receipt.request.kind == Kind::BaseConflict {
1842 self.repair_base_conflict(false).await?;
1843 }
1844 let _coordination = self.coordinate(&winner.worktree).await?;
1847 let rescued = git::rescue_commit(
1848 &winner.worktree,
1849 "magi: preserve interrupted granted repair",
1850 )
1851 .await?;
1852 self.state.note_withheld("repair-grant", &rescued.withheld);
1853 self.state
1856 .verification_history
1857 .push(crate::run::VerificationEvidence {
1858 superseded_by: None,
1859 archived_at: Some(Timestamp::now()),
1860 fixes_spent: Some(self.state.gate_fixes.len()),
1861 gate_head: self.state.gate_head.take(),
1862 gate_ran: std::mem::take(&mut self.state.gate_ran),
1863 gate: std::mem::take(&mut self.state.gate),
1864 e2e: self.state.head_e2e.take(),
1865 });
1866 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1867 self.state
1868 .verification_history
1869 .push(crate::run::VerificationEvidence {
1870 superseded_by: None,
1871 archived_at: Some(Timestamp::now()),
1872 fixes_spent: Some(self.state.gate_fixes.len()),
1873 gate_head: None,
1874 gate_ran: false,
1875 gate: Vec::new(),
1876 e2e: Some(e2e),
1877 });
1878 }
1879 self.state.repair_grants[index].review_start = self.state.reviews.len();
1880 self.state.repair_grants[index].head =
1881 git::rev_parse(&winner.worktree, "HEAD").await?;
1882 self.state.repair_grants[index].phase = Phase::Reviewing;
1883 if grant.receipt.request.kind == Kind::BaseConflict {
1884 self.state.base_sync_recovery.stop = None;
1885 if let Some(sync) = &mut self.state.base_sync {
1886 sync.tip = grant
1887 .receipt
1888 .request
1889 .tip
1890 .clone()
1891 .context("repair base tip missing")?;
1892 sync.conflict = None;
1893 sync.behind = 0;
1894 }
1895 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1896 }
1897 self.state.save()?;
1898 }
1899 Phase::Reviewing => {
1900 self.review_loop_pass().await?;
1901 if self.resource_halt()? {
1902 return Ok(());
1903 }
1904 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1905 let clean = self.state.reviews.len() > grant.review_start
1906 && self.state.reviews.last().is_some_and(|r| {
1907 r.head == head
1908 && r.clean
1909 && r.expected > 0
1910 && r.answered == r.expected
1911 && matches!(
1912 r.e2e_status(),
1913 E2eStatus::Passed | E2eStatus::NotConfigured
1914 )
1915 });
1916 if !clean {
1917 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1919 == Kind::Review
1920 && grant.spent < grant.receipt.request.rounds
1921 && self.state.reviews.last().is_some_and(|r| {
1922 r.fix.is_none()
1923 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1924 }) {
1925 Phase::Pending
1926 } else {
1927 Phase::Complete
1928 };
1929 self.state.status = RunStatus::Blocked;
1930 self.state.event(
1931 "repair-grant",
1932 "fresh review did not clear the repaired head; no publication",
1933 );
1934 self.state.save()?;
1935 if self.state.repair_grants[index].phase == Phase::Pending {
1936 continue;
1937 }
1938 return Ok(());
1939 }
1940 self.state.repair_grants[index].phase = Phase::Gating;
1941 self.state.save()?;
1942 }
1943 Phase::Gating => {
1944 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1945 self.state.reconcile_verification_head(&head);
1946 let wait = self.repository_wait();
1947 if self
1948 .state
1949 .reviews
1950 .last()
1951 .is_none_or(|r| r.head != head || !r.clean)
1952 {
1953 self.state.status = RunStatus::Blocked;
1954 self.state.repair_grants[index].phase = Phase::Complete;
1955 self.state.event(
1956 "repair-grant",
1957 "head changed since fresh review; repair held",
1958 );
1959 self.state.save()?;
1960 return Ok(());
1961 }
1962 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1963 return Ok(());
1964 }
1965 let outcomes = self.run_gate(&winner).await?;
1966 if verify_inconclusive(&outcomes) {
1967 return Ok(());
1968 }
1969 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1970 if after != head {
1971 self.state.repair_grants[index].review_start = self.state.reviews.len();
1972 self.state.repair_grants[index].head = after;
1973 self.state.repair_grants[index].phase = Phase::Reviewing;
1974 self.state.status = RunStatus::Blocked;
1975 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1976 self.state.save()?;
1977 return Ok(());
1978 }
1979 let passed = outcomes.iter().all(CommandOutcome::ok);
1980 if passed
1983 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1984 {
1985 return Ok(());
1986 }
1987 self.state.repair_grants[index].succeeded = passed;
1988 self.state.gate = outcomes;
1989 self.state.gate_ran = true;
1990 self.state.status = if passed {
1991 RunStatus::Gating
1992 } else {
1993 RunStatus::Blocked
1994 };
1995 self.state.repair_grants[index].phase = if !passed
1996 && grant.receipt.request.kind == Kind::Gate
1997 && grant.spent < grant.receipt.request.rounds
1998 {
1999 Phase::Pending
2000 } else {
2001 Phase::Complete
2002 };
2003 if passed && self.state.config.merge.mode == MergeMode::Pr {
2004 self.state.pr_publication_pending = true;
2006 } else if passed {
2007 self.state.status = RunStatus::Ready;
2008 }
2009 self.state.save()?;
2010 if !passed {
2011 continue;
2012 }
2013 if self.state.config.merge.mode != MergeMode::Pr {
2017 self.state.status = RunStatus::Ready;
2018 self.state.event(
2019 "repair-grant",
2020 "repair verified locally; grant carries no merge authorization",
2021 );
2022 self.state.save()?;
2023 return Ok(());
2024 }
2025 return self.merge().await;
2026 }
2027 Phase::Complete => return Ok(()),
2028 }
2029 }
2030 }
2031
2032 async fn execute_graph(&mut self) -> Result<()> {
2033 let archive_home = crate::run::home().canonicalize()?;
2037 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2038 let repo = self.state.repo.clone();
2039 let _coordination = self.coordinate(&repo).await?;
2040 ensure_setup_config(&mut self.state, &repo).await?;
2041 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2042 }
2043 self.state.configure_recovery_resume()?;
2044 if self
2045 .state
2046 .repair_grants
2047 .last()
2048 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2049 {
2050 return self.execute_granted_repair().await;
2051 }
2052 if self
2053 .state
2054 .repair_grants
2055 .last()
2056 .is_some_and(|g| !g.succeeded)
2057 {
2058 self.state.status = RunStatus::Blocked;
2059 self.state.save()?;
2060 return Ok(());
2061 }
2062 self.persist_candidates().await?;
2065 if self.state.base_sync_blocked() {
2068 self.sync_to_base().await?;
2069 if !self.state.base_sync_blocked()
2070 && self.state.status != RunStatus::AlreadyInBase
2071 && !self.resource_halt()?
2072 {
2073 self.finish_after_tally().await?;
2074 }
2075 return Ok(());
2076 }
2077 if self
2078 .state
2079 .merge_decision
2080 .as_ref()
2081 .is_some_and(|decision| !decision.closed)
2082 && self.state.status != RunStatus::Landing
2083 {
2084 if crate::merge_decision::resumable_hold(&self.state) {
2085 self.state.merge = Some(MergeOutcome {
2086 mode: MergeMode::Pr,
2087 ok: false,
2088 detail: self
2089 .state
2090 .merge_decision
2091 .as_ref()
2092 .unwrap()
2093 .target
2094 .pr
2095 .clone(),
2096 empty: false,
2097 });
2098 self.run_land().await?;
2099 self.settle_questions();
2100 return Ok(());
2101 }
2102 bail!(
2103 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2104 );
2105 }
2106 if self.state.config.graph.selected_implementer.is_some()
2107 && self.state.status == RunStatus::Blocked
2108 && self.state.tally.is_none()
2109 && self.state.candidate_preservation.is_empty()
2110 && self.state.interrupted_implementations.is_empty()
2111 {
2112 bail!(
2113 "selected implementation is held; explicitly release the owning task to retry it"
2114 );
2115 }
2116
2117 if self.state.merge_repairs.last().is_some_and(|r| {
2118 matches!(
2119 r.phase,
2120 crate::run::MergeRepairPhase::Requested
2121 | crate::run::MergeRepairPhase::Fixing
2122 | crate::run::MergeRepairPhase::Reviewing
2123 )
2124 }) {
2125 if self.repair_merge_review().await? {
2126 self.run_land().await?;
2127 }
2128 self.settle_questions();
2129 return Ok(());
2130 }
2131 if self.state.pr_publication_pending {
2134 self.merge().await?;
2135 if self.resource_halt()? {
2136 return Ok(());
2137 }
2138 self.state.save()?;
2139 self.settle_questions();
2140 return Ok(());
2141 }
2142 if self.state.status == RunStatus::Stalled {
2155 if self.recover_stall().await? {
2156 self.finish_after_tally().await?;
2157 } else {
2158 self.state.save()?;
2160 }
2161 return Ok(());
2162 }
2163 if self.state.status == RunStatus::Landing
2173 || crate::handover::landing_checkpoint(&self.state)
2174 {
2175 self.state.land_sync_pending = true;
2176 self.state.save()?;
2177 self.run_land().await?;
2178 self.settle_questions();
2183 return Ok(());
2184 }
2185 if self.state.github_text.is_some()
2190 && self.state.merge.is_none()
2191 && self.state.status == RunStatus::Gating
2192 {
2193 self.merge().await?;
2194 if self.resource_halt()? {
2195 return Ok(());
2196 }
2197 self.state.save()?;
2198 return Ok(());
2199 }
2200 if let Err(e) = self.prep().await {
2201 if e.is::<crate::repo_lock::ResourceBlocked>() {
2202 return Err(e);
2203 }
2204 self.state.status = RunStatus::Blocked;
2205 self.state
2206 .prep_failure
2207 .get_or_insert_with(|| format!("{e:#}"));
2208 self.state
2209 .event("prep", format!("preparation stopped: {e:#}"));
2210 self.state.save()?;
2211 return Err(e);
2212 }
2213 if self.park_here()? {
2214 return Ok(());
2215 }
2216 self.advise().await?;
2217 if self.park_here()? {
2218 return Ok(());
2219 }
2220 self.implement().await?;
2221 if self.park_here()? {
2222 return Ok(());
2223 }
2224 if self.state.status == RunStatus::VerifiedNoop {
2228 return Ok(());
2229 }
2230 self.judge().await?;
2231 if self.park_here()? {
2232 return Ok(());
2233 }
2234 self.deliberate().await?;
2235 if self.park_here()? {
2236 return Ok(());
2237 }
2238 self.vote().await?;
2239 if self.park_here()? {
2240 return Ok(());
2241 }
2242 self.tally()?;
2243 if self.state.status == RunStatus::Stalled {
2248 self.state.save()?;
2252 return Ok(());
2253 }
2254 self.finish_after_tally().await?;
2255 Ok(())
2256 }
2257
2258 fn stop_for_resource(&mut self) -> Result<()> {
2263 if self.pause.parked() || self.interrupt.parked() {
2266 self.park_here()?;
2267 return Ok(());
2268 }
2269 if self.state.resource_block.is_some() {
2270 self.state.status = RunStatus::Blocked;
2271 }
2272 self.state.save()
2273 }
2274
2275 fn resource_halt(&mut self) -> Result<bool> {
2277 if self.state.resource_block.is_none() {
2278 return Ok(false);
2279 }
2280 self.stop_for_resource()?;
2281 Ok(true)
2282 }
2283
2284 fn park_here(&mut self) -> Result<bool> {
2291 let parking = self.pause.parked() || self.interrupt.parked();
2292 if !parking && self.state.resource_block.is_some() {
2293 self.stop_for_resource()?;
2294 return Ok(true);
2295 }
2296 if !self.pause.parked() && !self.interrupt.parked() {
2301 return Ok(false);
2302 }
2303 if self.state.parked {
2304 return Ok(true);
2305 }
2306 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2307 if let Some(block) = &mut self.state.resource_block
2308 && matches!(
2309 block.cause,
2310 crate::run::ResourceBlockCause::Cancelled { .. }
2311 )
2312 {
2313 block.cause = crate::run::ResourceBlockCause::Cancelled {
2314 park: cause.clone(),
2315 };
2316 }
2317 let why = match cause.as_ref().map(ParkCause::label) {
2318 Some(reason) => format!(
2319 "parked after `{}` ({reason}) — resume to carry on from here",
2320 self.state.status.as_str()
2321 ),
2322 None => format!(
2323 "parked after `{}` — resume to carry on from here",
2324 self.state.status.as_str()
2325 ),
2326 };
2327 self.state.event("park", why);
2328 self.state
2329 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2330 self.state.parked = true;
2331 self.state.save()?;
2332 Ok(true)
2333 }
2334
2335 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2337 if self.state.prep_failure.is_none() {
2338 return Ok(());
2339 }
2340 let failed_at = self
2341 .state
2342 .events
2343 .iter()
2344 .rev()
2345 .find(|event| event.node == "prep")
2346 .map_or(self.state.updated_at, |event| event.at);
2347 anyhow::ensure!(
2348 approval_at > failed_at,
2349 "preparation retry requires a new explicit release after the failure"
2350 );
2351 self.state.prep_failure = None;
2352 self.state.prep_complete = Some(false);
2353 self.state
2354 .event("prep-retry", "operator approved retry of preparation");
2355 Ok(())
2356 }
2357
2358 pub fn on_pause(&mut self, pause: Pause) {
2360 self.pause = pause;
2361 }
2362
2363 pub fn watch_interrupt(&mut self, pause: Pause) {
2369 self.interrupt = pause;
2370 }
2371
2372 fn settle_questions(&mut self) {
2392 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2393 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2394 }
2395 }
2396
2397 async fn finish_after_tally(&mut self) -> Result<()> {
2400 self.graft().await?;
2403 if self.resource_halt()? {
2404 return Ok(());
2405 }
2406 self.fold_losers().await?;
2407 self.sync_to_base().await?;
2412 if self.hold_base_sync().await? || self.resource_halt()? {
2413 return Ok(());
2414 }
2415 if self.state.status == RunStatus::AlreadyInBase {
2416 return Ok(());
2417 }
2418 self.review_loop().await?;
2419 if self.resource_halt()? {
2420 return Ok(());
2421 }
2422 self.sync_to_base().await?;
2423 if self.hold_base_sync().await? || self.resource_halt()? {
2424 return Ok(());
2425 }
2426 if self.state.status == RunStatus::AlreadyInBase {
2427 return Ok(());
2428 }
2429 self.gate().await?;
2430 self.merge().await?;
2431 if self.resource_halt()? {
2432 return Ok(());
2433 }
2434 self.state.save()?;
2435 Ok(())
2436 }
2437
2438 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2445 let result = match ensure_setup_config(&mut self.state, repo).await {
2446 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2447 Err(e) => Err(e),
2448 };
2449 if let Err(e) = result {
2450 git::worktree_remove_clean(repo, wt).await.ok();
2451 if in_prep {
2452 self.rollback_prep(repo).await;
2453 }
2454 self.state.status = RunStatus::Blocked;
2455 self.state.event(
2456 "setup",
2457 format!("worktree setup failed in {}: {e:#}", wt.display()),
2458 );
2459 self.state.save()?;
2460 return Err(e);
2461 }
2462 Ok(())
2463 }
2464
2465 async fn rollback_prep(&mut self, repo: &Path) {
2467 let root = self.state.worktree_root();
2468 for c in std::mem::take(&mut self.state.candidates) {
2469 if matches!(
2470 git::worktree_remove_clean(repo, &c.worktree).await,
2471 Ok(true)
2472 ) {
2473 git::branch_delete(repo, &c.branch).await.ok();
2474 } else {
2475 self.state.candidates.push(c);
2476 }
2477 }
2478 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2479 for j in 1..=seats {
2480 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2481 .await
2482 .ok();
2483 }
2484 for k in 1..=self.state.config.graph.advisors {
2485 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2486 .await
2487 .ok();
2488 }
2489 }
2490
2491 async fn prep(&mut self) -> Result<()> {
2492 if let Some(reason) = &self.state.prep_failure {
2493 bail!(
2494 "candidate preparation previously failed: {reason}; correct the source before retrying"
2495 );
2496 }
2497 if self
2498 .state
2499 .prep_complete
2500 .unwrap_or(!self.state.candidates.is_empty())
2501 {
2502 return Ok(());
2503 }
2504 let repo = self.state.repo.clone();
2505 let _coordination = self.coordinate(&repo).await?;
2508 let base = self.state.base_commit.clone();
2509 let plan = self.state.sources.plan(&repo, &base).await?;
2510 self.rollback_prep(&repo).await;
2511 if !self.state.candidates.is_empty() {
2512 bail!(
2513 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2514 );
2515 }
2516 self.state.prep_complete = Some(false);
2517 self.state.status = RunStatus::Prep;
2518 self.state.save()?;
2519 let repo = self.state.repo.clone();
2520 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2521 let root = self.state.worktree_root();
2522 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2523
2524 let hooks_dir = self.state.dir().join("hooks");
2527 if self.state.config.blind.commit_msg_hook {
2528 std::fs::create_dir_all(&hooks_dir)
2529 .with_context(|| format!("create {}", hooks_dir.display()))?;
2530 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2531 let path = hooks_dir.join("commit-msg");
2532 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2533 make_executable(&path)?;
2534 git::acquire_worktree_config(&repo).await?;
2542 self.state.enabled_worktree_config = true;
2543 }
2544
2545 ensure_setup_config(&mut self.state, &repo).await?;
2548
2549 for (index, (spec, label)) in self
2550 .roles
2551 .implementers
2552 .clone()
2553 .into_iter()
2554 .zip(labels)
2555 .enumerate()
2556 {
2557 let branch = self.state.branch_for(label);
2558 let worktree = root.join(format!("cand-{label}"));
2559 self.state.event(
2563 "prep-intent",
2564 serde_json::json!({
2565 "branch": branch,
2566 "worktree": worktree,
2567 "start": start,
2568 "picks": plan.picks,
2569 })
2570 .to_string(),
2571 );
2572 self.state.save()?;
2573 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2574 if self.state.config.blind.commit_msg_hook {
2575 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2576 }
2577 git::local_exclude(&worktree, "/.magi/").await?;
2578 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2579 git::worktree_remove_clean(&repo, &worktree).await.ok();
2582 git::branch_delete(&repo, &branch).await.ok();
2583 self.rollback_prep(&repo).await;
2584 self.state.status = RunStatus::Blocked;
2585 self.state
2586 .event("prep", format!("worktree setup failed: {e:#}"));
2587 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2588 self.state.save()?;
2589 return Err(e);
2590 }
2591 self.state.candidates.push(Candidate {
2592 index,
2593 label,
2594 agent: spec.id.clone(),
2595 branch,
2596 worktree,
2597 summary: String::new(),
2598 stat: String::new(),
2599 files: 0,
2600 commits: 0,
2601 empty: false,
2602 failed: None,
2603 verified_noop: None,
2604 duration_ms: 0,
2605 folded: false,
2606 });
2607 self.state.save()?;
2608 for pick in &plan.picks {
2609 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2610 {
2611 self.state.status = RunStatus::Blocked;
2612 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2613 self.state
2614 .event("prep", format!("cannot apply referenced commit: {e}"));
2615 self.state.save()?;
2616 return Err(e);
2617 }
2618 }
2619 }
2620
2621 for j in 1..=self.roles.judges.len() {
2622 let wt = root.join(format!("judge-{j}"));
2623 if !wt.exists() {
2624 git::worktree_add_detached(&repo, &wt, &base).await?;
2625 self.setup_seat_worktree(&repo, &wt, true).await?;
2626 }
2627 }
2628
2629 if self.state.config.graph.advise {
2637 for k in 1..=self.state.config.graph.advisors {
2638 let wt = root.join(format!("advisor-{k}"));
2639 if !wt.exists() {
2640 git::worktree_add_detached(&repo, &wt, &base).await?;
2641 self.setup_seat_worktree(&repo, &wt, true).await?;
2642 }
2643 }
2644 }
2645
2646 let authors: Vec<&str> = self
2651 .roles
2652 .implementers
2653 .iter()
2654 .map(|a| a.id.as_str())
2655 .collect();
2656 let overlap: Vec<String> = self
2657 .roles
2658 .judges
2659 .iter()
2660 .enumerate()
2661 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2662 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2663 .collect();
2664 if !overlap.is_empty() {
2665 let note = format!(
2666 "{} also authored a candidate; blind, but the panel is less \
2667 independent than {} distinct agents would be",
2668 overlap.join(", "),
2669 self.roles.judges.len()
2670 );
2671 self.state.event("prep", note);
2672 }
2673
2674 self.state.event(
2675 "prep",
2676 format!(
2677 "{} candidates, {} judges, base {} ({})",
2678 self.state.candidates.len(),
2679 self.roles.judges.len(),
2680 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2681 self.state.base_branch
2682 ),
2683 );
2684 self.state.prep_complete = Some(true);
2685 self.state.status = RunStatus::Implementing;
2686 self.state.save()?;
2687 Ok(())
2688 }
2689
2690 async fn advise(&mut self) -> Result<()> {
2723 let implement_untouched = self
2724 .state
2725 .candidates
2726 .iter()
2727 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2728 if !self.state.config.graph.advise || self.state.advise_attempted {
2729 return Ok(());
2730 }
2731 if !implement_untouched {
2732 self.state.event(
2733 "advise",
2734 "skipping the design-deliberation stage: at least one \
2735 candidate already shows implementation progress, so this \
2736 run is past the point the stage exists to run before"
2737 .to_owned(),
2738 );
2739 self.state.advise_attempted = true;
2740 self.state.save()?;
2741 return Ok(());
2742 }
2743 let run_id = self.state.id.clone();
2744 let prompts = self.state.config.prompts.clone();
2745 let instruction = self.state.instruction.clone();
2746 let language = self.state.config.graph.language.clone();
2747 let root = self.state.worktree_root();
2748 let n = self.state.config.graph.advisors;
2749 let where_recorded = self.state.dir().join("run.json");
2750
2751 let seats = match self.state.config.advisors() {
2752 Ok(seats) if !seats.is_empty() => seats,
2753 Ok(_) => {
2754 self.state.event(
2755 "advise",
2756 format!(
2757 "[graph] advisors is 0; skipping the design-deliberation \
2758 stage and continuing without a synthesis brief (see {})",
2759 where_recorded.display()
2760 ),
2761 );
2762 self.state.advise_attempted = true;
2763 self.state.save()?;
2764 return Ok(());
2765 }
2766 Err(e) => {
2767 self.state.event(
2768 "advise",
2769 format!(
2770 "could not resolve advisor seats ({e:#}); continuing \
2771 without a design-deliberation brief (see {})",
2772 where_recorded.display()
2773 ),
2774 );
2775 self.state.advise_attempted = true;
2776 self.state.save()?;
2777 return Ok(());
2778 }
2779 };
2780
2781 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2782 let artifacts = agent::artifacts_dir(&self.state.dir());
2783 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2784
2785 let mut jobs = Vec::new();
2786 for (i, spec) in seats.iter().cloned().enumerate() {
2787 let seat_key = format!("advisor-{}", i + 1);
2788 let seat = self.seat(&seat_key, &spec.id);
2789 jobs.push(SeatJob {
2790 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2791 spec,
2792 seat,
2793 cwd: worktrees[i % worktrees.len()].clone(),
2794 timeout,
2795 allow_write: false,
2796 sessions: false,
2797 artifacts: artifacts.clone(),
2798 stem: seat_key,
2799 handover: None,
2800 });
2801 }
2802
2803 self.state.event(
2804 "advise",
2805 format!(
2806 "{} advisor seat(s) sketching a design in parallel",
2807 jobs.len()
2808 ),
2809 );
2810 let mut quota_losses = Vec::new();
2811 let cache = self.state.config.cache_dir();
2812 let ctx = WaveCtx {
2813 carry_seats: false,
2814 pauses: Some((&self.pause, &self.interrupt)),
2815 run: &run_id,
2816 node: "advise",
2817 prompts: &prompts,
2818 cache: cache.as_deref(),
2819 round: None,
2820 };
2821 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2822 let results = ask_json_wave::<Proposal>(
2823 jobs,
2824 Arc::clone(&self.sem),
2825 self.state.config.graph.retries,
2826 &advisor_roster,
2827 &ctx,
2828 &mut quota_losses,
2829 &mut self.state,
2830 &|p: &Proposal| p.validate(),
2831 )
2832 .await;
2833 if self.state.resource_block.is_some() {
2834 return self.stop_for_resource();
2835 }
2836 self.state.quota.extend(quota_losses);
2837
2838 let mut records = Vec::with_capacity(results.len());
2839 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2840 let agent_id = seat.agent.clone();
2841 self.state.seats.insert(seat.key.clone(), seat);
2842 match res {
2843 Ok((proposal, out)) => {
2844 self.state
2845 .event("advise", format!("advisor-{} proposed a design", i + 1));
2846 records.push(advise::AdvisorRecord::proposed(
2847 i + 1,
2848 agent_id,
2849 proposal,
2850 out.duration_ms,
2851 ));
2852 }
2853 Err(e) => {
2854 self.state.event(
2855 "advise",
2856 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2857 );
2858 records.push(advise::AdvisorRecord::failed(
2859 i + 1,
2860 agent_id,
2861 e.to_string(),
2862 ));
2863 }
2864 }
2865 }
2866
2867 let mut advice = advise::Advice {
2868 records,
2869 synthesis: None,
2870 };
2871 if advice.proposals().is_empty() {
2872 self.state.event(
2873 "advise",
2874 "no advisor produced a usable proposal; continuing without a \
2875 synthesis brief"
2876 .to_owned(),
2877 );
2878 } else {
2879 match self
2880 .synthesize_brief(
2881 &advice,
2882 &instruction,
2883 &language,
2884 &worktrees[0],
2885 &artifacts,
2886 &run_id,
2887 &prompts,
2888 cache.as_deref(),
2889 )
2890 .await
2891 {
2892 Ok(Some(text)) => {
2893 self.state.event(
2894 "advise",
2895 "synthesized a design brief for the implementer".to_owned(),
2896 );
2897 advice.synthesis = Some(text);
2898 }
2899 Ok(None) => {
2900 self.state.event(
2901 "advise",
2902 "the synthesis seat produced nothing usable; continuing \
2903 without a design brief"
2904 .to_owned(),
2905 );
2906 }
2907 Err(e) => {
2908 if e.is::<crate::repo_lock::ResourceBlocked>() {
2909 return Err(e);
2910 }
2911 self.state.event(
2912 "advise",
2913 format!("could not synthesize a design brief: {e:#}"),
2914 );
2915 }
2916 }
2917 }
2918 advise::apply_reflection(&mut advice);
2919
2920 self.state.advice = Some(advice);
2921 self.state.advise_attempted = true;
2922 self.state.save()?;
2923 Ok(())
2924 }
2925
2926 #[allow(clippy::too_many_arguments)]
2938 async fn synthesize_brief(
2939 &mut self,
2940 advice: &advise::Advice,
2941 instruction: &str,
2942 language: &str,
2943 cwd: &Path,
2944 artifacts: &Path,
2945 run_id: &str,
2946 prompts: &Prompts,
2947 cache: Option<&Path>,
2948 ) -> Result<Option<String>> {
2949 let chain = agent::pick_chain(
2950 &self.state.config.agents,
2951 self.state.config.roles.synthesizer.as_ref(),
2952 &agent::installed,
2953 "synthesizer",
2954 )?;
2955 let proposals = advice.proposals();
2956 let mut prompt = prompt::with_overlay(
2957 prompt::synthesize_brief(instruction, &proposals, language),
2958 prompts.overlay("advise"),
2959 );
2960 if cache.is_some() {
2961 prompt.push('\n');
2966 prompt.push_str(&prompt::build_cache_note("advise", false));
2967 }
2968 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2969 let mut last = None;
2974 for (n, spec) in chain.iter().enumerate() {
2975 if n > 0 {
2976 self.state
2977 .event("advise", format!("synthesis falling back to {}", spec.id));
2978 }
2979 let mut seat = self.seat("advise-synthesis", &spec.id);
2980 let wait = self.repository_wait();
2981 let outcome = wait
2982 .scope(agent::invoke(
2983 spec,
2984 &mut seat,
2985 &Invocation {
2986 coordination: self.state.config.coordination.as_ref(),
2987 fetched_base: None,
2988 cwd,
2989 prompt: &prompt,
2990 timeout,
2991 allow_write: false,
2992 sessions: false,
2993 artifacts,
2994 stem: &if n == 0 {
2995 "advise-synthesis".to_owned()
2996 } else {
2997 format!("advise-synthesis-{}", spec.id)
2998 },
2999 run: run_id,
3000 node: "advise",
3001 cache_dir: None,
3002 attachments: &[],
3003 writable: &[],
3004 },
3005 ))
3006 .await;
3007 if outcome.is_ok() {
3008 self.state.seats.insert(seat.key.clone(), seat);
3009 }
3010 let advance = agent::chain_advances(&outcome);
3011 last = Some(outcome);
3012 if !advance {
3013 break;
3014 }
3015 }
3016 let out = last.expect("a chain holds at least one agent")?;
3019 if !out.usable() {
3020 return Ok(None);
3021 }
3022 let text =
3023 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3024 Ok((!text.trim().is_empty()).then_some(text))
3025 }
3026
3027 async fn implement(&mut self) -> Result<()> {
3030 self.persist_candidates().await?;
3031 let run_id = self.state.id.clone();
3036 let prompts = self.state.config.prompts.clone();
3037 let todo: Vec<usize> = self
3038 .state
3039 .candidates
3040 .iter()
3041 .enumerate()
3042 .filter(|(_, c)| {
3043 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3044 && c.failed.is_none()
3045 && !c.empty
3046 })
3047 .map(|(i, _)| i)
3048 .collect();
3049 if todo.is_empty() {
3050 return self.after_implement();
3051 }
3052 self.state.status = RunStatus::Implementing;
3053
3054 let language = self.state.config.graph.language.clone();
3055 let github_language = self.state.config.graph.github_language.clone();
3056 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3057 let sessions = self.state.config.graph.sessions;
3058 let artifacts = agent::artifacts_dir(&self.state.dir());
3059 let brief = self
3063 .state
3064 .advice
3065 .as_ref()
3066 .and_then(|a| a.synthesis.as_deref())
3067 .map(str::to_owned);
3068 let attachments = self.state.attachments.clone();
3069
3070 let mut jobs = Vec::new();
3071 for &i in &todo {
3072 let (index, label, worktree) = {
3073 let c = &self.state.candidates[i];
3074 (c.index, c.label, c.worktree.clone())
3075 };
3076 let spec = self.roles.implementers[index].clone();
3077 let seat_key = format!("impl-{label}");
3078 let seat = self.seat(&seat_key, &spec.id);
3079 let instruction = seeded_instruction(&self.state);
3080 jobs.push(SeatJob {
3081 spec,
3082 seat,
3083 prompt: prompt::implement_in(
3084 &instruction,
3085 &worktree.to_string_lossy(),
3086 &language,
3087 &github_language,
3088 brief.as_deref(),
3089 &attachments,
3090 ),
3091 cwd: worktree,
3092 timeout,
3093 allow_write: true,
3094 sessions,
3095 artifacts: artifacts.clone(),
3096 stem: format!("impl-{label}"),
3097 handover: None,
3098 });
3099 }
3100
3101 self.state.event(
3102 "implement",
3103 format!("{} candidates in parallel", jobs.len()),
3104 );
3105 let mut sent = jobs.clone();
3111 let cache = self.state.config.cache_dir();
3112 let ctx = WaveCtx {
3113 carry_seats: false,
3114 pauses: Some((&self.pause, &self.interrupt)),
3115 run: &run_id,
3116 node: "implement",
3117 prompts: &prompts,
3118 cache: cache.as_deref(),
3119 round: None,
3120 };
3121 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3122 if !wave_blocked(&results) {
3123 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3124 .await;
3125 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3126 .await?;
3127 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3128 .await;
3129 }
3130 if !results.is_empty()
3133 && results
3134 .iter()
3135 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3136 {
3137 let losses: Vec<_> = results
3138 .iter()
3139 .map(|(_, seat, out)| {
3140 let AgentOutcome::Quota(o) = out else {
3141 unreachable!()
3142 };
3143 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3144 })
3145 .collect();
3146 self.state.quota.extend(losses.clone());
3147 for (_, seat, _) in results {
3148 self.state.seats.insert(seat.key.clone(), seat);
3149 }
3150 for &i in &todo {
3151 let label = self.state.candidates[i].label;
3152 self.state.interrupted_implementations.insert(label);
3153 self.state
3154 .candidate_preservation
3155 .insert(label, "machine outage awaiting persistence".into());
3156 }
3157 block_machine_outage(&mut self.state, "implement", &losses);
3160 self.state.save()?;
3161 self.persist_candidates().await?;
3162 for &i in &todo {
3163 self.state
3164 .interrupted_implementations
3165 .insert(self.state.candidates[i].label);
3166 }
3167 return self.stop_for_resource();
3168 }
3169 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3170
3171 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3172 && results
3173 .iter()
3174 .any(|(_, _, out)| FailClass::of(out).is_some());
3175 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3176 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3177 if self
3181 .state
3182 .candidate_preservation
3183 .contains_key(&self.state.candidates[i].label)
3184 {
3185 self.state.seats.insert(seat.key.clone(), seat);
3186 }
3187 continue;
3188 }
3189 let seat_key = seat.key.clone();
3190 let agent = seat.agent.clone();
3199 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3200 self.state.seats.insert(seat.key.clone(), seat);
3201 let label = self.state.candidates[i].label;
3202 self.state.interrupted_implementations.remove(&label);
3203
3204 let (summary, duration, failed, verified_claim) = match out {
3205 AgentOutcome::Ok(o) => {
3206 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3207 let failed = (!o.usable()).then(|| {
3208 if o.timed_out {
3209 "agent timed out".to_owned()
3210 } else {
3211 format!(
3212 "agent exited with {}",
3213 crate::notices::exit_status(o.exit_code)
3214 )
3215 }
3216 });
3217 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3218 (text, o.duration_ms, failed, verified_claim)
3219 }
3220 AgentOutcome::Dropped(o) => {
3226 let why = o
3227 .dropped
3228 .as_ref()
3229 .map(|d| d.why.as_str())
3230 .unwrap_or("the CLI ended the stream without delivering its answer");
3231 (
3232 String::new(),
3233 o.duration_ms,
3234 Some(format!("the CLI dropped the stream ({why})")),
3235 None,
3236 )
3237 }
3238 AgentOutcome::Quota(o) => {
3239 self.state.quota.push(QuotaLoss::from_report(
3240 seat_key,
3241 "implement",
3242 o.quota.as_ref(),
3243 ));
3244 (
3245 String::new(),
3246 o.duration_ms,
3247 Some("rate limited (quota); produced no change".to_owned()),
3248 None,
3249 )
3250 }
3251 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3252 (String::new(), 0, Some(e), None)
3253 }
3254 };
3255
3256 let c = &mut self.state.candidates[i];
3259 if !exhausted_the_fallback_chain {
3260 c.agent = agent;
3261 }
3262 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3263 c.duration_ms = duration;
3264 c.failed = failed;
3265 c.verified_noop = verified_claim;
3266 self.state
3267 .candidate_preservation
3268 .insert(label, "awaiting persistence".to_owned());
3269 }
3270 self.state.save()?;
3271 self.persist_candidates().await?;
3272
3273 if resource_blocked {
3274 return self.stop_for_resource();
3275 }
3276 if selected_failed {
3277 self.state.status = RunStatus::Blocked;
3278 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3279 self.state.save()?;
3280 bail!(
3281 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3282 );
3283 }
3284 self.after_implement()
3285 }
3286
3287 async fn persist_candidates(&mut self) -> Result<()> {
3289 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3290 for label in labels {
3291 let i = self
3292 .state
3293 .candidates
3294 .iter()
3295 .position(|c| c.label == label)
3296 .context("pending candidate is missing")?;
3297 let c = self.state.candidates[i].clone();
3298 let interrupted = self.state.interrupted_implementations.contains(&label);
3299 let result = async {
3300 let wait = crate::repo_lock::WaitPolicy::new(
3301 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3302 || false,
3303 );
3304 let _coordination =
3305 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3306 if let Err(e) = git::snapshot_uncommitted(
3309 &c.worktree,
3310 &self
3311 .state
3312 .dir()
3313 .join("artifacts")
3314 .join(format!("candidate-{label}-recovery")),
3315 )
3316 .await
3317 {
3318 self.state.event(
3319 "preserve",
3320 format!("candidate {label} snapshot incomplete: {e:#}"),
3321 );
3322 }
3323 let rescued = git::rescue_commit(
3324 &c.worktree,
3325 &format!("magi: candidate {label} (uncommitted work)"),
3326 )
3327 .await?;
3328 self.state.note_withheld("implement", &rescued.withheld);
3329 let base = &self.state.base_commit;
3330 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3331 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3332 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3333 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3334 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3335 if c.empty && commits > 0 && !patch.trim().is_empty()
3336 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3337 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3338 || self.state.tally.is_some())
3339 {
3340 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3344 }
3345 let c = &mut self.state.candidates[i];
3346 c.commits = commits;
3347 c.stat = stat;
3348 c.files = files;
3349 let no_changes = commits == 0 || patch.trim().is_empty();
3350 c.empty = no_changes && !interrupted;
3351 if interrupted && no_changes {
3352 c.failed = None;
3353 c.verified_noop = None;
3354 }
3355 if !c.empty {
3356 c.failed = None;
3357 c.verified_noop = None;
3358 }
3359 self.state.event(
3360 "preserve",
3361 format!("candidate {label}: {files} files, {commits} commits"),
3362 );
3363 Ok::<_, anyhow::Error>(())
3364 }
3365 .await;
3366 match result {
3367 Ok(()) => {
3368 self.state.candidate_preservation.remove(&label);
3369 if !interrupted || self.state.candidates[i].files > 0 {
3370 self.state.interrupted_implementations.remove(&label);
3371 }
3372 }
3373 Err(e) => {
3374 let reason = format!("candidate {label} preservation failed: {e:#}");
3375 self.state.event("preserve", &reason);
3376 self.state.candidate_preservation.insert(label, reason);
3377 }
3378 }
3379 if self.state.candidate_preservation.is_empty()
3380 && self.state.status == RunStatus::Blocked
3381 && self.state.candidates.iter().all(|c| c.failed.is_none())
3382 {
3383 self.state.status = RunStatus::Implementing;
3387 self.state.event(
3388 "preserve",
3389 "candidate persistence recovered; implementation can continue",
3390 );
3391 }
3392 self.state.save()?;
3393 }
3394 if !self.state.candidate_preservation.is_empty() {
3395 self.state.status = RunStatus::Blocked;
3396 self.state.save()?;
3397 bail!(
3398 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3399 );
3400 }
3401 Ok(())
3402 }
3403
3404 async fn resume_undelivered(
3432 &mut self,
3433 results: &mut [(usize, SeatState, AgentOutcome)],
3434 sent: &[SeatJob],
3435 prompts: &Prompts,
3436 run_id: &str,
3437 ) {
3438 for (wi, seat, out) in results.iter_mut() {
3439 let Some(dropped) = (match &*out {
3440 AgentOutcome::Dropped(o) => o.dropped.clone(),
3441 _ => None,
3442 }) else {
3443 continue;
3444 };
3445 let Some(job) = sent.get(*wi) else { continue };
3446 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3448 self.state.event(
3449 "implement",
3450 format!(
3451 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3452 work is in the tree",
3453 seat.key, dropped.output_tokens, dropped.why
3454 ),
3455 );
3456 continue;
3457 }
3458 if !has_context(&job.spec, seat, job.sessions) {
3466 self.state.event(
3467 "implement",
3468 format!(
3469 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3470 is no session left to resume",
3471 seat.key, dropped.output_tokens, dropped.why
3472 ),
3473 );
3474 continue;
3475 }
3476 self.state.event(
3477 "implement",
3478 format!(
3479 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3480 conversation",
3481 seat.key, dropped.output_tokens, dropped.why
3482 ),
3483 );
3484 let mut retry = job.clone();
3485 retry.seat = seat.clone();
3486 retry.prompt = prompt::resume_after_drop(&dropped.why);
3487 retry.timeout = retry_budget(job.timeout, true);
3488 retry.stem = format!("{}-resume", job.stem);
3489 let cache = self.state.config.cache_dir();
3490 let ctx = WaveCtx {
3491 carry_seats: false,
3492 pauses: Some((&self.pause, &self.interrupt)),
3493 run: run_id,
3494 node: "implement",
3495 prompts,
3496 cache: cache.as_deref(),
3497 round: None,
3498 };
3499 let (resumed_seat, resumed) =
3500 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3501 *seat = resumed_seat;
3502 *out = resumed;
3503 }
3504 }
3505
3506 async fn resume_seat_handovers(
3571 &mut self,
3572 results: &mut [(usize, SeatState, AgentOutcome)],
3573 sent: &mut [SeatJob],
3574 prompts: &Prompts,
3575 run_id: &str,
3576 ) -> Result<()> {
3577 let instruction = seeded_instruction(&self.state);
3578 let language = self.state.config.graph.language.clone();
3579 let github_language = self.state.config.graph.github_language.clone();
3580 let brief = self
3581 .state
3582 .advice
3583 .as_ref()
3584 .and_then(|a| a.synthesis.as_deref())
3585 .map(str::to_owned);
3586 let attachments = self.state.attachments.clone();
3587 for (wi, seat, out) in results.iter_mut() {
3588 let others: BTreeSet<String> = sent
3591 .iter()
3592 .enumerate()
3593 .filter(|(j, _)| j != wi)
3594 .map(|(_, j)| j.spec.id.clone())
3595 .collect();
3596 let Some(job) = sent.get_mut(*wi) else {
3597 continue;
3598 };
3599 let start = self
3604 .roles
3605 .implementer_roster
3606 .iter()
3607 .position(|s| s.id == job.spec.id)
3608 .unwrap_or(0);
3609 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3610 let mut fallback_attempt = 0usize;
3611 let mut prev: Option<FailClass> = None;
3612 while let Some(cur) = FailClass::of(&*out) {
3613 if !should_hand_over(prev.as_ref(), &cur) {
3614 break;
3615 }
3616 let Some(next) =
3617 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3618 .cloned()
3619 else {
3620 break;
3621 };
3622 tried.insert(next.id.clone());
3623 fallback_attempt += 1;
3624
3625 let coordination = self.coordinate(&job.cwd).await?;
3626 if let Ok(r) = git::rescue_commit(
3627 &job.cwd,
3628 &format!(
3629 "magi: candidate {} (uncommitted work before {} fallback)",
3630 seat.key,
3631 if cur == FailClass::Quota {
3632 "quota"
3633 } else {
3634 "handover"
3635 }
3636 ),
3637 )
3638 .await
3639 {
3640 self.state.note_withheld("implement", &r.withheld);
3641 }
3642 drop(coordination);
3643
3644 record_handover(
3645 &mut self.state,
3646 "implement",
3647 &seat.key,
3648 &seat.agent,
3649 &next.id,
3650 &cur,
3651 &fail_reason(&*out),
3652 );
3653 prev = Some(cur.clone());
3654
3655 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3656 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3657 job.spec = next.clone();
3665 let mut retry = job.clone();
3666 retry.seat = new_seat;
3667 retry.prompt = prompt::implement_in(
3668 &instruction,
3669 &job.cwd.to_string_lossy(),
3670 &language,
3671 &github_language,
3672 brief.as_deref(),
3673 &attachments,
3674 );
3675 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3676 let cache = self.state.config.cache_dir();
3677 let ctx = WaveCtx {
3678 carry_seats: false,
3679 pauses: Some((&self.pause, &self.interrupt)),
3680 run: run_id,
3681 node: "implement",
3682 prompts,
3683 cache: cache.as_deref(),
3684 round: None,
3685 };
3686 let (fallback_seat, fallback_out) = run_one(
3687 retry,
3688 Arc::clone(&self.sem),
3689 &ctx,
3690 &mut self.state,
3691 fallback_attempt,
3692 )
3693 .await;
3694 *seat = fallback_seat;
3695 *out = fallback_out;
3696 }
3697 }
3698 Ok(())
3699 }
3700
3701 async fn resume_unconfirmed_commands(
3725 &mut self,
3726 results: &mut [(usize, SeatState, AgentOutcome)],
3727 sent: &[SeatJob],
3728 prompts: &Prompts,
3729 run_id: &str,
3730 ) {
3731 for (wi, seat, out) in results.iter_mut() {
3732 let AgentOutcome::Ok(o) = &*out else {
3733 continue;
3734 };
3735 if !has_unconfirmed_command(&o.commands) {
3736 continue;
3737 }
3738 let Some(job) = sent.get(*wi) else { continue };
3739 if !has_context(&job.spec, seat, job.sessions) {
3740 self.state.event(
3741 "implement",
3742 format!(
3743 "{}: the reply named a command whose own CLI never confirmed the exit \
3744 status of, but there is no session left to resume",
3745 seat.key
3746 ),
3747 );
3748 continue;
3749 }
3750 self.state.event(
3751 "implement",
3752 format!(
3753 "{}: the reply named a command whose own CLI never confirmed the exit \
3754 status of; resuming the conversation",
3755 seat.key
3756 ),
3757 );
3758 let mut retry = job.clone();
3759 retry.seat = seat.clone();
3760 retry.prompt = prompt::resume_incomplete(
3761 "a command in your last reply had no confirmed exit status",
3762 );
3763 retry.timeout = retry_budget(job.timeout, true);
3764 retry.stem = format!("{}-confirm", job.stem);
3765 let cache = self.state.config.cache_dir();
3766 let ctx = WaveCtx {
3767 carry_seats: false,
3768 pauses: Some((&self.pause, &self.interrupt)),
3769 run: run_id,
3770 node: "implement",
3771 prompts,
3772 cache: cache.as_deref(),
3773 round: None,
3774 };
3775 let (resumed_seat, resumed) =
3776 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3777 *seat = resumed_seat;
3778 *out = resumed;
3779 }
3780 }
3781
3782 async fn continue_fix_report(
3803 &mut self,
3804 mut seat: SeatState,
3805 parse_err: String,
3806 job: &SeatJob,
3807 prompts: &Prompts,
3808 run_id: &str,
3809 round: usize,
3810 ) -> (
3811 SeatState,
3812 Option<FixReport>,
3813 Option<String>,
3814 ContinuationRecord,
3815 ) {
3816 let mut last_err = parse_err;
3817 let mut cumulative_wait_ms = 0u64;
3818 let mut attempts = 0usize;
3819 loop {
3820 if !has_context(&job.spec, &seat, job.sessions) {
3821 self.state.event(
3822 "fix",
3823 format!(
3824 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3825 session left to resume into"
3826 ),
3827 );
3828 let outcome = if attempts == 0 {
3829 ContinuationOutcome::NoSession
3830 } else {
3831 ContinuationOutcome::Exhausted
3832 };
3833 return (
3834 seat,
3835 None,
3836 Some(format!("unparsable fix report: {last_err}")),
3837 ContinuationRecord {
3838 attempts,
3839 cumulative_wait_ms,
3840 outcome,
3841 },
3842 );
3843 }
3844 if attempts >= MAX_FIX_CONTINUATIONS {
3845 self.state.event(
3846 "fix",
3847 format!(
3848 "round {round}: fixer's reply still had no adoption report after \
3849 {attempts} continuation(s) ({last_err}); giving up"
3850 ),
3851 );
3852 return (
3853 seat,
3854 None,
3855 Some(format!(
3856 "unparsable fix report after {attempts} continuation(s): {last_err}"
3857 )),
3858 ContinuationRecord {
3859 attempts,
3860 cumulative_wait_ms,
3861 outcome: ContinuationOutcome::Exhausted,
3862 },
3863 );
3864 }
3865 attempts += 1;
3866 self.state.event(
3867 "fix",
3868 format!(
3869 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3870 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3871 ),
3872 );
3873 let mut retry = job.clone();
3874 retry.seat = seat.clone();
3875 retry.prompt = prompt::resume_incomplete(&last_err);
3876 retry.timeout = retry_budget(job.timeout, true);
3877 retry.stem = format!("{}-continue{attempts}", job.stem);
3878 let cache = self.state.config.cache_dir();
3879 let ctx = WaveCtx {
3880 carry_seats: false,
3881 pauses: Some((&self.pause, &self.interrupt)),
3882 run: run_id,
3883 node: "fix",
3884 prompts,
3885 cache: cache.as_deref(),
3886 round: Some(round),
3887 };
3888 let (resumed_seat, resumed_out) = run_one(
3889 retry,
3890 Arc::clone(&self.sem),
3891 &ctx,
3892 &mut self.state,
3893 attempts,
3894 )
3895 .await;
3896 seat = resumed_seat;
3897 match resumed_out {
3898 AgentOutcome::Ok(o) => {
3899 cumulative_wait_ms += o.duration_ms;
3900 match verdict::extract_json::<FixReport>(&o.text) {
3901 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3902 self.state.event(
3903 "fix",
3904 format!(
3905 "round {round}: fixer's adoption report recovered after \
3906 {attempts} continuation(s)"
3907 ),
3908 );
3909 return (
3910 seat,
3911 Some(report),
3912 None,
3913 ContinuationRecord {
3914 attempts,
3915 cumulative_wait_ms,
3916 outcome: ContinuationOutcome::Resumed,
3917 },
3918 );
3919 }
3920 Ok(_) => {
3928 last_err = "the reply parsed, but it reported a command whose own CLI \
3929 never confirmed an exit status"
3930 .to_owned();
3931 }
3932 Err(e) => last_err = e.to_string(),
3933 }
3934 }
3935 AgentOutcome::Quota(o) => {
3936 cumulative_wait_ms += o.duration_ms;
3937 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3938 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3939 self.state.quota.push(loss);
3940 self.state.event(
3941 "fix",
3942 format!(
3943 "round {round}: continuation rate limited (quota); not retrying now"
3944 ),
3945 );
3946 return (
3947 seat,
3948 None,
3949 Some("rate limited (quota) while recovering the fix report".to_owned()),
3950 ContinuationRecord {
3951 attempts,
3952 cumulative_wait_ms,
3953 outcome: ContinuationOutcome::QuotaLost,
3954 },
3955 );
3956 }
3957 AgentOutcome::Dropped(o) => {
3958 cumulative_wait_ms += o.duration_ms;
3959 let why = o
3960 .dropped
3961 .as_ref()
3962 .map(|d| d.why.as_str())
3963 .unwrap_or("the CLI ended the stream without delivering its answer");
3964 last_err = format!("the CLI dropped the stream ({why})");
3965 }
3966 AgentOutcome::Failed(e) => last_err = e,
3967 AgentOutcome::ResourceBlocked(e) => {
3970 return (
3971 seat,
3972 None,
3973 Some(format!("the continuation was not started: {e}")),
3974 ContinuationRecord {
3975 attempts: attempts.saturating_sub(1),
3976 cumulative_wait_ms,
3977 outcome: ContinuationOutcome::NoSession,
3978 },
3979 );
3980 }
3981 }
3982 }
3983 }
3984
3985 fn after_implement(&mut self) -> Result<()> {
3986 if !self.state.candidate_preservation.is_empty() {
3987 bail!("candidate preservation incomplete; judging is unsafe");
3988 }
3989 if self.state.leaks.is_empty() {
3991 let cfg = self.state.config.blind.clone();
3992 let mut leaks = Vec::new();
3993 for c in &self.state.candidates {
3994 let Some(patch) =
3995 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3996 else {
3997 continue;
3998 };
3999 leaks.extend(blind::scan(
4000 &format!("candidate {} patch", c.label),
4001 &patch,
4002 &cfg.vendor_tokens,
4003 ));
4004 }
4005 if !leaks.is_empty() {
4006 let summary = leaks
4007 .iter()
4008 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4009 .collect::<Vec<_>>()
4010 .join(", ");
4011 match cfg.on_leak {
4012 LeakPolicy::Fail => {
4013 self.state.status = RunStatus::Failed;
4014 self.state
4015 .event("blind", format!("vendor text in a patch: {summary}"));
4016 self.state.leaks = leaks;
4017 self.state.save()?;
4018 self.settle_questions();
4019 bail!(
4020 "blind.on_leak = \"fail\" and vendor text reached a \
4021 judged patch: {summary}"
4022 );
4023 }
4024 LeakPolicy::Redact => self.state.event(
4025 "blind",
4026 format!("redacting vendor text for judging: {summary}"),
4027 ),
4028 LeakPolicy::Warn => self.state.event(
4029 "blind",
4030 format!("vendor text present in a judged patch (shown as-is): {summary}"),
4031 ),
4032 }
4033 self.state.leaks = leaks;
4034 }
4035 }
4036
4037 if self.state.viable().is_empty() {
4038 if self.state.all_candidates_verified_noop() {
4039 self.state.status = RunStatus::VerifiedNoop;
4050 self.state.save()?;
4051 self.settle_questions();
4052 return Ok(());
4053 }
4054 self.state.status = RunStatus::Failed;
4055 self.state.save()?;
4056 self.settle_questions();
4057 bail!("no candidate produced a change; nothing to judge");
4058 }
4059 self.state.status = RunStatus::Judging;
4060 self.state.save()?;
4061 Ok(())
4062 }
4063
4064 async fn judge(&mut self) -> Result<()> {
4067 let run_id = self.state.id.clone();
4072 let prompts = self.state.config.prompts.clone();
4073 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4074 return Ok(());
4075 }
4076 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4077 if viable.len() == 1 {
4078 self.state.judge_skipped = true;
4085 self.state.event(
4086 "judge",
4087 format!(
4088 "only candidate {} produced a change; judging skipped",
4089 viable[0].label
4090 ),
4091 );
4092 self.state.save()?;
4093 return Ok(());
4094 }
4095 self.state.status = RunStatus::Judging;
4096
4097 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4098 let language = self.state.config.graph.language.clone();
4099 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4100 let sessions = self.state.config.graph.sessions;
4101 let artifacts = agent::artifacts_dir(&self.state.dir());
4102 let root = self.state.worktree_root();
4103 let base_short = short(&self.state.base_commit);
4104
4105 let mut jobs = Vec::new();
4106 let mut orders = Vec::new();
4107 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4108 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4109 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4110 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4111 let seat_key = format!("judge-{}", j + 1);
4112 let seat = self.seat(&seat_key, &spec.id);
4113 jobs.push(SeatJob {
4114 prompt: prompt::judge(
4115 &self.state.instruction,
4116 &views,
4117 self.roles.judges.len(),
4118 &base_short,
4119 &language,
4120 ),
4121 spec,
4122 seat,
4123 cwd: root.join(format!("judge-{}", j + 1)),
4124 timeout,
4125 allow_write: false,
4126 sessions,
4127 artifacts: artifacts.clone(),
4128 stem: format!("judge-{}", j + 1),
4129 handover: None,
4130 });
4131 }
4132
4133 self.state.event(
4134 "judge",
4135 format!(
4136 "{} judges ranking {} candidates blind",
4137 jobs.len(),
4138 viable.len()
4139 ),
4140 );
4141 let labels_for_check = labels.clone();
4142 let mut quota_losses = Vec::new();
4143 let cache = self.state.config.cache_dir();
4144 let ctx = WaveCtx {
4145 carry_seats: false,
4146 pauses: Some((&self.pause, &self.interrupt)),
4147 run: &run_id,
4148 node: "judge",
4149 prompts: &prompts,
4150 cache: cache.as_deref(),
4151 round: None,
4152 };
4153 let results = ask_json_wave::<Ranking>(
4154 jobs,
4155 Arc::clone(&self.sem),
4156 self.state.config.graph.retries,
4157 &self.roles.judge_roster,
4158 &ctx,
4159 &mut quota_losses,
4160 &mut self.state,
4161 &move |r: &Ranking| r.validate(&labels_for_check),
4162 )
4163 .await;
4164 if self.state.resource_block.is_some() {
4165 return self.stop_for_resource();
4166 }
4167 self.state.quota.extend(quota_losses);
4168
4169 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4170 let agent_id = seat.agent.clone();
4171 self.state.seats.insert(seat.key.clone(), seat);
4172 let mut record = Judgement {
4173 judge: j + 1,
4174 seat: format!("judge-{}", j + 1),
4175 agent: agent_id,
4176 ranking: Vec::new(),
4177 reasons: BTreeMap::new(),
4178 confidence: None,
4179 strengths: BTreeMap::new(),
4180 order: orders[j].clone(),
4181 failed: None,
4182 duration_ms: 0,
4183 };
4184 match res {
4185 Ok((ranking, out)) => {
4186 record.ranking = ranking.normalized();
4187 record.reasons = ranking.reasons;
4188 record.strengths = ranking.strengths;
4189 record.confidence = ranking.confidence;
4190 record.duration_ms = out.duration_ms;
4191 self.state.event(
4192 "judge",
4193 format!(
4194 "judge {} ranked {}",
4195 j + 1,
4196 record.ranking.iter().collect::<String>()
4197 ),
4198 );
4199 }
4200 Err(e) => {
4201 record.failed = Some(e.to_string());
4202 self.state
4203 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4204 }
4205 }
4206 self.state.judgements.push(record);
4207 self.state.save()?;
4208 }
4209 Ok(())
4210 }
4211
4212 async fn deliberate(&mut self) -> Result<()> {
4215 if self.hold_base_sync().await? {
4216 return Ok(());
4217 }
4218 let run_id = self.state.id.clone();
4223 let prompts = self.state.config.prompts.clone();
4224 let rounds = self.state.config.graph.deliberate_rounds;
4225 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4227 return Ok(());
4228 }
4229 let tops: Vec<char> = self
4230 .state
4231 .judgements
4232 .iter()
4233 .filter_map(|j| j.ranking.first().copied())
4234 .collect();
4235 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4236 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4237 self.state.event(
4238 "deliberate",
4239 format!("judges agreed on {} outright; no deliberation", tops[0]),
4240 );
4241 }
4242 self.state.status = RunStatus::Voting;
4243 self.state.save()?;
4244 return Ok(());
4245 }
4246
4247 self.state.status = RunStatus::Deliberating;
4248 self.state.event(
4249 "deliberate",
4250 format!(
4251 "split: first choices were {} — opening {rounds} round(s)",
4252 tops.iter().collect::<String>()
4253 ),
4254 );
4255
4256 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4257 let language = self.state.config.graph.language.clone();
4258 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4259 let sessions = self.state.config.graph.sessions;
4260 let artifacts = agent::artifacts_dir(&self.state.dir());
4261 let root = self.state.worktree_root();
4262 let base_short = short(&self.state.base_commit);
4263
4264 for round in self.state.deliberation.len() + 1..=rounds {
4268 let mut turns: Vec<DeliberationTurn> = Vec::new();
4269 let mut round_losses = Vec::new();
4270 let mut asked = 0;
4271 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4272 if self.state.judgements[j].failed.is_some() {
4273 continue;
4274 }
4275 let seat_key = format!("judge-{}", j + 1);
4276 let spec = self.occupant(&seat_key, spec);
4277 let mut seat = self.seat(&seat_key, &spec.id);
4278 let transcript = self.transcript(&turns, j);
4279 let build = |context: Option<&str>| {
4280 prompt::deliberate(
4281 &self.state.instruction,
4282 context,
4283 &transcript,
4284 round,
4285 rounds,
4286 &language,
4287 )
4288 };
4289 let block = self.candidate_block(&viable, &base_short);
4290 let full = build(Some(&block));
4291 let text = if has_context(&spec, &seat, sessions) {
4292 build(None)
4293 } else {
4294 full.clone()
4295 };
4296 let job = SeatJob {
4297 spec,
4298 seat: seat.clone(),
4299 prompt: text,
4300 cwd: root.join(format!("judge-{}", j + 1)),
4301 timeout,
4302 allow_write: false,
4303 sessions,
4304 artifacts: artifacts.clone(),
4305 stem: format!("delib-{round}-judge-{}", j + 1),
4306 handover: Some(full),
4307 };
4308 let cache = self.state.config.cache_dir();
4309 let ctx = WaveCtx {
4310 carry_seats: false,
4311 pauses: Some((&self.pause, &self.interrupt)),
4312 run: &run_id,
4313 node: "deliberate",
4314 prompts: &prompts,
4315 cache: cache.as_deref(),
4316 round: None,
4317 };
4318 let mut losses = Vec::new();
4322 asked += 1;
4323 let mut results = ask_wave_with::<String>(
4324 vec![job],
4325 Arc::clone(&self.sem),
4326 0,
4327 &self.roles.judge_roster,
4328 &ctx,
4329 &mut losses,
4330 &mut self.state,
4331 &|text: &str| {
4332 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4333 },
4334 )
4335 .await;
4336 if self.state.resource_block.is_some() {
4337 return self.stop_for_resource();
4338 }
4339 round_losses.extend(losses.iter().cloned());
4340 self.state.quota.extend(losses);
4341 let (updated, res, _) = results.pop().expect("one job in, one result out");
4342 seat = updated;
4343 let agent_id = seat.agent.clone();
4344 self.state.seats.insert(seat.key.clone(), seat);
4345 let body = match res {
4346 Ok((body, _)) => body,
4347 Err(e) => {
4350 self.state
4351 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4352 continue;
4353 }
4354 };
4355 let tentative = verdict::extract_json::<Position>(&body)
4356 .ok()
4357 .and_then(|p| p.tentative)
4358 .and_then(|s| s.trim().chars().next())
4359 .map(|c| c.to_ascii_uppercase());
4360 self.state.event(
4361 "deliberate",
4362 format!(
4363 "round {round}: judge {} now favours {}",
4364 j + 1,
4365 tentative.map_or("—".to_owned(), |c| c.to_string())
4366 ),
4367 );
4368 turns.push(DeliberationTurn {
4369 judge: j + 1,
4370 agent: agent_id,
4371 body: blind::sanitize_prose(&body, &self.state.config.blind),
4372 tentative,
4373 });
4374 }
4375 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4376 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4377 return self.stop_for_resource();
4378 }
4379 self.state
4380 .deliberation
4381 .push(DeliberationRound { round, turns });
4382 self.state.save()?;
4383 }
4384
4385 self.state.status = RunStatus::Voting;
4386 self.state.save()?;
4387 Ok(())
4388 }
4389
4390 async fn vote(&mut self) -> Result<()> {
4393 if self.hold_base_sync().await? {
4394 return Ok(());
4395 }
4396 let run_id = self.state.id.clone();
4401 let prompts = self.state.config.prompts.clone();
4402 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4403 return Ok(());
4404 }
4405 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4406 if viable.len() == 1 {
4407 return Ok(());
4408 }
4409 self.state.status = RunStatus::Voting;
4410
4411 let language = self.state.config.graph.language.clone();
4412 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4413 let sessions = self.state.config.graph.sessions;
4414 let artifacts = agent::artifacts_dir(&self.state.dir());
4415 let root = self.state.worktree_root();
4416 let base_short = short(&self.state.base_commit);
4417 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4418
4419 let mut jobs = Vec::new();
4420 let mut seats_at = Vec::new();
4421 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4422 if self
4423 .state
4424 .judgements
4425 .get(j)
4426 .is_some_and(|r| r.failed.is_some())
4427 {
4428 continue;
4429 }
4430 let seat_key = format!("judge-{}", j + 1);
4431 let spec = self.occupant(&seat_key, spec);
4432 let seat = self.seat(&seat_key, &spec.id);
4433 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4434 let text = if has_context(&spec, &seat, sessions) {
4435 prompt::final_vote(&viable, &language)
4436 } else {
4437 full.clone()
4438 };
4439 jobs.push(SeatJob {
4440 spec,
4441 seat,
4442 prompt: text,
4443 cwd: root.join(format!("judge-{}", j + 1)),
4444 timeout,
4445 allow_write: false,
4446 sessions,
4447 artifacts: artifacts.clone(),
4448 stem: format!("vote-judge-{}", j + 1),
4449 handover: Some(full),
4450 });
4451 seats_at.push(j);
4452 }
4453
4454 self.state.event(
4455 "vote",
4456 format!(
4457 "collecting {} final votes one by one, privately",
4458 jobs.len()
4459 ),
4460 );
4461 let allowed = viable.clone();
4462 let mut quota_losses = Vec::new();
4463 let cache = self.state.config.cache_dir();
4464 let ctx = WaveCtx {
4465 carry_seats: false,
4466 pauses: Some((&self.pause, &self.interrupt)),
4467 run: &run_id,
4468 node: "vote",
4469 prompts: &prompts,
4470 cache: cache.as_deref(),
4471 round: None,
4472 };
4473 let results = ask_json_wave::<FinalVote>(
4474 jobs,
4475 Arc::clone(&self.sem),
4476 self.state.config.graph.retries,
4477 &self.roles.judge_roster,
4478 &ctx,
4479 &mut quota_losses,
4480 &mut self.state,
4481 &move |v: &FinalVote| match v.label() {
4482 Some(c) if allowed.contains(&c) => Ok(()),
4483 other => bail!("vote {other:?} is not one of {allowed:?}"),
4484 },
4485 )
4486 .await;
4487 if self.state.resource_block.is_some() {
4488 return self.stop_for_resource();
4489 }
4490 self.state.quota.extend(quota_losses);
4491
4492 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4493 let agent_id = seat.agent.clone();
4494 self.state.seats.insert(seat.key.clone(), seat);
4495 let initial = self
4496 .state
4497 .judgements
4498 .get(j)
4499 .and_then(|r| r.ranking.first().copied());
4500 let mut record = VoteRecord {
4501 judge: j + 1,
4502 agent: agent_id,
4503 vote: None,
4504 reason: String::new(),
4505 changed: false,
4506 };
4507 match res {
4508 Ok((v, _)) => {
4509 record.vote = v.label();
4510 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4511 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4512 self.state.event(
4513 "vote",
4514 format!(
4515 "judge {} voted {}{}",
4516 j + 1,
4517 record.vote.unwrap_or('?'),
4518 if record.changed { " (changed)" } else { "" }
4519 ),
4520 );
4521 }
4522 Err(e) => {
4523 self.state
4524 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4525 }
4526 }
4527 self.state.votes.push(record);
4528 self.state.save()?;
4529 }
4530 Ok(())
4531 }
4532
4533 fn tally(&mut self) -> Result<()> {
4536 if self.state.tally.is_some() {
4537 return Ok(());
4538 }
4539 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4540 let tops: Vec<char> = self
4541 .state
4542 .judgements
4543 .iter()
4544 .filter_map(|j| j.ranking.first().copied())
4545 .collect();
4546 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4547
4548 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4551 let mut cast: Vec<char> = Vec::new();
4552 for (i, j) in self.state.judgements.iter().enumerate() {
4553 let vote = self
4554 .state
4555 .votes
4556 .iter()
4557 .find(|v| v.judge == i + 1)
4558 .and_then(|v| v.vote)
4559 .or_else(|| j.ranking.first().copied());
4560 if let Some(v) = vote {
4561 *first_choice.entry(v).or_insert(0) += 1;
4562 cast.push(v);
4563 }
4564 }
4565
4566 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4567 for j in &self.state.judgements {
4568 let n = j.ranking.len();
4569 for (pos, label) in j.ranking.iter().enumerate() {
4570 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4571 }
4572 }
4573
4574 let best = first_choice.values().copied().max().unwrap_or(0);
4575 let mut leaders: Vec<char> = first_choice
4576 .iter()
4577 .filter(|(_, v)| **v == best)
4578 .map(|(k, _)| *k)
4579 .collect();
4580 let mut tie_break = None;
4581 if leaders.len() > 1 {
4582 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4583 let borda_leaders: Vec<char> = leaders
4584 .iter()
4585 .copied()
4586 .filter(|l| borda[l] == top_borda)
4587 .collect();
4588 tie_break = Some(if borda_leaders.len() == 1 {
4589 format!(
4590 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4591 leaders.len()
4592 )
4593 } else {
4594 format!(
4595 "{} way tie on both first-choice votes and Borda points, broken by label order",
4596 leaders.len()
4597 )
4598 });
4599 leaders = borda_leaders;
4600 leaders.sort_unstable();
4601 }
4602 let winner = *leaders
4603 .first()
4604 .or(viable.first())
4605 .context("no candidate to declare a winner from")?;
4606
4607 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4608 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4609 let deliberated = !self.state.deliberation.is_empty();
4610
4611 let quota_seats: std::collections::BTreeSet<&str> =
4615 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4616 let mut present = 0usize;
4617 for (i, j) in self.state.judgements.iter().enumerate() {
4618 if quota_seats.contains(j.seat.as_str()) {
4619 continue;
4620 }
4621 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4622 let voted = self
4623 .state
4624 .votes
4625 .iter()
4626 .any(|v| v.judge == i + 1 && v.vote.is_some());
4627 if ranked || voted {
4628 present += 1;
4629 }
4630 }
4631 let needs_quorum = viable.len() > 1;
4637 let judges_total = if needs_quorum {
4638 self.roles.judges.len()
4639 } else {
4640 0
4641 };
4642 let quorum = if needs_quorum {
4643 judges_total / 2 + 1
4644 } else {
4645 0
4646 };
4647 let met_quorum = !needs_quorum || present >= quorum;
4648 let uncontested = (!needs_quorum).then(|| {
4649 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4650 });
4651
4652 self.state.event(
4653 "tally",
4654 match &uncontested {
4655 Some(reason) => format!("winner {winner} — {reason}"),
4656 None => format!(
4657 "winner {winner} — votes {} | initial {} | {} changed | \
4658 {present}/{judges_total} judges{}",
4659 first_choice
4660 .iter()
4661 .map(|(k, v)| format!("{k}:{v}"))
4662 .collect::<Vec<_>>()
4663 .join(" "),
4664 if unanimous_initial {
4665 "unanimous"
4666 } else {
4667 "split"
4668 },
4669 changed_votes,
4670 if met_quorum {
4671 String::new()
4672 } else {
4673 format!(" — below quorum ({quorum} required)")
4674 },
4675 ),
4676 },
4677 );
4678 if !met_quorum {
4679 self.state.event(
4680 "stall",
4681 format!(
4682 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4683 the run stops here, resumable"
4684 ),
4685 );
4686 }
4687 self.state.tally = Some(Tally {
4688 first_choice,
4689 borda,
4690 winner,
4691 rankings: tops.len(),
4692 unanimous_initial,
4693 deliberated,
4694 changed_votes,
4695 unanimous_final,
4696 tie_break,
4697 judges: judges_total,
4698 present,
4699 quorum,
4700 met_quorum,
4701 uncontested,
4702 });
4703 self.state.status = if met_quorum {
4704 RunStatus::Reviewing
4705 } else {
4706 RunStatus::Stalled
4707 };
4708 self.state.save()?;
4709 Ok(())
4710 }
4711
4712 #[allow(clippy::too_many_lines)]
4733 async fn recover_stall(&mut self) -> Result<bool> {
4734 let run_id = self.state.id.clone();
4739 let prompts = self.state.config.prompts.clone();
4740 let quota_seats: BTreeSet<&str> =
4745 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4746 let absent: Vec<String> = self
4747 .state
4748 .judgements
4749 .iter()
4750 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4751 .map(|j| j.seat.clone())
4752 .collect();
4753 if absent.is_empty() {
4754 return Ok(false);
4755 }
4756 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4757 if viable.len() <= 1 {
4758 return Ok(false);
4759 }
4760 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4761 let language = self.state.config.graph.language.clone();
4762 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4763 let sessions = self.state.config.graph.sessions;
4764 let artifacts = agent::artifacts_dir(&self.state.dir());
4765 let root = self.state.worktree_root();
4766 let base_short = short(&self.state.base_commit);
4767 let candidates: Vec<Candidate> = viable.clone();
4768
4769 let mut positions: Vec<usize> = absent
4771 .iter()
4772 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4773 .collect();
4774 if positions.is_empty() {
4775 return Ok(false);
4776 }
4777 positions.sort_unstable();
4778 positions.dedup();
4779
4780 let mut judge_jobs = Vec::new();
4782 for &j in &positions {
4783 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4784 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4785 let seat_key = format!("judge-{}", j + 1);
4786 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4787 let seat = self.seat(&seat_key, &spec.id);
4788 judge_jobs.push(SeatJob {
4789 spec,
4790 seat,
4791 prompt: prompt::judge(
4792 &self.state.instruction,
4793 &views,
4794 self.roles.judges.len(),
4795 &base_short,
4796 &language,
4797 ),
4798 cwd: root.join(seat_key),
4799 timeout,
4800 allow_write: false,
4801 sessions,
4802 artifacts: artifacts.clone(),
4803 stem: format!("judge-{}-recover", j + 1),
4804 handover: None,
4805 });
4806 }
4807
4808 let labels_for_check = labels.clone();
4809 let mut judge_losses = Vec::new();
4810 let retries = self.state.config.graph.retries;
4811 let cache = self.state.config.cache_dir();
4812 let ctx = WaveCtx {
4813 carry_seats: false,
4814 pauses: Some((&self.pause, &self.interrupt)),
4815 run: &run_id,
4816 node: "judge",
4817 prompts: &prompts,
4818 cache: cache.as_deref(),
4819 round: None,
4820 };
4821 let results = ask_json_wave::<Ranking>(
4822 judge_jobs,
4823 Arc::clone(&self.sem),
4824 retries,
4825 &self.roles.judge_roster,
4826 &ctx,
4827 &mut judge_losses,
4828 &mut self.state,
4829 &move |r: &Ranking| r.validate(&labels_for_check),
4830 )
4831 .await;
4832 if self.state.resource_block.is_some() {
4833 self.stop_for_resource()?;
4834 return Ok(false);
4835 }
4836
4837 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4839 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4840 let agent_id = seat.agent.clone();
4841 self.state.seats.insert(seat.key.clone(), seat);
4842 let record = &mut self.state.judgements[j];
4843 match res {
4844 Ok((ranking, out)) => {
4845 record.agent = agent_id;
4846 record.ranking = ranking.normalized();
4847 record.reasons = ranking.reasons;
4848 record.strengths = ranking.strengths;
4849 record.confidence = ranking.confidence;
4850 record.failed = None;
4851 record.duration_ms = out.duration_ms;
4852 recovered.insert(j);
4853 self.state.event(
4854 "recover",
4855 format!("judge {} ranked again after the limit", j + 1),
4856 );
4857 }
4858 Err(e) => {
4859 self.state
4860 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4861 }
4862 }
4863 }
4864
4865 let mut vote_jobs = Vec::new();
4867 let mut vote_pos: Vec<usize> = Vec::new();
4868 for &j in &recovered {
4869 let seat_key = format!("judge-{}", j + 1);
4870 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4871 let seat = self.seat(&seat_key, &spec.id);
4872 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4873 let text = if has_context(&spec, &seat, sessions) {
4874 prompt::final_vote(&labels, &language)
4875 } else {
4876 full.clone()
4877 };
4878 vote_jobs.push(SeatJob {
4879 spec,
4880 seat,
4881 prompt: text,
4882 cwd: root.join(seat_key),
4883 timeout,
4884 allow_write: false,
4885 sessions,
4886 artifacts: artifacts.clone(),
4887 stem: format!("vote-judge-{}-recover", j + 1),
4888 handover: Some(full),
4889 });
4890 vote_pos.push(j);
4891 }
4892 let allowed = labels.clone();
4893 let mut vote_losses = Vec::new();
4894 let vote_retries = self.state.config.graph.retries;
4895 let vote_cache = self.state.config.cache_dir();
4896 let ctx = WaveCtx {
4897 carry_seats: false,
4898 pauses: Some((&self.pause, &self.interrupt)),
4899 run: &run_id,
4900 node: "vote",
4901 prompts: &prompts,
4902 cache: vote_cache.as_deref(),
4903 round: None,
4904 };
4905 let votes = ask_json_wave::<FinalVote>(
4906 vote_jobs,
4907 Arc::clone(&self.sem),
4908 vote_retries,
4909 &self.roles.judge_roster,
4910 &ctx,
4911 &mut vote_losses,
4912 &mut self.state,
4913 &move |v: &FinalVote| match v.label() {
4914 Some(c) if allowed.contains(&c) => Ok(()),
4915 other => bail!("vote {other:?} is not one of {allowed:?}"),
4916 },
4917 )
4918 .await;
4919 if self.state.resource_block.is_some() {
4920 self.stop_for_resource()?;
4921 return Ok(false);
4922 }
4923 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4924 let agent_id = seat.agent.clone();
4925 self.state.seats.insert(seat.key.clone(), seat);
4926 match res {
4927 Ok((v, _)) => {
4928 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4929 rec.vote = v.label();
4930 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4931 } else {
4932 self.state.votes.push(VoteRecord {
4933 judge: j + 1,
4934 agent: agent_id,
4935 vote: v.label(),
4936 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4937 changed: false,
4938 });
4939 }
4940 self.state.event(
4941 "recover",
4942 format!("judge {} voted again after the limit", j + 1),
4943 );
4944 }
4945 Err(e) => {
4946 self.state
4947 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4948 }
4949 }
4950 }
4951
4952 let recovered_keys: BTreeSet<String> = recovered
4956 .iter()
4957 .map(|&j| format!("judge-{}", j + 1))
4958 .collect();
4959 self.state
4960 .quota
4961 .retain(|q| !recovered_keys.contains(&q.seat));
4962 for loss in judge_losses.into_iter().chain(vote_losses) {
4966 if recovered_keys.contains(&loss.seat) {
4967 continue;
4968 }
4969 self.state.quota.retain(|q| q.seat != loss.seat);
4970 self.state.quota.push(loss);
4971 }
4972
4973 self.state.tally = None;
4975 self.tally()?;
4976 Ok(self
4977 .state
4978 .tally
4979 .as_ref()
4980 .map(|t| t.met_quorum)
4981 .unwrap_or(false))
4982 }
4983
4984 async fn graft(&mut self) -> Result<()> {
5000 let Err(e) = self.graft_inner().await else {
5001 return Ok(());
5002 };
5003 if e.is::<crate::repo_lock::ResourceBlocked>() {
5004 return Err(e);
5005 }
5006 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5007 self.state.status = RunStatus::Blocked;
5008 self.state.event("graft", format!("blocked: {e:#}"));
5009 self.state.save()?;
5010 return Err(e);
5011 }
5012 if let Some(rec) = self.state.graft.clone()
5017 && !rec.outcome.settled()
5018 {
5019 if let Some(winner) = self.state.winner().cloned() {
5020 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5021 if re.is::<crate::repo_lock::ResourceBlocked>() {
5022 return Err(re);
5023 }
5024 self.state.status = RunStatus::Blocked;
5025 self.state.event("graft", format!("blocked: {re:#}"));
5026 self.state.save()?;
5027 return Err(re);
5028 }
5029 }
5030 if let Some(g) = self.state.graft.as_mut() {
5031 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5032 }
5033 }
5034 self.state.event("graft", format!("skipped: {e:#}"));
5035 if let Err(e) = self.state.save() {
5036 tracing::warn!("save after graft: {e:#}");
5037 }
5038 Ok(())
5039 }
5040
5041 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5044 let _coordination = self.coordinate(wt).await?;
5045 crate::recovery::restore_repair(wt, pre)
5046 .await
5047 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5048 }
5049
5050 fn graft_skipped(&mut self, why: &str) {
5052 let message = format!("skipped: {why}");
5053 if !self
5054 .state
5055 .events
5056 .iter()
5057 .any(|e| e.node == "graft" && e.message == message)
5058 {
5059 self.state.event("graft", message);
5060 }
5061 }
5062
5063 async fn graft_inner(&mut self) -> Result<()> {
5064 if let Some(rec) = self.state.graft.clone()
5067 && !rec.outcome.settled()
5068 {
5069 if self.state.reviews.is_empty()
5073 && let Some(winner) = self.state.winner().cloned()
5074 {
5075 self.graft_restore(&winner.worktree, &rec.pre).await?;
5076 }
5077 if rec.outcome == GraftOutcome::WaitingForRestore {
5078 self.state.graft = None;
5079 self.state.save()?;
5080 } else {
5081 if let Some(g) = self.state.graft.as_mut() {
5082 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5083 }
5084 self.state.event(
5085 "graft",
5086 "rolled back: the previous round was interrupted; the winner is untouched",
5087 );
5088 self.state.save()?;
5089 return Ok(());
5090 }
5091 }
5092 let plan = match graft::plan(&self.state) {
5093 Ok(plan) => plan,
5094 Err(skip) => {
5095 if skip != graft::Skip::AlreadyDone {
5096 self.graft_skipped(skip.reason());
5097 self.state.save()?;
5098 }
5099 return Ok(());
5100 }
5101 };
5102 let Some(winner) = self.state.winner().cloned() else {
5103 return Ok(());
5104 };
5105 let wt = winner.worktree.clone();
5106 if !git::is_clean(&wt).await? {
5107 self.graft_skipped("the winner's worktree has uncommitted changes");
5108 self.state.save()?;
5109 return Ok(());
5110 }
5111 let pre = git::rev_parse(&wt, "HEAD").await?;
5112 self.state.graft = Some(GraftRecord {
5115 pre: pre.clone(),
5116 post: None,
5117 lines: None,
5118 limit: None,
5119 agent: None,
5120 outcome: GraftOutcome::Running,
5121 requested: graft::requested(&plan),
5122 adoption: None,
5123 adoption_error: None,
5124 });
5125 self.state.event(
5126 "graft",
5127 format!(
5128 "porting strengths of candidate(s) {} onto {}",
5129 plan.strengths
5130 .iter()
5131 .map(|(l, _, _)| l.to_string())
5132 .collect::<Vec<_>>()
5133 .join(", "),
5134 plan.winner
5135 ),
5136 );
5137 self.state.save()?;
5138
5139 let fix_prompt = prompt::graft(
5140 &self.state.instruction,
5141 plan.winner,
5142 &plan.strengths,
5143 &self.state.config.graph.language,
5144 );
5145 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5146 let sessions = self.state.config.graph.sessions;
5147 let artifacts = agent::artifacts_dir(&self.state.dir());
5148 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5149 let (_, seat, out) = self
5150 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5151 SeatJob {
5152 prompt: fix_prompt.clone(),
5153 spec,
5154 seat,
5155 cwd: wt.clone(),
5156 timeout,
5157 allow_write: true,
5158 sessions,
5159 artifacts: artifacts.clone(),
5160 stem: "graft".to_owned(),
5161 handover: None,
5162 }
5163 })
5164 .await;
5165 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5166 self.state.graft.as_mut().expect("graft record").outcome =
5169 GraftOutcome::WaitingForRestore;
5170 return self.stop_for_resource();
5171 }
5172 let mut why: Option<String> = None;
5173 match &out {
5174 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5175 why = Some("the fixer produced nothing usable".to_owned());
5176 }
5177 AgentOutcome::Ok(_) => {}
5178 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5179 AgentOutcome::Quota(o) => {
5180 self.state.quota.push(QuotaLoss::from_report(
5181 seat.key.clone(),
5182 "graft",
5183 o.quota.as_ref(),
5184 ));
5185 why = Some("rate limited (quota)".to_owned());
5186 }
5187 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5188 why = Some(format!(
5189 "the fixer failed: {}",
5190 e.lines()
5191 .next()
5192 .unwrap_or("")
5193 .chars()
5194 .take(160)
5195 .collect::<String>()
5196 ));
5197 }
5198 }
5199 let agent_id = seat.agent.clone();
5200 self.state.seats.insert(seat.key.clone(), seat);
5201
5202 let mut post = None;
5203 let mut lines = None;
5204 let _coordination = self.coordinate(&wt).await?;
5205 if why.is_none() {
5206 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5207 if let Ok(r) = &rescued {
5208 self.state.note_withheld("graft", &r.withheld);
5209 }
5210 let head = git::rev_parse(&wt, "HEAD").await?;
5211 let dirty = match &rescued {
5215 Ok(r) => git::status_porcelain(&wt)
5216 .await
5217 .map(|st| {
5218 st.lines().any(|l| {
5219 !r.withheld
5220 .iter()
5221 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5222 })
5223 })
5224 .unwrap_or(true),
5225 Err(_) => true,
5226 };
5227 if dirty {
5228 why = Some("the fixer left changes that could not be committed".to_owned());
5229 } else if head == pre {
5230 why = Some("the fixer changed nothing".to_owned());
5231 } else {
5232 lines = git::changed_lines(&wt, &pre, &head).await;
5233 post = Some(head);
5234 }
5235 }
5236 let (adoption, adoption_error) = match &out {
5237 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5238 _ => (None, None),
5239 };
5240 let requested = graft::requested(&plan);
5241 let outcome = match why {
5242 None => {
5243 self.state.event(
5244 "graft",
5245 format!(
5246 "applied: {}",
5247 lines.map_or_else(
5248 || "changed lines unavailable".to_owned(),
5249 |n| format!("{n} changed lines")
5250 )
5251 ),
5252 );
5253 GraftOutcome::Applied
5254 }
5255 Some(why) => {
5256 crate::recovery::restore_repair(&wt, &pre)
5259 .await
5260 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5261 self.state.event(
5262 "graft",
5263 format!("rolled back: {why}; the winner is untouched"),
5264 );
5265 GraftOutcome::RolledBack(why)
5266 }
5267 };
5268 self.state.graft = Some(GraftRecord {
5269 pre,
5270 post,
5271 lines,
5272 limit: None,
5273 agent: Some(agent_id),
5274 outcome,
5275 requested,
5276 adoption,
5277 adoption_error,
5278 });
5279 self.state.save()?;
5280 Ok(())
5281 }
5282
5283 async fn fold_losers(&mut self) -> Result<()> {
5286 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5287 return Ok(());
5288 };
5289 if !self.state.candidate_preservation.is_empty() {
5290 bail!("candidate preservation incomplete; refusing cleanup");
5291 }
5292 let repo = self.state.repo.clone();
5293 let _coordination = self.coordinate(&repo).await?;
5294 let mut folded = Vec::new();
5295 for i in 0..self.state.candidates.len() {
5296 let c = &self.state.candidates[i];
5297 if c.label == winner || c.folded {
5298 continue;
5299 }
5300 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5301 remove_preserved_worktree(&repo, &wt).await?;
5302 if git::branch_exists(&repo, &branch).await?
5303 && !git::branch_delete(&repo, &branch).await?
5304 {
5305 bail!("candidate branch removal refused");
5306 }
5307 self.state.candidates[i].folded = true;
5308 self.state.save()?;
5309 folded.push(label.to_string());
5310 }
5311 let root = self.state.worktree_root();
5313 for j in 1..=self.roles.judges.len() {
5314 let wt = root.join(format!("judge-{j}"));
5315 if wt.exists() {
5316 remove_preserved_worktree(&repo, &wt).await?;
5317 }
5318 }
5319 if self.state.config.graph.advise {
5322 for k in 1..=self.state.config.graph.advisors {
5323 let wt = root.join(format!("advisor-{k}"));
5324 if wt.exists() {
5325 remove_preserved_worktree(&repo, &wt).await?;
5326 }
5327 }
5328 }
5329 if !folded.is_empty() {
5330 self.state
5331 .event("fold", format!("folded candidates {}", folded.join(", ")));
5332 self.state.save()?;
5333 }
5334 Ok(())
5335 }
5336
5337 async fn hold_base_sync(&mut self) -> Result<bool> {
5340 self.hold_base_sync_as(None).await
5341 }
5342
5343 async fn hold_base_sync_kind(
5344 &mut self,
5345 kind: crate::base_sync_recovery::StopKind,
5346 ) -> Result<()> {
5347 self.hold_base_sync_as(Some(kind)).await?;
5348 Ok(())
5349 }
5350
5351 async fn hold_base_sync_as(
5352 &mut self,
5353 kind: Option<crate::base_sync_recovery::StopKind>,
5354 ) -> Result<bool> {
5355 if !self.state.base_sync_blocked() {
5356 self.state.base_sync_recovery.stop = None;
5357 return Ok(false);
5358 }
5359 let head = match self.state.winner() {
5360 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5361 .await
5362 .unwrap_or_default(),
5363 None => String::new(),
5364 };
5365 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5366 && !authorization.consumed
5367 && authorization.stop.head != head
5368 {
5369 authorization.refuse(format!(
5370 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5371 authorization.stop.head
5372 ));
5373 }
5374 self.state.record_base_sync_stop(&head);
5375 if let Some(kind) = kind {
5376 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5377 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5378 stop.kind = kind;
5379 }
5380 }
5381 self.state.save()?;
5382 Ok(true)
5383 }
5384
5385 async fn sync_to_base(&mut self) -> Result<()> {
5427 if self.state.status == RunStatus::AlreadyInBase {
5428 return Ok(());
5429 }
5430 self.hold_base_sync().await?;
5431 let conflicted = self
5432 .state
5433 .base_sync
5434 .as_ref()
5435 .is_some_and(|s| s.conflict.is_some());
5436 let Some(winner) = self.state.winner().cloned() else {
5437 return Ok(());
5438 };
5439
5440 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5441
5442 let repo = self.state.repo.clone();
5443 let remote = self.state.config.merge.remote.clone();
5444 let base_branch = self.state.base_branch.clone();
5445 let tracking = format!("{remote}/{base_branch}");
5446
5447 let fetched_base =
5448 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5449 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5454 return Ok(());
5455 };
5456
5457 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5458 self.state.ensure_base_episode();
5459 let mut pending = self
5460 .state
5461 .base_sync_recovery
5462 .operations
5463 .last()
5464 .filter(|o| !o.finished)
5465 .cloned();
5466 let scratch = self.state.dir().join("base-sync");
5467 if git::rebase_in_progress(&scratch).await {
5468 if pending.is_none()
5471 && self.state.base_sync_recovery.operations.is_empty()
5472 && !conflicted
5473 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5474 .await
5475 .ok()
5476 .as_deref()
5477 == Some(head.as_str())
5478 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5479 && git::is_clean(&winner.worktree).await?
5480 {
5481 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5482 if self.state.base_sync.is_none() {
5483 self.state.base_sync = Some(BaseSync {
5484 tip: saved_tip.clone(),
5485 behind: 0,
5486 attempts: 0,
5487 conflict: None,
5488 already_in: None,
5489 });
5490 }
5491 self.state.reserve_base_operation(&head, &saved_tip);
5492 self.state.event(
5493 "base-recovery",
5494 "imported standing legacy rebase as a spent synchronization reservation",
5495 );
5496 self.state.save()?;
5497 pending = self.state.base_sync_recovery.operations.last().cloned();
5498 }
5499 let expected = pending.as_ref().map(|op| op.tip.as_str());
5500 if expected.is_none()
5501 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5502 {
5503 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5504 if self.state.base_sync.is_none() {
5505 self.state.base_sync = Some(BaseSync {
5506 tip: tip.clone(),
5507 behind: 0,
5508 attempts: 0,
5509 conflict: None,
5510 already_in: None,
5511 });
5512 }
5513 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5514 self.state.event("land", why);
5515 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5516 .await?;
5517 return Ok(());
5518 }
5519 }
5520 if let Some(op) = &pending {
5521 let mut replay_proven =
5523 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5524 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5525 let wait = repository_wait(&self.state, None);
5526 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5527 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5528 Ok(true) => {
5529 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5530 self.state.save()?;
5531 git::worktree_remove(&repo, &scratch).await.ok();
5532 replay_proven = true;
5533 }
5534 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5535 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5536 }
5537 }
5538 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5539 if self.state.foreign_checkout.is_some() {
5540 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5541 .await
5542 .ok()
5543 .as_deref()
5544 == Some(op.head.as_str())
5545 && git::rev_parse(&repo, &format!("{head}^2"))
5546 .await
5547 .ok()
5548 .as_deref()
5549 == Some(op.tip.as_str());
5550 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5551 .await
5552 .is_ok_and(|m| m.is_empty())
5553 && let Ok(added) =
5554 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5555 {
5556 replay_proven = true;
5557 for commit in added {
5558 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5559 replay_proven = false;
5560 break;
5561 }
5562 }
5563 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5564 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5565 }
5566 }
5567 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5571 || (head != op.head
5572 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5573 .await?
5574 .is_empty())
5575 || (!git::is_clean(&winner.worktree).await?
5576 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5577 .await
5578 .is_ok_and(|o| o.ok()))
5579 || !replay_proven
5580 {
5581 let why = format!(
5582 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5583 op.id
5584 );
5585 let sync = self
5586 .state
5587 .base_sync
5588 .as_mut()
5589 .context("reserved synchronization has no lifetime accounting")?;
5590 sync.conflict = Some(why.clone());
5591 self.state.event("land", why);
5592 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5593 .await?;
5594 return Ok(());
5595 }
5596 if head != op.head {
5597 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5598 }
5599 tip = op.tip.clone();
5600 }
5601 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5602 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5603 if pending.is_none()
5604 && !conflicted
5605 && fetched_base
5606 && self
5607 .state
5608 .base_sync_recovery
5609 .episodes
5610 .last()
5611 .is_some_and(|e| e.boundary.is_some())
5612 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5613 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5614 && owner.execution_runs().last() == Some(&self.state.id)
5615 && owner.status != crate::queue::TaskStatus::Done
5616 && !owner.fresh_start
5617 && owner.review_branch.is_none()
5618 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5619 && let Some(approval) = owner.resumption.as_ref()
5620 && approval.instruction == owner.instruction
5621 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5622 && git::is_clean(&winner.worktree).await?
5623 && self.state.renew_base_episode(&task, approval, &head, &tip)
5624 {
5625 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5626 self.state.save()?;
5627 }
5628
5629 if (behind > 0 || conflicted || head == tip)
5638 && self
5639 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5640 .await?
5641 {
5642 return Ok(());
5643 }
5644 if conflicted && self.state.base_recovery_pending() {
5645 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5646 let owner =
5647 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5648 let owner_valid = owner.as_ref().is_ok_and(|t| {
5649 t.execution_runs().last() == Some(&self.state.id)
5650 && t.status != crate::queue::TaskStatus::Done
5651 && !t.fresh_start
5652 && t.review_branch.is_none()
5653 });
5654 let invalidation = if !owner_valid {
5655 Some("latest task ownership changed")
5656 } else if !owner.as_ref().is_ok_and(|t| {
5657 authorization
5658 .generation
5659 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5660 }) {
5661 Some("owner approval generation changed")
5662 } else if authorization.question.as_ref().is_some_and(|question| {
5663 !crate::base_recovery_control::decision_current(
5664 &crate::queue::Queue::at(crate::run::home().join("queue")),
5665 &self.state,
5666 &authorization.task,
5667 question,
5668 )
5669 }) {
5670 Some("recovery decision ownership, routing or stop evidence changed")
5671 } else if authorization.request.is_some()
5672 && !owner
5673 .as_ref()
5674 .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5675 {
5676 Some("owner decision changed")
5677 } else if let Some(scope) = &authorization.scope {
5678 if scope.tip != tip {
5679 Some("fetched base changed")
5680 } else if !self
5681 .state
5682 .base_sync_recovery
5683 .episodes
5684 .last()
5685 .is_some_and(|e| e.id == scope.episode)
5686 {
5687 Some("active synchronization interval changed")
5688 } else if !owner
5689 .as_ref()
5690 .is_ok_and(|t| t.instruction == scope.instruction)
5691 {
5692 Some("task instruction changed")
5693 } else {
5694 None
5695 }
5696 } else {
5697 None
5698 };
5699 if let Some(reason) = invalidation {
5700 let authorization = self
5701 .state
5702 .base_sync_recovery
5703 .authorizations
5704 .last_mut()
5705 .unwrap();
5706 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5707 self.state.save()?;
5708 return Ok(());
5709 }
5710 if !fetched_base
5711 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5712 || !git::is_clean(&winner.worktree).await?
5713 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5714 {
5715 return Ok(());
5716 }
5717 }
5718 let recovery = conflicted && self.state.consume_base_recovery(&head);
5719 if conflicted {
5720 self.state.save()?;
5722 if !recovery {
5723 return Ok(());
5724 }
5725 }
5726
5727 if behind == 0 {
5728 if let Some(from) = self
5737 .state
5738 .rebase_fixes
5739 .iter()
5740 .rev()
5741 .find_map(|r| r.from.clone())
5742 && from != head
5743 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5744 .await
5745 .is_ok_and(|o| o.ok())
5746 {
5747 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5748 }
5749 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5755 if let Some(why) = &conflict {
5756 self.state.status = RunStatus::Blocked;
5757 self.state.event("land", why.clone());
5758 }
5759 self.state.base_sync = Some(BaseSync {
5760 tip,
5761 behind: 0,
5762 attempts,
5763 conflict,
5764 already_in: None,
5765 });
5766 if pending.is_some() {
5767 self.state.finish_base_operation(Some(&head));
5768 }
5769 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5770 .await?;
5771 self.state.save()?;
5772 return Ok(());
5773 }
5774
5775 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5776 let why = format!(
5777 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5778 synchronization(s); synchronizing again would only race it",
5779 winner.branch
5780 );
5781 self.state.status = RunStatus::Blocked;
5782 self.state.base_sync = Some(BaseSync {
5783 tip,
5784 behind,
5785 attempts,
5786 conflict: Some(why.clone()),
5787 already_in: None,
5788 });
5789 self.state.event("land", why);
5790 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5791 .await?;
5792 self.state.save()?;
5793 return Ok(());
5794 }
5795
5796 if self.state.foreign_checkout.is_some() {
5797 let wait = repository_wait(&self.state, None);
5800 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5801 if self.state.base_sync.is_none() {
5802 self.state.base_sync = Some(BaseSync {
5803 tip: tip.clone(),
5804 behind,
5805 attempts: 0,
5806 conflict: None,
5807 already_in: None,
5808 });
5809 }
5810 self.state.reserve_base_operation(&head, &tip);
5811 self.state.save()?;
5812 let merged = git::git_raw(
5813 &winner.worktree,
5814 &[
5815 "-c",
5816 "core.hooksPath=/dev/null",
5817 "merge",
5818 "--no-ff",
5819 "--no-autostash",
5820 "--no-edit",
5821 "-m",
5822 &format!("Merge {tracking} before verifying"),
5823 &tip,
5824 ],
5825 )
5826 .await?;
5827 let conflict = if merged.ok() {
5828 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5829 self.state.event(
5830 "land",
5831 format!("merged {tracking} into {} before verifying", winner.branch),
5832 );
5833 None
5834 } else {
5835 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5836 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5837 }
5838 let why = format!(
5839 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5840 winner.branch,
5841 merged.stderr.chars().take(600).collect::<String>()
5842 );
5843 self.state.status = RunStatus::Blocked;
5844 self.state.event("land", &why);
5845 Some(why)
5846 };
5847 let synced = conflict.is_none();
5848 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5849 self.state
5850 .finish_base_operation(synced.then_some(fresh.as_str()));
5851 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5852 self.state.base_sync = Some(BaseSync {
5853 tip,
5854 behind: if synced { 0 } else { behind },
5855 attempts,
5856 conflict,
5857 already_in: None,
5858 });
5859 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5860 .await?;
5861 self.state.save()?;
5862 return Ok(());
5863 }
5864
5865 self.state.event(
5866 "land",
5867 format!(
5868 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5869 winner.branch
5870 ),
5871 );
5872 self.state.save()?;
5873
5874 let branch_tracking = format!("{remote}/{}", winner.branch);
5880 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5881 let remote_tip = if self.state.task_pr_branch().is_none()
5882 && matches!(&fetched_branch, Ok(o) if o.ok())
5883 {
5884 git::rev_parse(&repo, &branch_tracking).await.ok()
5885 } else {
5886 None
5887 };
5888 if let Some(theirs) = &remote_tip
5892 && !git::is_ancestor(&repo, theirs, &head).await
5893 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5894 .await
5895 .is_ok_and(|missing| missing.is_empty())
5896 {
5897 let why = format!(
5898 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5899 them",
5900 short(theirs),
5901 winner.branch
5902 );
5903 self.state.status = RunStatus::Blocked;
5904 self.state.base_sync = Some(BaseSync {
5905 tip,
5906 behind,
5907 attempts,
5908 conflict: Some(why.clone()),
5909 already_in: None,
5910 });
5911 self.state.event("land", why);
5912 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5913 .await?;
5914 self.state.save()?;
5915 return Ok(());
5916 }
5917
5918 if self.state.base_sync.is_none() {
5919 self.state.base_sync = Some(BaseSync {
5920 tip: tip.clone(),
5921 behind,
5922 attempts: 0,
5923 conflict: None,
5924 already_in: None,
5925 });
5926 }
5927 self.state.reserve_base_operation(&head, &tip);
5928 self.state.save()?;
5929 let scratch = self.state.dir().join("base-sync");
5930 let rebased =
5931 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5932 .await
5933 {
5934 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5935 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5936 Err(e) => Err(e),
5937 };
5938 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5939 let stop_kind = match &rebased {
5940 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5941 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5942 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5943 };
5944 match rebased {
5945 Ok(None) => {
5946 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5950 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5951 self.state.finish_base_operation(Some(&fresh));
5952 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5953 let mut conflict = None;
5954 if let Some(pinned) = &remote_tip {
5955 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5956 match pushed {
5957 Ok(o) if o.ok() => self.state.event(
5958 "land",
5959 format!("pushed rebased {} to {remote}", winner.branch),
5960 ),
5961 Ok(o) => {
5962 conflict = Some(format!(
5963 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5964 winner.branch,
5965 short(pinned),
5966 o.stderr.chars().take(600).collect::<String>()
5967 ));
5968 }
5969 Err(e) => {
5970 conflict = Some(format!(
5971 "rebased {} locally but could not push it: {e:#}",
5972 winner.branch
5973 ));
5974 }
5975 }
5976 }
5977 if let Some(why) = &conflict {
5978 self.state.status = RunStatus::Blocked;
5979 self.state.event("land", why.clone());
5980 }
5981 self.state.base_sync = Some(BaseSync {
5982 tip: tip.clone(),
5983 behind: 0,
5984 attempts,
5985 conflict,
5986 already_in: None,
5987 });
5988 self.state
5989 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5990 }
5991 Ok(Some(conflict)) => {
5992 self.state.finish_base_operation(None);
5993 let why = format!(
5994 "{} conflicts with {tracking} and did not rebase: {}",
5995 winner.branch,
5996 conflict.chars().take(600).collect::<String>()
5997 );
5998 self.state.status = RunStatus::Blocked;
5999 self.state.base_sync = Some(BaseSync {
6000 tip,
6001 behind,
6002 attempts,
6003 conflict: Some(why.clone()),
6004 already_in: None,
6005 });
6006 self.state.event("land", why);
6007 }
6008 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6012 self.state.save()?;
6013 return Ok(());
6014 }
6015 Err(e) => {
6016 self.state.finish_base_operation(None);
6017 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6018 self.state.status = RunStatus::Blocked;
6019 self.state.base_sync = Some(BaseSync {
6020 tip,
6021 behind,
6022 attempts,
6023 conflict: Some(why.clone()),
6024 already_in: None,
6025 });
6026 self.state.event("land", why);
6027 }
6028 }
6029 if !self.state.base_sync_blocked() {
6030 self.state.base_sync_recovery.stop = None;
6031 }
6032 self.hold_base_sync_kind(stop_kind).await?;
6033 self.state.save()?;
6034 Ok(())
6035 }
6036
6037 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6043 if self.state.task_pr_branch().is_some() {
6046 return None;
6047 }
6048 let repo = self.state.repo.clone();
6049 let remote = self.state.config.merge.remote.clone();
6050 let fetched = git::fetch(&repo, &remote, branch).await;
6051 if !matches!(&fetched, Ok(o) if o.ok()) {
6052 return None;
6053 }
6054 let branch_tracking = format!("{remote}/{branch}");
6055 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6056 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6057 return None;
6058 }
6059 if !crate::reconcile::origin_missing(&repo, head, &theirs)
6060 .await
6061 .is_ok_and(|missing| missing.is_empty())
6062 {
6063 return Some(format!(
6064 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6065 them",
6066 short(&theirs)
6067 ));
6068 }
6069 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6070 Ok(o) if o.ok() => {
6071 self.state
6072 .event("land", format!("pushed rebased {branch} to {remote}"));
6073 None
6074 }
6075 Ok(o) => Some(format!(
6076 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6077 someone may have pushed): {}",
6078 short(&theirs),
6079 o.stderr.chars().take(600).collect::<String>()
6080 )),
6081 Err(e) => Some(format!(
6082 "{branch} is rebased locally but could not be pushed: {e:#}"
6083 )),
6084 }
6085 }
6086
6087 async fn settle_already_in(
6104 &mut self,
6105 branch: &str,
6106 tip: &str,
6107 head: &str,
6108 attempts: usize,
6109 behind: usize,
6110 ) -> Result<bool> {
6111 let repo = self.state.repo.clone();
6112 let remote = self.state.config.merge.remote.clone();
6113 let start = self.state.base_commit.clone();
6114 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6115 Ok(Some(e)) => e,
6116 Ok(None) => return Ok(false),
6117 Err(e) => {
6118 tracing::warn!("already-in-base check for {branch}: {e:#}");
6119 return Ok(false);
6120 }
6121 };
6122 let mut verified = vec![head.to_owned()];
6123 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6124 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6125 if matches!(&fetched, Ok(o) if o.ok())
6126 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6127 && theirs != head
6128 {
6129 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6130 Ok(Some(_)) => verified.push(theirs),
6131 _ => return Ok(false),
6132 }
6133 }
6134 let base_branch = self.state.base_branch.clone();
6135 let message = format!(
6136 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6137 land",
6138 evidence.names(),
6139 evidence.proof.as_str()
6140 );
6141 let closed =
6142 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6143 .await;
6144 match closed {
6145 Ok(Ok(url)) => self
6146 .state
6147 .event("land", format!("closed {url}: superseded on {base_branch}")),
6148 Ok(Err(why)) => self
6149 .state
6150 .event("land", format!("did not close a pull request: {why}")),
6151 Err(e) => {
6152 let why = format!(
6153 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6154 not be closed ({e:#}); resume to retry"
6155 );
6156 self.state.status = RunStatus::Blocked;
6157 self.state.base_sync = Some(BaseSync {
6158 tip: tip.to_owned(),
6159 behind,
6160 attempts,
6161 conflict: Some(why.clone()),
6162 already_in: None,
6163 });
6164 self.state.event("land", why);
6165 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6166 .await?;
6167 self.state.save()?;
6168 return Ok(true);
6169 }
6170 }
6171 self.state.status = RunStatus::AlreadyInBase;
6172 self.state.base_sync_recovery.stop = None;
6173 self.state.base_sync = Some(BaseSync {
6174 tip: tip.to_owned(),
6175 behind,
6176 attempts,
6177 conflict: None,
6178 already_in: Some(evidence),
6179 });
6180 self.state.event("land", message);
6181 self.state.save()?;
6182 self.settle_questions();
6183 Ok(true)
6184 }
6185
6186 fn landing_base(&self) -> String {
6196 self.state
6197 .base_sync
6198 .as_ref()
6199 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6200 }
6201
6202 pub async fn fix_selected(
6235 &mut self,
6236 ids: &[String],
6237 reason: &str,
6238 allow_stale: bool,
6239 ) -> Result<()> {
6240 let reason = reason.trim();
6241 if reason.is_empty() {
6242 bail!("a fix request needs a reason — that is the operator's own record of why");
6243 }
6244 if ids.is_empty() {
6245 bail!("no finding id given");
6246 }
6247 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6248 bail!(
6249 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6250 has already concluded — can be given a targeted fix. A run still \
6251 in progress should simply be resumed; a `merged` run's branch has \
6252 already landed, so its answer is a fresh `magi review <branch>`, \
6253 not reopening this run's own record",
6254 self.state.id,
6255 self.state.status.as_str()
6256 );
6257 }
6258 let _claim = FixClaim::acquire(&self.state.dir())?;
6261 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6262 let Some(winner) = self.state.winner().cloned() else {
6263 bail!("run {} has no winning candidate to fix", self.state.id);
6264 };
6265 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6266 bail!(
6267 "branch `{}` no longer exists; this run cannot be extended",
6268 winner.branch
6269 );
6270 }
6271 let home = crate::run::home();
6272 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6273 bail!(
6274 "run {} is currently being worked on by another magi process",
6275 self.state.id
6276 );
6277 }
6278 let mut seen = BTreeSet::new();
6282 let mut findings = Vec::new();
6283 let mut missing = Vec::new();
6284 for id in ids {
6285 if !seen.insert(id.clone()) {
6286 continue;
6287 }
6288 match self.state.finding(id) {
6289 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6290 id: f.id.clone(),
6291 severity: f.severity,
6292 reviewer_vote: rec.vote,
6293 round: round.round,
6294 round_head: round.head.clone(),
6295 reviewer: rec.reviewer,
6296 agent: rec.agent.clone(),
6297 file: f.file.clone(),
6298 line: f.line,
6299 title: f.title.clone(),
6300 detail: f.detail.clone(),
6301 outcome: OperatorFixOutcome::Pending,
6302 }),
6303 None => missing.push(id.clone()),
6304 }
6305 }
6306 if !missing.is_empty() {
6307 bail!(
6308 "unknown finding id(s): {}; nothing was changed",
6309 missing.join(", ")
6310 );
6311 }
6312
6313 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6314 let stale_details: Vec<(String, String)> = findings
6315 .iter()
6316 .filter(|f| f.round_head != head_at_request)
6317 .map(|f| (f.id.clone(), f.round_head.clone()))
6318 .collect();
6319 let stale = !stale_details.is_empty();
6320 if stale && !allow_stale {
6321 bail!(
6322 "the branch has moved since some finding(s) were raised — {} — now \
6323 at {}; pass --allow-stale to fix anyway, or re-run review first",
6324 stale_details
6325 .iter()
6326 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6327 .collect::<Vec<_>>()
6328 .join(", "),
6329 short(&head_at_request)
6330 );
6331 }
6332
6333 let request = OperatorFixRequest {
6334 requested_at: Timestamp::now(),
6335 reason: reason.to_owned(),
6336 findings,
6337 head_at_request: head_at_request.clone(),
6338 allow_stale,
6339 stale,
6340 fix: None,
6341 result_head: None,
6342 follow_up_review_run: None,
6343 };
6344 self.state.event(
6345 "fix",
6346 format!(
6347 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6348 request.findings.len(),
6349 request
6350 .findings
6351 .iter()
6352 .map(|f| f.id.as_str())
6353 .collect::<Vec<_>>()
6354 .join(", "),
6355 ),
6356 );
6357 self.state.operator_fixes.push(request);
6364 self.state.save()?;
6365 let request_index = self.state.operator_fixes.len() - 1;
6366
6367 let repo = self.state.repo.clone();
6376 let coordination = self.coordinate(&repo).await?;
6377 if winner.worktree.exists() {
6378 let dirty = git::git(
6381 &winner.worktree,
6382 &["status", "--porcelain", "--untracked-files=all"],
6383 )
6384 .await?;
6385 let only_withheld = dirty.lines().all(|l| {
6386 l.strip_prefix("?? ")
6387 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6388 });
6389 if !only_withheld {
6390 bail!(
6391 "`{}` has uncommitted changes; refusing to touch it — commit or \
6392 discard them first",
6393 winner.worktree.display()
6394 );
6395 }
6396 git::worktree_remove(&self.state.repo, &winner.worktree)
6397 .await
6398 .ok();
6399 }
6400 let fix_worktree = self.state.worktree_root().join("operator-fix");
6401 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6402 git::git(
6403 &self.state.repo,
6404 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6405 )
6406 .await
6407 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6408 if !git::is_clean(&fix_worktree).await? {
6409 git::worktree_remove(&self.state.repo, &fix_worktree)
6410 .await
6411 .ok();
6412 bail!(
6413 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6414 winner.branch
6415 );
6416 }
6417 let repo_for_setup = self.state.repo.clone();
6418 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6419 Ok(()) => {
6420 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6421 }
6422 Err(e) => Err(e),
6423 };
6424 if let Err(e) = setup {
6425 git::worktree_remove(&self.state.repo, &fix_worktree)
6426 .await
6427 .ok();
6428 return Err(e);
6429 }
6430
6431 self.state.resource_block = None;
6434
6435 let run_id = self.state.id.clone();
6436 let prompts = self.state.config.prompts.clone();
6437 let language = self.state.config.graph.language.clone();
6438 let sessions = self.state.config.graph.sessions;
6439 let artifacts = agent::artifacts_dir(&self.state.dir());
6440 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6441 .findings
6442 .iter()
6443 .map(|f| Finding {
6444 id: f.id.clone(),
6445 severity: f.severity,
6446 file: f.file.clone(),
6447 line: f.line,
6448 title: f.title.clone(),
6449 detail: f.detail.clone(),
6450 })
6451 .collect();
6452 let fix_prompt = prompt::operator_fix_in(
6453 &self.state.instruction,
6454 &finding_list,
6455 reason,
6456 &stale_details,
6457 &head_at_request,
6458 &language,
6459 &self.state.config.graph.github_language,
6460 );
6461 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6462 drop(coordination);
6463 let (job, seat, out) = self
6464 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6465 prompt: fix_prompt.clone(),
6466 spec,
6467 seat,
6468 cwd: fix_worktree.clone(),
6469 timeout,
6470 allow_write: true,
6471 sessions,
6472 artifacts: artifacts.clone(),
6473 stem: "operator-fix".to_owned(),
6474 handover: None,
6475 })
6476 .await;
6477 let agent_id = seat.agent.clone();
6478
6479 let mut fix = FixRecord {
6480 agent: agent_id,
6481 addressed: Vec::new(),
6482 rejected: Vec::new(),
6483 notes: String::new(),
6484 committed: false,
6485 failed: None,
6486 duration_ms: 0,
6487 continuation: None,
6488 };
6489 let mut final_seat = seat.clone();
6490 match out {
6491 AgentOutcome::Ok(o) => {
6492 fix.duration_ms = o.duration_ms;
6493 let parsed = verdict::extract_json::<FixReport>(&o.text);
6494 let incomplete_reason = match &parsed {
6495 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6496 "the reply parsed, but it reported a command whose own CLI \
6497 never confirmed an exit status"
6498 .to_owned(),
6499 ),
6500 Ok(_) => None,
6501 Err(e) => Some(e.to_string()),
6502 };
6503 match incomplete_reason {
6504 None => {
6505 let report = parsed.expect("checked Ok above");
6506 fix.addressed = report.addressed;
6507 fix.rejected = report.rejected;
6508 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6509 }
6510 Some(reason) => {
6511 let (resumed_seat, resolved, failure, cont) = self
6512 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6513 .await;
6514 fix.duration_ms += cont.cumulative_wait_ms;
6515 fix.continuation = Some(cont);
6516 final_seat = resumed_seat;
6517 match resolved {
6518 Some(report) => {
6519 fix.addressed = report.addressed;
6520 fix.rejected = report.rejected;
6521 fix.notes =
6522 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6523 }
6524 None => fix.failed = failure,
6525 }
6526 }
6527 }
6528 }
6529 AgentOutcome::Dropped(o) => {
6530 fix.duration_ms = o.duration_ms;
6531 let why = o
6532 .dropped
6533 .as_ref()
6534 .map(|d| d.why.as_str())
6535 .unwrap_or("the CLI ended the stream without delivering its answer");
6536 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6537 }
6538 AgentOutcome::Quota(o) => {
6539 self.state.quota.push(QuotaLoss::from_report(
6540 final_seat.key.clone(),
6541 "fix",
6542 o.quota.as_ref(),
6543 ));
6544 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6545 }
6546 AgentOutcome::Failed(e) => fix.failed = Some(e),
6547 AgentOutcome::ResourceBlocked(e) => {
6548 fix.failed = Some(format!("the fixer was not started: {e}"));
6549 }
6550 }
6551 if fix.continuation.is_none() {
6554 fix.continuation = Some(ContinuationRecord::not_needed());
6555 }
6556 self.state.seats.insert(final_seat.key.clone(), final_seat);
6557
6558 let rescue_message = format!(
6559 "magi: operator-selected fix ({}) (uncommitted work)",
6560 self.state.operator_fixes[request_index]
6561 .findings
6562 .iter()
6563 .map(|f| f.id.as_str())
6564 .collect::<Vec<_>>()
6565 .join(", ")
6566 );
6567 let repo = self.state.repo.clone();
6568 let coordination = self.coordinate(&repo).await?;
6569 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6570 self.state.note_withheld("fix", &r.withheld);
6571 }
6572 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6573 fix.committed = after != head_at_request;
6574 git::worktree_remove(&self.state.repo, &fix_worktree)
6575 .await
6576 .ok();
6577 drop(coordination);
6578
6579 self.state.event(
6580 "fix",
6581 match &fix.failed {
6582 Some(reason) => format!(
6583 "operator fix: adoption report was lost ({reason}); {}",
6584 if fix.committed {
6585 "committed"
6586 } else {
6587 "NO new commit"
6588 }
6589 ),
6590 None => format!(
6591 "operator fix: {} addressed, {} rejected, {}",
6592 fix.addressed.len(),
6593 fix.rejected.len(),
6594 if fix.committed {
6595 "committed"
6596 } else {
6597 "NO new commit"
6598 }
6599 ),
6600 },
6601 );
6602
6603 for f in &mut self.state.operator_fixes[request_index].findings {
6610 f.outcome = if fix.failed.is_some() {
6611 OperatorFixOutcome::Unreported
6612 } else if fix.addressed.contains(&f.id) {
6613 OperatorFixOutcome::Addressed
6614 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6615 OperatorFixOutcome::Rejected { why: r.why.clone() }
6616 } else {
6617 OperatorFixOutcome::Unreported
6618 };
6619 }
6620
6621 let committed = fix.committed;
6622 if committed {
6623 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6624 }
6625 self.state.operator_fixes[request_index].fix = Some(fix);
6626 self.state.save()?;
6629 if self.state.resource_block.is_some() {
6630 return self.stop_for_resource();
6631 }
6632
6633 if committed {
6634 self.state.event(
6635 "fix",
6636 format!(
6637 "operator fix committed {}; opening a follow-up review-only run",
6638 short(&after)
6639 ),
6640 );
6641 let origin =
6644 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6645 match Self::review(
6646 &self.state.repo,
6647 &winner.branch,
6648 self.state.config.clone(),
6649 origin,
6650 )
6651 .await
6652 {
6653 Ok(mut follow_up) => {
6654 follow_up.state.event(
6655 "start",
6656 format!(
6657 "requested by an operator fix on run {} for finding(s) {}",
6658 self.state.id,
6659 self.state.operator_fixes[request_index]
6660 .findings
6661 .iter()
6662 .map(|f| f.id.as_str())
6663 .collect::<Vec<_>>()
6664 .join(", "),
6665 ),
6666 );
6667 follow_up.state.save()?;
6668 let follow_up_id = follow_up.state.id.clone();
6669 let adopted = match crate::direct::adopt(
6672 &follow_up.state,
6673 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6674 ) {
6675 Ok(a) => a,
6676 Err(e) => {
6677 self.state.event(
6680 "fix",
6681 format!(
6682 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6683 ),
6684 );
6685 self.state.save()?;
6686 return Ok(());
6687 }
6688 };
6689 let executed = follow_up.execute().await;
6690 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6691 if let Some(a) = adopted {
6692 a.finish(&follow_up.state, executed);
6693 }
6694 if let Some(e) = failure {
6695 self.state.event(
6696 "fix",
6697 format!(
6698 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6699 ),
6700 );
6701 }
6702 self.state.operator_fixes[request_index].follow_up_review_run =
6703 Some(follow_up_id);
6704 }
6705 Err(e) => {
6706 self.state.event(
6707 "fix",
6708 format!("committed the fix but could not open a follow-up review: {e:#}"),
6709 );
6710 }
6711 }
6712 self.state.save()?;
6713 }
6714
6715 Ok(())
6716 }
6717
6718 async fn ask_fixer(
6738 &mut self,
6739 winner: &Candidate,
6740 node: &'static str,
6741 round: Option<usize>,
6742 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6743 ) -> (SeatJob, SeatState, AgentOutcome) {
6744 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6745 self.ask_fixer_among(attempts, node, round, None, build)
6746 .await
6747 }
6748
6749 async fn ask_fixer_among(
6754 &mut self,
6755 attempts: Vec<(AgentSpec, String)>,
6756 node: &'static str,
6757 round: Option<usize>,
6758 restore: Option<(&Path, &str)>,
6759 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6760 ) -> (SeatJob, SeatState, AgentOutcome) {
6761 let run_id = self.state.id.clone();
6762 let prompts = self.state.config.prompts.clone();
6763 let cache = self.state.config.cache_dir();
6764 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6765 let mut last = None;
6766 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6767 let seat = self.seat(&key, &spec.id);
6768 let mut job = build(spec.clone(), seat);
6769 if i > 0 {
6770 job.stem = format!("{}-{}", job.stem, spec.id);
6771 }
6772 let ctx = WaveCtx {
6773 carry_seats: false,
6774 pauses: Some((&self.pause, &self.interrupt)),
6775 run: &run_id,
6776 node,
6777 prompts: &prompts,
6778 cache: cache.as_deref(),
6779 round,
6780 };
6781 let (seat, out) =
6782 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6783 let advances = match &out {
6786 AgentOutcome::Ok(o) => agent::output_advances(o),
6787 AgentOutcome::ResourceBlocked(_) => false,
6788 _ => true,
6789 };
6790 if let Some(next) = ids.get(i + 1)
6791 && advances
6792 {
6793 self.state.seats.insert(seat.key.clone(), seat.clone());
6794 let class =
6795 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6796 let reason = match &out {
6797 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6798 other => fail_reason(other),
6799 };
6800 if let Some((wt, rev)) = restore
6804 && let Err(e) = self.graft_restore(wt, rev).await
6805 {
6806 let why =
6807 format!("could not restore the worktree before the next fixer: {e:#}");
6808 tracing::warn!("{why}");
6809 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6810 AgentOutcome::ResourceBlocked(why)
6811 } else {
6812 AgentOutcome::Failed(why)
6813 };
6814 last = Some((job, seat, out));
6815 break;
6816 }
6817 record_handover(
6818 &mut self.state,
6819 node,
6820 &seat.key,
6821 &spec.id,
6822 next,
6823 &class,
6824 &reason,
6825 );
6826 continue;
6827 }
6828 last = Some((job, seat, out));
6829 break;
6830 }
6831 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6832 if let AgentOutcome::Quota(output) = &out
6833 && node != "graft"
6834 {
6835 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6840 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6841 self.state.quota.push(loss);
6842 self.state.seats.insert(seat.key.clone(), seat.clone());
6843 self.state.status = RunStatus::Blocked;
6844 return (
6845 job,
6846 seat,
6847 AgentOutcome::ResourceBlocked(
6848 "fixer unavailable; restore agent availability and resume".into(),
6849 ),
6850 );
6851 }
6852 (job, seat, out)
6853 }
6854
6855 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6866 use crate::recovery::{Mode, Repair};
6867 if !self.state.reviews.is_empty() {
6868 return Ok(());
6869 }
6870 let Some(mut rec) = self.state.recovery.clone() else {
6871 return Ok(());
6872 };
6873 if rec.mode != Mode::VerifyRepair {
6874 return Ok(());
6875 }
6876 let wt = winner.worktree.clone();
6877 let mut head = git::rev_parse(&wt, "HEAD").await?;
6878 match rec.repair.clone() {
6879 Repair::Done { .. } => return Ok(()),
6880 Repair::WaitingForRestore { before } => {
6881 self.graft_restore(&wt, &before).await?;
6882 head = before;
6883 rec.repair = Repair::NotStarted;
6884 self.state.recovery = Some(rec.clone());
6885 self.state.save()?;
6886 }
6887 Repair::Started { before } => {
6888 if let Err(e) = self.graft_restore(&wt, &before).await {
6892 if e.is::<crate::repo_lock::ResourceBlocked>() {
6893 return Err(e);
6894 }
6895 return self
6896 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6897 .await;
6898 }
6899 rec.repair = Repair::Done {
6900 before: before.clone(),
6901 after: before,
6902 committed: false,
6903 failed: Some("interrupted; rolled back".to_owned()),
6904 };
6905 self.state
6906 .event("recovery", "the repair was interrupted and rolled back");
6907 self.state.recovery = Some(rec);
6908 self.state.save()?;
6909 return Ok(());
6910 }
6911 Repair::NotStarted => {}
6912 }
6913 if crate::recovery::recheck(&mut rec, &head) {
6914 self.state.event("recovery", rec.summary());
6915 self.state.recovery = Some(rec);
6916 self.state.save()?;
6917 return Ok(());
6918 }
6919 rec.repair = Repair::Started {
6920 before: head.clone(),
6921 };
6922 self.state.recovery = Some(rec.clone());
6923 self.state.save()?;
6924
6925 let language = self.state.config.graph.language.clone();
6926 let sessions = self.state.config.graph.sessions;
6927 let artifacts = agent::artifacts_dir(&self.state.dir());
6928 let fix_prompt = prompt::recovery_fix(
6929 &self.state.instruction,
6930 &rec,
6931 self.state.config.graph.review_rounds,
6932 &language,
6933 );
6934 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6935 self.state.event(
6936 "recovery",
6937 format!(
6938 "repair: the fixer starts from the failed verification on {}",
6939 short(&head)
6940 ),
6941 );
6942 let (_job, seat, out) = self
6943 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6944 prompt: fix_prompt.clone(),
6945 spec,
6946 seat,
6947 cwd: wt.clone(),
6948 timeout,
6949 allow_write: true,
6950 sessions,
6951 artifacts: artifacts.clone(),
6952 stem: "fix-recovery".to_owned(),
6953 handover: None,
6954 })
6955 .await;
6956 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6957 rec.repair = Repair::WaitingForRestore { before: head };
6958 self.state.recovery = Some(rec);
6959 return self.stop_for_resource();
6960 }
6961 let failed = match &out {
6962 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6963 AgentOutcome::Quota(o) => {
6964 self.state.quota.push(QuotaLoss::from_report(
6965 seat.key.clone(),
6966 "fix",
6967 o.quota.as_ref(),
6968 ));
6969 Some("rate limited (quota); the fixer could not run".to_owned())
6970 }
6971 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6972 AgentOutcome::Failed(e) => Some(e.clone()),
6973 };
6974 self.state.seats.insert(seat.key.clone(), seat);
6975 let coordination = self.coordinate(&wt).await?;
6976 let failed = match failed {
6977 Some(why) => {
6979 if let Err(e) = git::restore_to(&wt, &head).await {
6980 return self
6981 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6982 .await;
6983 }
6984 Some(why)
6985 }
6986 None => {
6987 if let Ok(r) = git::rescue_commit(
6988 &wt,
6989 "magi: recovery repair of the failed verification (uncommitted work)",
6990 )
6991 .await
6992 {
6993 self.state.note_withheld("fix", &r.withheld);
6994 }
6995 None
6996 }
6997 };
6998 let after = git::rev_parse(&wt, "HEAD").await?;
6999 drop(coordination);
7000 self.state.event(
7001 "recovery",
7002 match &failed {
7003 Some(why) => format!("repair: nothing kept ({why})"),
7004 None if after != head => {
7005 format!("repair: committed {}", short(&after))
7006 }
7007 None => "repair: the fixer made no commit".to_owned(),
7008 },
7009 );
7010 rec.repair = Repair::Done {
7011 before: head.clone(),
7012 committed: after != head,
7013 after,
7014 failed,
7015 };
7016 self.state.recovery = Some(rec);
7017 self.state.save()?;
7018 Ok(())
7019 }
7020
7021 async fn stop_recovery(
7024 &mut self,
7025 mut rec: crate::recovery::RecoveryRecord,
7026 head: &str,
7027 why: String,
7028 ) -> Result<()> {
7029 rec.repair = crate::recovery::Repair::Started {
7031 before: head.to_owned(),
7032 };
7033 self.state.recovery = Some(rec);
7034 self.state
7035 .event("recovery", format!("repair could not be undone: {why}"));
7036 self.state.status = RunStatus::Blocked;
7037 self.state.save()?;
7038 bail!("recovery repair could not be rolled back: {why}")
7039 }
7040
7041 async fn review_loop(&mut self) -> Result<()> {
7042 if self.hold_base_sync().await? {
7043 return Ok(());
7044 }
7045 loop {
7046 if self
7047 .state
7048 .base_sync
7049 .as_ref()
7050 .is_some_and(|s| s.conflict.is_some())
7051 {
7052 return Ok(());
7053 }
7054 if self.catchup_repairable() {
7055 let record = self.state.reviews.last().expect("repairable round").clone();
7056 let winner = self
7057 .state
7058 .winner()
7059 .cloned()
7060 .context("catch-up repair has no winner")?;
7061 if !self.fix_review_round(&winner, record).await? {
7062 return Ok(());
7063 }
7064 }
7065 self.review_loop_pass().await?;
7066 if !self.catchup_repairable() {
7067 return Ok(());
7068 }
7069 }
7070 }
7071
7072 fn catchup_repairable(&self) -> bool {
7075 self.state.reviews.len() < self.state.config.graph.review_rounds
7076 && self.state.reviews.last().is_some_and(|r| {
7077 !r.clean
7078 && r.blocking == 0
7079 && r.fix.is_none()
7080 && r.e2e_status() == E2eStatus::Failed
7081 })
7082 }
7083
7084 async fn review_loop_pass(&mut self) -> Result<()> {
7085 if self
7090 .state
7091 .base_sync
7092 .as_ref()
7093 .is_some_and(|s| s.conflict.is_some())
7094 {
7095 return Ok(());
7096 }
7097 let run_id = self.state.id.clone();
7102 let prompts = self.state.config.prompts.clone();
7103 let Some(winner) = self.state.winner().cloned() else {
7104 return Ok(());
7105 };
7106 let granted_review = self
7107 .state
7108 .repair_grants
7109 .last()
7110 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7111 .cloned();
7112 let fresh_review = granted_review
7113 .as_ref()
7114 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7115 let max_rounds = granted_review
7116 .as_ref()
7117 .map_or(self.state.config.graph.review_rounds, |g| {
7118 g.review_start.saturating_add(1)
7119 });
7120 if let Some(status) =
7130 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7131 {
7132 if status == RunStatus::Gating {
7135 self.record_contested_handoff();
7136 }
7137 self.state.status = status;
7138 self.state.save()?;
7139 return Ok(());
7140 }
7141 self.state.status = RunStatus::Reviewing;
7142 if !fresh_review
7152 && self.state.reviews.last().is_some_and(|r| {
7153 r.e2e_status() == E2eStatus::ResourceBlocked
7154 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7155 })
7156 {
7157 let shell = self.state.config.shell();
7158 return self
7159 .stop_unresolved_review(
7160 "the last round's own verification never resolved",
7161 &shell,
7162 &winner.worktree,
7163 )
7164 .await;
7165 }
7166
7167 if granted_review.is_none() {
7168 self.recovery_repair(&winner).await?;
7169 }
7170 if self.resource_halt()? {
7171 return Ok(());
7172 }
7173
7174 let repo = self.state.repo.clone();
7175 let root = self.state.worktree_root();
7176 let language = self.state.config.graph.language.clone();
7177 let sessions = self.state.config.graph.sessions;
7178 let artifacts = self.merge_repair_artifacts();
7179 let base = self.landing_base();
7180 let base_short = short(&base);
7181 let reviewers = self.roles.reviewers.clone();
7182 let shell = self.state.config.shell();
7183
7184 for round in (self.state.reviews.len() + 1)..=max_rounds {
7185 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7186 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7187 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7188 let prev_verification = self
7197 .state
7198 .reviews
7199 .last()
7200 .and_then(|r| r.verification_summary(&head));
7201 let prior_section = match (&self.state.recovery, round) {
7204 (Some(rec), 1) if self.state.reviews.is_empty() => {
7205 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7206 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7207 Some(crate::recovery::head_of(
7208 &git::diff(&winner.worktree, reviewed, "HEAD")
7209 .await
7210 .unwrap_or_default(),
7211 crate::recovery::SINCE_DIFF_MAX,
7212 ))
7213 }
7214 _ => None,
7215 };
7216 Some(prompt::recovery_review(rec, since.as_deref()))
7217 }
7218 _ => None,
7219 };
7220
7221 let mut jobs = Vec::new();
7225 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7226 self.coordinate(&repo).await?
7227 } else {
7228 None
7229 };
7230 for (r, spec) in reviewers.iter().cloned().enumerate() {
7231 let wt = root.join(format!("review-{}", r + 1));
7232 if wt.exists() {
7235 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7236 } else {
7237 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7238 None
7239 } else {
7240 self.coordinate(&repo).await?
7241 };
7242 git::worktree_add_detached(&repo, &wt, &head).await?;
7243 }
7244 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7245 None
7246 } else {
7247 self.coordinate(&repo).await?
7248 };
7249 self.setup_seat_worktree(&repo, &wt, false).await?;
7250 let seat_key = format!("review-{}", r + 1);
7251 let spec = pick_start_spec(
7254 &self.roles.reviewer_roster,
7255 spec,
7256 self.state.seat_history.get(&seat_key),
7257 );
7258 let seat = self.seat(&seat_key, &spec.id);
7259 jobs.push(SeatJob {
7260 prompt: prompt::review(&prompt::ReviewCtx {
7261 instruction: &self.state.instruction,
7262 branch: &winner.branch,
7263 base_short: &base_short,
7264 stat: &stat,
7265 patch: &patch,
7266 verification: prev_verification.as_ref(),
7267 prior: prior_section.as_deref(),
7268 reviewers: reviewers.len(),
7269 round,
7270 rounds: max_rounds,
7271 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7274 lens: Lens::for_seat(r),
7275 language: &language,
7276 github_language: &self.state.config.graph.github_language,
7277 }),
7278 spec,
7279 seat,
7280 cwd: wt,
7281 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7282 allow_write: false,
7283 sessions,
7284 artifacts: artifacts.clone(),
7285 stem: format!("review-{round}-{}", r + 1),
7286 handover: None,
7287 });
7288 }
7289
7290 drop(coordination);
7291 self.state.event(
7292 "review",
7293 format!(
7294 "round {round}: {} reviewers on {}",
7295 jobs.len(),
7296 short(&head)
7297 ),
7298 );
7299 let mut quota_losses = Vec::new();
7300 let review_retries = self.state.config.graph.retries;
7301 let review_cache = self.state.config.cache_dir();
7302 let ctx = WaveCtx {
7303 carry_seats: true,
7304 pauses: Some((&self.pause, &self.interrupt)),
7305 run: &run_id,
7306 node: "review",
7307 prompts: &prompts,
7308 cache: review_cache.as_deref(),
7309 round: Some(round),
7310 };
7311 let results = ask_json_wave::<Review>(
7312 jobs,
7313 Arc::clone(&self.sem),
7314 review_retries,
7315 &self.roles.reviewer_roster,
7316 &ctx,
7317 &mut quota_losses,
7318 &mut self.state,
7319 &|_: &Review| Ok(()),
7320 )
7321 .await;
7322 if self.state.resource_block.is_some() {
7323 return self.stop_for_resource();
7324 }
7325 let round_quota_missing = quota_losses.len();
7329 self.state.quota.extend(quota_losses);
7330
7331 let mut records = Vec::new();
7332 let mut all_findings = Vec::new();
7333 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7334 let agent_id = seat.agent.clone();
7335 self.state.seats.insert(seat.key.clone(), seat);
7336 let mut record = ReviewRecord {
7337 reviewer: r + 1,
7338 agent: agent_id,
7339 summary: String::new(),
7340 findings: Vec::new(),
7341 vote: None,
7342 failed: None,
7343 duration_ms: 0,
7344 attempts,
7350 };
7351 match res {
7352 Ok((review, out)) => {
7353 record.summary =
7361 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7362 record.vote = Some(review.vote);
7363 record.duration_ms = out.duration_ms;
7364 for (n, mut f) in review.findings.into_iter().enumerate() {
7365 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7368 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7369 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7370 f.file = f
7376 .file
7377 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7378 all_findings.push(f.clone());
7379 record.findings.push(f);
7380 }
7381 self.state.event(
7382 "review",
7383 format!(
7384 "round {round}: reviewer {} voted {} with {} finding(s)",
7385 r + 1,
7386 review.vote.label(),
7387 record.findings.len()
7388 ),
7389 );
7390 }
7391 Err(e) => {
7392 record.failed = Some(e.to_string());
7393 self.state.event(
7394 "review",
7395 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7396 );
7397 }
7398 }
7399 records.push(record);
7400 }
7401
7402 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7409 let vote_split =
7410 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7411 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7412 if vote_split {
7413 self.state.event(
7414 "review",
7415 format!(
7416 "round {round}: votes split ({}) — one round of reconsideration",
7417 initial_votes
7418 .iter()
7419 .map(|v| v.label())
7420 .collect::<Vec<_>>()
7421 .join(", ")
7422 ),
7423 );
7424 let panel: Vec<ReviewSeatReport<'_>> = records
7427 .iter()
7428 .filter_map(|r| {
7429 r.vote.map(|vote| ReviewSeatReport {
7430 reviewer: r.reviewer,
7431 vote,
7432 summary: &r.summary,
7433 findings: &r.findings,
7434 })
7435 })
7436 .collect();
7437
7438 let mut jobs = Vec::new();
7439 let mut seats_at = Vec::new();
7440 for (r, spec) in reviewers.iter().cloned().enumerate() {
7441 if records[r].vote.is_none() {
7445 continue;
7446 }
7447 let wt = root.join(format!("review-{}", r + 1));
7448 let seat_key = format!("review-{}", r + 1);
7449 let spec = self.occupant(&seat_key, spec);
7450 let seat = self.seat(&seat_key, &spec.id);
7451 let build = |with_patch: bool| {
7459 prompt::review_reconsider(&ReviewReconsiderCtx {
7460 instruction: &self.state.instruction,
7461 reviewer: r + 1,
7462 lens: Lens::for_seat(r),
7463 panel: &panel,
7464 patch: with_patch.then_some(ReviewPatch {
7465 branch: &winner.branch,
7466 base_short: &base_short,
7467 stat: &stat,
7468 patch: &patch,
7469 }),
7470 round,
7471 rounds: max_rounds,
7472 language: &language,
7473 })
7474 };
7475 let full = build(true);
7476 let prompt = if has_context(&spec, &seat, sessions) {
7477 build(false)
7478 } else {
7479 full.clone()
7480 };
7481 jobs.push(SeatJob {
7482 prompt,
7483 spec,
7484 seat,
7485 cwd: wt,
7486 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7487 allow_write: false,
7488 sessions,
7489 artifacts: artifacts.clone(),
7490 stem: format!("review-{round}-reconsider-{}", r + 1),
7491 handover: Some(full),
7492 });
7493 seats_at.push(r);
7494 }
7495
7496 let mut recon_quota_losses = Vec::new();
7497 let recon_cache = self.state.config.cache_dir();
7498 let recon_ctx = WaveCtx {
7499 carry_seats: true,
7500 pauses: Some((&self.pause, &self.interrupt)),
7501 run: &run_id,
7502 node: "review",
7503 prompts: &prompts,
7504 cache: recon_cache.as_deref(),
7505 round: Some(round),
7506 };
7507 let recon_results = ask_json_wave::<ReviewRevote>(
7508 jobs,
7509 Arc::clone(&self.sem),
7510 review_retries,
7511 &self.roles.reviewer_roster,
7512 &recon_ctx,
7513 &mut recon_quota_losses,
7514 &mut self.state,
7515 &|_: &ReviewRevote| Ok(()),
7516 )
7517 .await;
7518 if self.state.resource_block.is_some() {
7519 return self.stop_for_resource();
7520 }
7521 self.state.quota.extend(recon_quota_losses);
7522
7523 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7524 let agent_id = seat.agent.clone();
7525 self.state.seats.insert(seat.key.clone(), seat);
7526 let mut rec = ReviewRevoteRecord {
7527 reviewer: r + 1,
7528 agent: agent_id,
7529 vote: None,
7530 reason: String::new(),
7531 failed: None,
7532 };
7533 match res {
7534 Ok((rv, _)) => {
7535 rec.vote = Some(rv.vote);
7536 rec.reason =
7537 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7538 self.state.event(
7539 "review",
7540 format!(
7541 "round {round}: reviewer {} revoted {}",
7542 r + 1,
7543 rv.vote.label()
7544 ),
7545 );
7546 }
7547 Err(e) => {
7548 rec.failed = Some(e.to_string());
7549 self.state.event(
7550 "review",
7551 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7552 );
7553 }
7554 }
7555 reconsideration.push(rec);
7556 }
7557 } else if initial_votes.len() > 1 {
7558 self.state.event(
7559 "review",
7560 format!(
7561 "round {round}: votes agreed ({}) — no reconsideration",
7562 initial_votes[0].label()
7563 ),
7564 );
7565 }
7566
7567 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7568 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7569 let defer_e2e =
7580 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7581 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7582 let reason =
7583 format!("{blocking} blocking finding(s) already required a fix this round");
7584 self.state.event(
7585 "verify",
7586 format!(
7587 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7588 {}); it will run once a round has none left",
7589 short(&head)
7590 ),
7591 );
7592 (Vec::new(), false, true, Some(reason))
7593 } else {
7594 let e2e_commands = self.state.config.verify.e2e.clone();
7595 let cache_dir = self.state.config.cache_dir();
7596 let context = format!("round {round}");
7597 let wait = self.repository_wait();
7598 let (e2e, verify_retried) = with_cache_lease(
7599 &mut self.state,
7600 cache_dir.as_deref(),
7601 "e2e",
7602 "e2e",
7603 &winner.worktree,
7604 &head,
7605 verify_timeout,
7606 &context,
7607 &wait,
7608 |state, budget| {
7609 let shell = shell.clone();
7610 let e2e_commands = e2e_commands.clone();
7611 let worktree = winner.worktree.clone();
7612 let context = context.clone();
7613 async move {
7614 run_e2e_with_retry(
7615 state,
7616 &shell,
7617 &e2e_commands,
7618 &worktree,
7619 budget,
7620 &context,
7621 )
7622 .await
7623 }
7624 },
7625 )
7626 .await?;
7627 (e2e, verify_retried, false, None)
7628 };
7629
7630 let expected = records.len();
7631 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7632 let incomplete = answered < expected;
7633 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7634 let policy = self.state.config.graph.incomplete_review;
7635 let clean = round_is_clean(
7636 blocking,
7637 e2e_ok,
7638 answered,
7639 expected,
7640 round_quota_missing,
7641 policy,
7642 );
7643
7644 let mut round_record = ReviewRound {
7645 round,
7646 head: head.clone(),
7647 verified_head: None,
7648 verified_at: None,
7649 reviews: records,
7650 e2e,
7651 verify_retried,
7652 e2e_deferred,
7653 e2e_defer_reason,
7654 fix: None,
7655 blocking,
7656 answered,
7657 expected,
7658 clean,
7659 progressed: false,
7660 vote_split,
7661 reconsideration,
7662 verdict: None,
7663 };
7664 round_record.verdict = ReviewVote::worst(
7668 round_record
7669 .final_votes()
7670 .into_iter()
7671 .map(|(_, _, vote)| vote),
7672 );
7673 if !matches!(
7684 round_record.e2e_status(),
7685 E2eStatus::Deferred | E2eStatus::NotConfigured
7686 ) {
7687 round_record.verified_head = Some(head.clone());
7688 round_record.verified_at = Some(Timestamp::now());
7689 }
7690
7691 if incomplete {
7692 let missing: Vec<String> = round_record
7693 .reviews
7694 .iter()
7695 .filter(|r| r.failed.is_some())
7696 .map(|r| format!("review-{}", r.reviewer))
7697 .collect();
7698 self.state.event(
7699 "review",
7700 format!(
7701 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7702 missing.join(", ")
7703 ),
7704 );
7705 }
7706
7707 if clean {
7708 self.state.event(
7709 "review",
7710 if incomplete && policy == IncompleteReviewPolicy::Warn {
7711 format!(
7712 "round {round}: clean (warn policy, incomplete panel) — no \
7713 blocking findings from the seats that answered, verification green"
7714 )
7715 } else if incomplete {
7716 format!(
7717 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7718 quorum) — no blocking findings from the seats that answered, \
7719 verification green",
7720 expected - answered
7721 )
7722 } else {
7723 format!("round {round}: clean — no blocking findings, verification green")
7724 },
7725 );
7726 self.state.reviews.push(round_record);
7727 self.state.status = RunStatus::Gating;
7728 self.state.save()?;
7729 return Ok(());
7730 }
7731
7732 if incomplete && blocking == 0 && e2e_ok {
7740 self.state.reviews.push(round_record);
7741 self.state.save()?;
7742 if round == max_rounds {
7743 self.state.status = RunStatus::Blocked;
7744 self.state.event(
7745 "review",
7746 format!(
7747 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7748 refusing to call it clean",
7749 expected - answered
7750 ),
7751 );
7752 return Ok(());
7753 }
7754 continue;
7755 }
7756
7757 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7769 self.state.reviews.push(round_record);
7770 return self
7771 .stop_unresolved_review(
7772 "the round's own verification could not run",
7773 &shell,
7774 &winner.worktree,
7775 )
7776 .await;
7777 }
7778
7779 if round == max_rounds {
7780 self.state.reviews.push(round_record);
7781 return self
7782 .stop_reviewing(
7783 &format!(
7784 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7785 ),
7786 &shell,
7787 &winner.worktree,
7788 )
7789 .await;
7790 }
7791
7792 if !self.fix_review_round(&winner, round_record).await? {
7793 return Ok(());
7794 }
7795 }
7796 Ok(())
7797 }
7798
7799 async fn fix_review_round(
7802 &mut self,
7803 winner: &Candidate,
7804 mut round_record: ReviewRound,
7805 ) -> Result<bool> {
7806 let round = round_record.round;
7807 let shell = self.state.config.shell();
7808 let step = self.run_fix_round(winner, &mut round_record).await?;
7809 if matches!(step, FixStep::Resource) {
7810 self.stop_for_resource()?;
7811 return Ok(false);
7812 }
7813 if self.state.reviews.len() < round {
7814 self.state.reviews.push(round_record);
7815 } else {
7816 self.state.reviews[round - 1] = round_record;
7817 }
7818 self.state.save()?;
7819
7820 if matches!(step, FixStep::Recorded { report_lost: true }) {
7833 self.stop_reviewing(
7834 "the fixer's adoption report never came back, even after resuming its \
7835 own seat; refusing to start another round against the same worktree \
7836 while that is unresolved",
7837 &shell,
7838 &winner.worktree,
7839 )
7840 .await?;
7841 return Ok(false);
7842 }
7843
7844 let streak = self
7845 .state
7846 .reviews
7847 .iter()
7848 .rev()
7849 .take_while(|r| !r.progressed)
7850 .count();
7851 if streak >= STAGNANT_LIMIT {
7852 self.stop_reviewing(
7853 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7854 &shell,
7855 &winner.worktree,
7856 )
7857 .await?;
7858 return Ok(false);
7859 }
7860 Ok(true)
7861 }
7862
7863 async fn run_fix_round(
7865 &mut self,
7866 winner: &Candidate,
7867 round_record: &mut ReviewRound,
7868 ) -> Result<FixStep> {
7869 let round = round_record.round;
7870 let max_rounds = self.state.config.graph.review_rounds;
7871 let run_id = self.state.id.clone();
7872 let prompts = self.state.config.prompts.clone();
7873 let language = self.state.config.graph.language.clone();
7874 let sessions = self.state.config.graph.sessions;
7875 let artifacts = self.merge_repair_artifacts();
7876 let base = self.landing_base();
7877 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7878 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7879 let this_round_verification = round_record.verification_summary(&head);
7880 let all_findings: Vec<_> = round_record
7881 .reviews
7882 .iter()
7883 .flat_map(|r| r.findings.iter().cloned())
7884 .collect();
7885 let blocking_findings: Vec<_> = all_findings
7888 .iter()
7889 .filter(|f| f.severity.blocks())
7890 .cloned()
7891 .collect();
7892 let fix_prompt = prompt::fix_in(
7893 &self.state.instruction,
7894 &blocking_findings,
7895 this_round_verification.as_ref(),
7896 round,
7897 max_rounds,
7898 &language,
7899 &self.state.config.graph.github_language,
7900 );
7901 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7902 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7903 let (job, seat, out) = self
7904 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7905 prompt: fix_prompt.clone(),
7906 spec,
7907 seat,
7908 cwd: winner.worktree.clone(),
7909 timeout,
7910 allow_write: true,
7911 sessions,
7912 artifacts: artifacts.clone(),
7913 stem: format!("fix-{round}"),
7914 handover: None,
7915 })
7916 .await;
7917 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7920 return Ok(FixStep::Resource);
7921 }
7922 let agent_id = seat.agent.clone();
7923
7924 let mut fix = FixRecord {
7925 agent: agent_id,
7926 addressed: Vec::new(),
7927 rejected: Vec::new(),
7928 notes: String::new(),
7929 committed: false,
7930 failed: None,
7931 duration_ms: 0,
7932 continuation: None,
7933 };
7934 let mut continuation = ContinuationRecord::not_needed();
7935 let mut final_seat = seat.clone();
7936 match out {
7937 AgentOutcome::Ok(o) => {
7938 fix.duration_ms = o.duration_ms;
7939 let parsed = verdict::extract_json::<FixReport>(&o.text);
7940 let incomplete_reason = match &parsed {
7947 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7948 "the reply parsed, but it reported a command whose own CLI never \
7949 confirmed an exit status"
7950 .to_owned(),
7951 ),
7952 Ok(_) => None,
7953 Err(e) => Some(e.to_string()),
7954 };
7955 match incomplete_reason {
7956 None => {
7957 let report = parsed.expect("checked Ok above");
7958 fix.addressed = report.addressed;
7959 fix.rejected = report.rejected;
7960 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7961 }
7962 Some(reason) => {
7963 let (resumed_seat, resolved, failure, cont) = self
7964 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7965 .await;
7966 fix.duration_ms += cont.cumulative_wait_ms;
7967 continuation = cont;
7968 final_seat = resumed_seat;
7969 match resolved {
7970 Some(report) => {
7971 fix.addressed = report.addressed;
7972 fix.rejected = report.rejected;
7973 fix.notes =
7974 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7975 }
7976 None => fix.failed = failure,
7977 }
7978 }
7979 }
7980 }
7981 AgentOutcome::Dropped(o) => {
7983 fix.duration_ms = o.duration_ms;
7984 let why = o
7985 .dropped
7986 .as_ref()
7987 .map(|d| d.why.as_str())
7988 .unwrap_or("the CLI ended the stream without delivering its answer");
7989 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7990 }
7991 AgentOutcome::Quota(o) => {
7992 self.state.quota.push(QuotaLoss::from_report(
7993 final_seat.key.clone(),
7994 "fix",
7995 o.quota.as_ref(),
7996 ));
7997 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7998 }
7999 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8000 }
8001 if self.state.resource_block.is_some() {
8002 self.state.seats.insert(final_seat.key.clone(), final_seat);
8003 return Ok(FixStep::Resource);
8004 }
8005 fix.continuation = Some(continuation);
8006 self.state.seats.insert(final_seat.key.clone(), final_seat);
8007 let coordination = self.coordinate(&winner.worktree).await?;
8008 if let Ok(r) = git::rescue_commit(
8009 &winner.worktree,
8010 &format!("magi: review round {round} fixes (uncommitted work)"),
8011 )
8012 .await
8013 {
8014 self.state.note_withheld("fix", &r.withheld);
8015 }
8016 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8017 fix.committed = after != before;
8018 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8026 drop(coordination);
8027 let progressed = diff_after != patch;
8028 let commit_note = if fix.committed {
8029 "committed"
8030 } else {
8031 "NO new commit"
8032 };
8033 let tree_note = if progressed {
8034 "changed vs base"
8035 } else {
8036 "unchanged vs base"
8037 };
8038 self.state.event(
8039 "fix",
8040 match &fix.failed {
8041 Some(reason) => {
8047 format!(
8048 "round {round}: fixer's adoption report was lost ({reason}); \
8049 {commit_note}, tree {tree_note}"
8050 )
8051 }
8052 None => format!(
8053 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8054 {tree_note}{}",
8055 fix.addressed.len(),
8056 fix.rejected.len(),
8057 if continuation.outcome == ContinuationOutcome::Resumed {
8058 format!(
8059 " (adoption report recovered after {} continuation(s))",
8060 continuation.attempts
8061 )
8062 } else {
8063 String::new()
8064 },
8065 ),
8066 },
8067 );
8068 round_record.fix = Some(fix);
8069 round_record.progressed = progressed;
8070 Ok(FixStep::Recorded {
8071 report_lost: matches!(
8072 continuation.outcome,
8073 ContinuationOutcome::Exhausted
8074 | ContinuationOutcome::QuotaLost
8075 | ContinuationOutcome::NoSession
8076 ),
8077 })
8078 }
8079
8080 fn record_contested_handoff(&mut self) {
8114 if self.state.contested_handoff.is_some() {
8115 return;
8116 }
8117 let Some(contested) = self
8118 .state
8119 .reviews
8120 .last()
8121 .and_then(ReviewRound::contested_handoff)
8122 else {
8123 return;
8124 };
8125 self.state.event(
8126 "review",
8127 format!(
8128 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8129 wait for the owner's approval",
8130 contested.findings.len(),
8131 contested.rejecters.len()
8132 ),
8133 );
8134 self.state.contested_handoff = Some(contested);
8135 }
8136
8137 async fn stop_unresolved_review(
8138 &mut self,
8139 why: &str,
8140 shell: &[String],
8141 worktree: &Path,
8142 ) -> Result<()> {
8143 self.stop_reviewing_inner(why, shell, worktree, true).await
8144 }
8145
8146 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8147 self.stop_reviewing_inner(why, shell, worktree, false).await
8148 }
8149
8150 async fn stop_reviewing_inner(
8151 &mut self,
8152 why: &str,
8153 shell: &[String],
8154 worktree: &Path,
8155 repairable: bool,
8156 ) -> Result<()> {
8157 let round_idx = self.state.reviews.len() - 1;
8158 let needs_catchup_run = matches!(
8166 self.state.reviews[round_idx].e2e_status(),
8167 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8168 );
8169 if needs_catchup_run {
8170 let round = self.state.reviews[round_idx].round;
8171 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8172 let commands = self.state.config.verify.e2e.clone();
8173 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8174 let cache_dir = self.state.config.cache_dir();
8175 let context = format!(
8176 "round {round}: verification unresolved, catching up before the final decision"
8177 );
8178 let wait = self.repository_wait();
8179 let (outcomes, verify_retried) = with_cache_lease(
8180 &mut self.state,
8181 cache_dir.as_deref(),
8182 "e2e",
8183 "e2e",
8184 worktree,
8185 &attempted_head,
8186 timeout,
8187 &context,
8188 &wait,
8189 |state, budget| {
8190 let shell = shell.to_vec();
8191 let commands = commands.clone();
8192 let context = context.clone();
8193 async move {
8194 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8195 .await
8196 }
8197 },
8198 )
8199 .await?;
8200 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8201 self.state.record_base_progress(&attempted_head);
8202 }
8203 let last = &mut self.state.reviews[round_idx];
8204 last.e2e = outcomes;
8205 last.verify_retried = verify_retried;
8206 last.verified_head = Some(attempted_head);
8213 last.verified_at = Some(Timestamp::now());
8214 if verify_inconclusive(&last.e2e) {
8215 self.state.save()?;
8222 return Ok(());
8223 }
8224 last.e2e_deferred = false;
8225 if last.answered == last.expected {
8230 last.clean = round_is_clean(
8231 last.blocking,
8232 last.e2e.iter().all(CommandOutcome::ok),
8233 last.answered,
8234 last.expected,
8235 0,
8236 self.state.config.graph.incomplete_review,
8237 );
8238 }
8239 }
8240 let last = &self.state.reviews[round_idx];
8241 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8242
8243 match last.e2e_status() {
8244 E2eStatus::Failed => {
8245 let red: Vec<String> = last
8246 .e2e
8247 .iter()
8248 .filter(|o| !o.ok())
8249 .map(|o| {
8250 format!(
8251 "`{}` -> {:?}\n{}",
8252 o.command,
8253 o.code,
8254 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8255 )
8256 })
8257 .collect();
8258 let can_repair = last.blocking == 0 && last.fix.is_none();
8259 self.state.reviews[round_idx].clean = false;
8260 self.state
8261 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8262 self.state.status = RunStatus::Blocked;
8263 if repairable
8264 && can_repair
8265 && self.state.reviews.len() < self.state.config.graph.review_rounds
8266 {
8267 self.state.status = RunStatus::Reviewing;
8268 }
8269 }
8270 E2eStatus::ResourceBlocked => {
8275 self.state.event(
8276 "review",
8277 format!(
8278 "{why}; e2e could not run (resource unavailable); not \
8279 deciding yet"
8280 ),
8281 );
8282 }
8283 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8284 self.state.event(
8285 "review",
8286 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8287 );
8288 self.record_contested_handoff();
8289 self.state.status = RunStatus::Gating;
8290 }
8291 }
8292 self.state.save()?;
8293 Ok(())
8294 }
8295
8296 async fn gate(&mut self) -> Result<()> {
8299 if self.hold_base_sync().await? {
8300 return Ok(());
8301 }
8302 if self.state.status == RunStatus::Failed
8314 || self
8315 .state
8316 .base_sync
8317 .as_ref()
8318 .is_some_and(|s| s.conflict.is_some())
8319 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8320 != Some(RunStatus::Gating)
8321 {
8322 return Ok(());
8323 }
8324 let Some(winner) = self.state.winner().cloned() else {
8325 return Ok(());
8326 };
8327 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8328 self.state.reconcile_verification_head(&head);
8329 self.state.save()?;
8330 let wait = self.repository_wait();
8331 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8332 return Ok(());
8333 }
8334 if self.state.gate_ran {
8335 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8346 self.state.status = RunStatus::Blocked;
8347 self.state.save()?;
8348 }
8349 return Ok(());
8350 }
8351 self.state.status = RunStatus::Gating;
8352 let mut outcomes = self.run_gate(&winner).await?;
8353 let mut attempted_head = self.state.gate_head.clone();
8354 loop {
8355 if verify_inconclusive(&outcomes) {
8366 self.state.save()?;
8367 return Ok(());
8368 }
8369 if outcomes.iter().all(CommandOutcome::ok) {
8370 break;
8371 }
8372 let fixes_before = self.state.gate_fixes.len();
8373 match self.gate_fix_round(&winner, &outcomes).await? {
8374 GateFix::Retry => {
8375 outcomes = self.run_gate(&winner).await?;
8376 attempted_head = self.state.gate_head.clone();
8377 }
8378 GateFix::Stop => {
8379 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8380 if self.state.gate_fixes.len() > fixes_before
8381 && attempted_head.as_deref() != Some(current.as_str())
8382 && verify_current_head(&mut self.state, &winner, &wait).await?
8383 {
8384 outcomes = self.run_gate(&winner).await?;
8387 attempted_head = self.state.gate_head.clone();
8388 } else {
8389 break;
8390 }
8391 }
8392 GateFix::Defer => {
8393 self.state.save()?;
8394 return Ok(());
8395 }
8396 }
8397 }
8398 let passed = outcomes.iter().all(CommandOutcome::ok);
8399 self.state.gate = outcomes;
8400 self.state.gate_ran = true;
8401 self.state.gate_head = attempted_head;
8402 if !self.state.gate.is_empty()
8403 && !verify_inconclusive(&self.state.gate)
8404 && let Some(head) = self.state.gate_head.clone()
8405 {
8406 self.state.record_base_progress(&head);
8407 }
8408 self.state.save()?;
8409 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8410 return Ok(());
8411 }
8412 if !passed {
8413 self.state.status = RunStatus::Blocked;
8414 let spent = self.state.gate_fixes.len();
8415 self.state.event(
8416 "gate",
8417 if spent == 0 {
8418 "gate failed; not merging".to_owned()
8419 } else {
8420 format!("gate failed after {spent} gate-fix round(s); not merging")
8421 },
8422 );
8423 }
8424 self.state.save()?;
8425 Ok(())
8426 }
8427
8428 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8438 let commands = state.config.verify.pre_gate.clone();
8439 if commands.is_empty() {
8440 return;
8441 }
8442 let shell = state.config.shell();
8443 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8444 let (outcomes, _) = run_commands(
8445 state,
8446 "pre_gate",
8447 "pre_gate",
8448 0,
8449 &shell,
8450 &commands,
8451 &winner.worktree,
8452 timeout,
8453 )
8454 .await;
8455 for o in &outcomes {
8456 if !o.ok() {
8457 tracing::warn!(
8458 "pre_gate `{}` failed ({:?}); the gate decides",
8459 o.command,
8460 o.code
8461 );
8462 }
8463 state.event(
8464 "pre_gate",
8465 format!(
8466 "`{}` -> {}",
8467 o.command,
8468 if o.ok() {
8469 "pass".to_owned()
8470 } else {
8471 format!(
8472 "FAIL ({:?})\n{}",
8473 o.code,
8474 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8475 )
8476 }
8477 ),
8478 );
8479 }
8480 state.pre_gate = outcomes;
8481 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8482 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8483 Ok(head) => {
8484 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8485 state.pre_gate_commit = Some(head);
8486 }
8487 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8488 },
8489 Ok(false) => {}
8490 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8491 }
8492 if let Err(e) = state.save() {
8493 tracing::warn!("could not persist the pre_gate record: {e:#}");
8494 }
8495 }
8496
8497 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8500 let wait = self.repository_wait();
8501 run_landing_gate(&mut self.state, winner, &wait).await
8502 }
8503
8504 async fn gate_fix_round(
8517 &mut self,
8518 winner: &Candidate,
8519 outcomes: &[CommandOutcome],
8520 ) -> Result<GateFix> {
8521 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8522 g.phase == crate::repair_budget::Phase::Fixing
8523 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8524 }) {
8525 self.state.gate_fixes.len().saturating_add(1)
8526 } else {
8527 self.state.config.graph.gate_fix_rounds
8528 };
8529 let spent = self.state.gate_fixes.len();
8530 if spent >= cap {
8531 if cap > 0 {
8532 self.state.event(
8533 "gate",
8534 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8535 );
8536 }
8537 return Ok(GateFix::Stop);
8538 }
8539 if !gate_fixable(outcomes) {
8540 self.state.event(
8541 "gate",
8542 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8543 command or similar); not spending a fix round on it",
8544 );
8545 return Ok(GateFix::Stop);
8546 }
8547 let min_free = self.state.config.disk.min_free_bytes;
8548 if min_free > 0 {
8549 match crate::disk::free_bytes(&winner.worktree) {
8550 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8551 Ok(free) => {
8552 self.state.event(
8553 "gate",
8554 format!(
8555 "only {free} bytes free ({min_free} required by `[disk] \
8556 min_free_bytes`); not spending a fix round on a failure the disk \
8557 may explain"
8558 ),
8559 );
8560 return Ok(GateFix::Stop);
8561 }
8562 Err(e) => {
8563 self.state.event(
8564 "gate",
8565 format!("free disk space could not be measured ({e:#}); no fix round"),
8566 );
8567 return Ok(GateFix::Stop);
8568 }
8569 }
8570 }
8571
8572 let attempt = spent + 1;
8573 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8574 let base = self.landing_base();
8575 let fix_prompt = prompt::gate_fix_in(
8576 &self.state.instruction,
8577 &failed,
8578 attempt,
8579 cap,
8580 &self.state.config.graph.language,
8581 &self.state.config.graph.github_language,
8582 );
8583 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8584 let sessions = self.state.config.graph.sessions;
8585 let artifacts = self.merge_repair_artifacts();
8586 self.state.event(
8587 "gate",
8588 format!("gate failed; gate-fix round {attempt} of {cap}"),
8589 );
8590 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8591 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8592 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8595 head: before.clone(),
8596 passed: false,
8597 outcomes: Vec::new(),
8598 });
8599 self.state.save()?;
8600 let (_, seat, out) = self
8601 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8602 prompt: fix_prompt.clone(),
8603 spec,
8604 seat,
8605 cwd: winner.worktree.clone(),
8606 timeout,
8607 allow_write: true,
8608 sessions,
8609 artifacts: artifacts.clone(),
8610 stem: format!("gate-fix-{attempt}"),
8611 handover: None,
8612 })
8613 .await;
8614 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8617 return Ok(GateFix::Defer);
8618 }
8619 let mut record = GateFixRecord {
8620 agent: seat.agent.clone(),
8621 failed,
8622 notes: String::new(),
8623 committed: false,
8624 error: None,
8625 };
8626 match out {
8627 AgentOutcome::Ok(o) => {
8628 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8631 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8632 }
8633 }
8634 AgentOutcome::Dropped(_) => {
8635 record.error = Some("the CLI dropped the stream".to_owned());
8636 }
8637 AgentOutcome::Quota(o) => {
8638 self.state.quota.push(QuotaLoss::from_report(
8639 seat.key.clone(),
8640 "gate-fix",
8641 o.quota.as_ref(),
8642 ));
8643 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8644 }
8645 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8646 }
8647 self.state.seats.insert(seat.key.clone(), seat);
8648 let coordination = self.coordinate(&winner.worktree).await?;
8649 if let Ok(r) = git::rescue_commit(
8650 &winner.worktree,
8651 &format!("magi: gate fix {attempt} (uncommitted work)"),
8652 )
8653 .await
8654 {
8655 self.state.note_withheld("gate-fix", &r.withheld);
8656 }
8657 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8658 record.committed = after != before;
8659 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8660 let note = record.error.clone();
8661 self.state.gate_fixes.push(record);
8662 self.state.save()?;
8663 if !changed {
8664 self.state.event(
8665 "gate",
8666 match note {
8667 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8668 None => format!("gate-fix round {attempt}: the tree did not change"),
8669 },
8670 );
8671 return Ok(GateFix::Stop);
8672 }
8673 self.state.event(
8674 "gate",
8675 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8676 );
8677
8678 drop(coordination);
8679 let wait = self.repository_wait();
8680 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8681 }
8682
8683 async fn guarded_pr_message(
8690 &mut self,
8691 winner: &Candidate,
8692 facts: Option<&BranchFacts>,
8693 posting: bool,
8694 ) -> Result<Option<PrMessage>> {
8695 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8696 let head = match final_head.as_deref() {
8697 Some(h) => crate::pr_outcome::FinalHead::At(h),
8698 None => crate::pr_outcome::FinalHead::Unknown,
8699 };
8700 let summarized = if posting {
8702 Some(self.summarize_change(winner, final_head.as_deref()).await)
8703 } else {
8704 None
8705 };
8706 if self.state.resource_block.is_some() {
8711 self.state.event(
8712 "pr-summary",
8713 "a shared resource was busy; nothing was published, resume to summarize again",
8714 );
8715 return Ok(None);
8716 }
8717 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8718 anyhow::bail!(
8719 "the final commit moved during public description generation; retry before publishing"
8720 );
8721 }
8722 let note = match (&summarized, &self.state.change_summary) {
8723 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8724 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8725 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8726 (None, _) => ChangeNote::NotAsked,
8727 };
8728 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8729 if !posting {
8730 let identity = crate::scrub::Identity::current();
8731 return Ok(Some(PrMessage {
8732 title: crate::scrub::scrub(&pr.title, &identity),
8733 body: crate::scrub::scrub(&pr.body, &identity),
8734 }));
8735 }
8736 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8739 if let Some(known) = self
8740 .state
8741 .github_text
8742 .clone()
8743 .filter(|g| g.fingerprint == raw_fp)
8744 {
8745 return self.settle_withheld(winner, &pr, known).await;
8746 }
8747 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8748 let mut violations =
8749 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8750 let identity = crate::scrub::Identity::current();
8751 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8752 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8753 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8754 {
8755 violations.push(crate::github_text::Violation::SensitiveData);
8756 }
8757 let has_language = violations
8761 .iter()
8762 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8763 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8764 self.state.event(
8765 "github-text",
8766 if has_language {
8767 format!("description rejected: {violations:?}; requesting one rewrite")
8768 } else {
8769 format!("description rejected: {violations:?}; asking the owner")
8770 },
8771 );
8772 let mut rewritten = false;
8773 let mut rejected_rewrite = None;
8774 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8775 {
8776 let output = text;
8777 if let Some(title) = summary_title(&output) {
8778 let raw_body = summary_without_title(&output);
8781 let clean_title = crate::scrub::scrub(&title, &identity);
8782 let mut body = crate::scrub::scrub(&raw_body, &identity);
8783 let redacted = clean_title != title || body != raw_body;
8784 let title = clean_title;
8785 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8786 if !complete {
8787 self.state.event(
8788 "github-text",
8789 "description rewrite omitted required findings",
8790 );
8791 }
8792 if !body.trim().is_empty() {
8793 body.push_str("\n\n");
8794 body.push_str(&public_pr_evidence(&self.state, head));
8795 }
8796 let marker = format!("magi:run/{}", self.state.id);
8797 if !body.trim().is_empty() && !body.contains(&marker) {
8798 body.push_str(&format!("\n\n{marker}"));
8799 }
8800 let rewrite = PrMessage {
8801 title: title.clone(),
8802 body: body.clone(),
8803 };
8804 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8805 let mut failed = crate::github_text::check_for(
8806 &self.state.config.graph,
8807 &title,
8808 &body,
8809 redecision,
8810 );
8811 if (!crate::github_text::shareable(&title)
8812 || !crate::github_text::shareable(&body))
8813 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8814 {
8815 failed.push(crate::github_text::Violation::SensitiveData);
8816 }
8817 if complete
8818 && !title.trim().is_empty()
8819 && !body.trim().is_empty()
8820 && !failed.is_empty()
8821 {
8822 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8823 title: title.clone(),
8824 body: body.clone(),
8825 failed: failed
8826 .iter()
8827 .map(|v| crate::github_text::category(*v).to_owned())
8828 .collect(),
8829 });
8830 }
8831 if complete
8832 && !title.trim().is_empty()
8833 && !raw_body.trim().is_empty()
8834 && failed.is_empty()
8835 {
8836 decision = redecision;
8837 pr = PrMessage { title, body };
8838 rewritten = true;
8839 self.state
8840 .event("github-text", "description rewrite passed");
8841 if redacted {
8842 self.state.event(
8843 "github-text",
8844 "sensitive data removed from description rewrite",
8845 );
8846 }
8847 }
8848 }
8849 }
8850 if !rewritten {
8851 self.state.event(
8852 "github-text",
8853 "description rewrite unavailable or rejected; asking the owner",
8854 );
8855 if let Some(w) =
8856 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8857 {
8858 self.retire_github_text_question("the withheld text changed");
8861 let record = crate::run::GithubTextAsk {
8862 rewrite: rejected_rewrite,
8863 fingerprint: raw_fp,
8864 title: w.title,
8865 body: w.body,
8866 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8867 question: None,
8868 asks: 0,
8869 resolved: false,
8870 chosen_title: None,
8871 choice: None,
8872 publication: None,
8873 };
8874 return self.settle_withheld(winner, &pr, record).await;
8875 }
8876 }
8877 }
8878 let (title, body) =
8879 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8880 Ok(Some(PrMessage { title, body }))
8881 }
8882
8883 async fn summarize_change(
8894 &mut self,
8895 winner: &Candidate,
8896 head: Option<&str>,
8897 ) -> std::result::Result<(), String> {
8898 let Some(head) = head else {
8899 return Err("the final commit could not be read".to_owned());
8900 };
8901 let remote = self.state.config.merge.remote.clone();
8902 let tracking = format!("{remote}/{}", self.state.base_branch);
8903 if !git::rev_exists(&winner.worktree, &tracking).await {
8904 return Err(format!("the remote base `{tracking}` could not be read"));
8905 }
8906 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8907 Ok(b) if !b.is_empty() => b,
8908 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8909 };
8910 let evidence = crate::pr_outcome::summary_evidence(
8911 &self.state,
8912 crate::pr_outcome::FinalHead::At(head),
8913 );
8914 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8915 if self
8916 .state
8917 .change_summary
8918 .as_ref()
8919 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8920 {
8921 return Ok(());
8922 }
8923 if self.state.change_summary.take().is_some() {
8924 self.state.event(
8925 "pr-summary",
8926 "the final diff or evidence changed; the earlier change summary was discarded",
8927 );
8928 }
8929 if let Err(e) = write_artifact(
8931 &self.state,
8932 &format!("public-description-{fingerprint}.md"),
8933 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8934 ) {
8935 tracing::warn!("could not save public description evidence: {e:#}");
8936 }
8937 let chain = match agent::pick_chain(
8938 &self.state.config.agents,
8939 self.state.config.roles.synthesizer.as_ref(),
8940 &agent::installed,
8941 "synthesizer",
8942 ) {
8943 Ok(chain) => chain,
8944 Err(e) => {
8945 tracing::warn!("no agent for the change summary: {e:#}");
8946 self.state.event("pr-summary", "no summary agent available");
8947 return Err("no agent was available to summarize it".to_owned());
8948 }
8949 };
8950 let subjects = git::subjects(&winner.worktree, &base, head)
8951 .await
8952 .unwrap_or_default();
8953 let stat = git::diff_stat(&winner.worktree, &base, head)
8954 .await
8955 .unwrap_or_default();
8956 let stat: String = stat.chars().take(6_000).collect();
8957 let mut prompt = crate::prompt::change_summary_in(
8958 &base,
8959 head,
8960 &subjects,
8961 stat.trim(),
8962 &self.state.config.graph.github_language,
8963 );
8964 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8965 let sessions = self.state.config.graph.sessions;
8966 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8967 let prompts = self.state.config.prompts.clone();
8968 let cache = self.state.config.cache_dir();
8969 let run = self.state.id.clone();
8970 for (i, spec) in chain.into_iter().enumerate() {
8971 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8972 let github_spec = match agent::for_github_text(
8973 &spec,
8974 &self.state.config.graph.github_language,
8975 &winner.worktree,
8976 ) {
8977 Ok(spec) => spec,
8978 Err(e) => {
8979 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8980 continue;
8981 }
8982 };
8983 let job = SeatJob {
8984 spec: github_spec,
8985 seat,
8986 cwd: winner.worktree.clone(),
8987 prompt: prompt.clone(),
8988 timeout,
8989 allow_write: false,
8990 sessions,
8991 artifacts: agent::artifacts_dir(&self.state.dir()),
8992 stem: if i == 0 {
8993 "change-summary".to_owned()
8994 } else {
8995 format!("change-summary-{}", spec.id)
8996 },
8997 handover: None,
8998 };
8999 let ctx = WaveCtx {
9000 run: &run,
9001 node: "pr-summary",
9002 prompts: &prompts,
9003 cache: cache.as_deref(),
9004 round: None,
9005 carry_seats: false,
9006 pauses: Some((&self.pause, &self.interrupt)),
9007 };
9008 let (seat, outcome) =
9009 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9010 self.state.seats.insert(seat.key.clone(), seat);
9011 let text = match outcome {
9012 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9013 AgentOutcome::ResourceBlocked(why) => {
9014 if self.state.resource_block.is_none() {
9017 self.state.resource_block = Some(crate::run::ResourceBlock {
9018 cause: crate::run::ResourceBlockCause::Other,
9019 node: "pr-summary".to_owned(),
9020 kind: "shared resource".to_owned(),
9021 detail: why,
9022 at: Timestamp::now(),
9023 });
9024 }
9025 return Err("a shared resource was busy".to_owned());
9026 }
9027 _ => continue,
9028 };
9029 let blind = &self.state.config.blind;
9030 let Some(title) = summary_title(&text) else {
9031 continue;
9032 };
9033 let body: String = summary_without_title(&text)
9034 .lines()
9035 .filter(|l| !l.contains("magi:run/"))
9036 .collect::<Vec<_>>()
9037 .join("\n");
9038 let body = blind::sanitize_prose(body.trim(), blind);
9039 let title = blind::sanitize_prose(&title, blind);
9040 if body.trim().is_empty() || title.trim().is_empty() {
9041 continue;
9042 }
9043 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9044 continue;
9045 }
9046 if git::rev_parse(&winner.worktree, "HEAD")
9048 .await
9049 .ok()
9050 .as_deref()
9051 != Some(head)
9052 {
9053 return Err("the branch moved while the summary was written".to_owned());
9054 }
9055 if git::merge_base(&winner.worktree, &tracking, head)
9056 .await
9057 .ok()
9058 .as_deref()
9059 != Some(base.as_str())
9060 || crate::pr_outcome::summary_evidence(
9061 &self.state,
9062 crate::pr_outcome::FinalHead::At(head),
9063 ) != evidence
9064 {
9065 return Err("the summary evidence changed while it was written".to_owned());
9066 }
9067 self.state.change_summary = Some(crate::run::ChangeSummary {
9068 head: head.to_owned(),
9069 base,
9070 evidence: fingerprint,
9071 title,
9072 body,
9073 });
9074 self.state.event(
9075 "pr-summary",
9076 format!("change summary written by {} for the final diff", spec.id),
9077 );
9078 if let Err(e) = self.state.save() {
9079 tracing::warn!("could not save the change summary: {e:#}");
9080 }
9081 return Ok(());
9082 }
9083 self.state
9084 .event("pr-summary", "no agent produced a change summary");
9085 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9086 }
9087
9088 async fn rewrite_pr_text(
9098 &mut self,
9099 winner: &Candidate,
9100 pr: &PrMessage,
9101 violations: &[crate::github_text::Violation],
9102 ) -> Option<String> {
9103 let prompt = format!(
9104 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9105 pr.title, pr.body
9106 );
9107 let sessions = self.state.config.graph.sessions;
9108 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9109 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9112 let key = format!("impl-{}", winner.label);
9113 if let Some(spec) = self
9114 .state
9115 .config
9116 .agents
9117 .iter()
9118 .find(|a| a.id == winner.agent)
9119 .cloned()
9120 && let Some(existing) = self.state.seats.get(&key)
9121 && existing.agent == spec.id
9122 && agent::has_session(spec.kind, existing, sessions)
9123 {
9124 attempts.push((spec, Some(existing.clone())));
9125 }
9126 match agent::pick_chain(
9127 &self.state.config.agents,
9128 self.state.config.roles.language_judge.as_ref(),
9129 &agent::installed,
9130 "language_judge",
9131 ) {
9132 Ok(chain) => {
9133 for spec in chain {
9134 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9135 continue;
9136 }
9137 attempts.push((spec, None));
9138 }
9139 }
9140 Err(e) if attempts.is_empty() => {
9141 tracing::warn!("no agent for the description rewrite: {e:#}");
9142 self.state
9143 .event("github-text", "no rewrite agent available");
9144 return None;
9145 }
9146 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9147 }
9148 let prompts = self.state.config.prompts.clone();
9149 let cache = self.state.config.cache_dir();
9150 let run = self.state.id.clone();
9151 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9152 let resumed = resumed_seat.is_some();
9153 let seat = resumed_seat.unwrap_or_else(|| {
9154 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9155 });
9156 let github_spec = match agent::for_github_text(
9157 &spec,
9158 &self.state.config.graph.github_language,
9159 &winner.worktree,
9160 ) {
9161 Ok(spec) => spec,
9162 Err(e) => {
9163 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9164 continue;
9165 }
9166 };
9167 let job = SeatJob {
9168 spec: github_spec,
9169 seat,
9170 cwd: winner.worktree.clone(),
9171 prompt: prompt.clone(),
9172 timeout: retry_budget(timeout_full, resumed),
9173 allow_write: false,
9174 sessions,
9175 artifacts: agent::artifacts_dir(&self.state.dir()),
9176 stem: if i == 0 {
9177 "github-text-rewrite".to_owned()
9178 } else {
9179 format!("github-text-rewrite-{}", spec.id)
9180 },
9181 handover: None,
9182 };
9183 let ctx = WaveCtx {
9184 run: &run,
9185 node: "github-text",
9186 prompts: &prompts,
9187 cache: cache.as_deref(),
9188 round: None,
9189 carry_seats: false,
9190 pauses: Some((&self.pause, &self.interrupt)),
9191 };
9192 let (seat, outcome) =
9193 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9194 self.state.seats.insert(seat.key.clone(), seat);
9195 match outcome {
9196 AgentOutcome::Ok(o)
9197 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9198 {
9199 return Some(o.text);
9200 }
9201 AgentOutcome::ResourceBlocked(_) => return None,
9202 _ => {}
9203 }
9204 }
9205 None
9206 }
9207
9208 fn retire_github_text_question(&mut self, why: &str) {
9210 let Some(id) = self
9211 .state
9212 .github_text
9213 .as_ref()
9214 .and_then(|g| g.question.clone())
9215 else {
9216 return;
9217 };
9218 let _ = ask::Questions::open().update(&id, |q| {
9219 q.abandon(why);
9220 Ok(())
9221 });
9222 }
9223
9224 async fn settle_withheld(
9228 &mut self,
9229 winner: &Candidate,
9230 pr: &PrMessage,
9231 mut g: crate::run::GithubTextAsk,
9232 ) -> Result<Option<PrMessage>> {
9233 use crate::github_text as gt;
9234 let store = ask::Questions::open();
9235 let timeout = self.state.config.graph.answer_timeout;
9236 if !g.resolved {
9237 let mut standing = g
9238 .question
9239 .as_ref()
9240 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9241 if let Some(q) = standing.as_ref()
9242 && q.status.open()
9243 && gt::expired(q, timeout)
9244 {
9245 let why = format!("no answer within {}s of asking", timeout.max(1));
9246 standing = store
9247 .update(&q.id, |q| {
9248 q.abandon(&why);
9249 Ok(())
9250 })
9251 .ok()
9252 .map(|(q, ())| q);
9253 }
9254 let mut ask_again: Option<Option<Vec<String>>> = None;
9255 match standing {
9256 None if g.asks <= 1 => {
9259 g.asks = 0;
9260 ask_again = Some(None);
9261 }
9262 None => {
9263 g.resolved = true;
9264 g.chosen_title = None;
9265 g.choice = Some(GithubTextChoice::Fallback);
9266 }
9267 Some(q) if q.status.open() => {
9268 self.park_on_github_text(&g)?;
9269 return Ok(None);
9270 }
9271 Some(q) => match gt::read_reply(&q) {
9272 gt::Reply::Fallback => {
9273 g.resolved = true;
9274 g.chosen_title = None;
9275 g.choice = Some(GithubTextChoice::Fallback);
9276 }
9277 gt::Reply::Original => {
9278 g.resolved = true;
9279 g.chosen_title = None;
9280 g.choice = Some(GithubTextChoice::Original);
9281 }
9282 gt::Reply::Rewrite => {
9283 g.resolved = true;
9284 g.chosen_title = None;
9285 g.choice = Some(if g.rewrite.is_some() {
9286 GithubTextChoice::Rewrite
9287 } else {
9288 GithubTextChoice::Fallback
9289 });
9290 }
9291 gt::Reply::Title(text) => {
9292 let first = text
9293 .lines()
9294 .map(str::trim)
9295 .find(|l| !l.is_empty())
9296 .unwrap_or("");
9297 let decision = if first.is_empty() {
9298 None
9299 } else {
9300 let candidate = PrMessage {
9301 title: first.to_owned(),
9302 body: String::new(),
9303 };
9304 self.judge_pr_language(&winner.worktree, &candidate).await
9305 };
9306 match gt::vet_title(
9307 &text,
9308 decision,
9309 self.state.config.graph.github_english(),
9310 ) {
9311 Ok(title) => {
9312 g.resolved = true;
9313 g.chosen_title = Some(title);
9314 g.choice = Some(GithubTextChoice::MyTitle);
9315 }
9316 Err(cats) if g.asks < 2 => {
9317 ask_again = Some(Some(
9318 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9319 ))
9320 }
9321 Err(_) => {
9322 g.resolved = true;
9323 g.chosen_title = None;
9324 g.choice = Some(GithubTextChoice::Fallback);
9325 }
9326 }
9327 }
9328 },
9329 }
9330 if let Some(rejected) = ask_again {
9331 let w = gt::Withheld {
9334 title: g.title,
9335 body: g.body,
9336 categories: g.categories.clone(),
9337 };
9338 let artifact = match crate::run::write_artifact(
9341 &self.state,
9342 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9343 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9344 ) {
9345 Ok(p) => Some(p),
9346 Err(e) => {
9347 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9348 None
9349 }
9350 };
9351 let (detail, comparison) = gt::question_content(
9352 &w,
9353 &pr.title,
9354 &pr.body,
9355 &self.state.id,
9356 rejected.as_deref(),
9357 artifact.as_deref(),
9358 g.rewrite.as_ref(),
9359 );
9360 let mut q = ask::Question::new(
9361 self.state.id.clone(),
9362 gt::ASK_NODE.to_owned(),
9363 gt::ASK_SEAT.to_owned(),
9364 gt::question_summary(&self.state.config.graph.language, &w),
9365 detail,
9366 gt::question_choices(&w, g.rewrite.as_ref()),
9367 );
9368 q.posting_comparison = Some(comparison);
9369 q.answer_timeout = timeout;
9370 g.question = Some(q.id.clone());
9373 g.asks += 1;
9374 self.state.github_text = Some(g.clone());
9375 self.state.event(
9376 "github-text",
9377 format!("asking the owner about the withheld text ({})", q.short()),
9378 );
9379 self.state.save()?;
9380 store
9381 .put_executable(&mut q)
9382 .context("file the posting-gate question")?;
9383 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9384 tracing::warn!(
9385 "could not notify about posting-gate question {}: {e:#}",
9386 q.short()
9387 );
9388 }
9389 self.park_on_github_text(&g)?;
9390 return Ok(None);
9391 }
9392 }
9393 let resolved = g.chosen_title.clone();
9394 let original = g.choice == Some(GithubTextChoice::Original);
9395 self.state.event(
9397 "github-text",
9398 if original {
9399 format!(
9400 "posting the original text for: {}",
9401 match (g.title, g.body) {
9402 (true, true) => "title, description",
9403 (true, false) => "title",
9404 _ => "description",
9405 }
9406 )
9407 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9408 "posting the saved rewrite title and description".to_owned()
9409 } else if resolved.is_some() {
9410 "posting the owner's replacement title".to_owned()
9411 } else {
9412 "posting the neutral text for the withheld fields".to_owned()
9413 },
9414 );
9415 self.state.github_text = Some(g.clone());
9416 self.state.save()?;
9417 if g.choice == Some(GithubTextChoice::Rewrite)
9418 && let Some(r) = g.rewrite.as_ref()
9419 {
9420 return Ok(Some(PrMessage {
9421 title: r.title.clone(),
9422 body: r.body.clone(),
9423 }));
9424 }
9425 let w = gt::Withheld {
9426 title: g.title,
9427 body: g.body,
9428 categories: g.categories.clone(),
9429 };
9430 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9431 if let Some(t) = resolved {
9432 title = t;
9433 }
9434 let english = gt::LanguageDecision {
9436 title_english: true,
9437 body_english: true,
9438 body_complete: true,
9439 };
9440 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9441 if original {
9442 if g.title {
9445 title = pr.title.clone();
9446 }
9447 if g.body {
9448 body = pr.body.clone();
9449 }
9450 }
9451 Ok(Some(PrMessage { title, body }))
9452 }
9453
9454 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9455 self.state.github_text = Some(g.clone());
9456 self.state.end_pass_parked(ParkCause::Question);
9457 self.state.parked = true;
9458 self.state.event(
9459 "github-text",
9460 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9461 );
9462 self.state.save()
9463 }
9464
9465 async fn judge_pr_language(
9468 &mut self,
9469 cwd: &Path,
9470 pr: &PrMessage,
9471 ) -> Option<crate::github_text::LanguageDecision> {
9472 self.state.config.roles.language_judge.as_ref()?;
9473 if !self.state.config.graph.language_gate() {
9474 return None;
9475 }
9476 let decision =
9477 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9478 self.state.event(
9479 "github-text",
9480 if decision.is_some() {
9481 "language decided by the language judge"
9482 } else {
9483 "language judge unavailable; using built-in heuristics"
9484 },
9485 );
9486 decision
9487 }
9488
9489 async fn merge(&mut self) -> Result<()> {
9490 if self.resource_halt()? || self.hold_base_sync().await? {
9491 return Ok(());
9492 }
9493 if self
9508 .state
9509 .base_sync
9510 .as_ref()
9511 .is_some_and(|s| s.conflict.is_some())
9512 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9513 != Some(RunStatus::Gating)
9514 || !self.state.gate_allows_progress()
9522 {
9523 return Ok(());
9524 }
9525 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9534 return Ok(());
9535 }
9536 let Some(winner) = self.state.winner().cloned() else {
9537 return Ok(());
9538 };
9539 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9540 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9541 self.state.status = RunStatus::Blocked;
9542 self.state.event(
9543 "gate",
9544 "winner changed after gate; fresh verification required",
9545 );
9546 self.state.save()?;
9547 return Ok(());
9548 }
9549 let wait = self.repository_wait();
9550 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9551 return Ok(());
9552 }
9553 let repo = self.state.repo.clone();
9554 let base = self.state.base_branch.clone();
9555 let mode = self.state.config.merge.mode;
9556 let style = self.state.config.merge.style;
9557 let facts = if is_review_run(&self.state) {
9558 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9559 let start = review_base(
9560 &repo,
9561 &self.state.config.merge.remote,
9562 &base,
9563 &self.state.base_commit,
9564 &winner.branch,
9565 )
9566 .await;
9567 branch_facts(&repo, &start, &winner.branch).await
9568 } else {
9569 None
9570 };
9571 let leaked = if is_review_run(&self.state) {
9574 leaked_subjects(&self.state, &winner.branch).await
9575 } else {
9576 Some(Vec::new())
9577 };
9578 let Some(pr) = self
9582 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9583 .await?
9584 else {
9585 return Ok(());
9586 };
9587 let message = pr.commit_message();
9588
9589 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9590 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9591 return Ok(());
9592 }
9593 let outcome = match mode {
9594 MergeMode::None => MergeOutcome {
9595 mode,
9596 ok: true,
9597 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9598 empty: false,
9599 },
9600 MergeMode::Pr | MergeMode::Local
9601 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9602 {
9603 MergeOutcome {
9604 mode,
9605 ok: false,
9606 detail: empty_candidate_detail(&self.state, &base),
9607 empty: true,
9608 }
9609 }
9610 MergeMode::Local => {
9611 let on = git::current_branch(&repo).await?;
9612 if on.as_deref() != Some(base.as_str()) {
9613 MergeOutcome {
9614 mode,
9615 ok: false,
9616 detail: format!(
9617 "{} has {} checked out, not the base branch {base}",
9618 repo.display(),
9619 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9620 ),
9621 empty: false,
9622 }
9623 } else if !git::is_clean(&repo).await? {
9624 MergeOutcome {
9625 mode,
9626 ok: false,
9627 detail: format!("{} is dirty; refusing to merge", repo.display()),
9628 empty: false,
9629 }
9630 } else {
9631 let out = match style {
9632 MergeStyle::Merge => {
9633 git::merge_no_ff(&repo, &winner.branch, &message).await?
9634 }
9635 MergeStyle::Squash => {
9636 git::merge_squash(&repo, &winner.branch, &message).await?
9637 }
9638 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9639 };
9640 MergeOutcome {
9641 mode,
9642 ok: out.ok(),
9643 detail: if out.ok() { out.stdout } else { out.stderr },
9644 empty: false,
9645 }
9646 }
9647 }
9648 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9649 self.state.pr_publication_pending = true;
9650 self.state.save()?;
9651 let out =
9652 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9653 self.state.pr_publication_pending = out.is_err();
9654 MergeOutcome {
9655 mode,
9656 ok: out.is_ok(),
9657 detail: match out {
9658 Ok(url) => url,
9659 Err(e) => format!("{e:#}"),
9660 },
9661 empty: false,
9662 }
9663 }
9664 MergeMode::Pr => {
9665 let remote = self.state.config.merge.remote.clone();
9666 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9667 let pushed = if self.state.foreign_checkout.is_some() {
9668 let refspec = format!("{delivery_head}:refs/heads/{public}");
9669 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9670 } else {
9671 git::push(&winner.worktree, &remote, &winner.branch).await?
9672 };
9673 if !pushed.ok() {
9674 MergeOutcome {
9675 mode,
9676 ok: false,
9677 detail: pushed.stderr,
9678 empty: false,
9679 }
9680 } else {
9681 if self.state.foreign_checkout.is_some() {
9682 self.state.pr_pushed_head = Some(delivery_head.clone());
9683 }
9684 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9690 let out = match pr_merge_plan(found) {
9691 PrPlan::Create => {
9692 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9693 .await
9694 }
9695 PrPlan::Adopt { url, title } => {
9696 self.state
9697 .event("merge", format!("Pr: adopted open pull request {url}"));
9698 if self.state.explicit_github_text_selection() {
9699 land::set_pr_text_verbatim(
9702 &winner.worktree,
9703 &url,
9704 &pr.title,
9705 &pr.body,
9706 )
9707 .await
9708 .map(|()| url)
9709 } else {
9710 if title != pr.title
9711 && (!is_review_run(&self.state)
9712 || leaked
9713 .as_deref()
9714 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9715 && let Err(e) =
9716 if self.state.github_text.as_ref().is_some_and(|g| {
9717 g.resolved
9718 && g.title
9719 && g.choice == Some(GithubTextChoice::Original)
9720 }) {
9721 land::set_pr_title_verbatim(
9723 &winner.worktree,
9724 &url,
9725 &pr.title,
9726 )
9727 .await
9728 } else {
9729 land::set_pr_title(
9730 &mut self.state,
9731 &winner.worktree,
9732 &url,
9733 &pr.title,
9734 )
9735 .await
9736 }
9737 {
9738 tracing::warn!("could not refresh title of {url}: {e:#}");
9739 self.state
9740 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9741 }
9742 Ok(url)
9743 }
9744 }
9745 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9746 };
9747 match out {
9748 Ok(url) => MergeOutcome {
9749 mode,
9750 ok: true,
9751 detail: url,
9752 empty: false,
9753 },
9754 Err(e) => MergeOutcome {
9755 mode,
9756 ok: false,
9757 detail: e.to_string(),
9758 empty: false,
9759 },
9760 }
9761 }
9762 }
9763 };
9764
9765 self.state.status = match (mode, outcome.ok) {
9766 (MergeMode::None, _) => RunStatus::Ready,
9767 (_, true) if outcome.empty => RunStatus::Ready,
9768 (MergeMode::Pr, true) => RunStatus::Ready,
9769 (_, true) => RunStatus::Merged,
9770 (_, false) => RunStatus::Blocked,
9771 };
9772 self.state.event(
9773 "merge",
9774 format!(
9775 "{:?}: {}",
9776 mode,
9777 outcome.detail.lines().next().unwrap_or("")
9778 ),
9779 );
9780 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9781 self.state.record_github_text_publication(&outcome.detail);
9782 }
9783 self.state.merge = Some(outcome);
9784 self.state.save()?;
9785
9786 if self.state.config.graph.land
9792 && mode == MergeMode::Pr
9793 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9794 {
9795 self.run_land().await?;
9796 }
9797 self.settle_questions();
9802 Ok(())
9803 }
9804
9805 async fn run_land(&mut self) -> Result<()> {
9816 crate::pr_head::ensure_owner(&self.state)?;
9817 let url = self
9818 .state
9819 .pr
9820 .as_ref()
9821 .map(|pr| pr.url.clone())
9822 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9823 .unwrap_or_default();
9824 let url = url.lines().next().unwrap_or("").trim().to_owned();
9825 if !url.starts_with("http") {
9826 return Ok(());
9827 }
9828 loop {
9831 match land::land(&mut self.state, &url).await {
9832 Ok(_)
9833 if self
9834 .state
9835 .merge_repairs
9836 .last()
9837 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9838 {
9839 if self.repair_merge_review().await? {
9840 continue;
9841 }
9842 }
9843 Ok(_) if self.state.resource_block.is_some() => {
9844 return self.stop_for_resource();
9845 }
9846 Ok(pr) if self.state.parked => {
9847 let _ = pr;
9851 }
9852 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9853 self.state.status = RunStatus::Superseded;
9854 self.state.save()?;
9855 }
9856 Ok(pr) => {
9857 self.state.status = match pr.state {
9858 land::PrLifecycle::Merged => RunStatus::Merged,
9859 _ => RunStatus::Blocked,
9860 };
9861 if bump::should_release_bump(self.state.status)
9868 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9869 {
9870 self.state
9876 .event("bump", format!("release bump skipped: {e:#}"));
9877 }
9878 if self.state.status == RunStatus::Merged {
9881 crate::followup::after_merge(&mut self.state, &pr.url).await;
9882 }
9883 self.state.save()?;
9884 }
9885 Err(e) => {
9886 if e.is::<crate::repo_lock::ResourceBlocked>() {
9887 return Err(e);
9888 }
9889 self.state.status = RunStatus::Blocked;
9890 self.state.event("land", format!("gave up: {e}"));
9891 self.state.save()?;
9892 }
9893 }
9894 break;
9895 }
9896 Ok(())
9897 }
9898
9899 async fn repair_merge_review(&mut self) -> Result<bool> {
9902 use crate::run::MergeRepairPhase as Phase;
9903 crate::pr_head::ensure_owner(&self.state)?;
9904 let i = self.state.merge_repairs.len() - 1;
9905 let request = self.state.merge_repairs[i].clone();
9906 if request.phase == Phase::Fixing {
9907 let restored = match self.state.winner().cloned() {
9908 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9909 None => Err(anyhow::anyhow!("repair has no winning branch")),
9910 };
9911 let why = match restored {
9912 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9913 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9914 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9915 };
9916 self.hold_merge_repair(i, why)?;
9917 return Ok(false);
9918 }
9919 if request.phase == Phase::Reviewing && self.state.gate_ran {
9924 let winner = self
9925 .state
9926 .winner()
9927 .context("repair has no winning branch")?;
9928 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9929 if self.state.gate_head.as_deref() != Some(head.as_str())
9930 || !git::is_clean(&winner.worktree).await?
9931 {
9932 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9933 return Ok(false);
9934 }
9935 }
9936 let failure = if i >= self.state.config.graph.land_rounds
9937 || self.state.config.graph.review_rounds == 0
9938 {
9939 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9940 } else {
9941 None
9942 };
9943 if let Some(why) = failure {
9944 self.hold_merge_repair(i, why)?;
9945 return Ok(false);
9946 }
9947 if request.phase == Phase::Requested {
9948 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9949 Ok(pr) => pr,
9950 Err(e) => {
9951 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9952 return Ok(false);
9953 }
9954 };
9955 self.state.merge_repairs[i].phase = Phase::Fixing;
9956 self.state.land_approval = None;
9957 self.state.land_armed_head = None;
9958 self.state.save()?;
9959 let evidence = serde_json::to_string_pretty(&request)?;
9962 let reason = format!(
9963 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9964 );
9965 let budget = self.state.config.graph.land_rounds;
9966 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9967 match result {
9968 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9969 Ok(land::Fixed::Deferred) => {
9970 let worktree = self
9974 .state
9975 .winner()
9976 .context("repair has no winning branch")?
9977 .worktree
9978 .clone();
9979 self.graft_restore(&worktree, &request.head).await?;
9980 self.state.merge_repairs[i].phase = Phase::Requested;
9981 self.stop_for_resource()?;
9982 return Ok(false);
9983 }
9984 Ok(land::Fixed::Failed(why)) => {
9985 self.hold_merge_repair(
9986 i,
9987 format!(
9988 "repair failed: {why}; inspect the branch and run an explicit review"
9989 ),
9990 )?;
9991 return Ok(false);
9992 }
9993 Err(e) => {
9994 self.hold_merge_repair(
9995 i,
9996 format!(
9997 "repair failed: {e:#}; inspect the branch and run an explicit review"
9998 ),
9999 )?;
10000 return Ok(false);
10001 }
10002 }
10003 self.begin_merge_repair_review(i)?;
10007 }
10008 self.review_loop().await?;
10009 self.gate().await?;
10010 if self.resource_halt()? || self.state.parked {
10011 return Ok(false);
10012 }
10013 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10014 != Some(RunStatus::Gating)
10015 || !self.state.gate_allows_progress()
10016 {
10017 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10018 return Ok(false);
10019 }
10020 let winner = self
10023 .state
10024 .winner()
10025 .cloned()
10026 .context("repair has no winning branch")?;
10027 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10028 if self.state.gate_head.as_deref() != Some(after.as_str())
10029 || !git::is_clean(&winner.worktree).await?
10030 {
10031 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10032 return Ok(false);
10033 }
10034 let expected = self
10035 .state
10036 .pr_pushed_head
10037 .as_deref()
10038 .unwrap_or(&request.head)
10039 .to_owned();
10040 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10041 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10044 return Ok(false);
10045 }
10046 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10047 let refspec = format!("{after}:refs/heads/{public}");
10048 let push = git::git_raw(
10049 &winner.worktree,
10050 &["push", &lease, &self.state.config.merge.remote, &refspec],
10051 )
10052 .await;
10053 match push {
10054 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10055 Ok(out) => {
10056 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10057 return Ok(false);
10058 }
10059 Err(e) => {
10060 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10061 return Ok(false);
10062 }
10063 }
10064 self.state.merge_repairs[i].phase = Phase::Complete;
10065 self.state.status = RunStatus::Landing;
10066 self.state.event(
10067 "repair",
10068 "new review and gate completed; fresh owner merge decision required",
10069 );
10070 self.state.save()?;
10071 Ok(true)
10072 }
10073
10074 fn merge_repair_artifacts(&self) -> PathBuf {
10075 let artifacts = agent::artifacts_dir(&self.state.dir());
10076 match self
10077 .state
10078 .merge_repairs
10079 .last()
10080 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10081 {
10082 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10083 None => artifacts,
10084 }
10085 }
10086
10087 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10088 if let Some(e2e) = self.state.head_e2e.take() {
10089 self.state
10090 .verification_history
10091 .push(crate::run::VerificationEvidence {
10092 superseded_by: None,
10093 archived_at: Some(Timestamp::now()),
10094 fixes_spent: Some(self.state.gate_fixes.len()),
10095 gate_head: None,
10096 gate_ran: false,
10097 gate: Vec::new(),
10098 e2e: Some(e2e),
10099 });
10100 }
10101 self.state.reviews.clear();
10102 self.state.gate.clear();
10103 self.state.gate_ran = false;
10104 self.state.gate_head = None;
10105 self.state.gate_fixes.clear();
10106 self.state.gate_repair_e2e = None;
10107 self.state.pre_gate.clear();
10108 self.state.pre_gate_commit = None;
10109 self.state.contested_handoff = None;
10110 self.state.land_reruns = None;
10111 self.state.land_approval = None;
10112 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10113 self.state.status = RunStatus::Reviewing;
10114 self.state.save()
10115 }
10116
10117 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10118 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10119 self.state.merge_repairs[index].failure = Some(why.clone());
10120 self.state.status = RunStatus::Blocked;
10121 self.state.event("repair", why);
10122 self.state.save()
10123 }
10124
10125 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10129 if let Some(existing) = self.state.seats.get(key)
10130 && existing.agent == agent
10131 {
10132 return existing.clone();
10133 }
10134 let fresh = if self.state.seats.contains_key(key) {
10137 handover_seat(key, agent, self.state.next_seat_seed())
10138 } else {
10139 SeatState::new(key, agent, self.state.seed)
10140 };
10141 self.state.seats.insert(key.to_owned(), fresh.clone());
10142 fresh
10143 }
10144
10145 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10151 match self.state.seats.get(key) {
10152 Some(s) if s.agent != spec.id => {
10153 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10154 }
10155 _ => spec,
10156 }
10157 }
10158
10159 fn view(&self, c: &Candidate) -> CandidateView {
10161 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10162 .unwrap_or_default();
10163 let (patch, _) = blind::sanitize_patch(
10164 &format!("candidate {} patch", c.label),
10165 &raw,
10166 &self.state.config.blind,
10167 );
10168 CandidateView {
10169 label: c.label,
10170 branch: c.branch.clone(),
10171 summary: c.summary.clone(),
10172 stat: c.stat.clone(),
10173 patch,
10174 }
10175 }
10176
10177 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10179 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10180 prompt::judge(
10181 "(see above)",
10182 &views,
10183 self.roles.judges.len(),
10184 base_short,
10185 "en",
10186 )
10187 }
10188
10189 fn vote_prompt_full(
10195 &self,
10196 j: usize,
10197 labels: &[char],
10198 language: &str,
10199 candidates: &[Candidate],
10200 base_short: &str,
10201 ) -> String {
10202 let mut text = format!(
10203 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10204 prompt::final_vote(labels, language),
10205 self.state.instruction,
10206 self.candidate_block(candidates, base_short)
10207 );
10208 if let Some(own) = self
10209 .state
10210 .judgements
10211 .get(j)
10212 .filter(|r| !r.ranking.is_empty())
10213 {
10214 let reasons = own
10215 .reasons
10216 .iter()
10217 .map(|(k, v)| format!("- {k}: {v}"))
10218 .collect::<Vec<_>>()
10219 .join("\n");
10220 text.push_str(&format!(
10221 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10222 own.ranking.iter().collect::<String>(),
10223 if reasons.is_empty() {
10224 ""
10225 } else {
10226 ", because:\n"
10227 }
10228 ));
10229 }
10230 if !self.state.deliberation.is_empty() {
10231 text.push_str("\n# What was argued before this vote\n");
10232 for t in self.transcript(&[], j) {
10233 text.push_str(&format!(
10234 "\n## {}{}\n\n{}\n",
10235 t.who,
10236 if t.is_self { " (you)" } else { "" },
10237 t.body.trim()
10238 ));
10239 }
10240 }
10241 text
10242 }
10243
10244 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10251 let mut turns = Vec::new();
10252 for j in &self.state.judgements {
10253 if j.ranking.is_empty() {
10254 continue;
10255 }
10256 let reasons = j
10257 .reasons
10258 .iter()
10259 .map(|(k, v)| format!("- {k}: {v}"))
10260 .collect::<Vec<_>>()
10261 .join("\n");
10262 turns.push(Turn {
10263 who: format!("Judge {} (opening ranking)", j.judge),
10264 is_self: j.judge == self_idx + 1,
10265 body: format!(
10266 "Ranked {}{}{reasons}",
10267 j.ranking.iter().collect::<String>(),
10268 if reasons.is_empty() {
10269 ""
10270 } else {
10271 ", because:\n"
10272 }
10273 ),
10274 });
10275 }
10276 for t in self
10277 .state
10278 .deliberation
10279 .iter()
10280 .flat_map(|r| r.turns.iter())
10281 .chain(current)
10282 {
10283 turns.push(Turn {
10284 who: format!("Judge {}", t.judge),
10285 is_self: t.judge == self_idx + 1,
10286 body: t.body.clone(),
10287 });
10288 }
10289 turns
10290 }
10291}
10292
10293fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10295 agent::has_session(spec.kind, seat, sessions)
10296}
10297
10298fn next_untried_in_roster<'a>(
10319 roster: &'a [AgentSpec],
10320 start: usize,
10321 tried: &BTreeSet<String>,
10322) -> Option<&'a AgentSpec> {
10323 roster
10324 .get(start + 1..)?
10325 .iter()
10326 .find(|s| !tried.contains(&s.id))
10327}
10328
10329fn pick_successor<'a>(
10350 roster: &'a [AgentSpec],
10351 start: usize,
10352 tried: &BTreeSet<String>,
10353 carried: Option<&BTreeSet<String>>,
10354 others: &BTreeSet<String>,
10355) -> Option<&'a AgentSpec> {
10356 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10357 roster
10358 .get(start + 1..)
10359 .and_then(|tail| {
10360 tail.iter()
10361 .filter(free)
10362 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10363 })
10364 .or_else(|| {
10365 carried
10366 .is_some()
10367 .then(|| roster.iter().find(free))
10368 .flatten()
10369 })
10370 .or_else(|| match carried {
10371 Some(c) => next_for_seat(roster, start, tried, c),
10372 None => next_untried_in_roster(roster, start, tried),
10373 })
10374}
10375
10376fn next_for_seat<'a>(
10386 roster: &'a [AgentSpec],
10387 start: usize,
10388 round_tried: &BTreeSet<String>,
10389 carried_failed: &BTreeSet<String>,
10390) -> Option<&'a AgentSpec> {
10391 roster
10392 .get(start + 1..)?
10393 .iter()
10394 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10395 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10396}
10397
10398fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10404 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10405 return spec;
10406 };
10407 let ok = |id: &str| !h.failed.contains(id);
10408 if let Some(last) = h.last_ok.as_deref()
10409 && ok(last)
10410 && let Some(s) = roster.iter().find(|s| s.id == last)
10411 {
10412 return s.clone();
10413 }
10414 if ok(&spec.id) {
10415 return spec;
10416 }
10417 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10418 roster
10419 .get(from + 1..)
10420 .into_iter()
10421 .flatten()
10422 .chain(roster.iter())
10423 .find(|s| ok(&s.id))
10424 .cloned()
10425 .unwrap_or(spec)
10426}
10427
10428pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10432 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10433}
10434
10435const TIMED_OUT: &str = "timed out";
10438
10439impl FailClass {
10440 fn of(out: &AgentOutcome) -> Option<Self> {
10442 match out {
10443 AgentOutcome::Ok(_) => None,
10444 AgentOutcome::Quota(_) => Some(Self::Quota),
10445 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10446 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10447 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10448 AgentOutcome::ResourceBlocked(_) => None,
10451 }
10452 }
10453
10454 fn stem_word(&self) -> &'static str {
10456 match self {
10457 Self::Quota => "quota",
10458 _ => "handover",
10459 }
10460 }
10461}
10462
10463fn failure_signature(msg: &str) -> String {
10467 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10468 if let Some(code) = line
10471 .strip_prefix("process exited with no usable output (exit code ")
10472 .and_then(|s| s.strip_suffix(')'))
10473 .and_then(|s| s.parse::<i32>().ok())
10474 {
10475 line = format!("exited with some({code}) and no usable output");
10476 } else if matches!(
10477 line.as_str(),
10478 "process exited with no usable output (no exit code reported)"
10479 | "process exited with no usable output (exit code unavailable)"
10480 ) {
10481 line = "exited with none and no usable output".to_owned();
10482 }
10483 let mut out = Vec::new();
10484 for word in line.split_whitespace() {
10485 if word.contains('/') || word.contains('\\') {
10486 out.push("<path>".to_owned());
10487 continue;
10488 }
10489 let mut w = String::new();
10490 let mut in_digits = false;
10491 for c in word.chars() {
10492 if c.is_ascii_digit() {
10493 if !in_digits {
10494 w.push('#');
10495 }
10496 in_digits = true;
10497 } else {
10498 in_digits = false;
10499 w.push(c);
10500 }
10501 }
10502 out.push(w);
10503 }
10504 out.join(" ").chars().take(120).collect()
10505}
10506
10507fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10514 match cur {
10515 FailClass::Quota | FailClass::Timeout => true,
10516 FailClass::Other(_) => prev != Some(cur),
10517 }
10518}
10519
10520fn fail_reason(out: &AgentOutcome) -> String {
10522 match out {
10523 AgentOutcome::Ok(_) => String::new(),
10524 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10525 AgentOutcome::Dropped(o) => format!(
10526 "the CLI dropped the stream ({})",
10527 o.dropped
10528 .as_ref()
10529 .map(|d| d.why.as_str())
10530 .unwrap_or("it ended without delivering its answer")
10531 ),
10532 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10533 e.lines().next().unwrap_or("").chars().take(160).collect()
10534 }
10535 }
10536}
10537
10538pub(crate) fn record_handover(
10541 state: &mut RunState,
10542 node: &str,
10543 seat: &str,
10544 from: &str,
10545 to: &str,
10546 class: &FailClass,
10547 reason: &str,
10548) {
10549 let message = if *class == FailClass::Quota {
10550 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10551 } else {
10552 format!("{seat}: handed over {from} -> {to} ({reason})")
10553 };
10554 state.event(node, message);
10555 state.handovers.push(Handover {
10556 at: Timestamp::now(),
10557 node: node.to_owned(),
10558 seat: seat.to_owned(),
10559 from: from.to_owned(),
10560 to: to.to_owned(),
10561 reason: reason.to_owned(),
10562 });
10563}
10564
10565fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10579 commands.iter().any(|c| c.exit_code.is_none())
10580}
10581
10582fn verified_noop_claim(
10595 usable: bool,
10596 commands: &[agent::CommandEvidence],
10597 text: &str,
10598) -> Option<String> {
10599 (usable && !has_unconfirmed_command(commands))
10600 .then(|| verdict::verified_noop(text))
10601 .flatten()
10602}
10603
10604fn short(commit: &str) -> String {
10605 commit.chars().take(7).collect()
10606}
10607
10608fn make_executable(path: &Path) -> Result<()> {
10609 #[cfg(unix)]
10610 {
10611 use std::os::unix::fs::PermissionsExt as _;
10612 let mut perms = std::fs::metadata(path)?.permissions();
10613 perms.set_mode(0o755);
10614 std::fs::set_permissions(path, perms)?;
10615 }
10616 #[cfg(not(unix))]
10617 {
10618 let _ = path;
10619 }
10620 Ok(())
10621}
10622
10623struct WaveCtx<'a> {
10630 run: &'a str,
10633 node: &'a str,
10635 prompts: &'a Prompts,
10636 cache: Option<&'a Path>,
10638 round: Option<usize>,
10641 carry_seats: bool,
10645 pauses: Option<(&'a Pause, &'a Pause)>,
10648}
10649
10650fn repository_wait(
10652 state: &RunState,
10653 pauses: Option<(&Pause, &Pause)>,
10654) -> crate::repo_lock::WaitPolicy {
10655 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10656 let inherited = crate::repo_lock::WaitPolicy::current();
10657 crate::repo_lock::WaitPolicy::new(
10658 Duration::from_secs(state.config.graph.timeout_resource_wait),
10659 move || {
10660 inherited.cancelled()
10661 || pauses
10662 .as_ref()
10663 .is_some_and(|(a, b)| a.parked() || b.parked())
10664 },
10665 )
10666}
10667
10668fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10669 if error.is::<crate::repo_lock::ResourceBlocked>() {
10670 state.resource_block = Some(crate::run::ResourceBlock {
10671 cause: crate::run::ResourceBlockCause::Other,
10672 node: "repository".into(),
10673 kind: if error.to_string().contains("disk pressure") {
10674 "disk pressure"
10675 } else if error.to_string().contains("disk admission") {
10676 "disk admission"
10677 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10678 "incremental cache admission"
10679 } else {
10680 "repository coordination"
10681 }
10682 .into(),
10683 detail: error.to_string(),
10684 at: Timestamp::now(),
10685 });
10686 state.event("resource", error.to_string());
10687 }
10688}
10689
10690async fn coordinate_operation(
10691 state: &mut RunState,
10692 cwd: &Path,
10693 wait: &crate::repo_lock::WaitPolicy,
10694) -> Result<Option<crate::repo_lock::SeatLease>> {
10695 let result =
10696 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10697 .await;
10698 if let Err(e) = &result {
10699 record_repository_block(state, e);
10700 }
10701 result
10702}
10703
10704async fn run_one(
10706 job: SeatJob,
10707 sem: Arc<Semaphore>,
10708 ctx: &WaveCtx<'_>,
10709 state: &mut RunState,
10710 attempt: usize,
10711) -> (SeatState, AgentOutcome) {
10712 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10713 .await
10714 .pop()
10715 .expect("one job in, one result out");
10716 (seat, out)
10717}
10718
10719async fn wave(
10725 jobs: Vec<SeatJob>,
10726 sem: Arc<Semaphore>,
10727 ctx: &WaveCtx<'_>,
10728 state: &mut RunState,
10729 attempt: usize,
10730) -> Vec<(usize, SeatState, AgentOutcome)> {
10731 let WaveCtx {
10732 run,
10733 node,
10734 prompts,
10735 cache,
10736 round,
10737 carry_seats: _,
10738 pauses,
10739 } = *ctx;
10740 for job in &jobs {
10741 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10742 }
10743 if let Err(e) = state.save() {
10744 tracing::warn!("could not persist in-progress seats: {e:#}");
10749 }
10750 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10766 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10770 let cache_guard = if let Some(cache_dir) = cache {
10771 if jobs_had_a_writer {
10772 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10773 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10774 match acquire_cache_lease(
10775 state,
10776 cache_dir,
10777 &owner,
10778 resource_limit,
10779 node,
10780 node,
10781 &cancel,
10782 )
10783 .await
10784 {
10785 Ok(Some(g)) => Some(g),
10786 Ok(None) => {
10789 if let Some(block) = &mut state.resource_block {
10790 block.cause = crate::run::ResourceBlockCause::Cancelled {
10791 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10792 };
10793 }
10794 return block_wave(
10795 jobs,
10796 state,
10797 node,
10798 attempt,
10799 "parked while waiting for the shared build cache",
10800 );
10801 }
10802 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10805 Err(e) => {
10806 let why = format!("{e:#}");
10810 state.event(
10811 "resource",
10812 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10813 );
10814 return block_wave(jobs, state, node, attempt, &why);
10815 }
10816 }
10817 } else {
10818 None
10819 }
10820 } else {
10821 None
10822 };
10823 state.seats_spawning(node, attempt);
10826 if let Err(e) = state.save() {
10827 tracing::warn!("could not persist the start of a wave: {e:#}");
10828 }
10829 let slot_limit = resource_limit
10833 + jobs
10834 .iter()
10835 .map(|j| j.timeout)
10836 .max()
10837 .unwrap_or_default()
10838 .saturating_mul(jobs.len() as u32);
10839 let mut set = tokio::task::JoinSet::new();
10840 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10843 let overlay = prompts.overlay(node);
10844 let repository_wait = repository_wait(state, pauses);
10845 for (i, mut job) in jobs.into_iter().enumerate() {
10846 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10847 if cache.is_some() {
10848 job.prompt.push('\n');
10849 job.prompt
10850 .push_str(&prompt::build_cache_note(node, job.allow_write));
10851 }
10852 let sem = Arc::clone(&sem);
10853 let run = run.to_owned();
10854 let node = node.to_owned();
10855 let attachments = if node == "implement" {
10858 state.attachments.clone()
10859 } else {
10860 Vec::new()
10861 };
10862 let cache = cache
10873 .filter(|_| job.allow_write && cache_guard.is_some())
10874 .map(Path::to_path_buf);
10875 let spawned_tx = spawned_tx.clone();
10876 let repository_wait = repository_wait.clone();
10877 let coordination = state.config.coordination.clone();
10878 let fetched_base = state.fetched_base.clone();
10879 set.spawn(async move {
10880 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10884 Ok(p) => p,
10885 Err(_) => {
10886 return (
10887 i,
10888 job.seat,
10889 AgentOutcome::ResourceBlocked(format!(
10890 "no parallel slot became free within {}s",
10891 slot_limit.as_secs()
10892 )),
10893 false,
10894 );
10895 }
10896 };
10897 spawned_tx.send(job.seat.key.clone()).ok();
10898 let mut seat = job.seat;
10899 let out = repository_wait
10900 .scope(agent::invoke(
10901 &job.spec,
10902 &mut seat,
10903 &Invocation {
10904 coordination: coordination.as_ref(),
10905 fetched_base: fetched_base.as_ref(),
10906 cwd: &job.cwd,
10907 prompt: &job.prompt,
10908 timeout: job.timeout,
10909 allow_write: job.allow_write,
10910 sessions: job.sessions,
10911 artifacts: &job.artifacts,
10912 stem: &job.stem,
10913 run: &run,
10914 node: &node,
10915 cache_dir: cache.as_deref(),
10916 attachments: &attachments,
10917 writable: &[],
10918 },
10919 ))
10920 .await;
10921 let interrupted = out
10922 .as_ref()
10923 .err()
10924 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10925 let out = match out {
10926 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10927 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10928 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10936 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10937 Ok(o) => AgentOutcome::Failed(format!(
10938 "process exited with no usable output ({})",
10939 crate::notices::exit_status(o.exit_code)
10940 )),
10941 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10942 AgentOutcome::ResourceBlocked(e.to_string())
10943 }
10944 Err(e) => AgentOutcome::Failed(e.to_string()),
10945 };
10946 (i, seat, out, interrupted)
10947 });
10948 }
10949 drop(spawned_tx);
10950 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10951 loop {
10952 let joined = tokio::select! {
10953 Some(key) = spawned_rx.recv() => {
10954 state.seat_spawned(&key);
10956 if let Err(e) = state.save() {
10957 tracing::warn!("could not persist a seat's start: {e:#}");
10958 }
10959 continue;
10960 }
10961 joined = set.join_next() => joined,
10962 };
10963 let Some(joined) = joined else {
10964 break;
10965 };
10966 let (i, seat, out, interrupted) = match joined {
10967 Ok(v) => v,
10968 Err(e) => {
10972 tracing::error!("agent task panicked: {e}");
10973 continue;
10974 }
10975 };
10976 state.seat_finished(&seat.key);
10977 if interrupted
10978 && node == "implement"
10979 && let Some(label) = state
10980 .candidates
10981 .iter()
10982 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10983 .map(|candidate| candidate.label)
10984 {
10985 state.candidate_preservation.insert(
10986 label,
10987 "interrupted implementation awaiting persistence".into(),
10988 );
10989 state.interrupted_implementations.insert(label);
10990 }
10991 if let AgentOutcome::ResourceBlocked(why) = &out
10992 && state.resource_block.is_none()
10993 {
10994 state.resource_block = Some(crate::run::ResourceBlock {
10995 cause: crate::run::ResourceBlockCause::Other,
10996 node: node.to_owned(),
10997 kind: if why.contains("disk pressure") {
10998 "disk pressure"
10999 } else if why.contains("disk admission") {
11000 "disk admission"
11001 } else if why.contains(crate::incremental::BLOCKED) {
11002 "incremental cache admission"
11003 } else if why.contains("repository coordination") {
11004 "repository coordination"
11005 } else {
11006 "parallel slot"
11007 }
11008 .to_owned(),
11009 detail: why.clone(),
11010 at: Timestamp::now(),
11011 });
11012 let activity = if interrupted {
11013 "agent execution interrupted"
11014 } else {
11015 "no agent was started"
11016 };
11017 state.event("resource", format!("{node}: {activity} - {why}"));
11018 }
11019 record_jobs(state, node, round, &seat.key, &out);
11020 if let Err(e) = state.save() {
11021 tracing::warn!("could not persist a seat's completion: {e:#}");
11022 }
11023 if collected.len() <= i {
11024 collected.resize_with(i + 1, || None);
11025 }
11026 collected[i] = Some((i, seat, out));
11027 }
11028 if state
11034 .active
11035 .values()
11036 .any(|a| a.node == node && a.attempt == attempt)
11037 {
11038 state
11039 .active
11040 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11041 if let Err(e) = state.save() {
11042 tracing::warn!("could not persist the end of a wave: {e:#}");
11043 }
11044 }
11045 if let Some(cache_dir) = cache
11052 && jobs_had_a_writer
11053 {
11054 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11055 }
11056 if let Some(guard) = cache_guard {
11057 guard.release();
11058 }
11059 collected.into_iter().flatten().collect()
11060}
11061
11062fn record_jobs(
11073 state: &mut RunState,
11074 node: &str,
11075 round: Option<usize>,
11076 seat: &str,
11077 out: &AgentOutcome,
11078) {
11079 let commands: &[agent::CommandEvidence] = match out {
11080 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11081 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11082 };
11083 let checked_at = Timestamp::now();
11084 for c in commands {
11085 state.jobs.push(JobRecord {
11086 node: node.to_owned(),
11087 round,
11088 seat: seat.to_owned(),
11089 id: c.id.clone(),
11090 description: c.description.clone(),
11091 checked_at,
11092 status: match c.exit_code {
11093 Some(0) => JobStatus::Completed,
11094 Some(_) => JobStatus::Failed,
11095 None => JobStatus::Unknown,
11096 },
11097 exit_code: c.exit_code,
11098 result_summary: c.result_summary.clone(),
11099 source: c.source.clone(),
11100 });
11101 }
11102}
11103
11104fn round_is_clean(
11125 blocking: usize,
11126 e2e_ok: bool,
11127 answered: usize,
11128 expected: usize,
11129 quota_missing: usize,
11130 policy: IncompleteReviewPolicy,
11131) -> bool {
11132 if blocking != 0 || !e2e_ok {
11133 return false;
11134 }
11135 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11136 return true;
11137 }
11138 answered > 0 && expected - answered <= quota_missing
11139}
11140
11141pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11165 if max_rounds == 0
11166 || reviews.last().is_some_and(|r| {
11167 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11168 })
11169 {
11170 return Some(RunStatus::Gating);
11171 }
11172 let last = reviews.last()?;
11173 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11174 if reviews.len() < max_rounds && !stagnant {
11175 return None;
11176 }
11177 if last.incomplete() && last.blocking == 0 {
11178 return Some(RunStatus::Blocked);
11179 }
11180 if last.e2e_status() == E2eStatus::ResourceBlocked
11181 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11182 {
11183 return None;
11184 }
11185 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11186 RunStatus::Gating
11187 } else {
11188 RunStatus::Blocked
11189 })
11190}
11191
11192pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11207 if nudged {
11208 (full / 4).max(Duration::from_secs(120)).min(full)
11209 } else {
11210 full
11211 }
11212}
11213
11214#[allow(clippy::too_many_arguments)]
11243async fn ask_json_wave<T>(
11244 jobs: Vec<SeatJob>,
11245 sem: Arc<Semaphore>,
11246 retries: usize,
11247 roster: &[AgentSpec],
11248 ctx: &WaveCtx<'_>,
11249 losses: &mut Vec<QuotaLoss>,
11250 state: &mut RunState,
11251 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11252) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11253where
11254 T: serde::de::DeserializeOwned + Send + 'static,
11255{
11256 ask_wave_with(
11257 jobs,
11258 sem,
11259 retries,
11260 roster,
11261 ctx,
11262 losses,
11263 state,
11264 &|text: &str| {
11265 let v = verdict::extract_json::<T>(text)?;
11266 validate(&v)?;
11267 Ok(v)
11268 },
11269 )
11270 .await
11271}
11272
11273#[allow(clippy::too_many_arguments)]
11282async fn ask_wave_with<T>(
11283 jobs: Vec<SeatJob>,
11284 sem: Arc<Semaphore>,
11285 retries: usize,
11286 roster: &[AgentSpec],
11287 ctx: &WaveCtx<'_>,
11288 losses: &mut Vec<QuotaLoss>,
11289 state: &mut RunState,
11290 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11291) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11292where
11293 T: Send + 'static,
11294{
11295 let n = jobs.len();
11296 let originals: Vec<SeatJob> = jobs;
11297 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11298 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11299 let mut nudges: Vec<usize> = vec![0; n];
11306 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11310 let mut tried: Vec<BTreeSet<String>> = specs
11311 .iter()
11312 .map(|s| BTreeSet::from([s.id.clone()]))
11313 .collect();
11314 let starts: Vec<usize> = specs
11315 .iter()
11316 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11317 .collect();
11318 let carry = ctx.carry_seats && !roster.is_empty();
11319 let carried: Vec<BTreeSet<String>> = originals
11322 .iter()
11323 .map(|j| {
11324 state
11325 .seat_history
11326 .get(&j.seat.key)
11327 .filter(|_| carry)
11328 .map(|h| h.failed.clone())
11329 .unwrap_or_default()
11330 })
11331 .collect();
11332 let mut prev: Vec<Option<FailClass>> = originals
11333 .iter()
11334 .map(|j| {
11335 state
11336 .seat_history
11337 .get(&j.seat.key)
11338 .filter(|_| carry)
11339 .and_then(|h| h.last_fail.clone())
11340 })
11341 .collect();
11342 let next_agent =
11343 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11344 let others: BTreeSet<String> = specs
11345 .iter()
11346 .enumerate()
11347 .filter(|(j, _)| *j != i)
11348 .map(|(_, s)| s.id.clone())
11349 .collect();
11350 pick_successor(
11351 roster,
11352 starts[i],
11353 tried,
11354 carry.then(|| &carried[i]),
11355 &others,
11356 )
11357 .cloned()
11358 };
11359 let mut fresh: Vec<Option<String>> = vec![None; n];
11360 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11361 let mut pending: Vec<usize> = (0..n).collect();
11362
11363 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11368 for round in 0..max_rounds {
11369 if pending.is_empty() {
11370 break;
11371 }
11372 let mut batch = Vec::with_capacity(pending.len());
11373 let mut renudged: Vec<&str> = Vec::new();
11374 for &i in &pending {
11375 let src = &originals[i];
11376 let full: &str = match &src.handover {
11381 Some(h) if specs[i].id != src.spec.id => h,
11382 _ => &src.prompt,
11383 };
11384 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11387 (
11388 full.to_owned(),
11389 src.timeout,
11390 format!("{}-{word}-{}", src.stem, specs[i].id),
11391 )
11392 } else if nudges[i] == 0 {
11393 (full.to_owned(), src.timeout, src.stem.clone())
11394 } else {
11395 renudged.push(src.seat.key.as_str());
11396 let why = done[i]
11397 .as_ref()
11398 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11399 .unwrap_or_else(|| "no parsable answer".to_owned());
11400 let nudge = prompt::nudge(&why);
11401 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11402 let prompt = if nudged {
11403 nudge
11404 } else {
11405 format!("{full}\n\n---\n\n{nudge}")
11406 };
11407 (
11408 prompt,
11409 retry_budget(src.timeout, nudged),
11410 format!("{}-retry{}", src.stem, nudges[i]),
11411 )
11412 };
11413 batch.push(SeatJob {
11414 spec: specs[i].clone(),
11415 seat: seats[i].clone(),
11416 cwd: src.cwd.clone(),
11417 prompt,
11418 timeout,
11419 allow_write: src.allow_write,
11420 sessions: src.sessions,
11421 artifacts: src.artifacts.clone(),
11422 stem,
11423 handover: None,
11424 });
11425 }
11426
11427 if !renudged.is_empty() {
11428 state.event(
11429 ctx.node,
11430 format!("retry {round}: re-asking {}", renudged.join(", ")),
11431 );
11432 }
11433 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11434 let mut still = Vec::new();
11435 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11436 seats[i] = seat;
11437 if let AgentOutcome::ResourceBlocked(why) = &out {
11440 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11441 continue;
11442 }
11443 let class = FailClass::of(&out);
11444 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11447 && nudges[i] < retries
11448 && !roster.is_empty();
11449 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11450 let next = should_hand_over(prev[i].as_ref(), &cur)
11451 .then(|| next_agent(i, &tried[i], &specs))
11452 .flatten();
11453 if carry {
11454 let h = state
11455 .seat_history
11456 .entry(originals[i].seat.key.clone())
11457 .or_default();
11458 h.failed.insert(specs[i].id.clone());
11459 h.last_fail = Some(cur.clone());
11460 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11461 h.last_ok = None;
11462 }
11463 if let Err(e) = state.save() {
11466 tracing::warn!("could not persist a seat's failure history: {e:#}");
11467 }
11468 }
11469 if let Some(next) = next {
11470 record_handover(
11471 state,
11472 ctx.node,
11473 &originals[i].seat.key,
11474 &specs[i].id,
11475 &next.id,
11476 &cur,
11477 &fail_reason(&out),
11478 );
11479 tried[i].insert(next.id.clone());
11480 prev[i] = Some(cur.clone());
11481 seats[i] =
11482 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11483 specs[i] = next;
11484 fresh[i] = Some(cur.stem_word().to_owned());
11485 nudges[i] = 0;
11486 done[i] = Some(Err(anyhow::anyhow!(
11487 "handed over after: {}",
11488 fail_reason(&out)
11489 )));
11490 still.push(i);
11491 continue;
11492 }
11493 }
11494 if carry
11495 && !nudge_first
11496 && let Some(cur) = class.clone()
11497 {
11498 let h = state
11500 .seat_history
11501 .entry(originals[i].seat.key.clone())
11502 .or_default();
11503 h.failed.insert(specs[i].id.clone());
11504 h.last_fail = Some(cur);
11505 }
11506 let parsed = match out {
11507 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11508 AgentOutcome::Quota(o) => {
11509 last_quota[i] = o.quota.clone();
11510 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11511 }
11512 AgentOutcome::Dropped(o) => {
11516 let why = o
11517 .dropped
11518 .as_ref()
11519 .map(|d| d.why.as_str())
11520 .unwrap_or("the CLI ended the stream without delivering its answer");
11521 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11522 }
11523 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11524 Err(anyhow::anyhow!(e))
11525 }
11526 };
11527 let quota = class == Some(FailClass::Quota);
11528 let failed = parsed.is_err();
11529 if !failed && matches!(ctx.node, "judge" | "vote") {
11530 state
11533 .quota
11534 .retain(|loss| loss.seat != originals[i].seat.key);
11535 }
11536 done[i] = Some(parsed);
11537 if carry && !failed {
11538 let h = state
11539 .seat_history
11540 .entry(originals[i].seat.key.clone())
11541 .or_default();
11542 h.failed.remove(&specs[i].id);
11543 h.last_ok = Some(specs[i].id.clone());
11544 h.last_fail = None;
11545 }
11546 let agent_failure = class.is_some()
11554 && !nudge_first
11555 && !roster.is_empty()
11556 && next_agent(i, &tried[i], &specs).is_some();
11557 if failed && !quota && !agent_failure && nudges[i] < retries {
11558 nudges[i] += 1;
11559 still.push(i);
11560 }
11561 }
11562 pending = still;
11563 }
11564
11565 for (i, q) in last_quota.into_iter().enumerate() {
11568 if let Some(quota) = q {
11569 losses.push(QuotaLoss::from_report(
11570 originals[i].seat.key.clone(),
11571 ctx.node,
11572 Some("a),
11573 ));
11574 }
11575 }
11576
11577 if state.status != RunStatus::Stalled
11580 && ctx.node != "deliberate"
11581 && n > 0
11582 && losses.len() == n
11583 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11584 {
11585 block_machine_outage(state, ctx.node, losses);
11586 state.quota.append(losses);
11587 }
11588
11589 seats
11590 .into_iter()
11591 .zip(done)
11592 .zip(nudges)
11593 .map(|((seat, res), attempts)| {
11594 (
11595 seat,
11596 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11597 attempts,
11598 )
11599 })
11600 .collect()
11601}
11602
11603pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11606 let mut auth: Vec<_> = losses
11607 .iter()
11608 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11609 .map(|loss| loss.agent.clone())
11610 .collect();
11611 auth.sort();
11612 auth.dedup();
11613 let (kind, detail) = if auth.is_empty() {
11614 (
11615 "agent usage limit",
11616 "All seats reached their agent usage limits; resume after usage is available"
11617 .to_owned(),
11618 )
11619 } else {
11620 (
11621 "agent authentication",
11622 format!(
11623 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11624 auth.join(", ")
11625 ),
11626 )
11627 };
11628 state.resource_block = Some(crate::run::ResourceBlock {
11629 node: node.to_owned(),
11630 kind: kind.to_owned(),
11631 cause: crate::run::ResourceBlockCause::Other,
11632 detail: detail.clone(),
11633 at: Timestamp::now(),
11634 });
11635 state.event(node, detail);
11636}
11637
11638fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11640 results
11641 .iter()
11642 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11643}
11644
11645fn block_wave(
11649 jobs: Vec<SeatJob>,
11650 state: &mut RunState,
11651 node: &str,
11652 attempt: usize,
11653 why: &str,
11654) -> Vec<(usize, SeatState, AgentOutcome)> {
11655 state.resource_wait = None;
11656 state
11657 .active
11658 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11659 if let Err(e) = state.save() {
11660 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11661 }
11662 jobs.into_iter()
11663 .enumerate()
11664 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11665 .collect()
11666}
11667
11668async fn acquire_cache_lease(
11681 state: &mut RunState,
11682 cache_dir: &Path,
11683 owner: &crate::cache::Owner,
11684 budget: Duration,
11685 node: &str,
11686 context: &str,
11687 cancel: &(dyn Fn() -> bool + Sync),
11688) -> Result<Option<crate::cache::Guard>> {
11689 let home = crate::run::home();
11690 let started = Instant::now();
11691 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11692 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11693 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11694 Err(e) => {
11695 state.event(
11696 "verify",
11697 format!("{context}: could not check the shared build cache: {e:#}"),
11698 );
11699 if let Err(e2) = state.save() {
11700 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11701 }
11702 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11703 }
11704 };
11705 state.event(
11706 "verify",
11707 format!(
11708 "{context}: waiting for the shared build cache at {} ({})",
11709 cache_dir.display(),
11710 busy.describe()
11711 ),
11712 );
11713 state.resource_wait = Some(crate::run::ResourceWait {
11716 node: context.to_owned(),
11717 kind: "shared build cache".to_owned(),
11718 holder: busy.describe(),
11719 since: Timestamp::now(),
11720 limit_secs: budget.as_secs(),
11721 });
11722 if let Err(e) = state.save() {
11723 tracing::warn!("could not persist a cache wait: {e:#}");
11724 }
11725 let remaining = budget.saturating_sub(started.elapsed());
11726 let waited = crate::cache::wait_for_unless(
11727 &home,
11728 cache_dir,
11729 owner,
11730 remaining,
11731 Duration::from_secs(5),
11732 cancel,
11733 )
11734 .await;
11735 state.resource_wait = None;
11736 let cause = match &waited {
11737 Ok(Some(_)) => None,
11738 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11739 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11740 Some(crate::run::ResourceBlockCause::Timeout {
11741 waited_secs: started.elapsed().as_secs(),
11742 limit_secs: budget.as_secs(),
11743 })
11744 }
11745 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11746 };
11747 if let Some(cause) = cause {
11748 let detail = match &waited {
11749 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11750 Err(error) => format!("{error:#}"),
11751 };
11752 state.resource_block = Some(crate::run::ResourceBlock {
11753 cause,
11754 node: node.to_owned(),
11755 kind: "shared build cache".into(),
11756 detail,
11757 at: Timestamp::now(),
11758 });
11759 state.event(
11760 "resource",
11761 state.resource_block.as_ref().unwrap().description(),
11762 );
11763 }
11764 if let Err(error) = state.save() {
11765 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11766 }
11767 waited
11768}
11769
11770#[derive(Debug)]
11775struct CacheCheckError(String);
11776
11777impl std::fmt::Display for CacheCheckError {
11778 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11779 f.write_str(&self.0)
11780 }
11781}
11782
11783impl std::error::Error for CacheCheckError {}
11784
11785fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11787 state
11788 .resource_block
11789 .get_or_insert_with(|| crate::run::ResourceBlock {
11790 cause: crate::run::ResourceBlockCause::Other,
11791 node: node.to_owned(),
11792 kind: "shared build cache".into(),
11793 detail: format!("{error:#}"),
11794 at: Timestamp::now(),
11795 });
11796 if let Err(error) = state.save() {
11797 tracing::warn!("could not persist cache resource block: {error:#}");
11798 }
11799}
11800
11801#[allow(clippy::too_many_arguments)]
11822async fn with_cache_lease<'s, F, Fut>(
11823 state: &'s mut RunState,
11824 cache_dir: Option<&Path>,
11825 node: &str,
11826 seat: &str,
11827 worktree: &Path,
11828 head: &str,
11829 budget: Duration,
11830 context: &str,
11831 wait: &crate::repo_lock::WaitPolicy,
11832 body: F,
11833) -> Result<(Vec<CommandOutcome>, bool)>
11834where
11835 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11836 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11837{
11838 let started = Instant::now();
11841 let home = crate::run::home();
11842 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11843 let guard = if let Some(cache_dir) = cache_dir {
11844 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11845 match acquire_cache_lease(
11846 state,
11847 cache_dir,
11848 &owner,
11849 resource_limit,
11850 node,
11851 context,
11852 &|| wait.cancelled(),
11853 )
11854 .await
11855 .and_then(|g| {
11856 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11857 }) {
11858 Ok(g) => Some(g),
11859 Err(e) => {
11860 record_cache_error(state, node, &e);
11861 return Ok((
11862 vec![CommandOutcome {
11863 command: "(waiting for the shared build cache)".into(),
11864 code: None,
11865 output_tail: e.to_string(),
11866 duration_ms: started.elapsed().as_millis() as u64,
11867 resource_blocked: true,
11868 }],
11869 false,
11870 ));
11871 }
11872 }
11873 } else {
11874 None
11875 };
11876 let _coordination = coordinate_operation(state, worktree, wait).await?;
11877 if let Some(cache_dir) = cache_dir {
11878 let identity = crate::cache::Identity::new(worktree, head);
11879 if let Err(e) = crate::cache::ensure_fresh(
11880 &home,
11881 cache_dir,
11882 &identity,
11883 &state.config.verify.cargo_manifests(),
11884 ) {
11885 record_cache_error(state, node, &e);
11886 state.event(
11887 "verify",
11888 format!(
11889 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11890 worktree.display(),
11891 short(head)
11892 ),
11893 );
11894 if let Some(guard) = guard {
11895 guard.release();
11896 }
11897 return Ok((
11898 vec![CommandOutcome {
11899 command: "(confirming the shared build cache is fresh)".into(),
11900 code: None,
11901 output_tail: e.to_string(),
11902 duration_ms: started.elapsed().as_millis() as u64,
11903 resource_blocked: true,
11904 }],
11905 false,
11906 ));
11907 }
11908 }
11909 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11910 if !timed_out_pids.is_empty() {
11911 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11912 }
11913 drop(_coordination);
11916 if let Some(guard) = guard {
11917 guard.release();
11918 }
11919 Ok((outcomes, retried))
11920}
11921
11922async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11934 wait_for_pids_with(
11935 pids,
11936 crate::proc::pid_alive,
11937 LEASE_RELEASE_POLL,
11938 LEASE_RELEASE_MAX_WAIT,
11939 )
11940 .await;
11941}
11942
11943async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11949 pids: &[u32],
11950 alive: F,
11951 poll: Duration,
11952 max_wait: Duration,
11953) {
11954 let deadline = Instant::now() + max_wait;
11955 loop {
11956 if pids.iter().all(|&pid| !alive(pid)) {
11957 return;
11958 }
11959 if Instant::now() >= deadline {
11960 return;
11961 }
11962 tokio::time::sleep(poll).await;
11963 }
11964}
11965
11966fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11973 outcomes.iter().any(|o| o.resource_blocked)
11974}
11975
11976pub(crate) enum GateFix {
11978 Retry,
11980 Stop,
11983 Defer,
11986}
11987
11988pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11996 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11997 red.peek().is_some()
11998 && red.all(|o| {
11999 !o.resource_blocked
12000 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12001 && !o.output_tail.trim().is_empty()
12002 })
12003}
12004
12005fn e2e_outcome_label(o: &CommandOutcome) -> String {
12009 if o.ok() {
12010 return "pass".to_owned();
12011 }
12012 let reason = if o.build_failed() {
12013 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12014 } else {
12015 format!("FAIL ({:?})", o.code)
12016 };
12017 format!(
12018 "{reason}\n{}",
12019 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12020 )
12021}
12022
12023async fn run_e2e_with_retry(
12031 state: &mut RunState,
12032 shell: &[String],
12033 commands: &[String],
12034 worktree: &Path,
12035 timeout: Duration,
12036 context: &str,
12037) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12038 let (mut e2e, mut timed_out_pids) = run_commands(
12039 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12040 )
12041 .await;
12042 for o in &e2e {
12043 state.event(
12044 "verify",
12045 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12046 );
12047 }
12048 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12052 if verify_retried {
12053 state.event(
12054 "verify",
12055 format!(
12056 "{context}: verify could not build/link, not a test result — retrying once \
12057 before concluding"
12058 ),
12059 );
12060 let retried = run_commands(
12061 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12062 )
12063 .await;
12064 e2e = retried.0;
12065 timed_out_pids.extend(retried.1);
12068 for o in &e2e {
12069 state.event(
12070 "verify",
12071 format!(
12072 "{context}: retry `{}` -> {}",
12073 o.command,
12074 e2e_outcome_label(o)
12075 ),
12076 );
12077 }
12078 }
12079 (e2e, verify_retried, timed_out_pids)
12080}
12081
12082#[allow(clippy::too_many_arguments)]
12098async fn run_commands(
12099 state: &mut RunState,
12100 node: &str,
12101 task: &str,
12102 attempt: usize,
12103 shell: &[String],
12104 commands: &[String],
12105 cwd: &Path,
12106 timeout: Duration,
12107) -> (Vec<CommandOutcome>, Vec<u32>) {
12108 if commands.is_empty() {
12109 return (Vec::new(), Vec::new());
12114 }
12115 let mut out = Vec::new();
12116 let mut timed_out_pids = Vec::new();
12117 let total = commands.len();
12118 for (idx, command) in commands.iter().enumerate() {
12119 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12120 if let Err(e) = state.save() {
12121 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12122 }
12123 let started = Instant::now();
12124 let mut cmd = tokio::process::Command::new(&shell[0]);
12125 cmd.quiet();
12126 for key in crate::worker::INVOCATION_ENV {
12130 cmd.env_remove(key);
12131 }
12132 cmd.args(&shell[1..])
12133 .arg(command)
12134 .current_dir(cwd)
12135 .stdin(std::process::Stdio::null())
12136 .stdout(std::process::Stdio::piped())
12137 .stderr(std::process::Stdio::piped())
12138 .kill_on_drop(true);
12139 let mut outputs = Vec::new();
12140 let protected = (|| -> Result<()> {
12141 crate::incremental::configure(&mut cmd, cwd)?;
12142 let mut paths = vec![cwd.to_owned()];
12143 if let Some(cache) = state.config.cache_dir() {
12144 paths.push(cache);
12145 }
12146 outputs.extend(crate::retention::OutputLease::admit(
12147 &crate::run::home(),
12148 &paths,
12149 state.config.disk.min_free_bytes,
12150 )?);
12151 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12152 outputs.push(guard);
12153 }
12154 if let Some(cache) = state.config.cache_dir() {
12155 outputs.push(crate::retention::OutputLease::cache(
12156 &crate::run::home(),
12157 &cache,
12158 false,
12159 )?);
12160 }
12161 Ok(())
12162 })();
12163 if let Err(e) = protected {
12164 let detail = format!("execution admission unavailable: {e:#}");
12165 state.resource_block = Some(crate::run::ResourceBlock {
12166 cause: crate::run::ResourceBlockCause::Other,
12167 node: node.to_owned(),
12168 kind: if detail.contains("disk admission") {
12169 "disk admission"
12170 } else {
12171 "output ownership"
12172 }
12173 .to_owned(),
12174 detail: detail.clone(),
12175 at: Timestamp::now(),
12176 });
12177 out.push(CommandOutcome {
12178 command: command.clone(),
12179 code: None,
12180 output_tail: detail,
12181 duration_ms: 0,
12182 resource_blocked: true,
12183 });
12184 break;
12185 }
12186 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12187 let (code, body) = match spawned {
12188 Ok(mut running) => {
12189 let pid = running.child().id();
12190 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12191 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12192 let mut paths = vec![cwd.to_owned()];
12193 if let Some(cache) = state.config.cache_dir() {
12194 paths.push(cache);
12195 }
12196 let mut pressure = None;
12197 let status = tokio::select! {
12198 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12199 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12200 pressure = Some(reason); None
12201 }
12202 };
12203 let (code, failure) = match status {
12204 Some(Ok(Ok(status))) => (
12205 status.code(),
12206 status
12207 .code()
12208 .is_none()
12209 .then(|| format!("command terminated: {status}")),
12210 ),
12211 Some(Ok(Err(error))) => {
12212 (None, Some(format!("failed waiting for command: {error}")))
12213 }
12214 Some(Err(_)) => {
12215 if !running.coordinated()
12219 && let Some(pid) = pid
12220 {
12221 timed_out_pids.push(pid);
12222 }
12223 (
12224 None,
12225 Some(format!("timed out after {}s", timeout.as_secs())),
12226 )
12227 }
12228 None => (None, None),
12229 };
12230 running.finish().await;
12231 if let Some(reason) = pressure {
12232 state.resource_block = Some(crate::run::ResourceBlock {
12233 cause: crate::run::ResourceBlockCause::Other,
12234 node: node.to_owned(),
12235 kind: "disk pressure".to_owned(),
12236 detail: reason.clone(),
12237 at: Timestamp::now(),
12238 });
12239 out.push(CommandOutcome {
12240 command: command.clone(),
12241 code: None,
12242 output_tail: reason,
12243 duration_ms: started.elapsed().as_millis() as u64,
12244 resource_blocked: true,
12245 });
12246 break;
12247 }
12248 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12249 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12250 if let Some(failure) = failure {
12251 body.push('\n');
12252 body.push_str(&failure);
12253 }
12254 (code, body)
12255 }
12256 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12257 };
12258 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12259 if !body.is_empty() {
12260 match crate::run::write_verification_artifact(state, task, &body) {
12264 Ok(name) if summary != body => {
12265 summary = format!("[full output: artifacts/{name}]\n{summary}");
12266 }
12267 Ok(_) => {}
12268 Err(e) => {
12269 let warning = format!("could not save the full {task} output: {e:#}");
12270 tracing::warn!("{warning}");
12271 state.event(node, warning);
12272 if summary != body {
12273 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12274 }
12275 }
12276 }
12277 }
12278 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12279 out.push(CommandOutcome {
12280 command: command.clone(),
12281 code,
12282 output_tail: summary,
12283 duration_ms: started.elapsed().as_millis() as u64,
12284 resource_blocked,
12285 });
12286 if resource_blocked {
12287 state.resource_block = Some(crate::run::ResourceBlock {
12288 cause: crate::run::ResourceBlockCause::Other,
12289 node: node.to_owned(),
12290 kind: "incremental cache admission".into(),
12291 detail: body,
12292 at: Timestamp::now(),
12293 });
12294 break;
12295 }
12296 }
12297 state.task_finished(task);
12298 if let Err(e) = state.save() {
12299 tracing::warn!("could not persist the end of {task}: {e:#}");
12300 }
12301 (out, timed_out_pids)
12302}
12303
12304fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12316 let repo = repo.display();
12317 match style {
12318 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12319 MergeStyle::Squash => {
12320 let subject = message
12323 .lines()
12324 .next()
12325 .unwrap_or(branch)
12326 .replace(['\\', '"', '$', '`'], "");
12327 format!(
12328 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12329 )
12330 }
12331 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12332 }
12333}
12334
12335const PR_TITLE_MAX: usize = 240;
12346
12347fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12358 let line = queue::first_line(text)?;
12359 let chars: Vec<char> = line.chars().collect();
12360 let (kept, mark) = if chars.len() <= max {
12361 (line.to_owned(), "")
12362 } else if let Some(end) = sentence_end(&chars, max) {
12363 (chars[..end].iter().collect::<String>(), "")
12364 } else {
12365 let room = max.saturating_sub(3);
12366 let cut = if chars[room].is_whitespace() {
12369 room
12370 } else {
12371 chars[..room]
12372 .iter()
12373 .rposition(|c| c.is_whitespace())
12374 .unwrap_or(room)
12375 };
12376 let head: String = chars[..cut].iter().collect();
12377 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12378 (head.to_owned(), "...")
12379 };
12380 if kept.is_empty() {
12381 return None;
12382 }
12383 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12384 return None;
12385 }
12386 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12387 return None;
12388 }
12389 Some(format!("{kept}{mark}"))
12390}
12391
12392fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12396 const MIN: usize = 20;
12397 for i in MIN..max.min(chars.len()) {
12398 if !matches!(chars[i], '.' | '!' | '?') {
12399 continue;
12400 }
12401 let Some(&next) = chars.get(i + 1) else {
12402 continue;
12403 };
12404 if !next.is_whitespace() {
12405 continue;
12406 }
12407 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12408 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12409 continue;
12410 }
12411 let word: String = chars[..i]
12412 .iter()
12413 .rev()
12414 .take_while(|c| !c.is_whitespace())
12415 .collect::<Vec<_>>()
12416 .into_iter()
12417 .rev()
12418 .collect();
12419 let word = word.to_ascii_lowercase();
12420 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12421 continue;
12422 }
12423 let end = chars[..i]
12424 .iter()
12425 .rposition(|c| !c.is_whitespace())
12426 .map_or(i, |p| p + 1);
12427 return Some(end);
12428 }
12429 None
12430}
12431
12432struct PrMessage {
12437 title: String,
12438 body: String,
12439}
12440
12441impl PrMessage {
12442 fn commit_message(&self) -> String {
12446 format!("{}\n\n{}", self.title, self.body)
12447 }
12448}
12449
12450fn title_marker(line: &str) -> Option<&str> {
12452 let line = line.trim();
12453 let head = line.get(..6)?;
12454 head.eq_ignore_ascii_case("title:")
12455 .then(|| line[6..].trim())
12456}
12457
12458fn summary_title(summary: &str) -> Option<String> {
12463 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12464 let raw = title_marker(first)?;
12465 if raw.is_empty() {
12466 return None;
12467 }
12468 let title = queue::title_from(raw, PR_TITLE_MAX);
12469 let lower = title.to_ascii_lowercase();
12470 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12471 return None;
12472 }
12473 Some(title)
12474}
12475
12476const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12478
12479pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12481
12482pub(crate) fn is_review_run(state: &RunState) -> bool {
12486 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12487}
12488
12489fn review_title(state: &RunState) -> Option<String> {
12494 english_subject(
12495 state.reviewed_commits.as_ref()?.first()?,
12496 state.config.graph.github_english(),
12497 )
12498}
12499
12500fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12503 let raw = raw.trim();
12504 if raw.is_empty() {
12505 return None;
12506 }
12507 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12508 return None;
12509 }
12510 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12511 return None;
12512 }
12513 let title = queue::title_from(raw, PR_TITLE_MAX);
12514 let lower = title.to_ascii_lowercase();
12515 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12516 return None;
12517 }
12518 Some(title)
12519}
12520
12521#[derive(Debug, Clone, PartialEq, Eq)]
12524struct BranchFacts {
12525 commits: Vec<(String, String)>,
12527 stat: String,
12529}
12530
12531const STAT_FILE_LINES: usize = 25;
12533const COMMITS_MAX_CHARS: usize = 20_000;
12535
12536async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12539 let commits = git::commit_log(repo, base, branch).await.ok()?;
12540 if commits.is_empty() {
12541 return None;
12542 }
12543 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12544 let lines: Vec<&str> = stat.lines().collect();
12545 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12546 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12547 let more = format!(" ... {omitted} more file(s)");
12548 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12549 kept.push(&more);
12550 kept.push(lines[lines.len() - 1]);
12551 kept.join("\n")
12552 } else {
12553 lines.join("\n")
12554 };
12555 Some(BranchFacts { commits, stat })
12556}
12557
12558fn markdown_safe(text: &str) -> String {
12561 text.replace("</details>", "</details>")
12562 .replace("\x60\x60\x60", "~~~")
12563}
12564
12565fn neutral_title(state: &RunState, winner: char) -> String {
12566 format!(
12567 "chore: land candidate {} of run {}",
12568 winner.to_ascii_uppercase(),
12569 state.id
12570 )
12571}
12572
12573pub(crate) async fn repair_landing_gate(
12576 state: &mut RunState,
12577 winner: &Candidate,
12578 outcomes: &[CommandOutcome],
12579) -> Result<GateFix> {
12580 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12581 return Ok(GateFix::Stop);
12582 }
12583 let mut runner = Runner {
12584 roles: state.config.resolve_roles()?,
12585 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12586 state: state.clone(),
12587 pause: Pause::new(),
12588 interrupt: Pause::new(),
12589 };
12590 let result = runner.gate_fix_round(winner, outcomes).await;
12591 *state = runner.state;
12592 result
12593}
12594
12595pub(crate) async fn verify_current_head(
12598 state: &mut RunState,
12599 winner: &Candidate,
12600 wait: &crate::repo_lock::WaitPolicy,
12601) -> Result<bool> {
12602 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12603 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12605 state
12606 .verification_history
12607 .push(crate::run::VerificationEvidence {
12608 superseded_by: Some(head.clone()),
12609 archived_at: Some(Timestamp::now()),
12610 fixes_spent: Some(state.gate_fixes.len()),
12611 gate_head: None,
12612 gate_ran: false,
12613 gate: Vec::new(),
12614 e2e: state.head_e2e.take(),
12615 });
12616 }
12617 if let Some(e) = &state.head_e2e
12618 && e.head == head
12619 {
12620 if e.passed {
12621 return Ok(true);
12622 }
12623 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12624 state.status = RunStatus::Blocked;
12625 state.save()?;
12626 return Ok(false);
12627 }
12628 } else if state.reviews.last().is_some_and(|r| {
12629 r.verified_head.as_deref() == Some(head.as_str())
12630 && (r.e2e_status() == E2eStatus::Passed
12631 || (r.e2e_status() == E2eStatus::NotConfigured
12632 && state.config.verify.e2e.is_empty()))
12633 }) || state
12634 .gate_repair_e2e
12635 .as_ref()
12636 .is_some_and(|e| e.head == head && e.passed)
12637 {
12638 return Ok(true);
12639 }
12640 if let Some(e) = &state.gate_repair_e2e
12641 && e.head == head
12642 && !e.passed
12643 && !e.outcomes.is_empty()
12644 && !verify_inconclusive(&e.outcomes)
12645 {
12646 state.status = RunStatus::Blocked;
12647 state.save()?;
12648 return Ok(false);
12649 }
12650 state.head_e2e = Some(crate::run::GateRepairE2e {
12651 head: head.clone(),
12652 passed: false,
12653 outcomes: Vec::new(),
12654 });
12655 state.save()?;
12656 let commands = state.config.verify.e2e.clone();
12657 let shell = state.config.shell();
12658 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12659 let cache_dir = state.config.cache_dir();
12660 let outcomes = if commands.is_empty() {
12661 Vec::new()
12662 } else {
12663 let (outcomes, _) = with_cache_lease(
12664 state,
12665 cache_dir.as_deref(),
12666 "e2e",
12667 "e2e",
12668 &winner.worktree,
12669 &head,
12670 timeout,
12671 "current head e2e",
12672 wait,
12673 |state, budget| async move {
12674 run_e2e_with_retry(
12675 state,
12676 &shell,
12677 &commands,
12678 &winner.worktree,
12679 budget,
12680 "current head e2e",
12681 )
12682 .await
12683 },
12684 )
12685 .await?;
12686 outcomes
12687 };
12688 let unresolved = verify_inconclusive(&outcomes);
12689 let passed = !unresolved
12690 && outcomes.iter().all(CommandOutcome::ok)
12691 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12692 if !outcomes.is_empty() && !unresolved {
12693 state.record_base_progress(&head);
12694 }
12695 state.head_e2e = Some(crate::run::GateRepairE2e {
12696 head: head.clone(),
12697 passed,
12698 outcomes,
12699 });
12700 if !passed && !unresolved {
12701 state.status = RunStatus::Blocked;
12702 state.event(
12703 "e2e",
12704 format!("verify.e2e failed on head {head}; nothing published"),
12705 );
12706 }
12707 state.save()?;
12708 Ok(passed)
12709}
12710
12711pub(crate) async fn check_gate_repair_e2e(
12714 state: &mut RunState,
12715 winner: &Candidate,
12716 wait: &crate::repo_lock::WaitPolicy,
12717) -> Result<GateFix> {
12718 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12719 return Ok(GateFix::Retry);
12720 }
12721 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12722 if state
12723 .gate_repair_e2e
12724 .as_ref()
12725 .is_some_and(|r| r.head == head && r.passed)
12726 {
12727 return Ok(GateFix::Retry);
12728 }
12729 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12730 head: head.clone(),
12731 passed: false,
12732 outcomes: Vec::new(),
12733 });
12734 state.save()?;
12735 let commands = state.config.verify.e2e.clone();
12736 let outcomes = if commands.is_empty() {
12737 Vec::new()
12738 } else {
12739 let shell = state.config.shell();
12740 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12741 let cache_dir = state.config.cache_dir();
12742 let (outcomes, _) = with_cache_lease(
12743 state,
12744 cache_dir.as_deref(),
12745 "e2e",
12746 "e2e",
12747 &winner.worktree,
12748 &head,
12749 timeout,
12750 "gate repair e2e",
12751 wait,
12752 |state, budget| async move {
12753 run_e2e_with_retry(
12754 state,
12755 &shell,
12756 &commands,
12757 &winner.worktree,
12758 budget,
12759 "gate repair e2e",
12760 )
12761 .await
12762 },
12763 )
12764 .await?;
12765 outcomes
12766 };
12767 let deferred = verify_inconclusive(&outcomes);
12768 let passed = outcomes.iter().all(CommandOutcome::ok)
12769 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12770 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12771 head,
12772 passed,
12773 outcomes,
12774 });
12775 state.save()?;
12776 if deferred {
12777 Ok(GateFix::Defer)
12778 } else if !passed {
12779 state.event(
12780 "gate",
12781 "verify.e2e failed after gate repair; nothing published",
12782 );
12783 Ok(GateFix::Stop)
12784 } else {
12785 Ok(GateFix::Retry)
12786 }
12787}
12788
12789pub(crate) async fn run_landing_gate(
12791 state: &mut RunState,
12792 winner: &Candidate,
12793 wait: &crate::repo_lock::WaitPolicy,
12794) -> Result<Vec<CommandOutcome>> {
12795 let shell = state.config.shell();
12796 let gate_commands = state.config.verify.gate.clone();
12797 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12798 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12799 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12800 Vec::new()
12801 } else {
12802 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12803 let cache_dir = state.config.cache_dir();
12804 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12805 let (outcomes, _) = with_cache_lease(
12806 state,
12807 cache_dir.as_deref(),
12808 "gate",
12809 "gate",
12810 &winner.worktree,
12811 &head,
12812 timeout,
12813 "final gate",
12814 wait,
12815 |state, budget| {
12816 let shell = shell.clone();
12817 let gate_commands = gate_commands.clone();
12818 let winner = winner.clone();
12819 async move {
12820 Runner::run_pre_gate(state, &winner).await;
12821 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12823 let (outcomes, timed_out_pids) = run_commands(
12824 state,
12825 "gate",
12826 "gate",
12827 0,
12828 &shell,
12829 &gate_commands,
12830 &winner.worktree,
12831 budget,
12832 )
12833 .await;
12834 (outcomes, false, timed_out_pids)
12835 }
12836 },
12837 )
12838 .await?;
12839 outcomes
12840 };
12841 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12842 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12843 let mut outcomes = outcomes;
12844 outcomes.push(CommandOutcome {
12845 command: "gate head stability".into(), code: None,
12846 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12847 duration_ms: 0, resource_blocked: false,
12848 });
12849 return Ok(outcomes);
12850 }
12851 if outcomes.is_empty() {
12852 state.event("gate", "no gate commands configured; not tested");
12857 }
12858 for o in &outcomes {
12859 state.event(
12860 "gate",
12861 format!(
12862 "`{}` -> {}",
12863 o.command,
12864 if o.ok() {
12865 "pass".to_owned()
12866 } else {
12867 format!(
12868 "FAIL ({:?})\n{}",
12869 o.code,
12870 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12871 )
12872 }
12873 ),
12874 );
12875 }
12876 Ok(outcomes)
12877}
12878
12879pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12885 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12886 ""
12887 } else {
12888 pr_title
12889 }
12890}
12891
12892pub fn landing_subject_source(state: &RunState) -> String {
12896 if is_review_run(state) {
12897 let winner = state.candidates.first().map_or('A', |c| c.label);
12898 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12899 }
12900 state.instruction.clone()
12901}
12902
12903fn summary_without_title(summary: &str) -> String {
12906 let mut lines = summary.trim().lines().peekable();
12907 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12908 lines.next();
12909 }
12910 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12911}
12912
12913#[cfg(test)]
12925fn pr_message(state: &RunState, winner: char) -> PrMessage {
12926 pr_message_with(state, winner, None)
12927}
12928
12929#[cfg(test)]
12932fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12933 let raw = pr_message_raw(
12934 state,
12935 winner,
12936 facts,
12937 crate::pr_outcome::FinalHead::Unreported,
12938 ChangeNote::NotAsked,
12939 );
12940 let id = crate::scrub::Identity::current();
12941 PrMessage {
12942 title: crate::scrub::scrub(&raw.title, &id),
12943 body: crate::scrub::scrub(&raw.body, &id),
12944 }
12945}
12946
12947#[derive(Debug, Clone, Copy)]
12949enum ChangeNote<'a> {
12950 NotAsked,
12952 Summary(&'a crate::run::ChangeSummary),
12954 Unavailable(&'a str),
12956}
12957
12958fn pr_message_raw(
12959 state: &RunState,
12960 winner: char,
12961 facts: Option<&BranchFacts>,
12962 head: crate::pr_outcome::FinalHead<'_>,
12963 change: ChangeNote<'_>,
12964) -> PrMessage {
12965 if !matches!(change, ChangeNote::NotAsked) {
12966 let (title, mut body) = match change {
12967 ChangeNote::Summary(c) => (
12968 c.title.clone(),
12969 format!("## Summary\n\n{}\n\n", c.body.trim()),
12970 ),
12971 ChangeNote::Unavailable(why) => (
12972 neutral_title(state, winner),
12973 format!(
12974 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12975 ),
12976 ),
12977 ChangeNote::NotAsked => unreachable!(),
12978 };
12979 body.push_str(&public_pr_evidence(state, head));
12980 body.push_str(&format!(
12981 "\n---\nmagi:run/{} magi:candidate-{}\n",
12982 state.id,
12983 winner.to_ascii_lowercase()
12984 ));
12985 return PrMessage { title, body };
12986 }
12987 pr_preview(state, winner, facts, head)
12988}
12989
12990fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12993 let mut body = crate::pr_outcome::review_rounds(state);
12994 body.push_str(&crate::pr_outcome::final_verification(state, head));
12995 let open = state.last_round_findings();
12996 if !open.is_empty() {
12997 body.push_str("## Open review findings\n\n");
12998 for f in open {
12999 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13000 }
13001 body.push('\n');
13002 }
13003 if let Some(c) = state.change_summary.as_ref()
13004 && head == crate::pr_outcome::FinalHead::At(&c.head)
13005 {
13006 body.push_str(&format!(
13007 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13008 c.base, c.head
13009 ));
13010 }
13011 body
13012}
13013
13014fn pr_preview(
13016 state: &RunState,
13017 winner: char,
13018 facts: Option<&BranchFacts>,
13019 head: crate::pr_outcome::FinalHead<'_>,
13020) -> PrMessage {
13021 let summary = state
13022 .candidates
13023 .iter()
13024 .find(|c| c.label == winner)
13025 .map(|c| c.summary.as_str())
13026 .unwrap_or_default();
13027 let review = is_review_run(state);
13030 let title = if review {
13031 facts
13032 .and_then(|f| {
13033 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13034 })
13035 .or_else(|| review_title(state))
13036 .or_else(|| {
13037 state
13038 .candidates
13039 .iter()
13040 .find(|c| c.label == winner)
13041 .filter(|c| !c.branch.starts_with("magi/"))
13042 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13043 })
13044 .unwrap_or_else(|| neutral_title(state, winner))
13045 } else {
13046 summary_title(summary).unwrap_or_else(|| {
13047 english_title(
13048 &state.instruction,
13049 PR_TITLE_MAX,
13050 state.config.graph.github_english(),
13051 )
13052 .unwrap_or_else(|| neutral_title(state, winner))
13053 })
13054 };
13055
13056 let mut body = String::new();
13057 let what = summary_without_title(summary);
13058 if !what.is_empty() {
13059 body.push_str("## Summary\n\n");
13060 body.push_str(&what);
13061 body.push_str("\n\n");
13062 }
13063
13064 body.push_str(&crate::pr_outcome::review_rounds(state));
13066 body.push_str(&crate::pr_outcome::final_verification(state, head));
13067 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13068
13069 let open = state.open_findings();
13070 if !open.is_empty() {
13071 body.push_str("## Open review findings\n\n");
13072 for f in &open {
13073 body.push_str(&format!(
13074 "- `{}` [{:?}] {}\n",
13075 f.id, f.severity, "Context retained in local review records."
13076 ));
13077 }
13078 body.push('\n');
13079 }
13080
13081 if let Some(fix) = fix
13082 && !fix.rejected.is_empty()
13083 {
13084 body.push_str("## Declined by the fixer\n\n");
13085 for r in &fix.rejected {
13086 body.push_str(&format!(
13087 "- `{}`: {}\n",
13088 r.id, "Reason retained in local review records."
13089 ));
13090 }
13091 body.push('\n');
13092 }
13093
13094 if review {
13095 body.push_str("## Commits under review\n\n");
13097 if let Some(facts) = facts {
13098 let mut left = COMMITS_MAX_CHARS;
13099 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13100 let mut entry = format!("- {}\n", markdown_safe(subject));
13101 for l in markdown_safe(text).lines() {
13102 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13103 }
13104 if left == 0 {
13105 body.push_str(&format!(
13106 "- ... {} more commit(s)\n",
13107 facts.commits.len() - i
13108 ));
13109 break;
13110 }
13111 if entry.len() > left {
13112 let mut end = left;
13115 while !entry.is_char_boundary(end) {
13116 end -= 1;
13117 }
13118 entry.truncate(end);
13119 entry.push_str("\n ... (truncated)\n");
13120 left = 0;
13121 } else {
13122 left -= entry.len();
13123 }
13124 body.push_str(&entry);
13125 }
13126 if !facts.stat.trim().is_empty() {
13127 body.push_str(&format!(
13128 "\n## Diff stat\n\n```\n{}\n```\n",
13129 markdown_safe(facts.stat.trim())
13130 ));
13131 }
13132 } else {
13133 match &state.reviewed_commits {
13134 Some(subjects) => {
13135 for s in subjects {
13136 body.push_str(&format!("- {}\n", s.trim()));
13137 }
13138 }
13139 None => {
13140 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13143 body.push_str(rest.trim());
13144 body.push('\n');
13145 }
13146 }
13147 }
13148 } else {
13149 let task = state.instruction.trim();
13150 let task = if task.is_empty() {
13151 "(empty task)"
13152 } else {
13153 task
13154 };
13155 body.push_str(&format!(
13156 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13157 task.replace("</details>", "</details>")
13158 ));
13159 }
13160
13161 body.push_str(&format!(
13162 "\n---\nmagi:run/{} magi:candidate-{}\n",
13163 state.id,
13164 winner.to_ascii_lowercase()
13165 ));
13166
13167 PrMessage { title, body }
13168}
13169
13170fn seeded_instruction(state: &RunState) -> String {
13173 let mut inputs = Vec::new();
13174 if let Some(start) = &state.sources.start_from {
13175 inputs.push(format!(
13176 "Start from `{}` at `{}` (including ancestry).",
13177 start.token, start.sha
13178 ));
13179 }
13180 for pick in &state.sources.pick_commits {
13181 inputs.push(format!(
13182 "Pick `{}` at `{}` (single commit).",
13183 pick.token, pick.sha
13184 ));
13185 }
13186 if inputs.is_empty() {
13187 state.instruction.clone()
13188 } else {
13189 format!(
13190 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13191 state.instruction,
13192 inputs.join("\n")
13193 )
13194 }
13195}
13196
13197async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13201 let base = &state.base_branch;
13202 let mut against = base.clone();
13203 if mode == MergeMode::Pr {
13204 let remote = &state.config.merge.remote;
13208 let tracking = format!("{remote}/{base}");
13209 let fetched = git::fetch(repo, remote, base).await;
13210 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13211 against = tracking;
13212 } else {
13213 return false;
13214 }
13215 }
13216 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13217}
13218
13219fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13222 let mut detail = format!(
13223 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13224 nothing to open a pull request for"
13225 );
13226 detail.push_str(
13227 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13228 );
13229 detail
13230}
13231
13232#[derive(Debug, PartialEq, Eq)]
13235enum PrPlan {
13236 Create,
13237 Adopt { url: String, title: String },
13238 Stop(String),
13239}
13240
13241fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13244 match found {
13245 Ok(land::OpenPr::None) => PrPlan::Create,
13246 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13247 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13248 "several open pull requests exist for this branch, not picking one: {}",
13249 urls.join(" ")
13250 )),
13251 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13252 }
13253}
13254
13255pub(crate) async fn gh_pr_create(
13257 cwd: &Path,
13258 base: &str,
13259 head: &str,
13260 title: &str,
13261 body: &str,
13262) -> Result<String> {
13263 let out = tokio::process::Command::new("gh")
13264 .args([
13265 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13266 ])
13267 .current_dir(cwd)
13268 .quiet()
13269 .stdin(std::process::Stdio::null())
13270 .output()
13271 .await
13272 .context("spawn gh")?;
13273 if out.status.success() {
13274 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13275 } else {
13276 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13277 }
13278}
13279
13280async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13282 match std::fs::symlink_metadata(path) {
13283 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13284 git::worktree_remove_clean(repo, path).await?;
13287 return Ok(());
13288 }
13289 Err(e) => return Err(e.into()),
13290 Ok(_) => {}
13291 }
13292 if !git::is_clean(path).await? {
13293 bail!(
13294 "worktree has uncommitted work; refusing removal: {}",
13295 path.display()
13296 );
13297 }
13298 if !git::worktree_remove_clean(repo, path).await? {
13299 bail!("worktree removal refused: {}", path.display());
13300 }
13301 Ok(())
13302}
13303
13304pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13313 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13314 bail!("refusing fold: {reason}");
13315 }
13316 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13317 bail!("candidate preservation incomplete; refusing cleanup");
13318 }
13319 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13323 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13324 let _driver = crate::driver::maintenance(home, &state.id)?;
13325 let mut output_guards = Vec::new();
13326 for entry in std::fs::read_dir(state.worktree_root())
13327 .into_iter()
13328 .flatten()
13329 .flatten()
13330 {
13331 if entry.path().join(".git").is_file() {
13332 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13333 }
13334 }
13335 let repo = state.repo.clone();
13336 let wait = repository_wait(state, None);
13339 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13340 Ok(lease) => lease,
13341 Err(e) => {
13342 state.save_under(home)?;
13343 return Err(e);
13344 }
13345 };
13346 let _source_outputs = if state.status == RunStatus::Merged
13349 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13350 {
13351 verify_merged_fold_sources(state, drop_winner, home).await?
13352 } else {
13353 Vec::new()
13354 };
13355 let root = state.worktree_root();
13356 let winner = state.tally.as_ref().map(|t| t.winner);
13357 let mut removed = Vec::new();
13358
13359 for i in 0..state.candidates.len() {
13360 let c = state.candidates[i].clone();
13361 let is_winner = Some(c.label) == winner;
13362 if is_winner && !drop_winner {
13363 continue;
13364 }
13365 let existed = c.worktree.exists();
13366 remove_preserved_worktree(&repo, &c.worktree).await?;
13369 if existed {
13370 removed.push(c.worktree.to_string_lossy().into_owned());
13371 }
13372 let handed_over = state.released_branches.contains(&c.branch);
13375 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13376 if !git::branch_delete(&repo, &c.branch).await? {
13377 bail!("candidate branch removal refused");
13378 }
13379 removed.push(c.branch.clone());
13380 }
13381 state.candidates[i].folded = true;
13382 }
13383
13384 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13385 let path = name.path();
13386 let keep = !drop_winner
13387 && winner.is_some_and(|w| {
13388 path.file_name()
13389 .is_some_and(|n| n == format!("cand-{w}").as_str())
13390 });
13391 if keep {
13392 continue;
13393 }
13394 remove_preserved_worktree(&repo, &path).await?;
13395 removed.push(path.to_string_lossy().into_owned());
13396 }
13397
13398 remove_if_empty(&root);
13407
13408 if state.enabled_worktree_config && drop_winner {
13409 git::release_worktree_config(&repo).await.ok();
13413 state.enabled_worktree_config = false;
13414 }
13415 state.save_under(home)?;
13416 Ok(removed)
13417}
13418
13419async fn verify_merged_fold_sources(
13420 state: &RunState,
13421 drop_winner: bool,
13422 home: &Path,
13423) -> Result<Vec<std::fs::File>> {
13424 let winner = state.tally.as_ref().map(|t| t.winner);
13425 let mut heads = BTreeSet::new();
13426 for candidate in &state.candidates {
13427 if Some(candidate.label) == winner && !drop_winner {
13428 continue;
13429 }
13430 anyhow::ensure!(
13431 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13432 "candidate recovery source is outside the inventoried run root: {}",
13433 candidate.worktree.display()
13434 );
13435 match std::fs::symlink_metadata(&candidate.worktree) {
13436 Ok(_) => {}
13437 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13438 Err(error) => return Err(error.into()),
13439 }
13440 if !state.released_branches.contains(&candidate.branch)
13441 && git::branch_exists(&state.repo, &candidate.branch).await?
13442 {
13443 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13444 }
13445 }
13446 let mut cargo_guards = Vec::new();
13447 let entries = match std::fs::read_dir(state.worktree_root()) {
13448 Ok(entries) => Some(entries),
13449 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13450 Err(error) => return Err(error.into()),
13451 };
13452 for entry in entries.into_iter().flatten() {
13453 let path = entry?.path();
13454 let metadata = std::fs::symlink_metadata(&path)?;
13455 anyhow::ensure!(
13456 metadata.is_dir() && !metadata.file_type().is_symlink(),
13457 "non-directory or symlink recovery source retained: {}",
13458 path.display()
13459 );
13460 if !drop_winner
13461 && winner.is_some_and(|w| {
13462 path.file_name()
13463 .is_some_and(|name| name == format!("cand-{w}").as_str())
13464 })
13465 {
13466 continue;
13467 }
13468 anyhow::ensure!(
13469 path.join(".git").is_file(),
13470 "unregistered recovery source retained: {}",
13471 path.display()
13472 );
13473 anyhow::ensure!(
13474 git::git(
13475 &path,
13476 &[
13477 "status",
13478 "--porcelain",
13479 "--untracked-files=all",
13480 "--ignored",
13481 "--",
13482 ".",
13483 ":(exclude)target"
13484 ]
13485 )
13486 .await?
13487 .is_empty(),
13488 "source changes or ignored recovery files retained: {}",
13489 path.display()
13490 );
13491 match std::fs::symlink_metadata(path.join("target")) {
13492 Ok(_) => {
13493 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13494 }
13495 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13496 Err(error) => return Err(error.into()),
13497 }
13498 for checkpoint in [
13499 "rebase-merge",
13500 "rebase-apply",
13501 "sequencer",
13502 "CHERRY_PICK_HEAD",
13503 "MERGE_HEAD",
13504 "REBASE_HEAD",
13505 "REVERT_HEAD",
13506 "BISECT_LOG",
13507 "AUTO_MERGE",
13508 "MERGE_MSG",
13509 ] {
13510 let location = git::git(
13511 &path,
13512 &[
13513 "rev-parse",
13514 "--path-format=absolute",
13515 "--git-path",
13516 checkpoint,
13517 ],
13518 )
13519 .await?;
13520 anyhow::ensure!(
13521 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13522 if e.kind() == std::io::ErrorKind::NotFound),
13523 "Git recovery checkpoint retained: {checkpoint}"
13524 );
13525 }
13526 heads.insert(git::rev_parse(&path, "HEAD").await?);
13527 }
13528 crate::fold_evidence::preserve(state, home, &heads).await?;
13529
13530 Ok(cargo_guards)
13531}
13532
13533fn remove_if_empty(dir: &Path) {
13544 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13545 std::fs::remove_dir(dir).ok();
13546 }
13547}
13548
13549pub fn worst_open(state: &RunState) -> Option<Severity> {
13551 state
13552 .reviews
13553 .last()?
13554 .reviews
13555 .iter()
13556 .flat_map(|r| r.findings.iter())
13557 .map(|f| f.severity)
13558 .max()
13559}
13560
13561async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13565 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13566 git::acquire_worktree_config(repo).await?;
13567 state.enabled_worktree_config = true;
13568 }
13569 Ok(())
13570}
13571
13572#[cfg(test)]
13573mod tests {
13574 #[tokio::test]
13575 async fn launcher_base_fetch_requires_remote_success() {
13576 let tmp = tempfile::tempdir().unwrap();
13577 let repo = tmp.path().join("repo");
13578 let remote = tmp.path().join("remote");
13579 std::fs::create_dir(&repo).unwrap();
13580 std::fs::create_dir(&remote).unwrap();
13581 git::git(&remote, &["init", "--bare", "-b", "main"])
13582 .await
13583 .unwrap();
13584 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13585 git::git(
13586 &repo,
13587 &[
13588 "-c",
13589 "user.name=Fixture",
13590 "-c",
13591 "user.email=fixture@example.invalid",
13592 "commit",
13593 "--allow-empty",
13594 "-m",
13595 "fixture",
13596 ],
13597 )
13598 .await
13599 .unwrap();
13600 git::git(
13601 &repo,
13602 &["remote", "add", "origin", remote.to_str().unwrap()],
13603 )
13604 .await
13605 .unwrap();
13606 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13607 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13608 assert_eq!(
13609 resolve_base(&repo, "main", "origin", None)
13610 .await
13611 .unwrap()
13612 .commit,
13613 head
13614 );
13615 #[cfg(unix)]
13616 {
13617 let policy: crate::config::Coordination = toml::from_str(
13618 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13619 ).unwrap();
13620 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13621 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13622 let held = crate::repo_lock::SeatLease::operation_with(
13623 &repo,
13624 Some(&policy),
13625 &crate::repo_lock::WaitPolicy::default(),
13626 )
13627 .await
13628 .unwrap();
13629 assert!(held.is_some());
13630 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13631 tokio::pin!(fetch);
13632 assert!(
13633 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13634 .await
13635 .is_err()
13636 );
13637 assert!(
13638 !repo.join(".git/FETCH_HEAD").exists(),
13639 "no fetch before admission"
13640 );
13641 drop(held);
13642 let fetched = fetch.await.unwrap();
13643 assert_eq!(fetched.commit, head);
13644 assert_eq!(fetched.remote, "origin");
13645 assert_eq!(fetched.branch, "main");
13646 }
13647 git::git(
13649 &repo,
13650 &[
13651 "remote",
13652 "set-url",
13653 "origin",
13654 tmp.path().join("missing").to_str().unwrap(),
13655 ],
13656 )
13657 .await
13658 .unwrap();
13659 let error = resolve_base(&repo, "main", "origin", None)
13660 .await
13661 .unwrap_err()
13662 .to_string();
13663 assert!(error.contains("cannot read origin/main"));
13664 assert!(error.contains("refusing to branch off"));
13665 }
13666
13667 #[test]
13668 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13669 let leaked = vec!["chore(deps): update a crate".to_owned()];
13670 let own = "fix(daemon): apply a chosen action";
13671 assert!(should_retitle("", own, &leaked));
13672 assert!(should_retitle(
13673 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13674 own,
13675 &leaked
13676 ));
13677 assert!(should_retitle(
13678 "chore: land candidate A of run 1",
13679 own,
13680 &leaked
13681 ));
13682 assert!(should_retitle(
13683 "magi: candidate A (uncommitted work)",
13684 own,
13685 &leaked
13686 ));
13687 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13688 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13689 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13690 }
13691
13692 #[test]
13693 fn pr_merge_plan_creates_adopts_or_stops() {
13694 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13695 assert_eq!(
13696 pr_merge_plan(Ok(land::OpenPr::One {
13697 url: "u".into(),
13698 title: "t".into()
13699 })),
13700 PrPlan::Adopt {
13701 url: "u".into(),
13702 title: "t".into()
13703 }
13704 );
13705 let PrPlan::Stop(many) =
13706 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13707 else {
13708 panic!("many must stop");
13709 };
13710 assert!(many.contains('a') && many.contains('b'));
13711 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13712 panic!("a failed lookup must stop");
13713 };
13714 assert!(err.contains("bad token"));
13715 }
13716
13717 use super::*;
13718 use crate::run::GateStatus;
13719 use std::collections::BTreeMap;
13720 use std::time::Duration;
13721
13722 fn conductor() -> AgentSpec {
13723 AgentSpec {
13724 id: "conductor".to_owned(),
13725 kind: crate::config::AgentKind::Command,
13726 model: None,
13727 command: vec!["true".to_owned()],
13728 extra_args: Vec::new(),
13729 env: BTreeMap::new(),
13730 prompt_delivery: None,
13731 max_turn_secs: None,
13732 }
13733 }
13734
13735 fn spec(id: &str) -> AgentSpec {
13736 AgentSpec {
13737 id: id.to_owned(),
13738 kind: crate::config::AgentKind::Command,
13739 model: None,
13740 command: vec!["true".to_owned()],
13741 extra_args: Vec::new(),
13742 env: BTreeMap::new(),
13743 prompt_delivery: None,
13744 max_turn_secs: None,
13745 }
13746 }
13747
13748 fn ids(xs: &[&str]) -> BTreeSet<String> {
13749 xs.iter().map(|s| (*s).to_owned()).collect()
13750 }
13751
13752 #[test]
13753 fn pick_successor_skips_an_agent_another_seat_holds() {
13754 let roster = [spec("a"), spec("b"), spec("c")];
13756 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13757 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13758 }
13759
13760 #[test]
13761 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13762 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13764 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13765 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13766 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13767 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13768 }
13769
13770 #[test]
13771 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13772 let roster = [spec("a"), spec("b")];
13773 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13774 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13775 let carried = ids(&["b"]);
13776 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13777 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13778 }
13779
13780 #[test]
13781 fn pick_successor_returns_none_without_any_untried_successor() {
13782 let roster = [spec("a"), spec("b")];
13783 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13784 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13785 let carried = ids(&["a"]);
13786 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13787 }
13788
13789 #[test]
13790 fn pick_successor_rescue_avoids_another_seats_occupant() {
13791 let roster = [spec("a"), spec("b"), spec("c")];
13793 let carried = ids(&["b", "c"]);
13794 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13795 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13796 }
13797
13798 #[test]
13799 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13800 let roster = [spec("a"), spec("b"), spec("c")];
13801 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13802 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13803 }
13804
13805 #[test]
13806 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13807 let roster = [spec("a"), spec("b"), spec("c")];
13808 let failed = ids(&["a", "b", "c"]);
13809 let mut tried = ids(&["b"]);
13811 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13812 assert_eq!(first.id, "a");
13813 tried.insert(first.id.clone());
13814 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13815 assert_eq!(second.id, "c");
13816 tried.insert(second.id.clone());
13817 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13818 }
13819
13820 #[test]
13821 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13822 let roster = [spec("a"), spec("b")];
13823 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13824 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13825 }
13826
13827 #[test]
13828 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13829 let roster = [spec("a"), spec("b"), spec("c")];
13830 let h = SeatHistory {
13831 failed: ids(&["a"]),
13832 last_ok: Some("b".to_owned()),
13833 last_fail: None,
13834 };
13835 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13836 let gone = SeatHistory {
13838 last_ok: Some("zzz".to_owned()),
13839 ..h.clone()
13840 };
13841 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13842 let failed = SeatHistory {
13843 failed: ids(&["a", "b"]),
13844 last_ok: Some("b".to_owned()),
13845 last_fail: None,
13846 };
13847 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13848 }
13849
13850 #[test]
13851 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13852 let first = SeatState::new("review-1", "alpha", 7);
13853 let next = handover_seat("review-1", "gamma", 7);
13854 assert_ne!(first.claude_session, next.claude_session);
13855 }
13856
13857 #[test]
13858 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13859 let mut state = state_with_summary("x", "y");
13860 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13861 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13862 assert_ne!(a.claude_session, b.claude_session);
13863 }
13864
13865 #[test]
13866 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13867 let roster = [spec("a"), spec("b")];
13868 let h = SeatHistory {
13869 failed: ids(&["a", "b"]),
13870 ..SeatHistory::default()
13871 };
13872 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13873 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13874 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13875 }
13876
13877 #[test]
13884 fn failure_signature_ignores_numbers_and_paths() {
13885 assert_eq!(
13886 failure_signature("exited with Some(2) and no usable output"),
13887 failure_signature("exited with Some(137) and no usable output")
13888 );
13889 assert_eq!(
13890 failure_signature("process exited with no usable output (exit code 2)"),
13891 failure_signature("process exited with no usable output (exit code 137)")
13892 );
13893 for (old, new) in [
13894 (
13895 "exited with Some(2) and no usable output",
13896 "process exited with no usable output (exit code 2)",
13897 ),
13898 (
13899 "exited with None and no usable output",
13900 "process exited with no usable output (exit code unavailable)",
13901 ),
13902 ] {
13903 let previous = FailClass::Other(failure_signature(old));
13904 let current = FailClass::Other(failure_signature(new));
13905 assert_eq!(previous, current, "resumption retains the failure class");
13906 assert!(!should_hand_over(Some(&previous), ¤t));
13907 }
13908 assert_eq!(
13909 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13910 failure_signature("cannot open /var/x.txt: denied")
13911 );
13912 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13913 }
13914
13915 #[test]
13916 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13917 let other = FailClass::Other("x".into());
13918 assert!(should_hand_over(None, &FailClass::Quota));
13919 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13920 assert!(should_hand_over(
13921 Some(&FailClass::Timeout),
13922 &FailClass::Timeout
13923 ));
13924 assert!(should_hand_over(None, &other));
13925 assert!(!should_hand_over(Some(&other), &other));
13926 assert!(should_hand_over(
13927 Some(&other),
13928 &FailClass::Other("y".into())
13929 ));
13930 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13932 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13933 }
13934
13935 #[test]
13936 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13937 let a = SeatState::new("judge-1", "alpha", 7);
13938 let b = handover_seat("judge-1", "beta", 7);
13939 assert_ne!(a.claude_session, b.claude_session);
13940 assert_eq!(b.turns, 0);
13941 }
13942
13943 #[test]
13944 fn a_timeout_is_classified_apart_from_other_failures() {
13945 assert_eq!(
13946 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13947 Some(FailClass::Timeout)
13948 );
13949 assert!(matches!(
13950 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13951 Some(FailClass::Other(_))
13952 ));
13953 }
13954
13955 #[test]
13956 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13957 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13958 let tried = BTreeSet::from(["beta".to_owned()]);
13959 let next = next_untried_in_roster(&roster, 1, &tried);
13962 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13963 }
13964
13965 #[test]
13966 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13967 let roster = vec![spec("alpha"), spec("beta")];
13968 let tried = BTreeSet::from(["beta".to_owned()]);
13969 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13973 }
13974
13975 #[test]
13976 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13977 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13978 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13979 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13983 }
13984
13985 #[test]
13986 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13987 let roster = vec![spec("a"), spec("a"), spec("b")];
13988 let tried = BTreeSet::from(["a".to_owned()]);
13989 let next = next_untried_in_roster(&roster, 0, &tried);
13990 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13991 }
13992
13993 #[test]
13994 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13995 let roster = vec![spec("a"), spec("b")];
13996 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13997 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13998 }
13999
14000 #[test]
14001 fn remove_if_empty_only_ever_takes_a_bare_directory() {
14002 let dir = tempfile::tempdir().unwrap();
14003 let bay = dir.path().join("ffff");
14004
14005 remove_if_empty(&bay);
14007 assert!(!bay.exists());
14008
14009 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14012 remove_if_empty(&bay);
14013 assert!(bay.exists(), "non-empty directory must survive");
14014
14015 std::fs::remove_dir(bay.join("cand-A")).unwrap();
14017 remove_if_empty(&bay);
14018 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14019 }
14020
14021 #[test]
14030 fn a_full_panel_that_found_nothing_is_clean() {
14031 assert!(round_is_clean(
14032 0,
14033 true,
14034 2,
14035 2,
14036 0,
14037 IncompleteReviewPolicy::Block
14038 ));
14039 }
14040
14041 #[test]
14042 fn a_missing_seat_is_never_clean_under_the_default_policy() {
14043 assert!(!round_is_clean(
14044 0,
14045 true,
14046 1,
14047 2,
14048 0,
14049 IncompleteReviewPolicy::Block
14050 ));
14051 }
14052
14053 #[test]
14054 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14055 assert!(!round_is_clean(
14056 1,
14057 true,
14058 1,
14059 2,
14060 0,
14061 IncompleteReviewPolicy::Warn
14062 ));
14063 }
14064
14065 #[test]
14066 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14067 assert!(round_is_clean(
14068 0,
14069 true,
14070 1,
14071 2,
14072 0,
14073 IncompleteReviewPolicy::Warn
14074 ));
14075 }
14076
14077 #[test]
14078 fn a_full_panel_with_an_open_finding_is_not_clean() {
14079 assert!(!round_is_clean(
14080 1,
14081 true,
14082 2,
14083 2,
14084 0,
14085 IncompleteReviewPolicy::Block
14086 ));
14087 }
14088
14089 #[test]
14090 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14091 assert!(!round_is_clean(
14092 0,
14093 false,
14094 2,
14095 2,
14096 0,
14097 IncompleteReviewPolicy::Block
14098 ));
14099 }
14100
14101 #[test]
14108 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14109 assert!(round_is_clean(
14112 0,
14113 true,
14114 1,
14115 2,
14116 1,
14117 IncompleteReviewPolicy::Block
14118 ));
14119 }
14120
14121 #[test]
14122 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14123 assert!(!round_is_clean(
14126 0,
14127 true,
14128 1,
14129 2,
14130 0,
14131 IncompleteReviewPolicy::Block
14132 ));
14133 }
14134
14135 #[test]
14136 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14137 assert!(!round_is_clean(
14138 1,
14139 true,
14140 1,
14141 2,
14142 1,
14143 IncompleteReviewPolicy::Block
14144 ));
14145 assert!(!round_is_clean(
14146 0,
14147 false,
14148 1,
14149 2,
14150 1,
14151 IncompleteReviewPolicy::Block
14152 ));
14153 }
14154
14155 #[test]
14156 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14157 assert!(!round_is_clean(
14161 0,
14162 true,
14163 0,
14164 2,
14165 2,
14166 IncompleteReviewPolicy::Block
14167 ));
14168 }
14169
14170 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14171 CommandOutcome {
14172 command: "test".to_owned(),
14173 code,
14174 output_tail: String::new(),
14175 duration_ms: 0,
14176 resource_blocked,
14177 }
14178 }
14179
14180 #[cfg(unix)]
14181 #[tokio::test]
14182 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14183 ask_test_home();
14184 let temp = tempfile::tempdir().unwrap();
14185 let mut runner = runner_at(RunStatus::Gating);
14186 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14187 let commands = vec![
14188 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14189 "touch must-not-run".to_owned(),
14190 ];
14191 let (outcomes, retried, pids) = run_e2e_with_retry(
14192 &mut runner.state,
14193 &shell,
14194 &commands,
14195 temp.path(),
14196 Duration::from_secs(5),
14197 "fixture",
14198 )
14199 .await;
14200 assert_eq!(outcomes.len(), 1);
14201 assert!(outcomes[0].resource_blocked);
14202 assert!(!outcomes[0].build_failed());
14203 assert!(!retried);
14204 assert!(pids.is_empty());
14205 assert!(!temp.path().join("must-not-run").exists());
14206 assert_eq!(
14207 runner.state.resource_block.as_ref().unwrap().kind,
14208 "incremental cache admission"
14209 );
14210 }
14211
14212 #[test]
14213 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14214 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14215 assert!(
14216 !verify_inconclusive(&[outcome(Some(1), false)]),
14217 "an ordinary failure is still evidence about the patch"
14218 );
14219 assert!(verify_inconclusive(&[outcome(None, true)]));
14220 assert!(
14221 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14222 "one inconclusive outcome taints the whole batch"
14223 );
14224 assert!(!verify_inconclusive(&[]));
14225 }
14226
14227 #[tokio::test]
14228 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14229 let calls = std::sync::atomic::AtomicUsize::new(0);
14233 let started = Instant::now();
14234 wait_for_pids_with(
14235 &[123],
14236 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14237 Duration::from_millis(5),
14238 Duration::from_secs(5),
14239 )
14240 .await;
14241 assert!(
14242 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14243 "must keep checking rather than deciding on the first answer"
14244 );
14245 assert!(
14246 started.elapsed() < Duration::from_secs(1),
14247 "must return the moment it is confirmed dead, not wait out the ceiling"
14248 );
14249 }
14250
14251 #[tokio::test]
14252 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14253 let started = Instant::now();
14254 wait_for_pids_with(
14255 &[123],
14256 |_| true, Duration::from_millis(5),
14258 Duration::from_millis(30),
14259 )
14260 .await;
14261 let elapsed = started.elapsed();
14262 assert!(
14263 elapsed >= Duration::from_millis(30),
14264 "must not give up before its own ceiling: {elapsed:?}"
14265 );
14266 assert!(
14267 elapsed < Duration::from_secs(1),
14268 "must not wait past its own ceiling either: {elapsed:?}"
14269 );
14270 }
14271
14272 #[tokio::test]
14273 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14274 let started = Instant::now();
14275 wait_for_pids_with(
14276 &[],
14277 |_| true,
14278 Duration::from_secs(5),
14279 Duration::from_secs(5),
14280 )
14281 .await;
14282 assert!(
14283 started.elapsed() < Duration::from_millis(200),
14284 "an empty pid list has nothing to confirm"
14285 );
14286 }
14287
14288 fn review_round(
14294 clean: bool,
14295 blocking: usize,
14296 answered: usize,
14297 expected: usize,
14298 progressed: bool,
14299 e2e_ok: bool,
14300 ) -> ReviewRound {
14301 ReviewRound {
14302 round: 1,
14303 head: "h".to_owned(),
14304 verified_head: None,
14305 verified_at: None,
14306 reviews: Vec::new(),
14307 e2e: vec![CommandOutcome {
14308 command: "test".to_owned(),
14309 code: Some(if e2e_ok { 0 } else { 1 }),
14310 output_tail: String::new(),
14311 duration_ms: 0,
14312 resource_blocked: false,
14313 }],
14314 verify_retried: false,
14315 e2e_deferred: false,
14316 e2e_defer_reason: None,
14317 fix: None,
14318 blocking,
14319 answered,
14320 expected,
14321 clean,
14322 progressed,
14323 vote_split: false,
14324 reconsideration: Vec::new(),
14325 verdict: None,
14326 }
14327 }
14328
14329 #[test]
14330 fn review_conclusion_is_none_when_nothing_has_run() {
14331 assert_eq!(review_conclusion(&[], 3), None);
14332 }
14333
14334 #[test]
14335 fn review_conclusion_is_none_while_rounds_remain() {
14336 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14337 assert_eq!(review_conclusion(&rounds, 3), None);
14338 }
14339
14340 #[test]
14341 fn review_conclusion_is_gating_once_a_round_is_clean() {
14342 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14343 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14344 }
14345
14346 #[test]
14347 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14348 let rounds = vec![
14349 review_round(false, 1, 2, 2, true, true),
14350 review_round(false, 1, 2, 2, true, true),
14351 ];
14352 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14353 }
14354
14355 #[test]
14356 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14357 let rounds = vec![
14358 review_round(false, 1, 2, 2, true, true),
14359 review_round(false, 1, 2, 2, true, false),
14360 ];
14361 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14362 }
14363
14364 #[test]
14365 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14366 let mut blocked = review_round(false, 1, 2, 2, true, false);
14373 blocked.e2e[0].resource_blocked = true;
14374 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14375 assert_eq!(review_conclusion(&rounds, 2), None);
14376 }
14377
14378 #[test]
14379 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14380 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14382 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14383 }
14384
14385 #[test]
14386 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14387 let rounds = vec![
14388 review_round(false, 1, 2, 2, false, true),
14389 review_round(false, 1, 2, 2, false, true),
14390 ];
14391 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14392 }
14393
14394 fn secs(n: u64) -> Duration {
14395 Duration::from_secs(n)
14396 }
14397
14398 #[tokio::test]
14401 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14402 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14403 if std::env::var_os(CHILD).is_none() {
14404 let output = std::process::Command::new(std::env::current_exe().unwrap())
14408 .args([
14409 "--exact",
14410 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14411 "--nocapture",
14412 ])
14413 .env(CHILD, "1")
14414 .env_remove("MAGI_WORKER_HOME")
14415 .env_remove("MAGI_WORKER_ID")
14416 .output()
14417 .unwrap();
14418 assert!(
14419 output.status.success()
14420 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14421 "{}\n{}",
14422 String::from_utf8_lossy(&output.stdout),
14423 String::from_utf8_lossy(&output.stderr)
14424 );
14425 return;
14426 }
14427 ask_test_home();
14428 let tmp = tempfile::tempdir().unwrap();
14429 let repo = tmp.path().join("repo");
14430 std::fs::create_dir_all(&repo).unwrap();
14431 init_repo(&repo);
14432 let g = |args: &[&str]| {
14433 let out = std::process::Command::new("git")
14434 .args(args)
14435 .current_dir(&repo)
14436 .quiet()
14437 .output()
14438 .unwrap();
14439 assert!(
14440 out.status.success(),
14441 "{}",
14442 String::from_utf8_lossy(&out.stderr)
14443 );
14444 };
14445 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14446 g(&["checkout", "-b", "historical"]);
14447 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14448 g(&["add", "."]);
14449 g(&["commit", "-m", "old alternative"]);
14450 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14451 g(&["checkout", "-b", "magi/task-selected", &base]);
14452 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14453 g(&["add", "."]);
14454 g(&["commit", "-m", "measurement infrastructure"]);
14455 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14456 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14457 g(&["add", "."]);
14458 g(&["commit", "-m", "dependent implementation"]);
14459 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14460 g(&["checkout", "main"]);
14461 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14462 g(&["add", "."]);
14463 g(&["commit", "-m", "unrelated base change"]);
14464 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14465 let origin = tmp.path().join("origin.git");
14466 g(&[
14467 "clone",
14468 "--bare",
14469 repo.to_str().unwrap(),
14470 origin.to_str().unwrap(),
14471 ]);
14472 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14473 g(&["fetch", "origin"]);
14474 let selection = crate::pr_recovery::Selection {
14475 url: "https://example.test/project/pull/1".into(),
14476 branch: "magi/task-selected".into(),
14477 head: head.clone(),
14478 source_head: None,
14479 source_run: None,
14480 source_proof: None,
14481 disposition: Some(crate::pr_recovery::Disposition::Review),
14482 instruction: "preserve measured behavior".into(),
14483 approved_at: jiff::Timestamp::now(),
14484 };
14485 let mut template = runner_at(RunStatus::Reviewing);
14486 template.state.repo = repo.clone();
14487 template.state.base_commit = current.clone();
14488 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14489 template.state.origin = Some(crate::run::Origin::queue("selected"));
14490 template.state.pr_head = Some(selection.branch.clone());
14491 template.state.pr_recovery = Some(selection.clone());
14492 let mut runner =
14493 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14494 .await
14495 .unwrap();
14496 let wt = runner.state.winner().unwrap().worktree.clone();
14497 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14498 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14499 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14500 assert!(runner.state.seeds.is_empty());
14501 assert_eq!(runner.state.instruction, "preserve measured behavior");
14502 runner.sync_to_base().await.unwrap();
14503 assert_eq!(
14504 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14505 "prerequisite"
14506 );
14507 assert_eq!(
14508 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14509 "measured implementation"
14510 );
14511 assert_eq!(
14512 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14513 "current main change"
14514 );
14515 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14516 assert_eq!(
14517 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14518 head
14519 );
14520
14521 let mut partial = runner_at(RunStatus::Reviewing);
14523 partial.state.repo = repo.clone();
14524 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14525 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14526 run: Some("exit 1".into()),
14527 ..Default::default()
14528 }];
14529 let selection = runner.state.pr_recovery.clone().unwrap();
14530 partial.state.pr_recovery = Some(selection.clone());
14531 partial.state.pr_head = Some(selection.branch.clone());
14532 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14533 let id = partial.state.id.clone();
14534 let error = match Runner::open_selected(
14535 &repo,
14536 partial.state,
14537 partial.roles,
14538 1,
14539 runner.state.base_commit.clone(),
14540 selection,
14541 )
14542 .await
14543 {
14544 Ok(_) => panic!("setup must fail"),
14545 Err(e) => e,
14546 };
14547 assert_eq!(
14548 error
14549 .downcast_ref::<crate::run::StartupFailure>()
14550 .unwrap()
14551 .run,
14552 id
14553 );
14554 let failed = RunState::load(&id).unwrap();
14555 assert_eq!(failed.status, RunStatus::Failed);
14556 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14557 assert!(Runner::resume(&id).is_err());
14558 runner.state.status = RunStatus::Ready;
14559 runner.state.driver_exited = true;
14560 runner.state.save().unwrap();
14561 let mut retry = runner_at(RunStatus::Reviewing);
14562 retry.state.repo = repo.clone();
14563 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14564 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14565 reselected.approved_at = jiff::Timestamp::now();
14566 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14567 retry.state.pr_head = Some(reselected.branch.clone());
14568 retry.state.pr_recovery = Some(reselected.clone());
14569 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14570 .await
14571 .unwrap();
14572 let restarted = Runner::open_selected(
14573 &repo,
14574 retry.state,
14575 retry.roles,
14576 1,
14577 runner.state.base_commit.clone(),
14578 reselected,
14579 )
14580 .await
14581 .unwrap();
14582 assert_eq!(
14583 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14584 .await
14585 .unwrap(),
14586 head
14587 );
14588 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14589 assert_eq!(
14590 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14591 head
14592 );
14593 }
14594
14595 #[tokio::test]
14596 #[cfg(unix)]
14597 async fn fallback_and_interrupted_restoration_require_coordination() {
14598 use std::os::fd::AsRawFd as _;
14599 ask_test_home();
14600 let tmp = tempfile::tempdir().unwrap();
14601 let mut runner = registered_runner(tmp.path()).await;
14602 runner.state.config.graph.timeout_resource_wait = 1;
14603 let winner = runner.state.candidates[0].clone();
14604 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14605 let leftover = winner.worktree.join("leftover");
14606 std::fs::write(&leftover, "untrusted edits").unwrap();
14607 let next = tmp.path().join("next-fixer");
14608 let mut failed = spec("first");
14609 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14610 let mut successor = spec("second");
14611 successor.command = vec![
14612 "sh".into(),
14613 "-c".into(),
14614 format!("touch '{}'; echo done", next.display()),
14615 ];
14616 let attempts = vec![
14617 (failed.clone(), "first".into()),
14618 (successor.clone(), "second".into()),
14619 ];
14620 let lock_path = runner.state.repo.join(".git/updater.lock");
14621 let external = std::fs::File::create(&lock_path).unwrap();
14622 assert_eq!(
14623 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14624 0
14625 );
14626 let artifacts = tmp.path().join("artifacts");
14627 let build = |spec, seat| SeatJob {
14628 spec,
14629 seat,
14630 cwd: winner.worktree.clone(),
14631 prompt: "fixture".into(),
14632 timeout: Duration::from_secs(10),
14633 allow_write: true,
14634 sessions: false,
14635 artifacts: artifacts.clone(),
14636 stem: "fix".into(),
14637 handover: None,
14638 };
14639 let (_, _, out) = runner
14640 .ask_fixer_among(
14641 attempts.clone(),
14642 "graft",
14643 None,
14644 Some((&winner.worktree, &head)),
14645 build,
14646 )
14647 .await;
14648 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14649 assert!(leftover.exists(), "contention must prevent clean");
14650 assert!(!next.exists());
14651 assert!(runner.state.handovers.is_empty());
14652 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14654 from_run: "earlier".into(),
14655 generation: 1,
14656 mode: crate::recovery::Mode::VerifyRepair,
14657 reasons: Vec::new(),
14658 reviewed_head: None,
14659 rounds: Vec::new(),
14660 open_findings: Vec::new(),
14661 evidence: None,
14662 criteria_inherited: true,
14663 repair: crate::recovery::Repair::Started {
14664 before: head.clone(),
14665 },
14666 });
14667 assert!(
14668 runner
14669 .recovery_repair(&winner)
14670 .await
14671 .unwrap_err()
14672 .is::<crate::repo_lock::ResourceBlocked>()
14673 );
14674 assert!(matches!(
14675 runner.state.recovery.as_ref().unwrap().repair,
14676 crate::recovery::Repair::Started { .. }
14677 ));
14678 assert!(leftover.exists());
14679 drop(external);
14680 runner.recovery_repair(&winner).await.unwrap();
14681 assert!(!leftover.exists());
14682 assert!(matches!(
14683 runner.state.recovery.as_ref().unwrap().repair,
14684 crate::recovery::Repair::Done { .. }
14685 ));
14686 std::fs::remove_file(&lock_path).unwrap();
14688 std::fs::create_dir(&lock_path).unwrap();
14689 std::fs::write(&leftover, "untrusted edits").unwrap();
14690 let (_, _, out) = runner
14691 .ask_fixer_among(
14692 attempts,
14693 "graft",
14694 None,
14695 Some((&winner.worktree, &head)),
14696 build,
14697 )
14698 .await;
14699 assert!(matches!(out, AgentOutcome::Failed(_)));
14700 assert!(leftover.exists());
14701 assert!(!next.exists());
14702 assert_eq!(
14703 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14704 head
14705 );
14706 }
14707
14708 #[cfg(unix)]
14709 async fn registered_runner(root: &Path) -> Runner {
14710 repository_runner(root, crate::config::test_coordination()).await
14711 }
14712
14713 async fn repository_runner(
14714 root: &Path,
14715 coordination: Option<crate::config::Coordination>,
14716 ) -> Runner {
14717 ask_test_home();
14718 let repo = root.join("repo");
14719 std::fs::create_dir_all(&repo).unwrap();
14720 init_repo(&repo);
14721 if coordination.is_some() {
14722 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14723 }
14724 let mut runner = runner_at(RunStatus::Prep);
14725 runner.state.repo = repo.clone();
14726 runner.state.config.coordination = coordination;
14727 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14728 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14729 runner.state.config.graph.advise = false;
14730 runner.roles.implementers = vec![spec("implement")];
14731 runner.roles.judges = vec![spec("judge")];
14732 runner.prep().await.unwrap();
14733 runner
14734 }
14735
14736 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14737
14738 #[tokio::test]
14739 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14740 let tmp = tempfile::tempdir().unwrap();
14741 let mut runner = repository_runner(tmp.path(), None).await;
14742 let prompts = runner.state.config.prompts.clone();
14743 let run = runner.state.id.clone();
14744 let ctx = WaveCtx {
14745 carry_seats: false,
14746 pauses: None,
14747 run: &run,
14748 node: "advise",
14749 prompts: &prompts,
14750 cache: None,
14751 round: None,
14752 };
14753 let jobs: Vec<_> = ["alpha", "beta"]
14754 .into_iter()
14755 .map(|id| SeatJob {
14756 spec: scripted(id, AUTH_OUTAGE),
14757 seat: SeatState::new(id, id, 7),
14758 prompt: "propose".into(),
14759 cwd: runner.state.candidates[0].worktree.clone(),
14760 timeout: Duration::from_secs(10),
14761 allow_write: false,
14762 sessions: false,
14763 artifacts: agent::artifacts_dir(&runner.state.dir()),
14764 stem: id.into(),
14765 handover: None,
14766 })
14767 .collect();
14768 let mut losses = Vec::new();
14769 let results = ask_wave_with(
14770 jobs.clone(),
14771 runner.sem.clone(),
14772 2,
14773 &[],
14774 &ctx,
14775 &mut losses,
14776 &mut runner.state,
14777 &|_: &str| Ok(()),
14778 )
14779 .await;
14780 assert!(
14781 results
14782 .iter()
14783 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14784 );
14785 assert_eq!(runner.state.quota.len(), 2);
14786 assert!(
14787 runner
14788 .state
14789 .quota
14790 .iter()
14791 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14792 );
14793 assert!(!runner.state.advise_attempted);
14794 runner.stop_for_resource().unwrap();
14795 assert_eq!(runner.state.status, RunStatus::Blocked);
14796 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14797 assert_eq!(
14798 runner.state.resource_block.as_ref().unwrap().kind,
14799 "agent authentication"
14800 );
14801 runner.state.resource_block = None;
14804 runner.state.quota.clear();
14805 let mut mixed = jobs;
14806 mixed[1].spec.command = vec!["false".into()];
14807 let results = ask_wave_with(
14808 mixed,
14809 runner.sem.clone(),
14810 0,
14811 &[],
14812 &ctx,
14813 &mut losses,
14814 &mut runner.state,
14815 &|_: &str| Ok(()),
14816 )
14817 .await;
14818 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14819 assert!(runner.state.resource_block.is_none());
14820 assert_eq!(losses.len(), 1);
14821 }
14822
14823 #[tokio::test]
14824 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14825 let tmp = tempfile::tempdir().unwrap();
14826 let mut runner = repository_runner(tmp.path(), None).await;
14827 let implementer = scripted("implement", AUTH_OUTAGE);
14828 runner.roles.implementers = vec![implementer.clone()];
14829 runner.state.config.agents = vec![implementer];
14830 runner.implement().await.unwrap();
14831 assert_eq!(runner.state.status, RunStatus::Blocked);
14832 assert_eq!(runner.state.quota.len(), 1);
14833 assert_eq!(
14834 runner.state.quota[0].cause,
14835 agent::QuotaCause::Authentication
14836 );
14837 let candidate = &runner.state.candidates[0];
14838 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14839 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14840 }
14841
14842 #[tokio::test]
14843 #[cfg(unix)]
14844 async fn parked_implementer_persists_completed_work() {
14845 for registered in [false, true] {
14846 let tmp = tempfile::tempdir().unwrap();
14847 let mut runner = registered_runner(tmp.path()).await;
14848 if !registered {
14849 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14850 }
14851 let marker = tmp.path().join("started");
14852 let release = tmp.path().join("release");
14853 let mut implementer = spec("implement");
14854 implementer.command = vec![
14855 "sh".into(),
14856 "-c".into(),
14857 format!(
14858 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14859 marker.display(),
14860 release.display()
14861 ),
14862 ];
14863 runner.roles.implementers = vec![implementer.clone()];
14864 runner.state.config.agents = vec![implementer];
14865 let pause = runner.pause.clone();
14866 let (result, ()) = tokio::join!(runner.implement(), async {
14867 tokio::time::timeout(Duration::from_secs(10), async {
14868 while !marker.exists() {
14869 tokio::time::sleep(Duration::from_millis(10)).await;
14870 }
14871 })
14872 .await
14873 .unwrap();
14874 pause.park();
14875 std::fs::write(&release, "go").unwrap();
14876 });
14877 result.unwrap();
14878 let candidate = &runner.state.candidates[0];
14879 assert!(candidate.commits > 0, "{candidate:?}");
14880 assert!(candidate.summary.contains("saved-result"));
14881 assert_eq!(candidate.files, 1);
14882 let before = candidate.commits;
14883 runner.implement().await.unwrap();
14884 assert_eq!(runner.state.candidates[0].commits, before);
14885 }
14886 }
14887
14888 #[tokio::test]
14889 #[cfg(unix)]
14890 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14891 let tmp = tempfile::tempdir().unwrap();
14892 let mut runner = registered_runner(tmp.path()).await;
14893 let winner = runner.state.candidates[0].clone();
14894 let cache = tmp.path().join("cache");
14895 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14896 runner.state.config.graph.timeout_resource_wait = 0;
14897 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14898 let crate::cache::AcquireOutcome::Acquired(held) =
14899 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14900 else {
14901 panic!("cache must be free")
14902 };
14903 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14904 from_run: "earlier".into(),
14905 generation: 1,
14906 mode: crate::recovery::Mode::VerifyRepair,
14907 reasons: Vec::new(),
14908 reviewed_head: None,
14909 rounds: Vec::new(),
14910 open_findings: Vec::new(),
14911 evidence: None,
14912 criteria_inherited: true,
14913 repair: crate::recovery::Repair::NotStarted,
14914 });
14915 runner.recovery_repair(&winner).await.unwrap();
14916 assert!(matches!(
14917 runner.state.recovery.as_ref().unwrap().repair,
14918 crate::recovery::Repair::WaitingForRestore { .. }
14919 ));
14920 drop(held);
14921 runner.state.resource_block = None;
14922 runner.state.config.graph.timeout_resource_wait = 1;
14923 runner.recovery_repair(&winner).await.unwrap();
14924 assert!(matches!(
14925 runner.state.recovery.as_ref().unwrap().repair,
14926 crate::recovery::Repair::Done { .. }
14927 ));
14928 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14929 }
14930
14931 #[tokio::test]
14932 #[cfg(unix)]
14933 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14934 use std::os::fd::AsRawFd as _;
14935 for graft in [false, true] {
14936 let tmp = tempfile::tempdir().unwrap();
14937 let mut runner = registered_runner(tmp.path()).await;
14938 runner.state.config.graph.timeout_resource_wait = 1;
14939 let winner = runner.state.candidates[0].clone();
14940 let marker = tmp.path().join("fixer-started");
14941 let mut fixer = spec("fixer");
14942 fixer.command = vec![
14943 "sh".into(),
14944 "-c".into(),
14945 format!("echo started >> '{}'; echo done", marker.display()),
14946 ];
14947 runner.roles.fixer = Some(vec![fixer]);
14948 if graft {
14949 let mut loser = winner.clone();
14950 loser.label = 'B';
14951 loser.index = 1;
14952 runner.state.candidates.push(loser);
14953 runner.state.config.graph.graft_runner_up = true;
14954 runner.state.tally = Some(Tally {
14955 first_choice: BTreeMap::new(),
14956 borda: BTreeMap::new(),
14957 winner: winner.label,
14958 rankings: 1,
14959 unanimous_initial: true,
14960 deliberated: false,
14961 changed_votes: 0,
14962 unanimous_final: true,
14963 tie_break: None,
14964 judges: 1,
14965 present: 1,
14966 quorum: 1,
14967 met_quorum: true,
14968 uncontested: None,
14969 });
14970 runner.state.judgements.push(crate::run::Judgement {
14971 judge: 1,
14972 seat: "judge-1".into(),
14973 agent: "judge".into(),
14974 ranking: vec![winner.label, 'B'],
14975 reasons: BTreeMap::new(),
14976 confidence: None,
14977 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14978 order: vec![0, 1],
14979 failed: None,
14980 duration_ms: 0,
14981 });
14982 } else {
14983 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14984 from_run: "earlier".into(),
14985 generation: 1,
14986 mode: crate::recovery::Mode::VerifyRepair,
14987 reasons: Vec::new(),
14988 reviewed_head: None,
14989 rounds: Vec::new(),
14990 open_findings: Vec::new(),
14991 evidence: None,
14992 criteria_inherited: true,
14993 repair: crate::recovery::Repair::NotStarted,
14994 });
14995 }
14996 let reader = crate::repo_lock::SeatLease::for_checkout(
14997 &winner.worktree,
14998 false,
14999 crate::config::test_coordination().as_ref(),
15000 )
15001 .await
15002 .unwrap();
15003 let pause = runner.pause.clone();
15004 let (result, ()) = tokio::join!(
15005 async {
15006 if graft {
15007 runner.graft().await
15008 } else {
15009 runner.recovery_repair(&winner).await
15010 }
15011 },
15012 async {
15013 tokio::time::sleep(Duration::from_millis(100)).await;
15014 pause.park();
15015 }
15016 );
15017 result.unwrap();
15018 assert!(!marker.exists(), "a blocked seat must not spawn");
15019 runner.state = RunState::load(&runner.state.id).unwrap();
15021 if graft {
15022 assert_eq!(
15023 runner.state.graft.as_ref().unwrap().outcome,
15024 GraftOutcome::WaitingForRestore
15025 );
15026 } else {
15027 assert!(matches!(
15028 runner.state.recovery.as_ref().unwrap().repair,
15029 crate::recovery::Repair::WaitingForRestore { .. }
15030 ));
15031 }
15032 drop(reader);
15033 runner.pause = Pause::new();
15034 runner.state.resource_block = None;
15035 let external =
15037 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15038 assert_eq!(
15039 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15040 0
15041 );
15042 let leftover = winner.worktree.join("leftover");
15043 std::fs::write(&leftover, "earlier fallback edits").unwrap();
15044 let result = if graft {
15045 runner.graft().await
15046 } else {
15047 runner.recovery_repair(&winner).await
15048 };
15049 assert!(
15050 result
15051 .unwrap_err()
15052 .is::<crate::repo_lock::ResourceBlocked>()
15053 );
15054 assert!(leftover.exists());
15055 assert!(!marker.exists());
15056 drop(external);
15057 runner.state.resource_block = None;
15058 let result = if graft {
15059 runner.graft().await
15060 } else {
15061 runner.recovery_repair(&winner).await
15062 };
15063 result.unwrap();
15064 assert!(!leftover.exists(), "restore must precede retry");
15065 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15066 if graft {
15067 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15068 } else {
15069 assert!(matches!(
15070 runner.state.recovery.as_ref().unwrap().repair,
15071 crate::recovery::Repair::Done { .. }
15072 ));
15073 }
15074 }
15075 }
15076
15077 #[tokio::test]
15078 #[cfg(unix)]
15079 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15080 use std::os::fd::AsRawFd as _;
15081 let tmp = tempfile::tempdir().unwrap();
15082 let mut runner = registered_runner(tmp.path()).await;
15083 runner.state.config.graph.timeout_resource_wait = 1;
15084 let winner = runner.state.candidates[0].clone();
15085 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15086 runner.state.tally = Some(Tally {
15087 first_choice: BTreeMap::new(),
15088 borda: BTreeMap::new(),
15089 winner: winner.label,
15090 rankings: 0,
15091 unanimous_initial: false,
15092 deliberated: false,
15093 changed_votes: 0,
15094 unanimous_final: false,
15095 tie_break: None,
15096 judges: 0,
15097 present: 0,
15098 quorum: 0,
15099 met_quorum: true,
15100 uncontested: Some("fixture".into()),
15101 });
15102 runner.state.graft = Some(GraftRecord {
15103 pre: pre.clone(),
15104 post: None,
15105 lines: None,
15106 limit: None,
15107 agent: None,
15108 outcome: GraftOutcome::Running,
15109 requested: Vec::new(),
15110 adoption: None,
15111 adoption_error: None,
15112 });
15113 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15114 git::commit_all(&winner.worktree, "unfinished graft")
15115 .await
15116 .unwrap();
15117 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15118 let leftover = winner.worktree.join("leftover");
15119 std::fs::write(&leftover, "untrusted edits").unwrap();
15120 let path = runner.state.repo.join(".git/updater.lock");
15121 let external = std::fs::File::create(&path).unwrap();
15122 assert_eq!(
15123 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15124 0
15125 );
15126 assert!(
15127 runner
15128 .graft()
15129 .await
15130 .unwrap_err()
15131 .is::<crate::repo_lock::ResourceBlocked>()
15132 );
15133 assert!(matches!(
15134 runner.state.graft.as_ref().unwrap().outcome,
15135 GraftOutcome::Running
15136 ));
15137 assert_eq!(
15138 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15139 unfinished
15140 );
15141 assert!(leftover.exists());
15142 drop(external);
15143 std::fs::remove_file(&path).unwrap();
15144 std::fs::create_dir(&path).unwrap();
15145 assert!(runner.graft().await.is_err());
15146 assert!(matches!(
15147 runner.state.graft.as_ref().unwrap().outcome,
15148 GraftOutcome::Running
15149 ));
15150 assert_eq!(
15151 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15152 unfinished
15153 );
15154 assert!(leftover.exists());
15155 std::fs::remove_dir(&path).unwrap();
15156 runner.graft().await.unwrap();
15157 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15158 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15159 assert!(!leftover.exists());
15160 }
15161
15162 #[tokio::test]
15163 #[cfg(unix)]
15164 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15165 use std::os::fd::AsRawFd as _;
15166 let tmp = tempfile::tempdir().unwrap();
15167 let mut runner = registered_runner(tmp.path()).await;
15168 runner.state.prep_complete = Some(false);
15169 runner.state.config.graph.timeout_resource_wait = 1;
15170 runner.state.save().unwrap();
15171 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15172 assert_eq!(
15173 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15174 0
15175 );
15176 runner.execute().await.unwrap();
15177 let disk = RunState::load(&runner.state.id).unwrap();
15178 assert!(disk.driver_exited);
15179 assert_eq!(disk.driver_pid, None);
15180 assert_eq!(runner.state.status, RunStatus::Blocked);
15181 assert_eq!(
15182 runner.state.resource_block.as_ref().unwrap().kind,
15183 "repository coordination"
15184 );
15185 assert!(runner.state.prep_failure.is_none());
15186 assert!(runner.state.seats.is_empty());
15187 runner.pause.park();
15188 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15191 .await
15192 .unwrap()
15193 .unwrap();
15194 assert!(runner.state.parked);
15195 assert!(runner.state.prep_failure.is_none());
15196 let disk = RunState::load(&runner.state.id).unwrap();
15197 assert!(disk.driver_exited);
15198 assert_eq!(disk.driver_pid, None);
15199 }
15200
15201 #[tokio::test]
15202 #[cfg(unix)]
15203 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15204 use std::os::fd::AsRawFd as _;
15205 let tmp = tempfile::tempdir().unwrap();
15206 let mut runner = registered_runner(tmp.path()).await;
15207 runner.state.prep_complete = Some(false);
15208 runner.state.config.graph.timeout_resource_wait = 30;
15209 runner.state.save().unwrap();
15210 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15211 assert_eq!(
15212 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15213 0
15214 );
15215 assert!(
15216 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15217 .await
15218 .is_err()
15219 );
15220 let disk = RunState::load(&runner.state.id).unwrap();
15221 assert!(disk.driver_exited);
15222 assert_eq!(disk.driver_pid, None);
15223 assert!(disk.status.resumable());
15224 assert!(runner.state.driver_exited);
15225 assert_eq!(runner.state.driver_pid, None);
15226 runner.state.save().unwrap();
15227 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15228 }
15229
15230 #[tokio::test]
15231 #[cfg(unix)]
15232 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15233 let tmp = tempfile::tempdir().unwrap();
15234 let mut runner = registered_runner(tmp.path()).await;
15235 let winner = runner.state.candidates[0].clone();
15236 let cache = tmp.path().join("cache");
15237 let started = tmp.path().join("pre-gate-started");
15238 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15239 runner.state.config.verify.gate =
15240 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15241 let home = crate::run::home();
15242 let owner =
15243 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15244 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15245 crate::cache::AcquireOutcome::Acquired(g) => g,
15246 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15247 };
15248 let mut gate = Box::pin(runner.run_gate(&winner));
15249 assert!(
15250 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15251 .await
15252 .is_err()
15253 );
15254 assert!(!started.exists(), "pre-gate must wait for the cache too");
15255 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15259 let editor = crate::repo_lock::SeatLease::checkout_with(
15260 &winner.worktree,
15261 true,
15262 crate::config::test_coordination().as_ref(),
15263 &wait,
15264 )
15265 .await
15266 .unwrap();
15267 drop(editor);
15268 guard.release();
15269 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15270 .await
15271 .unwrap()
15272 .unwrap();
15273 drop(gate);
15274 assert!(outcomes.iter().all(CommandOutcome::ok));
15275 assert!(started.exists());
15276 }
15277
15278 #[tokio::test]
15279 #[cfg(unix)]
15280 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15281 use std::os::fd::AsRawFd as _;
15282
15283 ask_test_home();
15284 let tmp = tempfile::tempdir().unwrap();
15285 let repo = tmp.path().join("repo");
15286 std::fs::create_dir_all(&repo).unwrap();
15287 init_repo(&repo);
15288 let common = repo.join(".git");
15289 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15290 let mut runner = runner_at(RunStatus::Prep);
15291 runner.state.repo = repo.clone();
15292 runner.state.config.coordination = crate::config::test_coordination();
15293 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15294 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15295 runner.state.config.graph.advise = false;
15296 runner.roles.implementers = vec![spec("implement")];
15297 runner.roles.judges = vec![spec("judge")];
15298 runner.prep().await.unwrap();
15299 let winner = runner.state.candidates[0].clone();
15300 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15303 let completed = tmp.path().join("seat-completed");
15304 runner.roles.implementers[0].command = vec![
15305 "sh".into(),
15306 "-c".into(),
15307 format!("touch '{}'; echo completed", completed.display()),
15308 ];
15309 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15310 assert_eq!(
15311 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15312 0
15313 );
15314 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15315 let mut implement = Box::pin(runner.implement());
15316 tokio::select! {
15317 result = &mut implement => panic!("rescue must wait: {result:?}"),
15318 result = tokio::time::timeout(Duration::from_secs(10), async {
15319 while !completed.exists() {
15320 tokio::time::sleep(Duration::from_millis(20)).await;
15321 }
15322 }) => result.unwrap(),
15323 }
15324 assert!(
15325 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15326 .await
15327 .is_err()
15328 );
15329 assert_eq!(
15330 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15331 before
15332 );
15333 assert_eq!(
15334 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15335 0
15336 );
15337 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15338 .await
15339 .unwrap()
15340 .unwrap();
15341 drop(implement);
15342 assert_ne!(
15343 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15344 before
15345 );
15346 assert!(git::is_clean(&winner.worktree).await.unwrap());
15347
15348 let started = tmp.path().join("pre-gate-started");
15349 let proceed = tmp.path().join("pre-gate-proceed");
15350 let checked = tmp.path().join("gate-ran");
15351 runner.state.config.verify.pre_gate = vec![format!(
15352 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15353 started.display(),
15354 proceed.display()
15355 )];
15356 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15357 assert_eq!(
15358 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15359 0
15360 );
15361 let mut gate = Box::pin(runner.run_gate(&winner));
15362 assert!(
15363 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15364 .await
15365 .is_err()
15366 );
15367 assert!(!started.exists());
15368 assert!(!checked.exists());
15369 assert!(git::is_clean(&winner.worktree).await.unwrap());
15370 assert_eq!(
15371 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15372 0
15373 );
15374 tokio::select! {
15375 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15376 result = tokio::time::timeout(Duration::from_secs(10), async {
15377 while !started.exists() {
15378 tokio::time::sleep(Duration::from_millis(20)).await;
15379 }
15380 }) => result.unwrap(),
15381 }
15382 assert_eq!(
15383 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15384 -1,
15385 "mechanical edits must hold the real lock"
15386 );
15387 std::fs::write(proceed, "ready").unwrap();
15388 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15389 .await
15390 .unwrap()
15391 .unwrap();
15392 drop(gate);
15393 assert!(outcomes.iter().all(CommandOutcome::ok));
15394 assert!(checked.exists());
15395 assert!(runner.state.pre_gate_commit.is_some());
15396 assert!(git::is_clean(&winner.worktree).await.unwrap());
15397 assert_eq!(
15398 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15399 0,
15400 "gate completion must release coordination"
15401 );
15402 drop(outsider);
15403 std::fs::remove_file(common.join("updater.lock")).unwrap();
15404 std::fs::create_dir(common.join("updater.lock")).unwrap();
15405 std::fs::remove_file(&started).unwrap();
15406 std::fs::remove_file(&checked).unwrap();
15407 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15408 let error = runner.run_gate(&winner).await.unwrap_err();
15409 assert!(
15410 error
15411 .to_string()
15412 .contains("acquire mandatory repository lock")
15413 );
15414 assert!(
15415 !started.exists(),
15416 "failed acquisition must prevent pre-gate"
15417 );
15418 assert!(!checked.exists(), "failed acquisition must prevent gate");
15419 assert!(git::is_clean(&winner.worktree).await.unwrap());
15420 assert_eq!(
15421 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15422 head
15423 );
15424 }
15425
15426 #[tokio::test]
15427 #[cfg(unix)]
15428 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15429 use std::os::fd::AsRawFd as _;
15430 use std::time::Duration;
15431
15432 ask_test_home();
15433 let tmp = tempfile::tempdir().unwrap();
15434 let repo = tmp.path().join("repo");
15435 std::fs::create_dir_all(&repo).unwrap();
15436 init_repo(&repo);
15437 let common = repo.join(".git");
15438 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15439 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15440 assert_eq!(
15441 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15442 0
15443 );
15444 let mut runner = runner_at(RunStatus::Prep);
15445 runner.state.repo = repo.clone();
15446 runner.state.config.coordination = crate::config::test_coordination();
15447 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15448 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15449 runner.state.config.graph.advise = true;
15450 runner.state.config.graph.advisors = 1;
15451 runner.roles.implementers = vec![spec("implement")];
15452 runner.roles.judges = vec![spec("judge")];
15453 let root = runner.state.worktree_root();
15454 let started = tmp.path().join("setup-started");
15455 let proceed = tmp.path().join("setup-proceed");
15456 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15457 run: Some(format!(
15458 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15459 started.display(),
15460 proceed.display()
15461 )),
15462 ..Default::default()
15463 }];
15464 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15465 let config = git::git(&repo, &["config", "--local", "--list"])
15466 .await
15467 .unwrap();
15468 let mut prep = Box::pin(runner.prep());
15469 assert!(
15470 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15471 .await
15472 .is_err()
15473 );
15474 assert!(
15475 !root.exists(),
15476 "contention must prevent even checkout creation"
15477 );
15478 assert!(!started.exists(), "contention must prevent setup");
15479 assert_eq!(
15480 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15481 branches
15482 );
15483 assert_eq!(
15484 git::git(&repo, &["config", "--local", "--list"])
15485 .await
15486 .unwrap(),
15487 config
15488 );
15489 drop(lock);
15490 tokio::select! {
15491 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15492 result = tokio::time::timeout(Duration::from_secs(10), async {
15493 while !started.exists() {
15494 tokio::time::sleep(Duration::from_millis(20)).await;
15495 }
15496 }) => result.unwrap(),
15497 }
15498 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15499 assert_eq!(
15500 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15501 -1
15502 );
15503 assert!(
15504 tokio::time::timeout(
15505 Duration::from_millis(100),
15506 crate::repo_lock::SeatLease::for_checkout(
15507 &repo,
15508 true,
15509 crate::config::test_coordination().as_ref()
15510 )
15511 )
15512 .await
15513 .is_err(),
15514 "preparation must also exclude local editors"
15515 );
15516 std::fs::write(proceed, "ready").unwrap();
15517 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15518 .await
15519 .unwrap()
15520 .unwrap();
15521 drop(prep);
15522 assert_eq!(runner.state.prep_complete, Some(true));
15523 assert!(root.join("judge-1").exists());
15524 assert!(root.join("advisor-1").exists());
15525 assert_eq!(
15526 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15527 0
15528 );
15529 }
15530
15531 #[tokio::test]
15532 #[cfg(unix)]
15533 async fn preparation_lock_failure_prevents_repository_changes() {
15534 ask_test_home();
15535 let tmp = tempfile::tempdir().unwrap();
15536 let repo = tmp.path().join("repo");
15537 std::fs::create_dir_all(&repo).unwrap();
15538 init_repo(&repo);
15539 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15540 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15543 let mut runner = runner_at(RunStatus::Prep);
15544 runner.state.repo = repo.clone();
15545 runner.state.config.coordination = crate::config::test_coordination();
15546 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15547 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15548 runner.roles.implementers = vec![spec("implement")];
15549 let error = runner.prep().await.unwrap_err();
15550 assert!(
15551 error
15552 .to_string()
15553 .contains("acquire mandatory repository lock")
15554 );
15555 assert!(!runner.state.worktree_root().exists());
15556 assert!(runner.state.candidates.is_empty());
15557 assert!(!repo.join(".git/config.worktree").exists());
15558 assert_eq!(
15559 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15560 "* main"
15561 );
15562 }
15563
15564 #[tokio::test]
15565 async fn preparation_records_creation_intent_before_git_can_fail() {
15566 ask_test_home();
15567 let tmp = tempfile::tempdir().unwrap();
15568 let repo = tmp.path().join("repo");
15569 std::fs::create_dir_all(&repo).unwrap();
15570 init_repo(&repo);
15571 let mut runner = runner_at(RunStatus::Prep);
15572 runner.state.repo = repo.clone();
15573 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15574 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15575 runner.state.config.blind.commit_msg_hook = false;
15576 runner.roles.implementers = vec![spec("one")];
15577 let label = blind::assign_labels(1, runner.state.seed)[0];
15578 let branch = runner.state.branch_for(label);
15579 git::git(&repo, &["branch", &branch]).await.unwrap();
15582 assert!(runner.prep().await.is_err());
15583 let saved = RunState::load(&runner.state.id).unwrap();
15584 assert!(saved.candidates.is_empty());
15585 let event = saved
15586 .events
15587 .iter()
15588 .find(|e| e.node == "prep-intent")
15589 .unwrap();
15590 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15591 assert_eq!(intent["branch"], branch);
15592 assert_eq!(intent["start"], runner.state.base_commit);
15593 assert_eq!(
15594 intent["worktree"],
15595 runner
15596 .state
15597 .worktree_root()
15598 .join(format!("cand-{label}"))
15599 .to_string_lossy()
15600 .as_ref()
15601 );
15602 assert_eq!(intent["picks"], serde_json::json!([]));
15603 }
15604
15605 #[tokio::test]
15606 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15607 ask_test_home();
15608 let tmp = tempfile::tempdir().unwrap();
15609 let repo = tmp.path().join("repo");
15610 std::fs::create_dir_all(&repo).unwrap();
15611 init_repo(&repo);
15612 let mut runner = runner_at(RunStatus::Prep);
15613 runner.state.repo = repo.clone();
15614 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15615 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15616 runner.state.sources.pick_commits.push(refs::Input {
15617 token: "missing".into(),
15618 sha: "0".repeat(40),
15619 provenance: refs::Provenance::ExplicitCli,
15620 });
15621 assert!(runner.prep().await.is_err());
15622 assert!(runner.state.candidates.is_empty());
15623 assert!(!runner.state.worktree_root().exists());
15624 }
15625
15626 #[tokio::test]
15627 async fn prose_and_legacy_inference_never_import_into_candidates() {
15628 ask_test_home();
15629 let tmp = tempfile::tempdir().unwrap();
15630 let repo = tmp.path().join("repo");
15631 std::fs::create_dir_all(&repo).unwrap();
15632 init_repo(&repo);
15633 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15634 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15635 .await
15636 .unwrap();
15637 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15638 git::git(&repo, &["add", "."]).await.unwrap();
15639 git::git(&repo, &["commit", "-m", "historical implementation"])
15640 .await
15641 .unwrap();
15642 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15643 git::git(&repo, &["checkout", "main"]).await.unwrap();
15644 for instruction in [
15645 format!("Example: magi/history/A at {source}"),
15646 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15647 format!("Never import magi/history/A or {source}"),
15648 format!("Conversation answer: I mentioned {source} only as background"),
15649 ] {
15650 let mut runner = runner_at(RunStatus::Prep);
15651 runner.state.repo = repo.clone();
15652 runner.state.base_commit = base.clone();
15653 runner.state.instruction = instruction.clone();
15654 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15655 runner.state.config.blind.commit_msg_hook = false;
15656 runner.state.seeds = vec![refs::Seed {
15657 token: "magi/history/A".into(),
15658 sha: source.clone(),
15659 kind: refs::SeedKind::Unmerged,
15660 branch: true,
15661 detail: String::new(),
15662 }];
15663 runner.roles.implementers = vec![spec("one")];
15664 assert_eq!(seeded_instruction(&runner.state), instruction);
15665 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15667 legacy.as_object_mut().unwrap().remove("sources");
15668 runner.state = serde_json::from_value(legacy).unwrap();
15669 assert_eq!(runner.state.sources, refs::Sources::default());
15670 runner.state.save().unwrap();
15671 runner.state = RunState::load(&runner.state.id).unwrap();
15672 runner.prep().await.unwrap();
15673 for candidate in &runner.state.candidates {
15674 assert_eq!(
15675 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15676 base
15677 );
15678 assert!(!candidate.worktree.join("unwanted").exists());
15679 }
15680 }
15681 }
15682
15683 #[tokio::test]
15684 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15685 ask_test_home();
15686 let tmp = tempfile::tempdir().unwrap();
15687 let repo = tmp.path().join("repo");
15688 std::fs::create_dir_all(&repo).unwrap();
15689 init_repo(&repo);
15690 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15691 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15692 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15693 git::git(&repo, &["add", "."]).await.unwrap();
15694 git::git(&repo, &["commit", "-m", "required seed"])
15695 .await
15696 .unwrap();
15697 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15698 git::git(&repo, &["checkout", "main"]).await.unwrap();
15699 let mut runner = runner_at(RunStatus::Prep);
15700 runner.state.repo = repo.clone();
15701 runner.state.base_commit = base.clone();
15702 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15703 runner.state.config.graph.advise = false;
15704 runner.state.config.blind.commit_msg_hook = false;
15705 runner.state.sources =
15706 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15707 .await
15708 .unwrap();
15709 runner.roles.implementers = vec![spec("one"), spec("two")];
15710 runner.roles.judges = vec![spec("judge")];
15711 let label = blind::assign_labels(2, runner.state.seed)[0];
15712 let branch = runner.state.branch_for(label);
15713 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15714 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15715 .await
15716 .unwrap();
15717 runner.state.candidates.push(Candidate {
15718 index: 0,
15719 label,
15720 agent: "one".into(),
15721 branch,
15722 worktree,
15723 summary: String::new(),
15724 stat: String::new(),
15725 files: 0,
15726 commits: 0,
15727 empty: false,
15728 failed: None,
15729 verified_noop: None,
15730 duration_ms: 0,
15731 folded: false,
15732 });
15733 runner.state.save().unwrap();
15734 runner.state = RunState::load(&runner.state.id).unwrap();
15736 assert_eq!(runner.state.prep_complete, Some(false));
15737 runner.prep().await.unwrap();
15738 assert_eq!(runner.state.prep_complete, Some(true));
15739 assert_eq!(runner.state.candidates.len(), 2);
15740 for candidate in &runner.state.candidates {
15741 assert_eq!(
15742 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15743 git::tree_of(&repo, &seed).await.unwrap()
15744 );
15745 assert_eq!(
15746 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15747 "required commit"
15748 );
15749 }
15750 assert!(runner.state.worktree_root().join("judge-1").exists());
15751 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15752 runner.prep().await.unwrap();
15753 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15754
15755 runner.state.prep_complete = Some(false);
15756 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15757 run: Some(format!(
15758 "test -f '{}'",
15759 tmp.path().join("dependency-ready").display()
15760 )),
15761 ..Default::default()
15762 }];
15763 assert!(runner.prep().await.is_err());
15764 assert!(runner.state.prep_failure.is_some());
15765 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15766 runner.state = RunState::load(&runner.state.id).unwrap();
15767 assert!(
15768 runner.prep().await.is_err(),
15769 "correcting the cause alone must not auto-retry"
15770 );
15771 let failed_at = runner.state.events.last().unwrap().at;
15772 assert!(
15773 runner.retry_preparation(failed_at).is_err(),
15774 "old approvals cannot unlock the hold"
15775 );
15776 let mut task = crate::queue::Task::new(
15777 "retry".into(),
15778 "retry setup".into(),
15779 repo.clone(),
15780 crate::queue::Source::Human,
15781 );
15782 task.start(runner.state.id.clone());
15783 task.hold_machine(Some("setup failed".into()));
15784 task.release_operator("cli", None);
15785 runner
15786 .retry_preparation(task.resumption.as_ref().unwrap().at)
15787 .unwrap();
15788 runner.prep().await.unwrap();
15789 assert!(runner.state.prep_failure.is_none());
15790 assert_eq!(runner.state.prep_complete, Some(true));
15791 assert_eq!(runner.state.candidates.len(), 2);
15792 assert!(runner.state.worktree_root().join("judge-1").exists());
15793 for candidate in &runner.state.candidates {
15794 assert_eq!(
15795 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15796 git::tree_of(&repo, &seed).await.unwrap()
15797 );
15798 }
15799 }
15800
15801 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15802 let mut heads = Vec::new();
15803 for candidate in candidates {
15804 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15805 }
15806 heads
15807 }
15808
15809 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15810 ask_test_home();
15811 let tmp = tempfile::tempdir().unwrap();
15812 let repo = tmp.path().join("repo");
15813 std::fs::create_dir(&repo).unwrap();
15814 init_repo(&repo);
15815 let mut runner = runner_at(RunStatus::Reviewing);
15816 runner.state.repo = repo.clone();
15817 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15818 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15819 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15820 let branch = format!("candidate-{label}");
15821 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15822 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15823 .await
15824 .unwrap();
15825 if label == 'B' {
15826 std::fs::write(worktree.join("change"), "winner").unwrap();
15827 git::rescue_commit(&worktree, "candidate change")
15828 .await
15829 .unwrap();
15830 }
15831 runner.state.candidates.push(Candidate {
15832 index,
15833 label,
15834 agent: "alpha".into(),
15835 branch,
15836 worktree,
15837 summary: "completed reply".into(),
15838 stat: String::new(),
15839 files: usize::from(label == 'B'),
15840 commits: usize::from(label == 'B'),
15841 empty: label == 'A',
15842 failed: None,
15843 verified_noop: None,
15844 duration_ms: 1,
15845 folded: false,
15846 });
15847 }
15848 runner.state.judge_skipped = true;
15849 runner.state.tally = Some(Tally {
15850 first_choice: BTreeMap::new(),
15851 borda: BTreeMap::new(),
15852 winner: 'B',
15853 rankings: 0,
15854 unanimous_initial: false,
15855 deliberated: false,
15856 changed_votes: 0,
15857 unanimous_final: false,
15858 tie_break: None,
15859 judges: 0,
15860 present: 0,
15861 quorum: 0,
15862 met_quorum: true,
15863 uncontested: Some("only B produced a change".into()),
15864 });
15865 runner.state.save().unwrap();
15866 (tmp, runner)
15867 }
15868
15869 #[tokio::test]
15870 #[cfg(unix)]
15871 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15872 for (solo, failed_checkpoint) in
15873 [(true, false), (false, false), (true, true), (false, true)]
15874 {
15875 let (_tmp, mut runner) = preservation_resume_fixture().await;
15876 if solo {
15877 runner.state.candidates.retain(|c| c.label == 'A');
15878 }
15879 runner.state.candidates[0].empty = false;
15880 runner.state.candidates[0].summary.clear();
15881 runner.state.tally = None;
15882 runner.state.judge_skipped = false;
15883 runner.state.status = RunStatus::Implementing;
15884 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15885 let mut implementer = spec("alpha");
15886 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15887 runner.roles.implementers = vec![implementer];
15888 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15889 if failed_checkpoint {
15890 std::fs::create_dir_all(&patch).unwrap();
15891 }
15892 runner.state.save().unwrap();
15893 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15894 drop(pressure);
15895 runner.state = RunState::load(&runner.state.id).unwrap();
15896 assert_eq!(runner.state.status, RunStatus::Blocked);
15897 assert!(runner.state.interrupted_implementations.contains(&'A'));
15898 assert_eq!(runner.state.candidates[0].commits, 0);
15899 assert!(!runner.state.candidates[0].empty);
15900 assert!(runner.state.candidates[0].failed.is_none());
15901 if failed_checkpoint {
15902 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15903 std::fs::remove_dir(patch).unwrap();
15904 }
15905 runner.state.resource_block = None;
15906 runner.state.config.disk.min_free_bytes = 0;
15907 runner.roles.implementers[0].command = vec![
15908 "sh".into(),
15909 "-c".into(),
15910 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15911 ];
15912 runner.implement().await.unwrap();
15913 assert!(runner.state.interrupted_implementations.is_empty());
15914 assert!(runner.state.candidate_preservation.is_empty());
15915 assert_eq!(runner.state.candidates[0].commits, 1);
15916 assert!(!runner.state.candidates[0].empty);
15917 assert!(
15918 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15919 .unwrap()
15920 .contains("retried empty interrupted seat")
15921 );
15922 if !solo {
15923 assert_eq!(runner.state.candidates[1].commits, 1);
15924 }
15925 }
15926 }
15927
15928 #[tokio::test]
15929 #[cfg(unix)]
15930 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15931 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15932 let (_tmp, mut runner) = preservation_resume_fixture().await;
15933 runner.state.candidates.retain(|c| c.label == 'A');
15934 runner.state.candidates[0].empty = false;
15935 runner.state.candidates[0].summary.clear();
15936 runner.state.tally = None;
15937 runner.state.judge_skipped = false;
15938 runner.state.status = RunStatus::Implementing;
15939 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15942 let wt = runner.state.candidates[0].worktree.clone();
15943 std::fs::write(
15944 wt.join("interrupted-source"),
15945 "retain implementation work\n",
15946 )
15947 .unwrap();
15948 if committed {
15949 git::rescue_commit(&wt, "implementation checkpoint")
15950 .await
15951 .unwrap();
15952 }
15953 let lock = git::git(
15954 &wt,
15955 &[
15956 "rev-parse",
15957 "--path-format=absolute",
15958 "--git-path",
15959 "index.lock",
15960 ],
15961 )
15962 .await
15963 .unwrap();
15964 if block_rescue {
15965 std::fs::write(&lock, "fixture-owned lock").unwrap();
15966 }
15967 let mut implementer = spec("alpha");
15968 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15969 runner.roles.implementers = vec![implementer];
15970 runner.state.save().unwrap();
15971 let result = runner.implement().await;
15972 drop(pressure);
15973 assert_eq!(result.is_err(), block_rescue);
15974 assert_eq!(
15975 runner.state.resource_block.as_ref().unwrap().kind,
15976 "disk pressure"
15977 );
15978 assert!(
15979 runner
15980 .state
15981 .events
15982 .iter()
15983 .any(|e| e.message.contains("agent execution interrupted"))
15984 );
15985 assert!(
15986 runner
15987 .state
15988 .dir()
15989 .join("artifacts/candidate-A-recovery/staged.patch")
15990 .exists()
15991 );
15992 if block_rescue {
15993 let saved = RunState::load(&runner.state.id).unwrap();
15994 assert!(saved.candidate_preservation.contains_key(&'A'));
15995 assert!(!saved.candidates[0].empty);
15996 assert_eq!(
15997 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15998 "retain implementation work\n"
15999 );
16000 std::fs::remove_file(lock).unwrap();
16001 runner.state = saved;
16002 } else {
16003 assert_eq!(runner.state.status, RunStatus::Blocked);
16004 assert_eq!(runner.state.candidates[0].commits, 1);
16005 }
16006 runner.roles.implementers.clear();
16009 runner.state.config.disk.min_free_bytes = 0;
16010 runner.state.resource_block = None;
16011 runner.implement().await.unwrap();
16012 assert!(runner.state.candidate_preservation.is_empty());
16013 assert_eq!(runner.state.candidates[0].commits, 1);
16014 assert!(!runner.state.candidates[0].empty);
16015 assert!(git::is_clean(&wt).await.unwrap());
16016 assert!(
16017 crate::run::read_artifact(&runner.state, "cand-A.patch")
16018 .unwrap()
16019 .contains("retain implementation work")
16020 );
16021 }
16022 }
16023
16024 #[tokio::test]
16025 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16026 let (_tmp, mut runner) = preservation_resume_fixture().await;
16027 runner.implement().await.unwrap();
16029 assert!(runner.state.candidate_preservation.is_empty());
16030 assert!(
16031 !runner
16032 .state
16033 .dir()
16034 .join("artifacts/candidate-A-recovery")
16035 .exists()
16036 );
16037 runner.fold_losers().await.unwrap();
16038 assert!(!runner.state.candidates[0].worktree.exists());
16039 for schema in [14, crate::run::SCHEMA] {
16040 runner.state.schema = schema;
16041 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16042 runner.state.save().unwrap();
16043 runner.state = RunState::load(&runner.state.id).unwrap();
16044 runner.implement().await.unwrap();
16045 assert!(runner.state.candidate_preservation.is_empty());
16046 assert!(runner.state.candidates[0].folded);
16047 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16048 }
16049 }
16050
16051 #[tokio::test]
16052 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16053 let (_tmp, mut runner) = preservation_resume_fixture().await;
16054 runner.state.candidates.retain(|c| c.label == 'A');
16055 runner.state.tally = None;
16056 runner.state.judge_skipped = false;
16057 let candidate = &mut runner.state.candidates[0];
16058 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16059 candidate.failed = Some("deployment validation unavailable".into());
16060 std::fs::write(
16061 candidate.worktree.join("repair"),
16062 "future launcher policy\n",
16063 )
16064 .unwrap();
16065 runner
16066 .state
16067 .candidate_preservation
16068 .insert('A', "awaiting persistence".into());
16069 runner.persist_candidates().await.unwrap();
16070 let candidate = &runner.state.candidates[0];
16071 assert!(!candidate.empty);
16072 assert_eq!(candidate.commits, 1);
16073 assert_eq!(candidate.files, 1);
16074 assert!(candidate.failed.is_none());
16075 assert!(candidate.summary.contains("EPERM"));
16076 assert!(
16077 crate::run::read_artifact(&runner.state, "cand-A.patch")
16078 .unwrap()
16079 .contains("future launcher policy")
16080 );
16081 assert!(runner.state.candidate_preservation.is_empty());
16082 }
16083
16084 #[tokio::test]
16085 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16086 let (_tmp, mut runner) = preservation_resume_fixture().await;
16087 let wt = runner.state.candidates[0].worktree.clone();
16088 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16089 runner.state.schema = 14;
16090 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16091 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16092 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16093 for _ in 0..2 {
16094 assert!(runner.implement().await.is_err());
16095 assert_eq!(runner.state.status, RunStatus::Blocked);
16096 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16097 assert!(
16098 runner.state.candidates[0].empty,
16099 "old verdict must remain unusable"
16100 );
16101 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16102 assert!(runner.state.judge_skipped);
16103 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16104 assert!(
16105 crate::run::read_artifact(&runner.state, "cand-A.patch")
16106 .unwrap()
16107 .contains("owned edits")
16108 );
16109 assert!(wt.exists());
16110 assert_eq!(
16111 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16112 .await
16113 .unwrap(),
16114 1
16115 );
16116 runner.state = RunState::load(&runner.state.id).unwrap();
16117 }
16118 }
16119
16120 #[tokio::test]
16121 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16122 let (_tmp, mut runner) = preservation_resume_fixture().await;
16123 let wt = runner.state.candidates[0].worktree.clone();
16124 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16125 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16126 let index_lock = PathBuf::from(
16127 git::git(
16128 &wt,
16129 &[
16130 "rev-parse",
16131 "--path-format=absolute",
16132 "--git-path",
16133 "index.lock",
16134 ],
16135 )
16136 .await
16137 .unwrap(),
16138 );
16139 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16140 runner.state.schema = 14;
16141 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16142 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16143 assert!(runner.implement().await.is_err());
16144 assert!(runner.after_implement().is_err());
16145 assert!(runner.fold_losers().await.is_err());
16146 assert_eq!(
16147 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16148 "failed seat edits"
16149 );
16150 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16151 runner.state = RunState::load(&runner.state.id).unwrap();
16152 std::fs::remove_file(index_lock).unwrap();
16153 assert!(runner.implement().await.is_err());
16155 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16156 assert!(runner.state.candidates[0].empty);
16157 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16158 assert!(
16159 crate::run::read_artifact(&runner.state, "cand-A.patch")
16160 .unwrap()
16161 .contains("failed seat edits")
16162 );
16163 assert!(wt.exists());
16164 }
16165
16166 #[tokio::test]
16167 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16168 let (_tmp, mut runner) = preservation_resume_fixture().await;
16169 let repo = runner.state.repo.clone();
16170 let common = git::git(
16171 &repo,
16172 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16173 )
16174 .await
16175 .unwrap();
16176 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16177 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16178 assert!(runner.fold_losers().await.is_err());
16179 assert!(!runner.state.candidates[0].worktree.exists());
16180 assert!(!runner.state.candidates[0].folded);
16181 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16182 runner.state = RunState::load(&runner.state.id).unwrap();
16183 std::fs::remove_file(branch_lock).unwrap();
16184 let judge = runner.state.worktree_root().join("judge-1");
16185 git::worktree_add_detached(&repo, &judge, "main")
16186 .await
16187 .unwrap();
16188 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16189 runner.roles.judges = vec![spec("judge")];
16190 assert!(runner.fold_losers().await.is_err());
16191 runner.state = RunState::load(&runner.state.id).unwrap();
16192 assert!(
16193 runner.state.candidates[0].folded,
16194 "candidate progress saved before judge failure"
16195 );
16196 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16197 assert!(judge.join("uncommitted").exists());
16198 std::fs::remove_file(judge.join("uncommitted")).unwrap();
16199 runner.fold_losers().await.unwrap();
16200 assert!(!judge.exists());
16201 runner.fold_losers().await.unwrap();
16202 }
16203
16204 #[tokio::test]
16205 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16206 let (_tmp, mut runner) = preservation_resume_fixture().await;
16207 runner.state.candidates.retain(|c| c.label == 'B');
16208 runner.state.tally = None;
16209 runner.state.judge_skipped = false;
16210 runner.state.prep_complete = Some(true);
16211 runner.state.config.graph.advise = false;
16212 runner.state.config.graph.selected_implementer = Some("alpha".into());
16213 let wt = runner.state.candidates[0].worktree.clone();
16214 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16215 runner
16216 .state
16217 .candidate_preservation
16218 .insert('B', "awaiting persistence".into());
16219 let index_lock = git::git(
16220 &wt,
16221 &[
16222 "rev-parse",
16223 "--path-format=absolute",
16224 "--git-path",
16225 "index.lock",
16226 ],
16227 )
16228 .await
16229 .unwrap();
16230 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16231 runner.state.save().unwrap();
16232 assert!(runner.execute_graph().await.is_err());
16233 assert_eq!(runner.state.status, RunStatus::Blocked);
16234 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16235 std::fs::remove_file(index_lock).unwrap();
16236 for _ in 0..2 {
16239 runner.state = RunState::load(&runner.state.id).unwrap();
16240 runner.pause.park();
16241 runner.execute_graph().await.unwrap();
16242 assert_eq!(runner.state.status, RunStatus::Implementing);
16243 assert!(runner.state.candidate_preservation.is_empty());
16244 assert_eq!(runner.state.candidates[0].commits, 2);
16245 assert!(runner.state.tally.is_none());
16246 }
16247 runner.implement().await.unwrap();
16248 assert_eq!(runner.state.status, RunStatus::Judging);
16249 runner.state.status = RunStatus::Blocked;
16251 runner.state.candidates[0].failed = Some("implementation failed".into());
16252 runner.state.save().unwrap();
16253 let error = runner.execute_graph().await.unwrap_err();
16254 assert!(
16255 error
16256 .to_string()
16257 .contains("selected implementation is held")
16258 );
16259 }
16260
16261 #[tokio::test]
16262 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16263 let (_tmp, mut runner) = preservation_resume_fixture().await;
16264 let repo = runner.state.repo.clone();
16265 let candidate = runner.state.candidates[0].clone();
16266 let registered = git::worktree_holding(&repo, &candidate.branch)
16269 .await
16270 .unwrap()
16271 .unwrap();
16272 assert_eq!(
16273 registered.canonicalize().unwrap(),
16274 candidate.worktree.canonicalize().unwrap()
16275 );
16276 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16277 assert_eq!(
16278 git::worktree_holding(&repo, &candidate.branch)
16279 .await
16280 .unwrap(),
16281 Some(registered)
16282 );
16283 assert!(
16284 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16285 "the stale registration initially keeps the branch checked out"
16286 );
16287 runner.fold_losers().await.unwrap();
16288 assert!(runner.state.candidates[0].folded);
16289 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16290 assert_eq!(
16291 git::worktree_holding(&repo, &candidate.branch)
16292 .await
16293 .unwrap(),
16294 None
16295 );
16296 runner.state = RunState::load(&runner.state.id).unwrap();
16297 runner.fold_losers().await.unwrap();
16298 }
16299
16300 #[tokio::test]
16301 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16302 let (_tmp, mut runner) = preservation_resume_fixture().await;
16303 let repo = runner.state.repo.clone();
16304 let candidate = runner.state.candidates[0].clone();
16305 let winner = runner.state.candidates[1].clone();
16306 let registered = git::worktree_holding(&repo, &candidate.branch)
16307 .await
16308 .unwrap()
16309 .unwrap();
16310 assert_eq!(
16311 registered.canonicalize().unwrap(),
16312 candidate.worktree.canonicalize().unwrap()
16313 );
16314 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16315 assert_eq!(
16316 git::worktree_holding(&repo, &candidate.branch)
16317 .await
16318 .unwrap(),
16319 Some(registered)
16320 );
16321 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16322
16323 for _ in 0..2 {
16324 fold_run(&mut runner.state, false, &crate::run::home())
16325 .await
16326 .unwrap();
16327 runner.state = RunState::load(&runner.state.id).unwrap();
16328 assert!(runner.state.candidates[0].folded);
16329 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16330 assert_eq!(
16331 git::worktree_holding(&repo, &candidate.branch)
16332 .await
16333 .unwrap(),
16334 None
16335 );
16336 assert!(!runner.state.candidates[1].folded);
16337 assert!(winner.worktree.exists());
16338 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16339 }
16340 }
16341
16342 #[tokio::test]
16343 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16344 ask_test_home();
16345 let tmp = tempfile::tempdir().unwrap();
16346 let repo = tmp.path().join("repo");
16347 std::fs::create_dir(&repo).unwrap();
16348 init_repo(&repo);
16349 let mut runner = runner_at(RunStatus::Implementing);
16350 runner.state.repo = repo.clone();
16351 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16352 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16353 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16354 let branch = format!("candidate-{label}");
16355 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16356 git::worktree_add_branch(&repo, &wt, &branch, "main")
16357 .await
16358 .unwrap();
16359 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16360 git::git(&wt, &["add", "README.md"]).await.unwrap();
16361 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16362 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16363 runner.state.candidates.push(Candidate {
16364 index,
16365 label,
16366 agent: "alpha".to_owned(),
16367 branch,
16368 worktree: wt,
16369 summary: "completed implementation".to_owned(),
16370 stat: String::new(),
16371 files: 0,
16372 commits: 0,
16373 empty: false,
16374 failed: None,
16375 verified_noop: None,
16376 duration_ms: 42,
16377 folded: false,
16378 });
16379 runner
16380 .state
16381 .candidate_preservation
16382 .insert(label, "awaiting persistence".to_owned());
16383 }
16384 let wt = runner.state.candidates[0].worktree.clone();
16385 let index_lock = git::git(
16386 &wt,
16387 &[
16388 "rev-parse",
16389 "--path-format=absolute",
16390 "--git-path",
16391 "index.lock",
16392 ],
16393 )
16394 .await
16395 .unwrap();
16396 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16397 runner.state.save().unwrap();
16398 assert!(runner.persist_candidates().await.is_err());
16399 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16400 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16401 assert_eq!(
16402 runner.state.candidates[1].commits, 1,
16403 "other results are still collected"
16404 );
16405 assert!(
16406 !runner.state.candidates[0].empty,
16407 "failed rescue is not no change"
16408 );
16409 assert_eq!(
16410 runner.state.candidates[0].summary,
16411 "completed implementation"
16412 );
16413 assert!(runner.after_implement().is_err());
16414 let home = tmp.path().join("fold-home");
16415 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16416 assert!(wt.join("new-file").exists());
16417 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16418 assert!(recovery.join("staged.patch").exists());
16419 assert!(recovery.join("unstaged.patch").exists());
16420 assert_eq!(
16421 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16422 "untracked edits\n"
16423 );
16424 runner.state.begin_pass();
16425 runner.state.end_pass_parked(ParkCause::Upgrade {
16426 from: "1.0".into(),
16427 to: Some("1.1".into()),
16428 });
16429 runner.state.save().unwrap();
16430 runner.state = RunState::load(&runner.state.id).unwrap();
16431 assert!(matches!(
16432 runner.state.passes.last().unwrap().end,
16433 Some(crate::run::PassEnd::Parked {
16434 cause: ParkCause::Upgrade { .. }
16435 })
16436 ));
16437 std::fs::remove_file(index_lock).unwrap();
16438 runner.implement().await.unwrap();
16440 assert!(runner.state.candidate_preservation.is_empty());
16441 assert!(
16442 runner
16443 .state
16444 .candidates
16445 .iter()
16446 .all(|c| c.commits == 1 && !c.empty)
16447 );
16448 assert!(git::is_clean(&wt).await.unwrap());
16449 assert!(
16450 crate::run::read_artifact(&runner.state, "cand-A.patch")
16451 .unwrap()
16452 .contains("later tracked edits")
16453 );
16454 }
16455
16456 #[tokio::test]
16457 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16458 ask_test_home();
16459 let tmp = tempfile::tempdir().unwrap();
16460 let repo = tmp.path().join("repo");
16461 std::fs::create_dir(&repo).unwrap();
16462 init_repo(&repo);
16463 let wt = tmp.path().join("candidate");
16464 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16465 .await
16466 .unwrap();
16467 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16468 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16469 assert!(wt.join("uncommitted").exists());
16470 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16471 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16472 let mut runner = runner_at(RunStatus::Implementing);
16473 runner.state.repo = repo.clone();
16474 runner.state.base_commit = "missing-base".to_owned();
16475 runner.state.candidates.push(Candidate {
16476 index: 0,
16477 label: 'A',
16478 agent: "alpha".into(),
16479 branch: "candidate".into(),
16480 worktree: wt.clone(),
16481 summary: "completed".into(),
16482 stat: String::new(),
16483 files: 0,
16484 commits: 0,
16485 empty: false,
16486 failed: None,
16487 verified_noop: None,
16488 duration_ms: 1,
16489 folded: false,
16490 });
16491 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16492 assert!(
16493 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16494 .await
16495 .is_err()
16496 );
16497 assert!(!runner.state.candidates[0].folded);
16498 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16499 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16500 runner
16501 .state
16502 .candidate_preservation
16503 .insert('A', "pending".into());
16504 assert!(runner.persist_candidates().await.is_err());
16505 assert!(!runner.state.candidates[0].empty);
16506 assert!(!runner.state.candidates[0].folded);
16507 assert!(wt.exists());
16508 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16509 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16510 std::fs::create_dir_all(&patch_path).unwrap();
16511 assert!(
16512 runner.persist_candidates().await.is_err(),
16513 "artifact failure stays pending"
16514 );
16515 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16516 assert!(!runner.state.candidates[0].empty);
16517 std::fs::remove_dir(patch_path).unwrap();
16518 runner.persist_candidates().await.unwrap();
16519 assert!(
16520 runner.state.candidates[0].empty,
16521 "only successful inspection confirms no change"
16522 );
16523 }
16524
16525 fn init_repo(dir: &Path) {
16526 let run = |args: &[&str]| {
16527 let out = std::process::Command::new("git")
16528 .args(args)
16529 .current_dir(dir)
16530 .quiet()
16531 .output()
16532 .expect("spawn git");
16533 assert!(
16534 out.status.success(),
16535 "git {args:?} failed: {}",
16536 String::from_utf8_lossy(&out.stderr)
16537 );
16538 };
16539 run(&["init", "-b", "main"]);
16540 run(&["config", "user.name", "magi test"]);
16541 run(&["config", "user.email", "magi@example.com"]);
16542 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16543 run(&["add", "-A"]);
16544 run(&["commit", "-m", "init"]);
16545 }
16546
16547 fn ask_test_home() {
16555 crate::run::pin_test_home();
16556 }
16557
16558 #[test]
16559 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16560 ask_test_home();
16561 let mut runner = runner_at(RunStatus::Prep);
16562 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16563 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16566 let error = guard
16567 .finish(
16568 &mut runner.state,
16569 Err(
16570 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16571 .into(),
16572 ),
16573 )
16574 .unwrap_err();
16575 assert_eq!(
16576 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16577 std::io::ErrorKind::PermissionDenied
16578 );
16579 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16580 let message = format!("{error:#}");
16581 assert!(message.contains("original refusal"));
16582 assert!(!message.contains("could not save terminal"));
16583 assert_eq!(
16584 error.chain().count(),
16585 2,
16586 "only startup attribution wraps the original error"
16587 );
16588 let saved = RunState::load(&runner.state.id).unwrap();
16589 assert_eq!(saved.status, RunStatus::Blocked);
16590 assert!(!saved.startup.unwrap().complete);
16591 }
16592
16593 #[test]
16594 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16595 ask_test_home();
16596 let mut runner = runner_at(RunStatus::Prep);
16597 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16598 let id = runner.state.id.clone();
16599 drop(guard);
16600 let saved = RunState::load(&id).unwrap();
16601 assert_eq!(saved.status, RunStatus::Blocked);
16602 assert!(!saved.startup.as_ref().unwrap().complete);
16603 assert!(
16604 saved
16605 .startup
16606 .unwrap()
16607 .failure
16608 .unwrap()
16609 .contains("interrupted")
16610 );
16611 assert!(Runner::resume(&id).is_err());
16612 }
16613
16614 #[test]
16615 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16616 ask_test_home();
16617 let mut runner = runner_at(RunStatus::Prep);
16618 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16619 let error = guard
16620 .finish(
16621 &mut runner.state,
16622 Err(
16623 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16624 .into(),
16625 ),
16626 )
16627 .unwrap_err();
16628 assert_eq!(
16629 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16630 std::io::ErrorKind::PermissionDenied
16631 );
16632 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16633 let mut task = crate::queue::Task::new(
16634 "review".into(),
16635 "criteria".into(),
16636 runner.state.repo.clone(),
16637 crate::queue::Source::Human,
16638 );
16639 task.record_startup_failure(failure.clone());
16640 task.record_startup_failure(failure.clone());
16641 assert_eq!(task.startup_failures.len(), 1);
16642 assert!(task.runs.is_empty());
16643 assert_eq!(task.attempts, 0);
16644 }
16645
16646 #[cfg(unix)]
16647 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16648 ask_test_home();
16649 let tmp = tempfile::tempdir().unwrap();
16650 let repo = tmp.path().join("repo");
16651 std::fs::create_dir(&repo).unwrap();
16652 init_repo(&repo);
16653 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16654 let origin = tmp.path().join("origin.git");
16655 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16656 .await
16657 .unwrap();
16658 git::git(
16659 &repo,
16660 &["remote", "add", "origin", origin.to_str().unwrap()],
16661 )
16662 .await
16663 .unwrap();
16664 git::git(&repo, &["checkout", "-b", "fix/retained"])
16665 .await
16666 .unwrap();
16667 std::fs::write(repo.join("retained"), "retained change").unwrap();
16668 git::git(&repo, &["add", "."]).await.unwrap();
16669 git::git(&repo, &["commit", "-m", "retained implementation"])
16670 .await
16671 .unwrap();
16672 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16673 git::git(&repo, &["checkout", "main"]).await.unwrap();
16674 std::fs::write(repo.join("base-change"), "new base").unwrap();
16675 git::git(&repo, &["add", "."]).await.unwrap();
16676 git::git(&repo, &["commit", "-m", "base moved"])
16677 .await
16678 .unwrap();
16679 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16680 git::git(&repo, &["checkout", "fix/retained"])
16681 .await
16682 .unwrap();
16683 let mut runner = runner_at(RunStatus::Voting);
16684 runner.state.repo = repo.clone();
16685 runner.state.base_commit = base;
16686 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16687 runner.state.config.disk.min_free_bytes = 0;
16688 runner.state.config.graph.review_rounds = 1;
16689 runner.state.config.verify.e2e = vec!["true".into()];
16690 runner.state.config.verify.gate = vec!["true".into()];
16691 runner.state.candidates = vec![Candidate {
16692 index: 0,
16693 label: 'A',
16694 agent: "fixture".into(),
16695 branch: "fix/retained".into(),
16696 worktree: repo.clone(),
16697 summary: String::new(),
16698 stat: String::new(),
16699 files: 1,
16700 commits: 1,
16701 empty: false,
16702 failed: None,
16703 verified_noop: None,
16704 duration_ms: 0,
16705 folded: false,
16706 }];
16707 runner.state.tally = Some(Tally {
16708 first_choice: BTreeMap::new(),
16709 borda: BTreeMap::new(),
16710 winner: 'A',
16711 rankings: 0,
16712 unanimous_initial: true,
16713 deliberated: false,
16714 changed_votes: 0,
16715 unanimous_final: true,
16716 tie_break: None,
16717 judges: 0,
16718 present: 0,
16719 quorum: 0,
16720 met_quorum: true,
16721 uncontested: Some("review-only".into()),
16722 });
16723 let mut task = crate::queue::Task::new(
16724 "fixture".into(),
16725 "fixture".into(),
16726 repo,
16727 crate::queue::Source::Human,
16728 );
16729 task.link_run(&runner.state.id);
16730 crate::queue::Queue::open().put(&mut task).unwrap();
16731 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16732 runner.state.base_sync = Some(BaseSync {
16733 tip: String::new(),
16734 behind: 1,
16735 attempts: BASE_SYNC_ROUNDS,
16736 conflict: None,
16737 already_in: None,
16738 });
16739 runner.sync_to_base().await.unwrap();
16740 assert_eq!(runner.state.status, RunStatus::Blocked);
16741 (tmp, runner, head, task.id)
16742 }
16743
16744 #[cfg(unix)]
16745 #[tokio::test]
16746 async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16747 for routed in [false, true] {
16748 let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16749 let home = crate::run::home();
16750 let queue = crate::queue::Queue::open();
16751 let task = queue.get(&owner).unwrap();
16752 runner.state.task_instruction = Some(task.instruction.clone());
16753 runner.state.driver_exited = true;
16754 runner.state.save().unwrap();
16755 let questions = crate::ask::Questions::at(home.join("questions"));
16756 let mut q = crate::ask::Question::new(
16757 runner.state.id.clone(),
16758 "base-recovery".into(),
16759 "owner".into(),
16760 "Recover bounded base movement?".into(),
16761 String::new(),
16762 vec!["recover".into()],
16763 );
16764 q.actions.insert(
16765 "recover".into(),
16766 crate::ask::ChoiceAction::RecoverBase {
16767 run: runner.state.id.clone(),
16768 stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16769 episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16770 },
16771 );
16772 q.routed_task = Some(owner.clone());
16773 q.answer(crate::ask::Answer::Choice("recover".into()))
16774 .unwrap();
16775 questions.put(&mut q).unwrap();
16776 crate::base_recovery_control::recover(
16777 &queue,
16778 &home,
16779 crate::base_recovery_control::Request {
16780 run: runner.state.id.clone(),
16781 head: head.clone(),
16782 actor: "decisions".into(),
16783 request: q.id.clone(),
16784 decision: Some((owner.clone(), q.answered_at.unwrap())),
16785 expected_stop: Some((
16786 runner.state.base_sync_recovery.stop.clone().unwrap(),
16787 runner.state.base_sync_recovery.episodes.last().unwrap().id,
16788 )),
16789 },
16790 )
16791 .await
16792 .unwrap();
16793 if routed {
16794 questions
16795 .update(&q.id, |q| {
16796 q.routed_task = Some("another-owner".into());
16797 Ok(())
16798 })
16799 .unwrap();
16800 } else {
16801 let mut task = queue.get(&owner).unwrap();
16802 task.hold_manual(Some("new owner decision".into()));
16803 task.status = crate::queue::TaskStatus::Queued;
16805 queue.put(&mut task).unwrap();
16806 }
16807 runner.state = RunState::load(&runner.state.id).unwrap();
16808 runner.sync_to_base().await.unwrap();
16809 let approval = runner
16810 .state
16811 .base_sync_recovery
16812 .authorizations
16813 .last()
16814 .unwrap();
16815 assert!(approval.consumed);
16816 assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
16817 "routing"
16818 } else {
16819 "owner decision"
16820 }));
16821 assert_eq!(
16822 runner.state.base_sync.as_ref().unwrap().attempts,
16823 BASE_SYNC_ROUNDS
16824 );
16825 assert_eq!(
16826 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16827 head
16828 );
16829 }
16830 }
16831
16832 #[cfg(unix)]
16833 #[tokio::test]
16834 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16835 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16836 if std::env::var_os(CHILD).is_none() {
16837 let home = tempfile::tempdir().unwrap();
16838 let output = std::process::Command::new(std::env::current_exe().unwrap())
16839 .args([
16840 "--exact",
16841 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16842 "--nocapture",
16843 ])
16844 .env(CHILD, "1")
16845 .env("MAGI_HOME", home.path())
16846 .env("MAGI_WORKER_HOME", home.path())
16847 .env("MAGI_WORKER_ID", "abcdef")
16848 .output()
16849 .unwrap();
16850 assert!(
16851 output.status.success()
16852 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16853 "{}\n{}",
16854 String::from_utf8_lossy(&output.stdout),
16855 String::from_utf8_lossy(&output.stderr)
16856 );
16857 return;
16858 }
16859 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16860 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16861 let home = crate::run::home();
16862 let queue = crate::queue::Queue::open();
16863 let mut owner = queue.get(&task).unwrap();
16864 owner.release_operator("test-release", None);
16865 owner.start(runner.state.id.clone());
16866 queue.put(&mut owner).unwrap();
16867 runner.state.driver_exited = true;
16868 runner.state.task_instruction = Some(owner.instruction.clone());
16869 runner.state.save().unwrap();
16870 let _claim = queue.claim(&task).unwrap();
16871 let reserve = |id: &str| {
16872 let directory = home.join("workers/jobs").join(id);
16873 std::fs::create_dir_all(&directory).unwrap();
16874 let job = crate::worker::Job {
16875 id: id.into(),
16876 sequence: 1,
16877 kind: "task".into(),
16878 key: format!("task:{task}"),
16879 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16880 generation: "fixture-generation".into(),
16881 version: None,
16882 status: crate::worker::Status::Running,
16883 result: None,
16884 pid: Some(std::process::id()),
16885 created_at: 0,
16886 finished_at: None,
16887 };
16888 std::fs::write(
16889 directory.join("job.json"),
16890 serde_json::to_vec(&job).unwrap(),
16891 )
16892 .unwrap();
16893 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16894 };
16895 let _own = reserve("abcdef");
16896 let other = reserve("fedcba");
16897 assert!(
16898 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16899 .await
16900 .is_err()
16901 );
16902 assert!(
16903 RunState::load(&runner.state.id)
16904 .unwrap()
16905 .base_sync_recovery
16906 .authorizations
16907 .is_empty()
16908 );
16909 drop(other);
16910 assert!(crate::daemon::is_working_on(
16911 &home,
16912 &runner.state.id,
16913 Timestamp::now()
16914 ));
16915 assert!(
16916 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16917 );
16918 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16919 .await
16920 .unwrap();
16921 assert!(
16922 RunState::load(&runner.state.id)
16923 .unwrap()
16924 .base_recovery_pending()
16925 );
16926 }
16927
16928 #[cfg(unix)]
16929 #[tokio::test]
16930 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16931 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16932 let home = crate::run::home();
16933 runner.state.driver_pid = Some(std::process::id());
16934 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16935 runner.state.driver_exited = false;
16936 runner.state.begin_pass();
16937 runner.state.save().unwrap();
16938 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16939 driver.record(&runner.state).unwrap();
16940 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16941 drop(driver);
16943 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16944 assert_eq!(
16945 runner.state.liveness_under(&home, false),
16946 crate::run::Liveness::Dead
16947 );
16948 let queue = crate::queue::Queue::open();
16949 let _claim = queue.claim(&task).unwrap();
16950 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16951 .await
16952 .unwrap();
16953 let disk = RunState::load(&runner.state.id).unwrap();
16954 assert!(!disk.driver_exited);
16955 assert!(disk.base_recovery_pending());
16956 }
16957
16958 #[cfg(unix)]
16959 #[tokio::test]
16960 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16961 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16962 if std::env::var_os(CHILD).is_none() {
16963 use std::os::unix::fs::PermissionsExt;
16964 let tmp = tempfile::tempdir().unwrap();
16965 let gh = tmp.path().join("gh");
16966 std::fs::write(&gh, r#"#!/bin/sh
16967case "$1 $2" in
16968 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16969 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16970 *) exit 2 ;;
16971esac
16972"#).unwrap();
16973 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16974 let mut paths = vec![tmp.path().to_path_buf()];
16975 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16976 let output = std::process::Command::new(std::env::current_exe().unwrap())
16977 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16978 .env(CHILD, "1")
16979 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16980 .env("PATH", std::env::join_paths(paths).unwrap())
16981 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16982 .output().unwrap();
16983 assert!(
16984 output.status.success()
16985 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16986 "{}\n{}",
16987 String::from_utf8_lossy(&output.stdout),
16988 String::from_utf8_lossy(&output.stderr)
16989 );
16990 return;
16991 }
16992 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16993 let repo = runner.state.repo.clone();
16994 git::git(&repo, &["checkout", "main"]).await.unwrap();
16995 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16996 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16997 git::git(&repo, &["checkout", "fix/retained"])
16998 .await
16999 .unwrap();
17000 runner.state.pr = Some(crate::run::PrRecord {
17001 url: "https://example.test/project/pull/1".into(),
17002 number: 1,
17003 state: "open".into(),
17004 checks: "unknown".into(),
17005 round: 0,
17006 rounds: 1,
17007 red_at_merge: Vec::new(),
17008 });
17009 runner.sync_to_base().await.unwrap();
17010 assert_eq!(runner.state.status, RunStatus::Blocked);
17011 assert_eq!(
17012 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17013 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17014 );
17015 assert!(runner.state.base_sync_closure_pending());
17016 assert!(crate::handover::can_resume(&runner.state));
17017 let queue = crate::queue::Queue::open();
17018 assert!(!crate::base_sync_recovery::held_task(
17019 &queue,
17020 &queue.get(&task).unwrap()
17021 ));
17022 assert!(runner.state.authorize_base_sync(&task).is_err());
17023 runner.state.base_sync_recovery.stop = None;
17025 runner.state.status = RunStatus::Voting;
17026 runner.state.gate_ran = true;
17027 runner.state.config.graph.review_rounds = 0;
17028 runner.state.save().unwrap();
17029 assert!(crate::handover::can_resume(&runner.state));
17030 runner.state = RunState::load(&runner.state.id).unwrap();
17031 runner.execute_graph().await.unwrap();
17032 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17033 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17034 assert_eq!(
17035 runner.state.base_sync.as_ref().unwrap().attempts,
17036 BASE_SYNC_ROUNDS
17037 );
17038 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17039 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17040 assert!(runner.state.reviews.is_empty());
17041 assert!(runner.state.gate.is_empty());
17042 }
17043
17044 #[cfg(unix)]
17045 #[tokio::test]
17046 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17047 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17048 runner.state.driver_exited = true;
17049 runner.state.save().unwrap();
17050 let home = crate::run::home();
17051 let queue = crate::queue::Queue::open();
17052 let _claim = queue.claim(&task).unwrap();
17053 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17054 assert!(
17055 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17056 .await
17057 .is_err()
17058 );
17059 drop(lease);
17060 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17061 assert!(
17062 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17063 .await
17064 .is_err()
17065 );
17066 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17067 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17068 .await
17069 .unwrap();
17070 let disk = RunState::load(&runner.state.id).unwrap();
17071 assert!(disk.base_recovery_pending());
17072 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17073 assert!(
17074 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17075 .await
17076 .is_err()
17077 );
17078 assert_eq!(
17079 RunState::load(&runner.state.id)
17080 .unwrap()
17081 .base_sync_recovery
17082 .authorizations
17083 .len(),
17084 1
17085 );
17086 let mut owner = queue.get(&task).unwrap();
17087 owner.runs.push("20260101-000000-ffff".into());
17088 queue.put(&mut owner).unwrap();
17089 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17090 .await
17091 .unwrap_err();
17092 assert!(error.to_string().contains("run task ownership changed"));
17093 }
17094
17095 #[cfg(unix)]
17096 #[tokio::test]
17097 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17098 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17099 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17100 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17101 legacy["status"] = serde_json::json!("voting");
17102 runner.state = serde_json::from_value(legacy).unwrap();
17103 runner.state.save().unwrap();
17104 for _ in 0..3 {
17105 runner.execute_graph().await.unwrap();
17106 assert_eq!(runner.state.status, RunStatus::Blocked);
17107 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17108 assert!(!crate::handover::can_resume(&runner.state));
17109 assert_eq!(
17110 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17111 head
17112 );
17113 assert!(runner.state.reviews.is_empty());
17114 assert!(!runner.state.gate_ran);
17115 }
17116 let notice = crate::notices::run_ended(&runner.state).unwrap();
17117 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17118 runner.state.begin_pass();
17119 runner.state.status = RunStatus::Voting;
17120 runner.deliberate().await.unwrap();
17121 runner.vote().await.unwrap();
17122 runner.finish_after_tally().await.unwrap();
17123 assert_eq!(runner.state.status, RunStatus::Blocked);
17124 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17125 assert_eq!(
17126 crate::notices::run_ended(&runner.state).unwrap().since,
17127 notice.since
17128 );
17129 }
17130
17131 #[cfg(unix)]
17132 #[tokio::test]
17133 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17134 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17135 for detail in [
17136 "fix/retained conflicts with origin/main and did not rebase: conflict",
17137 "rebased fix/retained locally but origin refused the push",
17138 "unknown failure",
17139 ] {
17140 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17141 runner.state.record_base_sync_stop(&head);
17142 assert!(runner.state.authorize_base_sync(&task).is_err());
17143 runner.state.status = RunStatus::Voting;
17144 runner.execute_graph().await.unwrap();
17145 assert_eq!(runner.state.status, RunStatus::Blocked);
17146 assert!(!crate::handover::can_resume(&runner.state));
17147 assert_eq!(
17148 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17149 head
17150 );
17151 }
17152 }
17153
17154 #[cfg(unix)]
17155 #[tokio::test]
17156 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17157 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17158 runner.state.authorize_base_sync(&task).unwrap();
17159 assert!(runner.state.consume_base_recovery(&head));
17160 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17161 runner.state.reserve_base_operation(&head, &tip);
17162 runner.state.save().unwrap();
17163 runner.state = RunState::load(&runner.state.id).unwrap();
17164 assert!(!runner.state.consume_base_recovery(&head));
17165 runner.sync_to_base().await.unwrap();
17166 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17167 assert_eq!(runner.state.base_episode_used(), 1);
17168 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17169 assert!(
17170 runner
17171 .state
17172 .base_sync_recovery
17173 .operations
17174 .last()
17175 .unwrap()
17176 .finished
17177 );
17178 assert!(!runner.state.base_sync_blocked());
17179 runner.sync_to_base().await.unwrap();
17180 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17181 }
17182
17183 #[cfg(unix)]
17184 #[tokio::test]
17185 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17186 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17187 let original = runner
17189 .state
17190 .base_sync
17191 .as_ref()
17192 .unwrap()
17193 .conflict
17194 .clone()
17195 .unwrap();
17196 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17197 original
17198 .replace("synchronization(s)", "rebase(s)")
17199 .replace("synchronizing", "rebasing"),
17200 );
17201 runner.state.base_sync_recovery = Default::default();
17202 runner.state.record_base_sync_stop(&head);
17203 assert_eq!(
17204 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17205 crate::base_sync_recovery::StopKind::Exhausted
17206 );
17207 runner.state.authorize_base_sync(&task).unwrap();
17208 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17209 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17210 .as_object_mut()
17211 .unwrap();
17212 auth.remove("interval");
17213 auth.remove("scope");
17214 runner.state = serde_json::from_value(legacy).unwrap();
17215 assert!(runner.state.consume_base_recovery(&head));
17216 assert_eq!(runner.state.base_episode_used(), 3);
17217 runner.sync_to_base().await.unwrap();
17218 assert_eq!(runner.state.base_episode_used(), 4);
17219 runner.state.resource_block = Some(crate::run::ResourceBlock {
17220 node: "gate".into(),
17221 kind: "cache".into(),
17222 detail: "legacy wait".into(),
17223 at: Timestamp::now(),
17224 cause: crate::run::ResourceBlockCause::Unrecorded,
17225 });
17226 runner.state.save().unwrap();
17227 assert!(
17228 runner
17229 .state
17230 .base_sync_recovery
17231 .episodes
17232 .last()
17233 .unwrap()
17234 .boundary
17235 .is_none()
17236 );
17237 runner.state.resource_block = None;
17238 advance_base_fixture(&runner).await;
17239 runner.sync_to_base().await.unwrap();
17240 assert!(runner.state.base_sync_blocked());
17241 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17242 }
17243
17244 #[cfg(unix)]
17245 async fn advance_base_fixture(runner: &Runner) {
17246 let repo = &runner.state.repo;
17247 for args in [
17248 vec!["checkout", "main"],
17249 vec!["commit", "--allow-empty", "-m", "base advanced"],
17250 vec!["push", "origin", "main"],
17251 vec!["checkout", "fix/retained"],
17252 ] {
17253 git::git(repo, &args).await.unwrap();
17254 }
17255 }
17256
17257 #[cfg(unix)]
17258 fn fresh_base_interval_fixture(runner: &mut Runner) {
17259 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17260 runner.state.base_sync.as_mut().unwrap().conflict = None;
17261 runner.state.base_sync_recovery = Default::default();
17262 runner.state.status = RunStatus::Voting;
17263 }
17264
17265 #[cfg(unix)]
17266 #[tokio::test]
17267 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17268 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17269 fresh_base_interval_fixture(&mut runner);
17270 runner.state.begin_pass();
17271 runner.sync_to_base().await.unwrap();
17272 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17273 runner.state.resource_block = Some(crate::run::ResourceBlock {
17274 node: "gate".into(),
17275 kind: "build cache".into(),
17276 detail: "busy".into(),
17277 at: Timestamp::now(),
17278 cause: crate::run::ResourceBlockCause::Timeout {
17279 waited_secs: 1,
17280 limit_secs: 1,
17281 },
17282 });
17283 runner.state.save().unwrap();
17284 let queue = crate::queue::Queue::open();
17285 let mut owner = queue.get(&task).unwrap();
17286 runner.state.task_instruction = Some(owner.instruction.clone());
17287 owner.release_operator("fixture-owner", None);
17288 queue.put(&mut owner).unwrap();
17289 let approval = owner.resumption.clone().unwrap();
17290 runner.state.resource_block = None;
17291 runner.state.save().unwrap();
17292 runner.state = RunState::load(&runner.state.id).unwrap();
17293 advance_base_fixture(&runner).await;
17294 runner.sync_to_base().await.unwrap();
17295 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17296 assert_eq!(runner.state.base_episode_used(), 1);
17297 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17298 assert!(
17299 runner.state.base_sync_recovery.episodes[0]
17300 .boundary
17301 .as_ref()
17302 .unwrap()
17303 .consumed
17304 );
17305 assert!(
17306 !runner
17307 .state
17308 .renew_base_episode(&task, &approval, &head, "base")
17309 );
17310 for _ in 0..3 {
17311 owner.release_operator("fixture-owner", None);
17312 queue.put(&mut owner).unwrap();
17313 runner.state = RunState::load(&runner.state.id).unwrap();
17314 runner.state.begin_pass();
17315 runner.sync_to_base().await.unwrap();
17316 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17317 assert_eq!(runner.state.base_episode_used(), 1);
17318 }
17319 runner.state.resource_block = Some(crate::run::ResourceBlock {
17320 node: "gate".into(),
17321 kind: "disk pressure".into(),
17322 detail: "free space required".into(),
17323 at: Timestamp::now(),
17324 cause: crate::run::ResourceBlockCause::Other,
17325 });
17326 runner.state.end_pass_parked(ParkCause::Interrupt {
17327 reason: "disk pressure".into(),
17328 });
17329 runner.state.save().unwrap();
17330 owner.release_operator("fixture-owner-after-disk", None);
17331 queue.put(&mut owner).unwrap();
17332 runner.state.resource_block = None;
17333 advance_base_fixture(&runner).await;
17334 runner.sync_to_base().await.unwrap();
17335 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17336 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17337 assert_eq!(runner.state.base_episode_used(), 1);
17338 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17339 }
17340
17341 #[cfg(unix)]
17342 #[tokio::test]
17343 async fn base_recovery_continuous_advancement_stops_at_four() {
17344 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17345 fresh_base_interval_fixture(&mut runner);
17346 for spent in 1..=BASE_SYNC_ROUNDS {
17347 runner.sync_to_base().await.unwrap();
17348 assert!(!runner.state.base_sync_blocked());
17349 assert_eq!(runner.state.base_episode_used(), spent);
17350 advance_base_fixture(&runner).await;
17351 }
17352 runner.sync_to_base().await.unwrap();
17353 assert_eq!(
17354 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17355 crate::base_sync_recovery::StopKind::Exhausted
17356 );
17357 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17358 assert!(stop.unverified_sync);
17359 let action = runner.state.base_sync_next_action();
17360 assert!(action.contains("never reached its gate"));
17361 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17362 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17363 runner.state = RunState::load(&runner.state.id).unwrap();
17364 runner.sync_to_base().await.unwrap();
17365 assert_eq!(runner.state.base_episode_used(), 4);
17366 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17367 }
17368
17369 #[cfg(unix)]
17370 #[tokio::test]
17371 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17372 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17373 runner.state.ensure_base_episode();
17374 runner.state.record_base_progress(&head);
17376 runner.state.base_sync_recovery.stop = None;
17377 runner.state.record_base_sync_stop(&head);
17378 assert!(
17379 !runner
17380 .state
17381 .base_sync_recovery
17382 .stop
17383 .as_ref()
17384 .unwrap()
17385 .unverified_sync
17386 );
17387
17388 runner.state.finish_base_operation(Some(&head));
17389 runner.state.gate_ran = true;
17391 runner.state.gate_head = Some("older-head".into());
17392 runner.state.base_sync_recovery.stop = None;
17393 runner.state.record_base_sync_stop(&head);
17394 runner.state.save().unwrap();
17395 runner.state = RunState::load(&runner.state.id).unwrap();
17396 assert!(
17397 runner
17398 .state
17399 .base_sync_recovery
17400 .stop
17401 .as_ref()
17402 .unwrap()
17403 .unverified_sync
17404 );
17405
17406 runner.state.gate_head = Some(head.clone());
17407 runner.state.base_sync_recovery.stop = None;
17408 runner.state.record_base_sync_stop(&head);
17409 assert!(
17410 !runner
17411 .state
17412 .base_sync_recovery
17413 .stop
17414 .as_ref()
17415 .unwrap()
17416 .unverified_sync
17417 );
17418 assert!(
17419 !runner
17420 .state
17421 .base_sync_next_action()
17422 .contains("never reached its gate")
17423 );
17424
17425 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17427 legacy["base_sync_recovery"]["stop"]
17428 .as_object_mut()
17429 .unwrap()
17430 .remove("unverified_sync");
17431 legacy["base_sync_recovery"]["episodes"][0]
17432 .as_object_mut()
17433 .unwrap()
17434 .remove("last_sync_head");
17435 runner.state = serde_json::from_value(legacy).unwrap();
17436 assert!(
17437 !runner
17438 .state
17439 .base_sync_recovery
17440 .stop
17441 .as_ref()
17442 .unwrap()
17443 .unverified_sync
17444 );
17445 assert!(
17446 runner.state.base_sync_recovery.episodes[0]
17447 .last_sync_head
17448 .is_none()
17449 );
17450 }
17451
17452 #[cfg(unix)]
17453 #[tokio::test]
17454 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17455 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17456 runner.state.authorize_base_sync(&task).unwrap();
17457 assert!(
17458 runner
17459 .state
17460 .base_sync_next_action()
17461 .contains("Approval recorded")
17462 );
17463 assert!(
17464 runner
17465 .state
17466 .base_sync_next_action()
17467 .contains("4/4 in episode 0, 4 lifetime")
17468 );
17469 assert!(runner.state.consume_base_recovery(&head));
17470 assert!(
17471 runner
17472 .state
17473 .base_sync_next_action()
17474 .contains("0/4 in episode 1, 4 lifetime")
17475 );
17476 }
17477
17478 #[cfg(unix)]
17479 #[tokio::test]
17480 async fn base_recovery_changed_head_invalidates_without_spending() {
17481 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17482 runner.state.authorize_base_sync(&task).unwrap();
17483 git::git(
17484 &runner.state.repo,
17485 &["commit", "--allow-empty", "-m", "external head change"],
17486 )
17487 .await
17488 .unwrap();
17489 runner.sync_to_base().await.unwrap();
17490 let a = runner
17491 .state
17492 .base_sync_recovery
17493 .authorizations
17494 .last()
17495 .unwrap();
17496 assert!(a.consumed);
17497 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17498 runner.state = RunState::load(&runner.state.id).unwrap();
17499 assert!(
17500 runner
17501 .state
17502 .base_sync_next_action()
17503 .contains("Approval invalidated")
17504 );
17505 assert!(!runner.state.base_recovery_pending());
17506 assert!(runner.state.base_sync_blocked());
17507 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17508 }
17509
17510 #[cfg(unix)]
17511 #[tokio::test]
17512 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17513 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17514 runner.state.driver_exited = true;
17515 runner.state.save().unwrap();
17516 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17517 .await
17518 .unwrap();
17519 runner.state = RunState::load(&runner.state.id).unwrap();
17520 advance_base_fixture(&runner).await;
17521 runner.sync_to_base().await.unwrap();
17522 let a = runner
17523 .state
17524 .base_sync_recovery
17525 .authorizations
17526 .last()
17527 .unwrap();
17528 assert!(
17529 a.consumed
17530 && a.invalidated
17531 .as_deref()
17532 .unwrap()
17533 .contains("fetched base changed")
17534 );
17535 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17536 assert_eq!(
17537 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17538 head
17539 );
17540 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17541 .await
17542 .unwrap();
17543 runner.state = RunState::load(&runner.state.id).unwrap();
17544 runner.sync_to_base().await.unwrap();
17545 assert!(!runner.state.base_sync_blocked());
17546 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17547 }
17548
17549 #[cfg(unix)]
17550 #[tokio::test]
17551 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17552 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17553 runner.state.authorize_base_sync(&task).unwrap();
17554 assert!(runner.state.consume_base_recovery(&head));
17555 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17556 runner.state.reserve_base_operation(&head, &tip);
17557 runner.state.save().unwrap();
17558 let scratch = runner.state.dir().join("base-sync");
17559 assert_eq!(
17560 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17561 .await
17562 .unwrap(),
17563 crate::rebase::Rebased::Applied
17564 );
17565 runner.state = RunState::load(&runner.state.id).unwrap();
17566 runner.sync_to_base().await.unwrap();
17567 assert!(
17568 runner
17569 .state
17570 .base_sync_recovery
17571 .operations
17572 .last()
17573 .unwrap()
17574 .finished
17575 );
17576 assert_eq!(runner.state.base_episode_used(), 1);
17577 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17578 assert!(!runner.state.base_sync_blocked());
17579 }
17580
17581 #[cfg(unix)]
17582 #[tokio::test]
17583 async fn base_recovery_crash_preserves_untracked_files() {
17584 for ignored in [false, true] {
17585 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17586 runner.state.authorize_base_sync(&task).unwrap();
17587 assert!(runner.state.consume_base_recovery(&head));
17588 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17589 runner.state.reserve_base_operation(&head, &tip);
17590 runner.state.save().unwrap();
17591 let scratch = runner.state.dir().join("base-sync");
17592 assert_eq!(
17593 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17594 .await
17595 .unwrap(),
17596 crate::rebase::Rebased::Applied
17597 );
17598 let repo = &runner.state.repo;
17599 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17600 assert_ne!(result, head);
17601 if ignored {
17602 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17603 .await
17604 .unwrap();
17605 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17606 }
17607 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17608 assert!(
17609 git::git_raw(repo, &["diff", "--quiet", &head])
17610 .await
17611 .unwrap()
17612 .ok()
17613 );
17614 runner.state = RunState::load(&runner.state.id).unwrap();
17616 runner.sync_to_base().await.unwrap();
17617 assert!(runner.state.base_sync_blocked());
17618 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17619 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17620 assert_eq!(
17621 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17622 result
17623 );
17624 assert_eq!(
17625 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17626 "retained work\n"
17627 );
17628 assert!(
17629 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17630 .await
17631 .unwrap()
17632 .ok()
17633 );
17634 }
17635 }
17636
17637 #[cfg(unix)]
17638 #[tokio::test]
17639 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17640 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17641 runner.state.authorize_base_sync(&task).unwrap();
17642 assert!(runner.state.consume_base_recovery(&head));
17643 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17644 runner.state.reserve_base_operation(&head, &tip);
17645 runner.state.save().unwrap();
17646 let scratch = runner.state.dir().join("base-sync");
17647 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17648 .await
17649 .unwrap();
17650 git::sync_to_head(&runner.state.repo).await.unwrap();
17651 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17652 git::git(&runner.state.repo, &["add", "foreign"])
17653 .await
17654 .unwrap();
17655 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17656 .await
17657 .unwrap();
17658 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17659 runner.state = RunState::load(&runner.state.id).unwrap();
17660 runner.sync_to_base().await.unwrap();
17661 assert!(runner.state.base_sync_blocked());
17662 assert!(
17663 !runner
17664 .state
17665 .base_sync_recovery
17666 .operations
17667 .last()
17668 .unwrap()
17669 .finished
17670 );
17671 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17672 assert_eq!(
17673 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17674 foreign_head
17675 );
17676 }
17677
17678 #[cfg(unix)]
17679 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17680 {
17681 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17682 fresh_base_interval_fixture(&mut runner);
17683 let repo = &runner.state.repo;
17684 git::git(repo, &["checkout", "main"]).await.unwrap();
17685 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17686 git::git(repo, &["add", "retained"]).await.unwrap();
17687 git::git(repo, &["commit", "-m", "upstream retained change"])
17688 .await
17689 .unwrap();
17690 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17691 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17692 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17693 let scratch = runner.state.dir().join("base-sync");
17694 assert!(matches!(
17695 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17696 .await
17697 .unwrap(),
17698 git::RebaseStart::Conflicted(_)
17699 ));
17700 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17701 agent: "fixture".into(),
17702 paths: vec!["retained".into()],
17703 from: Some(head.clone()),
17704 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17705 index_outside: None,
17706 finished: false,
17707 error: None,
17708 launcher: None,
17709 infra: None,
17710 outage: None,
17711 });
17712 (tmp, runner, head, tip, scratch)
17713 }
17714
17715 #[cfg(unix)]
17716 #[tokio::test]
17717 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17718 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17719 std::fs::write(
17722 scratch.join("retained"),
17723 "upstream and retained resolution\n",
17724 )
17725 .unwrap();
17726 runner.state.resource_block = Some(crate::run::ResourceBlock {
17727 node: "rebase".into(),
17728 kind: "repository coordination".into(),
17729 detail: "busy".into(),
17730 at: Timestamp::now(),
17731 cause: crate::run::ResourceBlockCause::Other,
17732 });
17733 runner.state.save().unwrap();
17734 let newer = git::git(
17737 &runner.state.repo,
17738 &[
17739 "commit-tree",
17740 "main^{tree}",
17741 "-p",
17742 "main",
17743 "-m",
17744 "base advanced during wait",
17745 ],
17746 )
17747 .await
17748 .unwrap();
17749 git::git(
17750 &runner.state.repo,
17751 &[
17752 "push",
17753 "origin",
17754 &format!("{}:refs/heads/main", newer.trim()),
17755 ],
17756 )
17757 .await
17758 .unwrap();
17759 runner.state = RunState::load(&runner.state.id).unwrap();
17760 runner.state.resource_block = None;
17761 runner.sync_to_base().await.unwrap();
17762 assert!(!runner.state.base_sync_blocked());
17763 let op = &runner.state.base_sync_recovery.operations[0];
17764 assert_eq!(op.head, head);
17765 assert_eq!(op.tip, tip);
17766 assert!(op.finished);
17767 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17768 assert_eq!(runner.state.rebase_fixes.len(), 1);
17769 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17770 .await
17771 .unwrap();
17772 assert_ne!(latest, tip);
17773 runner.state = RunState::load(&runner.state.id).unwrap();
17774 runner.sync_to_base().await.unwrap();
17775 assert!(!runner.state.base_sync_blocked());
17776 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17777 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17778 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17779 }
17780
17781 #[cfg(unix)]
17782 #[tokio::test]
17783 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17784 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17785 runner.state.reserve_base_operation(&head, &tip);
17786 runner.state.save().unwrap();
17787 std::fs::write(
17788 scratch.join("retained"),
17789 "upstream and retained resolution\n",
17790 )
17791 .unwrap();
17792 git::git(&scratch, &["add", "retained"]).await.unwrap();
17793 git::git(
17794 &scratch,
17795 &["-c", "core.editor=true", "rebase", "--continue"],
17796 )
17797 .await
17798 .unwrap();
17799 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17800 assert!(
17801 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17802 .await
17803 .unwrap()
17804 .is_empty()
17805 );
17806 assert_eq!(
17807 crate::rebase::finish(
17808 &mut runner.state,
17809 &scratch,
17810 "fix/retained",
17811 &head,
17812 &tip,
17813 &["retained".into()],
17814 ""
17815 )
17816 .await
17817 .unwrap(),
17818 crate::rebase::Rebased::Applied
17819 );
17820 assert!(!scratch.exists());
17821 runner.state = RunState::load(&runner.state.id).unwrap();
17823 let op = &runner.state.base_sync_recovery.operations[0];
17824 assert!(!op.finished);
17825 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17826 runner.sync_to_base().await.unwrap();
17827 assert!(!runner.state.base_sync_blocked());
17828 assert!(runner.state.base_sync_recovery.operations[0].finished);
17829 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17830 assert_eq!(runner.state.rebase_fixes.len(), 1);
17831 let winner = runner.state.winner().unwrap().clone();
17832 let wait = runner.repository_wait();
17833 assert!(
17834 verify_current_head(&mut runner.state, &winner, &wait)
17835 .await
17836 .unwrap()
17837 );
17838 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17839 }
17840
17841 #[cfg(unix)]
17842 #[tokio::test]
17843 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17844 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17845 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17846 runner.state.reserve_base_operation(&head, &tip);
17847 runner.state.save().unwrap();
17848 std::fs::write(
17849 scratch.join("retained"),
17850 "upstream and retained resolution\n",
17851 )
17852 .unwrap();
17853 git::git(&scratch, &["add", "retained"]).await.unwrap();
17854 git::git(
17855 &scratch,
17856 &["-c", "core.editor=true", "rebase", "--continue"],
17857 )
17858 .await
17859 .unwrap();
17860 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17861 assert!(!git::rebase_in_progress(&scratch).await);
17862 assert!(
17863 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17864 .await
17865 .unwrap()
17866 .is_empty()
17867 );
17868 match damage {
17869 Some("record") => {
17870 let path =
17871 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17872 .await
17873 .unwrap();
17874 std::fs::write(scratch.join(path), "").unwrap();
17875 }
17876 Some("foreign") => {
17877 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17878 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17879 git::git(&scratch, &["commit", "-m", "foreign work"])
17880 .await
17881 .unwrap();
17882 }
17883 Some("dirty") => {
17884 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17885 }
17886 _ => {}
17887 }
17888 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17889 runner.state = RunState::load(&runner.state.id).unwrap();
17891 assert!(
17892 runner.state.base_sync_recovery.operations[0]
17893 .validated_head
17894 .is_none()
17895 );
17896 runner.sync_to_base().await.unwrap();
17897 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17898 assert_eq!(runner.state.base_episode_used(), 1);
17899 assert_eq!(runner.state.rebase_fixes.len(), 1);
17900 assert_eq!(
17901 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17902 retained_head
17903 );
17904 if damage.is_some() {
17905 assert!(runner.state.base_sync_blocked());
17906 assert!(scratch.exists());
17907 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17908 assert!(
17909 runner.state.base_sync_recovery.operations[0]
17910 .validated_head
17911 .is_none()
17912 );
17913 } else {
17914 assert!(!runner.state.base_sync_blocked());
17915 assert!(!scratch.exists());
17916 assert!(runner.state.base_sync_recovery.operations[0].finished);
17917 assert_eq!(
17918 runner.state.base_sync_recovery.operations[0]
17919 .validated_head
17920 .as_deref(),
17921 Some(result.as_str())
17922 );
17923 let winner = runner.state.winner().unwrap().clone();
17924 let wait = runner.repository_wait();
17925 assert!(
17926 verify_current_head(&mut runner.state, &winner, &wait)
17927 .await
17928 .unwrap()
17929 );
17930 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17931 runner.state = RunState::load(&runner.state.id).unwrap();
17932 runner.sync_to_base().await.unwrap();
17933 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17934 }
17935 }
17936 }
17937
17938 #[cfg(unix)]
17939 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17940 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17941 runner.state.prep_complete = Some(true);
17943 runner.state.base_sync = None;
17944 runner.state.base_sync_recovery = Default::default();
17945 runner.state.config.graph.gate_fix_rounds = 1;
17946 let mut review = review_round(true, 0, 1, 1, false, true);
17947 review.head = head.clone();
17948 review.verified_head = Some(head.clone());
17949 runner.state.reviews = vec![review];
17950 runner.state.gate = vec![CommandOutcome {
17951 command: "old gate".into(),
17952 code: Some(1),
17953 output_tail: "old failure".into(),
17954 duration_ms: 0,
17955 resource_blocked: false,
17956 }];
17957 runner.state.gate_head = Some(head.clone());
17958 runner.state.gate_ran = true;
17959 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17960 agent: "fixture".into(),
17961 failed: runner.state.gate.clone(),
17962 notes: "spent".into(),
17963 committed: false,
17964 error: None,
17965 }];
17966 runner.state.land_approval = Some(crate::run::LandApproval {
17967 question: "old approval".into(),
17968 head: head.clone(),
17969 });
17970 runner.state.status = RunStatus::Blocked;
17971 (tmp, runner, head)
17972 }
17973
17974 #[cfg(unix)]
17975 #[tokio::test]
17976 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17977 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17978 let (tmp, mut runner, old) = resumed_gate_fixture().await;
17979 let phases = tmp.path().join("phases");
17980 runner.state.config.verify.e2e = vec![format!(
17981 "echo e2e >> '{}'; {}",
17982 phases.display(),
17983 if e2e_ok {
17984 "true"
17985 } else {
17986 "echo new-e2e-failure; false"
17987 }
17988 )];
17989 runner.state.config.verify.gate = vec![format!(
17990 "echo gate >> '{}'; {}",
17991 phases.display(),
17992 if gate_ok {
17993 "true"
17994 } else {
17995 "echo new-gate-failure; false"
17996 }
17997 )];
17998 runner.state.save().unwrap();
17999 runner.state = RunState::load(&runner.state.id).unwrap();
18000 runner.sync_to_base().await.unwrap();
18001 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18002 assert_ne!(head, old);
18003 runner.state = RunState::load(&runner.state.id).unwrap();
18005 runner.execute_graph().await.unwrap();
18006 assert_eq!(runner.state.gate_fixes.len(), 1);
18007 assert_eq!(runner.state.reviews.len(), 1);
18008 assert_eq!(runner.state.reviews[0].head, old);
18009 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18010 assert_eq!(runner.state.verification_history.len(), 1);
18011 let history = &runner.state.verification_history[0];
18012 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18013 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18014 assert!(history.archived_at.is_some());
18015 assert_eq!(history.fixes_spent, Some(1));
18016 assert_eq!(history.gate[0].output_tail, "old failure");
18017 let archive_events: Vec<_> = runner
18018 .state
18019 .events
18020 .iter()
18021 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18022 .collect();
18023 assert_eq!(archive_events.len(), 1);
18024 assert!(archive_events[0].message.contains(&short(&old)));
18025 assert!(archive_events[0].message.contains(&short(&head)));
18026 assert!(
18027 archive_events[0]
18028 .message
18029 .contains("1 gate-fix round(s) already spent stay spent")
18030 );
18031 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18032 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18033 if e2e_ok {
18034 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18035 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18036 } else {
18037 assert!(!runner.state.gate_ran);
18038 }
18039 assert_eq!(
18040 runner.state.status,
18041 if e2e_ok && gate_ok {
18042 RunStatus::Ready
18043 } else {
18044 RunStatus::Blocked
18045 }
18046 );
18047 if !e2e_ok || !gate_ok {
18048 let cause = runner.state.verification_blocking_cause().unwrap();
18049 assert!(cause.contains(&head));
18050 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18051 runner.state.event("land", "rebased successfully");
18053 let notice = crate::notices::run_ended(&runner.state).unwrap();
18054 assert!(!format!("{notice:?}").contains("rebased successfully"));
18055 assert!(runner.state.merge.is_none());
18056 }
18057 let before = std::fs::read_to_string(&phases).unwrap();
18058 runner.state.save().unwrap();
18059 runner.state = RunState::load(&runner.state.id).unwrap();
18060 runner.review_loop().await.unwrap();
18061 runner.gate().await.unwrap();
18062 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18063 assert_eq!(runner.state.verification_history.len(), 1);
18064 let history = &runner.state.verification_history[0];
18065 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18066 assert!(history.archived_at.is_some());
18067 assert_eq!(history.fixes_spent, Some(1));
18068 }
18069 }
18070
18071 #[cfg(unix)]
18072 #[tokio::test]
18073 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18074 let (tmp, mut runner, head) = resumed_gate_fixture().await;
18075 let marker = tmp.path().join("gate-ran");
18076 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18077 runner.review_loop().await.unwrap();
18078 runner.gate().await.unwrap();
18079 assert_eq!(runner.state.status, RunStatus::Blocked);
18080 assert!(!marker.exists());
18081 assert!(runner.state.verification_history.is_empty());
18082 runner.state.gate_head = None;
18083 runner.gate().await.unwrap();
18084 assert!(marker.exists());
18085 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18086 assert_eq!(runner.state.verification_history.len(), 1);
18087 assert_eq!(runner.state.gate_fixes.len(), 1);
18088 }
18089
18090 #[cfg(unix)]
18091 #[tokio::test]
18092 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18093 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18094 runner.sync_to_base().await.unwrap();
18095 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18096 runner.state.reconcile_verification_head(&head);
18097 for outcomes in [
18099 Vec::new(),
18100 vec![CommandOutcome {
18101 command: "true".into(),
18102 code: None,
18103 output_tail: "cache unavailable".into(),
18104 duration_ms: 0,
18105 resource_blocked: true,
18106 }],
18107 ] {
18108 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18109 head: head.clone(),
18110 passed: false,
18111 outcomes,
18112 });
18113 runner.state.gate_ran = false;
18114 runner.state.gate.clear();
18115 runner.state.save().unwrap();
18116 runner.state = RunState::load(&runner.state.id).unwrap();
18117 runner.merge().await.unwrap();
18118 assert!(runner.state.merge.is_none());
18119 runner.review_loop().await.unwrap();
18120 runner.gate().await.unwrap();
18121 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18122 assert!(runner.state.gate_allows_progress());
18123 assert_eq!(runner.state.gate_fixes.len(), 1);
18124 assert!(runner.state.merge.is_none());
18125 }
18126 git::git(
18128 &runner.state.repo,
18129 &["commit", "--allow-empty", "-m", "external update"],
18130 )
18131 .await
18132 .unwrap();
18133 runner.merge().await.unwrap();
18134 assert!(runner.state.merge.is_none());
18135 assert_eq!(runner.state.status, RunStatus::Blocked);
18136 }
18137
18138 #[cfg(unix)]
18139 #[tokio::test]
18140 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18141 for waiting_on_gate in [false, true] {
18142 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18143 runner.sync_to_base().await.unwrap();
18144 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18145 runner.state.reconcile_verification_head(&head);
18146 if waiting_on_gate {
18147 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18148 head: head.clone(),
18149 passed: true,
18150 outcomes: Vec::new(),
18151 });
18152 }
18153 let cache = tmp.path().join("cache");
18154 runner.state.config.verify.e2e =
18155 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18156 runner.state.config.verify.gate = vec!["true".into()];
18157 runner.state.config.graph.timeout_resource_wait = 1;
18158 let owner =
18159 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18160 let guard =
18161 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18162 crate::cache::AcquireOutcome::Acquired(g) => g,
18163 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18164 };
18165 runner.review_loop().await.unwrap();
18166 runner.gate().await.unwrap();
18167 assert!(!runner.state.gate_ran);
18168 assert_eq!(runner.state.gate_fixes.len(), 1);
18169 assert!(runner.state.resource_block.is_some());
18170 runner.merge().await.unwrap();
18171 assert!(runner.state.merge.is_none());
18172 drop(guard);
18173 runner.state = RunState::load(&runner.state.id).unwrap();
18174 runner.state.resource_block = None;
18176 runner.review_loop().await.unwrap();
18177 runner.gate().await.unwrap();
18178 assert!(runner.state.gate_allows_progress());
18179 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18180 assert_eq!(runner.state.gate_fixes.len(), 1);
18181 }
18182 }
18183
18184 #[cfg(unix)]
18185 #[tokio::test]
18186 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18187 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18188 runner.sync_to_base().await.unwrap();
18189 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18190 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18191 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18193 runner.review_loop().await.unwrap();
18194 runner.gate().await.unwrap();
18195 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18196 assert_ne!(before, after);
18197 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18198 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18199 assert!(!runner.state.gate_allows_progress());
18200 runner.merge().await.unwrap();
18201 assert!(runner.state.merge.is_none());
18202 assert!(
18203 runner
18204 .state
18205 .verification_history
18206 .iter()
18207 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18208 );
18209
18210 runner.state.reviews[0].clean = false;
18212 runner.state.reviews[0].answered = 0;
18213 runner.state.config.verify.e2e = vec!["true".into()];
18214 runner.review_loop().await.unwrap();
18215 runner.gate().await.unwrap();
18216 runner.merge().await.unwrap();
18217 assert_eq!(runner.state.status, RunStatus::Blocked);
18218 assert!(runner.state.merge.is_none());
18219 assert!(
18220 runner
18221 .state
18222 .verification_blocking_cause()
18223 .unwrap()
18224 .contains("review blocked")
18225 );
18226 }
18227
18228 #[cfg(unix)]
18229 #[tokio::test]
18230 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18231 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18232 runner.state.gate_head = None;
18233 runner.state.config.verify.gate =
18234 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18235 runner.review_loop().await.unwrap();
18236 runner.gate().await.unwrap();
18237 assert_ne!(
18238 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18239 old
18240 );
18241 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18242 assert!(!runner.state.gate_allows_progress());
18243 assert_eq!(runner.state.status, RunStatus::Blocked);
18244 runner.merge().await.unwrap();
18245 assert!(runner.state.merge.is_none());
18246 }
18247
18248 #[cfg(unix)]
18249 #[tokio::test]
18250 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18251 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18252 let mut old_review = review_round(true, 0, 1, 1, false, true);
18253 old_review.head = head.clone();
18254 old_review.verified_head = Some(head.clone());
18255 runner.state.reviews = vec![old_review];
18256 runner.state.gate = vec![CommandOutcome {
18257 command: "old gate".into(),
18258 code: None,
18259 output_tail: "timed out after 2400s".into(),
18260 duration_ms: 2_400_000,
18261 resource_blocked: false,
18262 }];
18263 runner.state.gate_ran = true;
18264 runner.state.gate_head = Some(head.clone());
18265 runner.state.land_approval = Some(crate::run::LandApproval {
18266 question: "old-head-approval".into(),
18267 head: head.clone(),
18268 });
18269 runner.state.authorize_base_sync(&task).unwrap();
18270 runner.sync_to_base().await.unwrap();
18271 assert!(!runner.state.base_sync_blocked());
18272 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18273 assert_ne!(head, fresh);
18274 assert_eq!(runner.state.reviews.len(), 1);
18275 assert!(!runner.state.gate_ran);
18276 let saved = &runner.state.base_sync_recovery.authorizations[0];
18277 assert_eq!(saved.reviews[0].head, head);
18278 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18279 assert!(saved.gate_ran);
18280 assert!(!saved.gate[0].ok());
18281 runner.review_loop().await.unwrap();
18282 runner.gate().await.unwrap();
18283 assert_eq!(runner.state.reviews[0].head, head);
18284 assert_eq!(
18285 runner.state.reviews[0].verified_head.as_deref(),
18286 Some(head.as_str())
18287 );
18288 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18289 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18290 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18291 assert!(runner.state.gate_ran);
18292 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18293 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18294 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18295 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18296 assert_eq!(
18297 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18298 "retained change"
18299 );
18300 }
18301
18302 #[cfg(unix)]
18303 #[tokio::test]
18304 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18305 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18306 let queue = crate::queue::Queue::open();
18307 let mut owner = queue.get(&task).unwrap();
18308 owner.release_operator("test-resume", None);
18309 queue.put(&mut owner).unwrap();
18310 let generation = owner.resumption.as_ref().unwrap().generation;
18311 runner.state.task_instruction = Some(owner.instruction.clone());
18312 runner.state.driver_exited = true;
18313 runner.state.save().unwrap();
18314 let home = crate::run::home();
18315 let _claim = queue.claim(&task).unwrap();
18316 owner.resumption.as_mut().unwrap().at =
18317 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18318 queue.put(&mut owner).unwrap();
18319 assert!(
18320 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18321 .await
18322 .is_err(),
18323 "an approval from before the stop grants nothing"
18324 );
18325 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18326 queue.put(&mut owner).unwrap();
18327 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18328 .await
18329 .unwrap();
18330 assert!(
18331 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18332 .await
18333 .is_err()
18334 );
18335 runner.state = RunState::load(&runner.state.id).unwrap();
18336 assert_eq!(
18337 runner.state.base_sync_recovery.authorizations[0].generation,
18338 Some(generation)
18339 );
18340 runner.sync_to_base().await.unwrap();
18341 let repo = runner.state.repo.clone();
18342 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18343 git::git(&repo, &["checkout", "main"]).await.unwrap();
18344 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18345 git::git(&repo, &["add", "."]).await.unwrap();
18346 git::git(&repo, &["commit", "-m", "move base again"])
18347 .await
18348 .unwrap();
18349 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18350 git::git(&repo, &["checkout", "fix/retained"])
18351 .await
18352 .unwrap();
18353 runner.sync_to_base().await.unwrap();
18354 assert!(!runner.state.base_sync_blocked());
18355 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18356 assert_eq!(runner.state.base_episode_used(), 2);
18357 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18358 assert_ne!(fresh, head);
18359 assert!(!runner.state.base_recovery_pending());
18360 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18362 queue.put(&mut owner).unwrap();
18363 assert!(
18364 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18365 .await
18366 .is_err()
18367 );
18368 runner.state = RunState::load(&runner.state.id).unwrap();
18369 runner.sync_to_base().await.unwrap();
18370 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18371 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18372 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18373 }
18374
18375 #[cfg(unix)]
18376 #[tokio::test]
18377 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18378 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18379 runner.state.base_sync = None;
18380 runner.state.status = RunStatus::Blocked;
18381 let mut second = runner.state.candidates[0].clone();
18382 second.label = 'B';
18383 runner.state.candidates.push(second);
18384 assert!(runner.state.votes.is_empty());
18385 assert!(runner.state.deliberation.is_empty());
18386 runner.deliberate().await.unwrap();
18387 runner.vote().await.unwrap();
18388 assert_eq!(runner.state.status, RunStatus::Blocked);
18389 assert!(runner.state.votes.is_empty());
18390 assert!(runner.state.deliberation.is_empty());
18391 }
18392
18393 fn runner_at(status: RunStatus) -> Runner {
18396 let mut state = RunState::new(
18397 PathBuf::from("/nonexistent/repo"),
18398 "main".to_owned(),
18399 "deadbeef".to_owned(),
18400 "task".to_owned(),
18401 Config::default(),
18402 );
18403 state.config.disk.min_free_bytes = 0;
18405 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18408 state.id = format!(
18409 "20000101-000000-{:04x}",
18410 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18411 );
18412 state.status = status;
18413 Runner {
18414 state,
18415 roles: ResolvedRoles {
18416 implementers: Vec::new(),
18417 judges: Vec::new(),
18418 reviewers: Vec::new(),
18419 fixer: None,
18420 conductor: conductor(),
18421 implementer_roster: Vec::new(),
18422 judge_roster: Vec::new(),
18423 reviewer_roster: Vec::new(),
18424 },
18425 sem: Arc::new(Semaphore::new(1)),
18426 pause: Pause::new(),
18427 interrupt: Pause::new(),
18428 }
18429 }
18430
18431 #[tokio::test]
18432 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18433 crate::run::pin_test_home();
18434 let temp = tempfile::tempdir().unwrap();
18435 let root = temp.path().canonicalize().unwrap();
18436 let repo = root.join("repo");
18437 std::fs::create_dir(&repo).unwrap();
18438 init_repo(&repo);
18439 git::git(&repo, &["config", "core.autocrlf", "false"])
18440 .await
18441 .unwrap();
18442 let mut runner = runner_at(RunStatus::Blocked);
18443 runner.state.repo = repo.clone();
18444 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18445 runner.state.config.agents = vec![conductor()];
18446 runner.state.config.graph.selected_implementer = Some("conductor".into());
18447 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18448 let (shell, command, setup_output) = if cfg!(windows) {
18449 (
18450 vec!["cmd".into(), "/C".into()],
18451 "echo ready>seat-ready",
18452 "ready\r\n",
18453 )
18454 } else {
18455 (
18456 vec!["sh".into(), "-c".into()],
18457 "printf ready > seat-ready",
18458 "ready",
18459 )
18460 };
18461 runner.state.config.verify.shell = Some(shell);
18462 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18463 run: Some(command.into()),
18464 ..Default::default()
18465 }];
18466 runner.state.save().unwrap();
18467 let path = runner.state.worktree_root().join("judge-1");
18468 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18469 .await
18470 .unwrap();
18471 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18472 git::git(&path, &["add", "."]).await.unwrap();
18473 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18474 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18475 .await
18476 .unwrap();
18477 git::git(&path, &["checkout", "--detach", &head])
18478 .await
18479 .unwrap();
18480 let evidence = crate::fold_evidence::snapshot(
18481 &runner.state,
18482 &crate::run::home().canonicalize().unwrap(),
18483 &head,
18484 64 * 1024 * 1024,
18485 )
18486 .await
18487 .unwrap();
18488 let mut contents = BTreeMap::new();
18489 contents.insert(
18490 repo.join("logs/HEAD"),
18491 format!("{head} {head} fixture\n").into_bytes(),
18492 );
18493 let record = serde_json::json!({
18494 "version":1, "run":runner.state.id, "source":path, "head":head,
18495 "evidence":evidence, "admin":{}, "admin_contents":contents,
18496 "history":{head.clone():evidence}, "phase":"evicted"
18497 });
18498 std::fs::write(
18499 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18500 serde_json::to_vec(&record).unwrap(),
18501 )
18502 .unwrap();
18503 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18504 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18505 .await
18506 .unwrap();
18507 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18508 assert!(
18509 !git::git_raw(&repo, &["cat-file", "-e", &head])
18510 .await
18511 .unwrap()
18512 .ok()
18513 );
18514 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18515 let error = resumed.execute().await.unwrap_err();
18516 assert!(
18517 error
18518 .to_string()
18519 .contains("selected implementation is held"),
18520 "{error:#}"
18521 );
18522 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18523 assert_eq!(
18524 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18525 setup_output
18526 );
18527 assert_eq!(
18528 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18529 "unrelated seat"
18530 );
18531 let error = resumed.execute().await.unwrap_err();
18532 assert!(
18533 error
18534 .to_string()
18535 .contains("selected implementation is held"),
18536 "{error:#}"
18537 );
18538 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18539 git::git(&path, &["add", "."]).await.unwrap();
18540 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18541 let next = git::git(
18542 &path,
18543 &[
18544 "commit-tree",
18545 &next_tree,
18546 "-p",
18547 &head,
18548 "-m",
18549 "next review checkpoint",
18550 ],
18551 )
18552 .await
18553 .unwrap();
18554 git::git(&path, &["checkout", "--detach", &next])
18555 .await
18556 .unwrap();
18557 resumed.state.reviews.push(
18558 serde_json::from_value(serde_json::json!({
18559 "round":1, "head":next, "reviews":[]
18560 }))
18561 .unwrap(),
18562 );
18563 resumed.state.save().unwrap();
18564 let error = resumed.execute().await.unwrap_err();
18565 assert!(
18566 error
18567 .to_string()
18568 .contains("selected implementation is held"),
18569 "{error:#}"
18570 );
18571 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18572 }
18573
18574 #[tokio::test]
18575 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18576 crate::run::pin_test_home();
18577 let mut runner = runner_at(RunStatus::Gating);
18578 runner.state.save().unwrap();
18579 let id = runner.state.id.clone();
18580 let passes = runner.state.passes.len();
18581 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18582 let mut claiming = Box::pin(runner.claim_driver());
18583 std::future::poll_fn(|cx| {
18584 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18585 std::task::Poll::Ready(())
18586 })
18587 .await;
18588 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18589 drop(owner);
18590 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18591 .await
18592 .unwrap()
18593 .unwrap();
18594 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18595 assert!(execution.runner.state.resource_block.is_none());
18596 assert_eq!(
18597 execution.runner.state.liveness(false),
18598 crate::run::Liveness::Live
18599 );
18600 drop(execution);
18601 assert!(RunState::load(&id).unwrap().driver_exited);
18602 }
18603
18604 #[tokio::test]
18605 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18606 crate::run::pin_test_home();
18607 let mut runner = runner_at(RunStatus::Gating);
18608 runner.state.save().unwrap();
18609 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18612 assert!(runner.claim_driver().await.is_err());
18613 assert!(runner.state.driver_exited);
18614 assert_eq!(runner.state.driver_pid, None);
18615 let disk = RunState::load(&runner.state.id).unwrap();
18616 assert!(disk.driver_exited);
18617 assert_eq!(disk.driver_pid, None);
18618 assert_eq!(disk.status, RunStatus::Gating);
18619 }
18620
18621 #[tokio::test]
18622 async fn an_early_graph_error_releases_its_driver() {
18623 crate::run::pin_test_home();
18624 let mut runner = runner_at(RunStatus::Blocked);
18625 runner.state.config.graph.selected_implementer = Some("selected".into());
18626 runner.state.save().unwrap();
18627 assert!(runner.execute().await.is_err());
18628 let disk = RunState::load(&runner.state.id).unwrap();
18629 assert!(disk.driver_exited);
18630 assert_eq!(disk.driver_pid, None);
18631 assert_eq!(disk.status, RunStatus::Blocked);
18632 }
18633
18634 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18635 crate::run::MergeRepair {
18636 question: "repair-question".into(),
18637 head: "old-head".into(),
18638 pr: "https://github.com/o/r/pull/42".into(),
18639 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18640 gate: vec![outcome(Some(0), false)],
18641 gate_head: Some("old-head".into()),
18642 gate_fixes: Vec::new(),
18643 pre_gate: Vec::new(),
18644 phase,
18645 failure: None,
18646 }
18647 }
18648
18649 #[tokio::test]
18650 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18651 crate::run::pin_test_home();
18652 let mut runner = runner_at(RunStatus::Landing);
18653 runner.state.merge_repairs.push(merge_repair_request(
18654 crate::run::MergeRepairPhase::Requested,
18655 ));
18656 runner.state.save().unwrap();
18657 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18658 assert!(
18659 runner
18660 .execute()
18661 .await
18662 .unwrap_err()
18663 .to_string()
18664 .contains("already running")
18665 );
18666 assert_eq!(
18667 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18668 crate::run::MergeRepairPhase::Requested
18669 );
18670 drop(claim);
18671 let mut disk = runner.state.clone();
18672 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18673 disk.save().unwrap();
18674 assert!(
18675 runner
18676 .execute()
18677 .await
18678 .unwrap_err()
18679 .to_string()
18680 .contains("decision changed")
18681 );
18682 assert_eq!(
18683 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18684 crate::run::MergeRepairPhase::Complete
18685 );
18686 }
18687
18688 #[test]
18689 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18690 crate::run::pin_test_home();
18691 let mut runner = runner_at(RunStatus::Landing);
18692 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18693 runner.state.reviews = request.reviews.clone();
18694 runner.state.gate = request.gate.clone();
18695 runner.state.gate_ran = true;
18696 runner.state.land_approval = Some(crate::run::LandApproval {
18697 question: "old".into(),
18698 head: "old-head".into(),
18699 });
18700 runner.state.merge_repairs.push(request);
18701 runner.begin_merge_repair_review(0).unwrap();
18702 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18703 assert!(!runner.state.gate_status().ok());
18704 assert!(runner.state.land_approval.is_none());
18705 let disk = RunState::load(&runner.state.id).unwrap();
18706 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18707 assert!(disk.merge_repairs[0].gate[0].ok());
18708 assert_eq!(
18709 disk.merge_repairs[0].phase,
18710 crate::run::MergeRepairPhase::Reviewing
18711 );
18712 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18713 assert!(disk.reviews.is_empty());
18714 }
18715
18716 #[tokio::test]
18717 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18718 crate::run::pin_test_home();
18719 for phase in [
18720 crate::run::MergeRepairPhase::Fixing,
18721 crate::run::MergeRepairPhase::Requested,
18722 ] {
18723 let mut runner = runner_at(RunStatus::Landing);
18724 runner.state.config.graph.land_rounds = 0;
18725 runner.state.merge_repairs.push(merge_repair_request(phase));
18726 runner.state.save().unwrap();
18727 assert!(!runner.repair_merge_review().await.unwrap());
18728 assert_eq!(runner.state.status, RunStatus::Blocked);
18729 assert_eq!(
18730 runner.state.merge_repairs[0].phase,
18731 crate::run::MergeRepairPhase::Failed
18732 );
18733 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18734 assert!(
18735 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18736 "interrupted"
18737 } else {
18738 "budget exhausted"
18739 })
18740 );
18741 assert!(runner.state.merge.is_none());
18742 }
18743 }
18744
18745 #[test]
18749 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18750 crate::run::pin_test_home();
18751 let mut runner = runner_at(RunStatus::Implementing);
18752 let interrupt = Pause::new();
18753 runner.watch_interrupt(interrupt.clone());
18754
18755 interrupt.park_because("task a1b2 asked to run first");
18756
18757 assert!(runner.park_here().expect("park_here"));
18758 assert!(runner.state.parked);
18759 let last = runner.state.events.last().expect("a park event");
18760 assert_eq!(last.node, "park");
18761 assert!(
18762 last.message.contains("task a1b2 asked to run first"),
18763 "expected the interrupt reason in {:?}",
18764 last.message
18765 );
18766 }
18767
18768 #[test]
18771 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18772 crate::run::pin_test_home();
18773 let mut runner = runner_at(RunStatus::Implementing);
18774 runner.state.begin_pass();
18775 let shutdown = Pause::new();
18776 runner.on_pause(shutdown.clone());
18777 let upgrade = ParkCause::Upgrade {
18778 from: "0.121.0".to_owned(),
18779 to: Some("0.121.1".to_owned()),
18780 };
18781 shutdown.park_for(upgrade.clone());
18782 shutdown.park_for(ParkCause::Operator);
18783
18784 assert!(runner.park_here().expect("park_here"));
18785 assert_eq!(
18786 runner.state.passes[0].end,
18787 Some(crate::run::PassEnd::Parked { cause: upgrade })
18788 );
18789 let last = runner.state.events.last().expect("a park event");
18790 assert!(last.message.contains("0.121.0"), "{}", last.message);
18791 }
18792
18793 #[tokio::test]
18794 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18795 crate::run::pin_test_home();
18796 let mut runner = runner_at(RunStatus::Reviewing);
18797 runner.state.save().unwrap();
18798 let mut execution = runner.claim_driver().await.expect("first execution");
18799 assert_eq!(execution.runner.state.passes.len(), 1);
18800 execution
18801 .driver
18802 .finish(&mut execution.runner.state)
18803 .await
18804 .unwrap();
18805 }
18806
18807 #[tokio::test]
18808 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18809 crate::run::pin_test_home();
18810 let mut runner = runner_at(RunStatus::Reviewing);
18811 runner.state.config.agents = vec![conductor()];
18812 runner.state.config.graph.timeout_resource_wait = 0;
18813 runner.state.save().unwrap();
18814 let id = runner.state.id.clone();
18815 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18816 assert!(runner.claim_driver().await.is_err());
18817 drop(owner);
18818 drop(runner);
18819 let mut resumed = Runner::resume(&id).unwrap();
18820 assert_eq!(resumed.state.driver_claimed, Some(false));
18821 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18822 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18823 let mut execution = resumed
18824 .claim_driver()
18825 .await
18826 .expect("retry initial claim after reload");
18827 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18828 assert_eq!(execution.runner.state.passes.len(), 1);
18829 execution
18830 .driver
18831 .finish(&mut execution.runner.state)
18832 .await
18833 .unwrap();
18834 }
18835
18836 #[tokio::test]
18837 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18838 crate::run::pin_test_home();
18839 let mut runner = runner_at(RunStatus::Reviewing);
18840 runner.state.config.agents = vec![conductor()];
18841 runner.state.save().unwrap();
18842 let id = runner.state.id.clone();
18843 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18844 let mut resumed = Runner::resume(&id).unwrap();
18845 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18846 assert!(resumed.claim_driver().await.is_err());
18847 assert!(RunState::load(&id).unwrap().passes.is_empty());
18848 drop(lease);
18849 let mut execution = resumed.claim_driver().await.unwrap();
18850 execution
18851 .driver
18852 .finish(&mut execution.runner.state)
18853 .await
18854 .unwrap();
18855 }
18856
18857 #[tokio::test]
18858 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18859 crate::run::pin_test_home();
18860 let mut runner = runner_at(RunStatus::Reviewing);
18861 let state = &mut runner.state;
18862 state.config.agents = vec![conductor()];
18863 state.driver_claimed = None;
18864 state.driver_pid = Some(std::process::id());
18865 state.driver_started_at = None;
18866 state.begin_pass();
18867 state.save().unwrap();
18868 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18869 std::fs::write(
18870 state.dir().join("driver.json"),
18871 serde_json::to_vec(&serde_json::json!({
18872 "pid": state.driver_pid, "started_at": null,
18873 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18874 }))
18875 .unwrap(),
18876 )
18877 .unwrap();
18878 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18879 drop(lease);
18880 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18881 let mut resumed = Runner::resume(&state.id).unwrap();
18882 assert!(!resumed.state.driver_exited);
18883 let mut execution = resumed
18884 .claim_driver()
18885 .await
18886 .expect("lease proves previous execution ended");
18887 assert_eq!(execution.runner.state.passes.len(), 2);
18888 execution
18889 .driver
18890 .finish(&mut execution.runner.state)
18891 .await
18892 .unwrap();
18893 }
18894
18895 #[tokio::test]
18896 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18897 crate::run::pin_test_home();
18898 let mut runner = runner_at(RunStatus::Reviewing);
18899 let mut json = serde_json::to_value(&runner.state).unwrap();
18900 json.as_object_mut().unwrap().remove("driver_claimed");
18901 runner.state = serde_json::from_value(json).unwrap();
18902 assert_eq!(runner.state.driver_claimed, None);
18903 runner.state.save().unwrap();
18904 let mut execution = runner
18905 .claim_driver()
18906 .await
18907 .expect("the execution lease, not a missing marker, excludes a second driver");
18908 assert_eq!(execution.runner.state.passes.len(), 1);
18909 execution
18910 .driver
18911 .finish(&mut execution.runner.state)
18912 .await
18913 .unwrap();
18914 }
18915
18916 #[tokio::test]
18917 async fn driver_admission_refuses_a_proven_live_driver() {
18918 crate::run::pin_test_home();
18919 let mut runner = runner_at(RunStatus::Reviewing);
18920 let pid = std::process::id();
18921 runner.state.driver_pid = Some(pid);
18922 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18923 assert!(runner.state.driver_started_at.is_some());
18924 runner.state.save().unwrap();
18925 let error = runner
18926 .claim_driver()
18927 .await
18928 .err()
18929 .expect("a live driver must refuse");
18930 assert!(error.to_string().contains("live driver"), "{error:#}");
18931 let saved = RunState::load(&runner.state.id).unwrap();
18932 assert_eq!(saved.driver_pid, Some(pid));
18933 assert_eq!(saved.status, RunStatus::Reviewing);
18934 assert!(saved.passes.is_empty());
18935 }
18936
18937 #[test]
18938 fn parking_resource_halt_persists_the_pass_once() {
18939 crate::run::pin_test_home();
18940 let mut runner = runner_at(RunStatus::Reviewing);
18941 runner.state.begin_pass();
18942 runner.state.resource_block = Some(crate::run::ResourceBlock {
18943 node: "fix".into(),
18944 kind: "shared build cache".into(),
18945 detail: "cancelled".into(),
18946 at: Timestamp::now(),
18947 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18948 });
18949 let cause = ParkCause::Upgrade {
18950 from: "old".into(),
18951 to: Some("new".into()),
18952 };
18953 runner.pause.park_for(cause.clone());
18954 runner.stop_for_resource().unwrap();
18957 assert!(runner.resource_halt().unwrap());
18958 assert!(runner.park_here().unwrap());
18959 let saved = RunState::load(&runner.state.id).unwrap();
18960 assert!(saved.parked);
18961 assert_eq!(saved.status, RunStatus::Reviewing);
18962 assert!(saved.passes[0].ended_at.is_some());
18963 assert_eq!(
18964 saved.passes[0].end,
18965 Some(crate::run::PassEnd::Parked {
18966 cause: cause.clone()
18967 })
18968 );
18969 assert_eq!(
18970 saved.resource_block.unwrap().cause,
18971 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18972 );
18973 assert_eq!(
18974 saved
18975 .events
18976 .iter()
18977 .filter(|event| event.node == "park")
18978 .count(),
18979 1
18980 );
18981 }
18982
18983 #[test]
18991 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18992 crate::run::pin_test_home();
18993 let mut runner = runner_at(RunStatus::Implementing);
18994 let shutdown = Pause::new();
18995 runner.on_pause(shutdown.clone());
18996 let interrupt = Pause::new();
18997 runner.watch_interrupt(interrupt.clone());
18998
18999 assert!(!runner.park_here().expect("park_here"));
19001 assert!(!runner.state.parked);
19002
19003 interrupt.park_because("test");
19005 assert!(!shutdown.parked());
19006 assert!(runner.park_here().expect("park_here"));
19007 }
19008
19009 #[tokio::test]
19023 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19024 crate::run::pin_test_home();
19025 let mut runner = runner_at(RunStatus::Implementing);
19026 let interrupt = Pause::new();
19027 runner.watch_interrupt(interrupt.clone());
19028
19029 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19030 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19031
19032 let node = async move {
19036 started_tx.send(()).expect("send started");
19037 finish_rx.await.expect("recv finish");
19038 "node finished"
19039 };
19040
19041 let interrupter = async move {
19042 started_rx.await.expect("recv started");
19043 interrupt.park_because("higher-priority task waiting");
19045 tokio::task::yield_now().await;
19049 finish_tx.send(()).expect("send finish");
19050 };
19051
19052 let (node_result, ()) = tokio::join!(node, interrupter);
19053 assert_eq!(
19054 node_result, "node finished",
19055 "the in-flight call ran to completion"
19056 );
19057
19058 assert!(runner.park_here().expect("park_here"));
19061 assert!(runner.state.parked);
19062 }
19063
19064 #[test]
19070 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19071 crate::run::pin_test_home();
19072 let mut runner = runner_at(RunStatus::Judging);
19073 runner.state.config.agents = vec![conductor()];
19077 runner.state.candidates = vec![Candidate {
19078 index: 0,
19079 label: 'A',
19080 agent: "alpha".to_owned(),
19081 branch: "magi/x/A".to_owned(),
19082 worktree: PathBuf::from("/nonexistent/worktree"),
19083 summary: "did the thing".to_owned(),
19084 stat: "1 file changed".to_owned(),
19085 files: 1,
19086 commits: 1,
19087 empty: false,
19088 failed: None,
19089 verified_noop: None,
19090 duration_ms: 1234,
19091 folded: false,
19092 }];
19093 let run_id = runner.state.id.clone();
19094
19095 let interrupt = Pause::new();
19096 runner.watch_interrupt(interrupt.clone());
19097 interrupt.park_because("task c3d4 asked to run first");
19098 assert!(runner.park_here().expect("park_here"));
19099
19100 let resumed = Runner::resume(&run_id).expect("resume");
19101 assert_eq!(resumed.state.candidates.len(), 1);
19102 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19103 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19104 assert_eq!(resumed.state.status, runner.state.status);
19105 assert!(
19106 resumed.state.parked,
19107 "still parked until `execute` actually walks the graph again"
19108 );
19109 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19110 }
19111
19112 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19114 let mut q = ask::Question::new(
19115 run.to_owned(),
19116 "implement".to_owned(),
19117 "impl-A".to_owned(),
19118 "Which storage backend should the cache use?".to_owned(),
19119 String::new(),
19120 vec!["SQLite".to_owned(), "Redis".to_owned()],
19121 );
19122 store.put(&mut q).unwrap();
19123 q
19124 }
19125
19126 #[test]
19127 fn a_failed_runs_open_question_is_abandoned() {
19128 ask_test_home();
19129 let store = ask::Questions::open();
19130 let mut runner = runner_at(RunStatus::Failed);
19131 let run = runner.state.id.clone();
19132 let q = ask_open_question(&store, &run);
19133
19134 runner.settle_questions();
19135
19136 let back = store.get(&q.id).unwrap();
19137 assert!(
19138 !back.status.open(),
19139 "the seat that asked died with the run; nobody is left to read an answer"
19140 );
19141 assert!(
19142 back.detail.contains(&run) && back.detail.contains("failed"),
19143 "the reason names what the run became, not just that it is gone: {}",
19144 back.detail
19145 );
19146 }
19147
19148 #[test]
19149 fn a_merged_runs_open_question_is_abandoned_too() {
19150 ask_test_home();
19151 let store = ask::Questions::open();
19152 for status in [RunStatus::Merged, RunStatus::Ready] {
19155 let mut runner = runner_at(status);
19156 let run = runner.state.id.clone();
19157 let q = ask_open_question(&store, &run);
19158
19159 runner.settle_questions();
19160
19161 let back = store.get(&q.id).unwrap();
19162 assert!(
19163 !back.status.open(),
19164 "{status:?} run's question must not outlive the run"
19165 );
19166 }
19167 }
19168
19169 #[test]
19170 fn a_still_resumable_runs_open_question_is_left_alone() {
19171 ask_test_home();
19172 let store = ask::Questions::open();
19173 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19179 let mut runner = runner_at(status);
19180 let run = runner.state.id.clone();
19181 let q = ask_open_question(&store, &run);
19182
19183 runner.settle_questions();
19184
19185 let back = store.get(&q.id).unwrap();
19186 assert!(
19187 back.status.open(),
19188 "{status:?} is still alive; the question must still be waiting"
19189 );
19190 }
19191 }
19192
19193 #[test]
19194 fn settle_questions_never_touches_an_already_answered_question() {
19195 ask_test_home();
19196 let store = ask::Questions::open();
19197 let mut runner = runner_at(RunStatus::Failed);
19198 let run = runner.state.id.clone();
19199 let mut q = ask_open_question(&store, &run);
19200 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19201 .unwrap();
19202 store.put(&mut q).unwrap();
19203
19204 runner.settle_questions();
19209 runner.settle_questions();
19210
19211 let back = store.get(&q.id).unwrap();
19212 assert_eq!(
19213 back.status,
19214 ask::QuestionStatus::Answered,
19215 "a real answer is a decision on record, never overwritten by a sweep"
19216 );
19217 }
19218
19219 #[tokio::test]
19230 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19231 crate::run::pin_test_home();
19232 let tmp = tempfile::tempdir().expect("tempdir");
19233 let repo = tmp.path().join("repo");
19234 std::fs::create_dir_all(&repo).unwrap();
19235 init_repo(&repo);
19236
19237 let mut config = Config::default();
19238 config.graph.worktree_root = Some(tmp.path().join("wt"));
19239
19240 let mut state = RunState::new(
19241 repo.clone(),
19242 "main".to_owned(),
19243 "deadbeef".to_owned(),
19244 "task".to_owned(),
19245 config,
19246 );
19247 let root = state.worktree_root();
19248 let wt_a = root.join("cand-A");
19249 let wt_b = root.join("cand-B");
19250 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19251 .await
19252 .expect("worktree A");
19253 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19254 .await
19255 .expect("worktree B");
19256
19257 state.candidates = vec![
19258 Candidate {
19259 index: 0,
19260 label: 'A',
19261 agent: "alpha".to_owned(),
19262 branch: "magi/x/A".to_owned(),
19263 worktree: wt_a.clone(),
19264 summary: String::new(),
19265 stat: String::new(),
19266 files: 0,
19267 commits: 0,
19268 empty: false,
19269 failed: None,
19270 verified_noop: None,
19271 duration_ms: 0,
19272 folded: false,
19273 },
19274 Candidate {
19275 index: 1,
19276 label: 'B',
19277 agent: "beta".to_owned(),
19278 branch: "magi/x/B".to_owned(),
19279 worktree: wt_b.clone(),
19280 summary: String::new(),
19281 stat: String::new(),
19282 files: 0,
19283 commits: 0,
19284 empty: false,
19285 failed: None,
19286 verified_noop: None,
19287 duration_ms: 0,
19288 folded: false,
19289 },
19290 ];
19291 state.tally = Some(Tally {
19292 first_choice: BTreeMap::from([('A', 1)]),
19293 borda: BTreeMap::new(),
19294 winner: 'A',
19295 rankings: 1,
19296 unanimous_initial: true,
19297 deliberated: false,
19298 changed_votes: 0,
19299 unanimous_final: true,
19300 tie_break: None,
19301 judges: 1,
19302 present: 1,
19303 quorum: 1,
19304 met_quorum: true,
19305 uncontested: None,
19306 });
19307 state.status = RunStatus::Ready;
19308
19309 fold_run(&mut state, false, &crate::run::home())
19310 .await
19311 .expect("fold_run");
19312
19313 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19314 assert!(
19315 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19316 "the unmerged winner's branch survives"
19317 );
19318 assert!(
19319 !state.candidates[0].folded,
19320 "the winner is not marked folded"
19321 );
19322
19323 assert!(!wt_b.exists(), "the loser's worktree is removed");
19324 assert!(
19325 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19326 "the loser's branch is removed"
19327 );
19328 assert!(state.candidates[1].folded, "the loser is marked folded");
19329 }
19330
19331 #[tokio::test]
19332 #[cfg(unix)]
19333 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19334 use std::os::fd::AsRawFd as _;
19335 use std::time::Duration;
19336
19337 let tmp = tempfile::tempdir().unwrap();
19338 let repo = tmp.path().join("repo");
19339 std::fs::create_dir_all(&repo).unwrap();
19340 init_repo(&repo);
19341 let common = repo.join(".git");
19342 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19343 let home = tmp.path().join("home");
19344 let mut config = Config::default();
19345 config.coordination = crate::config::test_coordination();
19346 config.graph.worktree_root = Some(tmp.path().join("wt"));
19347 config.graph.timeout_resource_wait = 1;
19348 let mut state = RunState::new(
19349 repo.clone(),
19350 "main".into(),
19351 "deadbeef".into(),
19352 "task".into(),
19353 config,
19354 );
19355 let root = state.worktree_root();
19356 let candidate = root.join("cand-A");
19357 let reviewer = root.join("review-1");
19358 let branch = state.branch_for('A');
19359 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19360 .await
19361 .unwrap();
19362 git::worktree_add_detached(&repo, &reviewer, "main")
19363 .await
19364 .unwrap();
19365 git::acquire_worktree_config(&repo).await.unwrap();
19366 state.enabled_worktree_config = true;
19367 state.status = RunStatus::Ready;
19368 state.candidates.push(Candidate {
19369 index: 0,
19370 label: 'A',
19371 agent: "implement".into(),
19372 branch: branch.clone(),
19373 worktree: candidate.clone(),
19374 summary: String::new(),
19375 stat: String::new(),
19376 files: 0,
19377 commits: 0,
19378 empty: false,
19379 failed: None,
19380 verified_noop: None,
19381 duration_ms: 0,
19382 folded: false,
19383 });
19384 state.save_under(&home).unwrap();
19385 let config_before = std::fs::read(common.join("config")).unwrap();
19386 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19387 .await
19388 .unwrap();
19389 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19392 assert_eq!(
19393 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19394 0
19395 );
19396 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19397 assert!(
19398 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19399 .await
19400 .is_err()
19401 );
19402 assert!(candidate.exists());
19403 assert!(reviewer.exists());
19404 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19405 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19406 assert_eq!(
19407 git::git(&repo, &["worktree", "list", "--porcelain"])
19408 .await
19409 .unwrap(),
19410 worktrees_before
19411 );
19412 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19413 .await
19414 .unwrap()
19415 .unwrap_err();
19416 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19417 drop(fold);
19418 assert!(candidate.exists() && reviewer.exists());
19419 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19420 assert_eq!(
19421 state.resource_block.as_ref().unwrap().kind,
19422 "repository coordination"
19423 );
19424 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19425 drop(external);
19426 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19427 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19428 .await
19429 .unwrap()
19430 .unwrap();
19431 drop(fold);
19432 assert!(removed.contains(&branch));
19433 assert!(!root.exists());
19434 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19435 assert!(state.candidates[0].folded);
19436 assert!(!state.enabled_worktree_config);
19437 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19438 let enabled = git::git_raw(
19439 &repo,
19440 &["config", "--local", "--get", "extensions.worktreeConfig"],
19441 )
19442 .await
19443 .unwrap();
19444 assert!(!enabled.ok(), "the last config reference was released");
19445 }
19446
19447 #[tokio::test]
19450 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19451 let tmp = tempfile::tempdir().expect("tempdir");
19452 let repo = tmp.path().join("repo");
19453 std::fs::create_dir_all(&repo).unwrap();
19454 init_repo(&repo);
19455 let home = tmp.path().join("home");
19456
19457 let mut config = Config::default();
19458 config.graph.worktree_root = Some(tmp.path().join("wt"));
19459 let mut state = RunState::new(
19460 repo.clone(),
19461 "main".to_owned(),
19462 "deadbeef".to_owned(),
19463 "task".to_owned(),
19464 config,
19465 );
19466 git::git(&repo, &["branch", "magi/x/A", "main"])
19468 .await
19469 .expect("branch");
19470 state.candidates = vec![Candidate {
19471 index: 0,
19472 label: 'A',
19473 agent: "alpha".to_owned(),
19474 branch: "magi/x/A".to_owned(),
19475 worktree: state.worktree_root().join("cand-A"),
19476 summary: String::new(),
19477 stat: String::new(),
19478 files: 0,
19479 commits: 0,
19480 empty: false,
19481 failed: None,
19482 verified_noop: None,
19483 duration_ms: 0,
19484 folded: true,
19485 }];
19486 state.released_to = Some("20260901-000000-new1".to_owned());
19487 state.released_branches = vec!["magi/x/A".to_owned()];
19488
19489 fold_run(&mut state, true, &home).await.expect("fold_run");
19490 state.status = RunStatus::Merged;
19491 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19492 state.merge = Some(crate::run::MergeOutcome {
19493 mode: MergeMode::Pr,
19494 ok: true,
19495 detail: "confirmed merged".into(),
19496 empty: false,
19497 });
19498 fold_run(&mut state, true, &home)
19499 .await
19500 .expect("merged fold");
19501
19502 assert!(
19503 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19504 "the handed-over branch survives a fold"
19505 );
19506 }
19507
19508 #[tokio::test]
19512 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19513 let tmp = tempfile::tempdir().expect("tempdir");
19514 let repo = tmp.path().join("repo");
19515 std::fs::create_dir_all(&repo).unwrap();
19516 init_repo(&repo);
19517 let run = |args: &[&str]| {
19518 let out = std::process::Command::new("git")
19519 .quiet()
19520 .args(args)
19521 .current_dir(&repo)
19522 .output()
19523 .expect("spawn git");
19524 assert!(out.status.success(), "git {args:?}");
19525 };
19526 run(&["branch", "magi/x/A"]);
19527 run(&["checkout", "-q", "-b", "magi/x/B"]);
19528 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19529 run(&["add", "-A"]);
19530 run(&["commit", "-q", "-m", "work"]);
19531 run(&["checkout", "-q", "main"]);
19532
19533 let probe = RunState::new(
19537 repo.clone(),
19538 "main".to_owned(),
19539 "deadbeef".to_owned(),
19540 "task".to_owned(),
19541 Config::default(),
19542 );
19543 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19544 let bare = tmp.path().join("origin.git");
19545 let out = std::process::Command::new("git")
19546 .quiet()
19547 .args(["init", "-q", "--bare"])
19548 .arg(&bare)
19549 .output()
19550 .expect("spawn git");
19551 assert!(out.status.success(), "git init --bare");
19552 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19553 run(&["push", "-q", "origin", "main"]);
19554
19555 let mut state = RunState::new(
19556 repo.clone(),
19557 "main".to_owned(),
19558 "deadbeef".to_owned(),
19559 "task".to_owned(),
19560 Config::default(),
19561 );
19562 state.seeds = vec![refs::Seed {
19563 token: "magi/27b2/A".to_owned(),
19564 kind: refs::SeedKind::Unresolved,
19565 sha: String::new(),
19566 branch: true,
19567 detail: "no branch or commit named magi/27b2/A".to_owned(),
19568 }];
19569
19570 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19571 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19572 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19573 let detail = empty_candidate_detail(&state, "main");
19574 assert!(detail.starts_with("empty candidate"), "{detail}");
19575 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19576 assert!(detail.contains("--start-from"), "{detail}");
19577 }
19578
19579 #[tokio::test]
19588 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19589 let tmp = tempfile::tempdir().expect("tempdir");
19590 let repo = tmp.path().join("repo");
19591 std::fs::create_dir_all(&repo).unwrap();
19592 init_repo(&repo);
19593
19594 let mut config = Config::default();
19595 config.merge.mode = MergeMode::Local;
19596
19597 let mut state = RunState::new(
19598 repo.clone(),
19599 "main".to_owned(),
19600 "deadbeef".to_owned(),
19601 "task".to_owned(),
19602 config,
19603 );
19604 state.candidates = vec![Candidate {
19605 index: 0,
19606 label: 'A',
19607 agent: "alpha".to_owned(),
19608 branch: "does-not-exist".to_owned(),
19609 worktree: repo.clone(),
19610 summary: String::new(),
19611 stat: String::new(),
19612 files: 0,
19613 commits: 0,
19614 empty: false,
19615 failed: None,
19616 verified_noop: None,
19617 duration_ms: 0,
19618 folded: false,
19619 }];
19620 state.tally = Some(Tally {
19621 first_choice: BTreeMap::from([('A', 1)]),
19622 borda: BTreeMap::new(),
19623 winner: 'A',
19624 rankings: 1,
19625 unanimous_initial: true,
19626 deliberated: false,
19627 changed_votes: 0,
19628 unanimous_final: true,
19629 tie_break: None,
19630 judges: 0,
19631 present: 0,
19632 quorum: 0,
19633 met_quorum: true,
19634 uncontested: Some("only candidate A produced a change".to_owned()),
19635 });
19636 state.reviews = vec![ReviewRound {
19637 round: 1,
19638 head: "deadbeef".to_owned(),
19639 verified_head: None,
19640 verified_at: None,
19641 reviews: Vec::new(),
19642 e2e: Vec::new(),
19643 fix: None,
19644 blocking: 0,
19645 answered: 0,
19646 expected: 0,
19647 clean: true,
19648 verify_retried: false,
19649 e2e_deferred: false,
19650 e2e_defer_reason: None,
19651 progressed: false,
19652 vote_split: false,
19653 reconsideration: Vec::new(),
19654 verdict: None,
19655 }];
19656 state.gate = vec![CommandOutcome {
19657 command: "test".to_owned(),
19658 code: Some(0),
19659 output_tail: String::new(),
19660 duration_ms: 0,
19661 resource_blocked: false,
19662 }];
19663 state.gate_ran = true;
19664 state.status = RunStatus::Ready;
19669 state.merge = Some(MergeOutcome {
19670 mode: MergeMode::Local,
19671 ok: false,
19672 detail: "already concluded".to_owned(),
19673 empty: false,
19674 });
19675
19676 let mut runner = Runner {
19677 state,
19678 roles: ResolvedRoles {
19679 implementers: Vec::new(),
19680 judges: Vec::new(),
19681 reviewers: Vec::new(),
19682 fixer: None,
19683 conductor: conductor(),
19684 implementer_roster: Vec::new(),
19685 judge_roster: Vec::new(),
19686 reviewer_roster: Vec::new(),
19687 },
19688 sem: Arc::new(Semaphore::new(1)),
19689 pause: Pause::new(),
19690 interrupt: Pause::new(),
19691 };
19692
19693 runner.merge().await.expect("merge");
19694
19695 assert_eq!(
19696 runner.state.status,
19697 RunStatus::Ready,
19698 "a concluded run's status must not change on reentry"
19699 );
19700 assert_eq!(
19701 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19702 Some("already concluded"),
19703 "merge must not run again once the node already recorded an outcome"
19704 );
19705 }
19706
19707 #[tokio::test]
19716 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19717 let tmp = tempfile::tempdir().expect("tempdir");
19718 let repo = tmp.path().join("repo");
19719 std::fs::create_dir_all(&repo).unwrap();
19720 init_repo(&repo);
19721
19722 let mut config = Config::default();
19723 config.merge.mode = MergeMode::Local;
19724
19725 let mut state = RunState::new(
19726 repo.clone(),
19727 "main".to_owned(),
19728 "deadbeef".to_owned(),
19729 "task".to_owned(),
19730 config,
19731 );
19732 state.candidates = vec![Candidate {
19733 index: 0,
19734 label: 'A',
19735 agent: "alpha".to_owned(),
19736 branch: "does-not-exist".to_owned(),
19737 worktree: repo.clone(),
19738 summary: String::new(),
19739 stat: String::new(),
19740 files: 0,
19741 commits: 0,
19742 empty: false,
19743 failed: None,
19744 verified_noop: None,
19745 duration_ms: 0,
19746 folded: false,
19747 }];
19748 state.tally = Some(Tally {
19749 first_choice: BTreeMap::from([('A', 1)]),
19750 borda: BTreeMap::new(),
19751 winner: 'A',
19752 rankings: 1,
19753 unanimous_initial: true,
19754 deliberated: false,
19755 changed_votes: 0,
19756 unanimous_final: true,
19757 tie_break: None,
19758 judges: 0,
19759 present: 0,
19760 quorum: 0,
19761 met_quorum: true,
19762 uncontested: Some("only candidate A produced a change".to_owned()),
19763 });
19764 state.reviews = vec![ReviewRound {
19765 round: 1,
19766 head: "deadbeef".to_owned(),
19767 verified_head: None,
19768 verified_at: None,
19769 reviews: Vec::new(),
19770 e2e: Vec::new(),
19771 fix: None,
19772 blocking: 0,
19773 answered: 0,
19774 expected: 0,
19775 clean: true,
19776 verify_retried: false,
19777 e2e_deferred: false,
19778 e2e_defer_reason: None,
19779 progressed: false,
19780 vote_split: false,
19781 reconsideration: Vec::new(),
19782 verdict: None,
19783 }];
19784 state.gate = Vec::new();
19786 state.gate_ran = false;
19787 state.status = RunStatus::Gating;
19788
19789 let mut runner = Runner {
19790 state,
19791 roles: ResolvedRoles {
19792 implementers: Vec::new(),
19793 judges: Vec::new(),
19794 reviewers: Vec::new(),
19795 fixer: None,
19796 conductor: conductor(),
19797 implementer_roster: Vec::new(),
19798 judge_roster: Vec::new(),
19799 reviewer_roster: Vec::new(),
19800 },
19801 sem: Arc::new(Semaphore::new(1)),
19802 pause: Pause::new(),
19803 interrupt: Pause::new(),
19804 };
19805
19806 runner.merge().await.expect("merge");
19807
19808 assert!(
19809 runner.state.merge.is_none(),
19810 "an empty gate must never be read as a passing one: {:?}",
19811 runner.state.merge
19812 );
19813 }
19814
19815 #[tokio::test]
19822 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19823 let tmp = tempfile::tempdir().expect("tempdir");
19824 let repo = tmp.path().join("repo");
19825 std::fs::create_dir_all(&repo).unwrap();
19826 init_repo(&repo);
19827
19828 let config = Config::default();
19830
19831 let mut state = RunState::new(
19832 repo.clone(),
19833 "main".to_owned(),
19834 "deadbeef".to_owned(),
19835 "task".to_owned(),
19836 config,
19837 );
19838 state.candidates = vec![Candidate {
19839 index: 0,
19840 label: 'A',
19841 agent: "alpha".to_owned(),
19842 branch: "does-not-exist".to_owned(),
19843 worktree: repo.clone(),
19844 summary: String::new(),
19845 stat: String::new(),
19846 files: 0,
19847 commits: 0,
19848 empty: false,
19849 failed: None,
19850 verified_noop: None,
19851 duration_ms: 0,
19852 folded: false,
19853 }];
19854 state.tally = Some(Tally {
19855 first_choice: BTreeMap::from([('A', 1)]),
19856 borda: BTreeMap::new(),
19857 winner: 'A',
19858 rankings: 1,
19859 unanimous_initial: true,
19860 deliberated: false,
19861 changed_votes: 0,
19862 unanimous_final: true,
19863 tie_break: None,
19864 judges: 0,
19865 present: 0,
19866 quorum: 0,
19867 met_quorum: true,
19868 uncontested: Some("only candidate A produced a change".to_owned()),
19869 });
19870 state.reviews = vec![ReviewRound {
19871 round: 1,
19872 head: "deadbeef".to_owned(),
19873 verified_head: None,
19874 verified_at: None,
19875 reviews: Vec::new(),
19876 e2e: Vec::new(),
19877 fix: None,
19878 blocking: 0,
19879 answered: 0,
19880 expected: 0,
19881 clean: true,
19882 verify_retried: false,
19883 e2e_deferred: false,
19884 e2e_defer_reason: None,
19885 progressed: false,
19886 vote_split: false,
19887 reconsideration: Vec::new(),
19888 verdict: None,
19889 }];
19890
19891 let mut runner = Runner {
19892 state,
19893 roles: ResolvedRoles {
19894 implementers: Vec::new(),
19895 judges: Vec::new(),
19896 reviewers: Vec::new(),
19897 fixer: None,
19898 conductor: conductor(),
19899 implementer_roster: Vec::new(),
19900 judge_roster: Vec::new(),
19901 reviewer_roster: Vec::new(),
19902 },
19903 sem: Arc::new(Semaphore::new(1)),
19904 pause: Pause::new(),
19905 interrupt: Pause::new(),
19906 };
19907
19908 runner.gate().await.expect("gate");
19909 assert!(
19910 runner.state.gate_ran,
19911 "zero configured commands is still a real attempt, not an unrun gate"
19912 );
19913 assert!(runner.state.gate.is_empty());
19914 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19915 assert_ne!(
19916 runner.state.status,
19917 RunStatus::Blocked,
19918 "a gate with nothing to check must not read as failed"
19919 );
19920
19921 runner.merge().await.expect("merge");
19922 assert_eq!(
19923 runner.state.status,
19924 RunStatus::Ready,
19925 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19926 );
19927 }
19928
19929 #[tokio::test]
19940 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19941 crate::run::pin_test_home();
19942 let home = crate::run::home();
19943
19944 let tmp = tempfile::tempdir().expect("tempdir");
19945 let repo = tmp.path().join("repo");
19946 std::fs::create_dir_all(&repo).unwrap();
19947 init_repo(&repo);
19948 let cache_dir = tmp.path().join("target");
19951
19952 let mut config = Config::default();
19953 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19954 config.graph.timeout_verify = Some(2);
19958 config.graph.timeout_resource_wait = 0;
19959
19960 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19961 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19962 .expect("no io error acquiring directly")
19963 {
19964 crate::cache::AcquireOutcome::Acquired(g) => g,
19965 crate::cache::AcquireOutcome::Busy(b) => {
19966 panic!("expected the direct acquire to win the lease first: {b:?}")
19967 }
19968 };
19969
19970 let mut state = RunState::new(
19971 repo.clone(),
19972 "main".to_owned(),
19973 "deadbeef".to_owned(),
19974 "task".to_owned(),
19975 config,
19976 );
19977 state.candidates = vec![Candidate {
19978 index: 0,
19979 label: 'A',
19980 agent: "alpha".to_owned(),
19981 branch: "does-not-exist".to_owned(),
19982 worktree: repo.clone(),
19983 summary: String::new(),
19984 stat: String::new(),
19985 files: 0,
19986 commits: 0,
19987 empty: false,
19988 failed: None,
19989 verified_noop: None,
19990 duration_ms: 0,
19991 folded: false,
19992 }];
19993 state.tally = Some(Tally {
19994 first_choice: BTreeMap::from([('A', 1)]),
19995 borda: BTreeMap::new(),
19996 winner: 'A',
19997 rankings: 1,
19998 unanimous_initial: true,
19999 deliberated: false,
20000 changed_votes: 0,
20001 unanimous_final: true,
20002 tie_break: None,
20003 judges: 0,
20004 present: 0,
20005 quorum: 0,
20006 met_quorum: true,
20007 uncontested: Some("only candidate A produced a change".to_owned()),
20008 });
20009 state.reviews = vec![ReviewRound {
20010 round: 1,
20011 head: "deadbeef".to_owned(),
20012 verified_head: None,
20013 verified_at: None,
20014 reviews: Vec::new(),
20015 e2e: Vec::new(),
20016 fix: None,
20017 blocking: 0,
20018 answered: 0,
20019 expected: 0,
20020 clean: true,
20021 verify_retried: false,
20022 e2e_deferred: false,
20023 e2e_defer_reason: None,
20024 progressed: false,
20025 vote_split: false,
20026 reconsideration: Vec::new(),
20027 verdict: None,
20028 }];
20029
20030 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20031 state.reviews[0].e2e = vec![CommandOutcome {
20032 command: state.config.verify.e2e[0].clone(),
20033 code: Some(0),
20034 output_tail: String::new(),
20035 duration_ms: 0,
20036 resource_blocked: false,
20037 }];
20038 let mut runner = Runner {
20039 state,
20040 roles: ResolvedRoles {
20041 implementers: Vec::new(),
20042 judges: Vec::new(),
20043 reviewers: Vec::new(),
20044 fixer: None,
20045 conductor: conductor(),
20046 implementer_roster: Vec::new(),
20047 judge_roster: Vec::new(),
20048 reviewer_roster: Vec::new(),
20049 },
20050 sem: Arc::new(Semaphore::new(1)),
20051 pause: Pause::new(),
20052 interrupt: Pause::new(),
20053 };
20054
20055 runner.gate().await.expect("gate");
20056 assert!(runner.state.resource_block.is_none());
20057 assert!(
20058 !runner
20059 .state
20060 .events
20061 .iter()
20062 .any(|event| { event.message.contains("waiting for the shared build cache") }),
20063 "a gate with nothing to run must never request the held cache lease"
20064 );
20065 assert!(
20066 runner.state.gate_ran,
20067 "zero commands is still a real, immediate attempt"
20068 );
20069 assert!(runner.state.gate.is_empty());
20070 assert_ne!(
20071 runner.state.status,
20072 RunStatus::Blocked,
20073 "must not read as resource-blocked on a lease it never asked for"
20074 );
20075 }
20076
20077 #[cfg(unix)]
20080 #[tokio::test]
20081 async fn verification_is_standalone_inside_a_live_worker() {
20082 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20083 if std::env::var_os(CHILD).is_none() {
20084 let home = tempfile::tempdir().unwrap();
20085 let generation = crate::worker::pin_current(home.path()).unwrap();
20086 let output = std::process::Command::new(std::env::current_exe().unwrap())
20087 .args([
20088 "--exact",
20089 "graph::tests::verification_is_standalone_inside_a_live_worker",
20090 "--nocapture",
20091 ])
20092 .env(CHILD, "1")
20093 .env("MAGI_HOME", home.path())
20094 .env("MAGI_WORKER_HOME", home.path())
20095 .env("MAGI_WORKER_ID", "abcdef")
20096 .env("MAGI_WORKER_GENERATION", &generation.id)
20097 .env("MAGI_WORKER_EXE", &generation.executable)
20098 .env("MAGI_RUN", "verification-parent")
20099 .env("MAGI_NODE", "gate")
20100 .env("MAGI_SEAT", "worker-fixture")
20101 .env("MAGI_PROCESS_OWNER", "fixture-owner")
20102 .env("MAGI_NO_AUTOUPDATE", "0")
20103 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20104 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20105 .env(
20106 "VERIFY_EXPECT_TOOLCHAIN",
20107 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20108 )
20109 .env(
20110 "VERIFY_EXPECT_CACHE",
20111 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20112 )
20113 .output()
20114 .unwrap();
20115 assert!(
20116 output.status.success()
20117 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20118 "{}\n{}",
20119 String::from_utf8_lossy(&output.stdout),
20120 String::from_utf8_lossy(&output.stderr)
20121 );
20122 return;
20123 }
20124 let home = crate::run::home();
20125 let directory = home.join("workers/jobs/abcdef");
20126 std::fs::create_dir_all(&directory).unwrap();
20127 let job = crate::worker::Job {
20128 id: "abcdef".into(),
20129 sequence: 1,
20130 kind: "task".into(),
20131 key: "task:verification".into(),
20132 payload: serde_json::json!({}),
20133 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20134 version: Some(env!("CARGO_PKG_VERSION").into()),
20135 status: crate::worker::Status::Running,
20136 result: None,
20137 pid: Some(std::process::id()),
20138 created_at: 0,
20139 finished_at: None,
20140 };
20141 std::fs::write(
20142 directory.join("job.json"),
20143 serde_json::to_vec(&job).unwrap(),
20144 )
20145 .unwrap();
20146 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20147 let assert_worker = || {
20148 assert_eq!(
20149 crate::worker::current_job(&home).unwrap().unwrap().id,
20150 job.id
20151 );
20152 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20153 assert_eq!(
20154 crate::worker::helper_executable(&home).unwrap(),
20155 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20156 );
20157 assert!(
20158 crate::updater::Checker::new(&crate::config::Update {
20159 mode: crate::config::UpdateMode::Notify,
20160 interval: None,
20161 auto_restart: false,
20162 })
20163 .is_none(),
20164 "the driver remains pinned, even while tests are standalone"
20165 );
20166 };
20167 assert_worker();
20168 let repo = home.join("repo");
20169 std::fs::create_dir_all(&repo).unwrap();
20170 init_repo(&repo);
20171 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20174 test -z "${MAGI_WORKER_GENERATION+x}" &&
20175 test -z "${MAGI_WORKER_HOME+x}" &&
20176 test -z "${MAGI_WORKER_EXE+x}" &&
20177 test "$MAGI_HOME/repo" -ef . &&
20178 test "$MAGI_RUN" = verification-parent &&
20179 test "$MAGI_NODE" = gate &&
20180 test "$MAGI_SEAT" = worker-fixture &&
20181 test -n "$MAGI_PROCESS_OWNER" &&
20182 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20183 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20184 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20185 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20186 assert!(
20188 !std::process::Command::new("sh")
20189 .args(["-c", probe])
20190 .current_dir(&repo)
20191 .status()
20192 .unwrap()
20193 .success()
20194 );
20195 let unrelated_home = tempfile::tempdir().unwrap();
20196 assert!(
20197 crate::worker::current_job(unrelated_home.path()).is_err(),
20198 "an inherited worker id cannot be resolved in a unit fixture's home"
20199 );
20200 let mut runner = runner_at(RunStatus::Gating);
20201 runner.state.repo = repo.clone();
20202 runner.state.config.disk.min_free_bytes = 0;
20203 runner.state.config.coordination = crate::config::test_coordination();
20204 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20205 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20206 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20207 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20208 let winner = Candidate {
20209 index: 0,
20210 label: 'A',
20211 agent: "fixture".into(),
20212 branch: "main".into(),
20213 worktree: repo.clone(),
20214 summary: String::new(),
20215 stat: String::new(),
20216 files: 0,
20217 commits: 0,
20218 empty: false,
20219 failed: None,
20220 verified_noop: None,
20221 duration_ms: 0,
20222 folded: false,
20223 };
20224 let (e2e, retried, pids) = run_e2e_with_retry(
20225 &mut runner.state,
20226 &["sh".into(), "-c".into()],
20227 &[format!("{probe} && printf initial")],
20228 &repo,
20229 Duration::from_secs(10),
20230 "initial",
20231 )
20232 .await;
20233 assert!(!retried && pids.is_empty());
20234 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20235 assert_eq!(e2e[0].output_tail, "initial");
20236 let gate = runner.run_gate(&winner).await.unwrap();
20237 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20238 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20241 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20242 passed: false,
20243 outcomes: Vec::new(),
20244 });
20245 runner.state.save().unwrap();
20246 let mut resumed = RunState::load(&runner.state.id).unwrap();
20247 let wait = crate::repo_lock::WaitPolicy::default();
20248 assert!(matches!(
20249 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20250 .await
20251 .unwrap(),
20252 GateFix::Retry
20253 ));
20254 let saved = RunState::load(&resumed.id).unwrap();
20255 let repair = saved.gate_repair_e2e.unwrap();
20256 assert!(repair.passed);
20257 assert_eq!(repair.outcomes.len(), 1);
20258 assert!(repair.outcomes[0].ok());
20259 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20260 .await
20261 .unwrap();
20262 assert_eq!(landing.len(), 1);
20263 assert!(landing[0].ok(), "{landing:?}");
20264 assert_eq!(
20265 std::fs::read_to_string(repo.join("phases")).unwrap(),
20266 "pre\ngate\ne2e\npre\ngate\n"
20267 );
20268 assert_worker();
20269 }
20270
20271 #[cfg(unix)]
20272 #[tokio::test]
20273 async fn verification_keeps_the_panic_and_the_full_output() {
20274 crate::run::pin_test_home();
20275 let tmp = tempfile::tempdir().unwrap();
20276 let mut config = Config::default();
20277 config.disk.min_free_bytes = 0;
20278 let mut state = RunState::new(
20279 tmp.path().to_owned(),
20280 "main".into(),
20281 "base".into(),
20282 "verification evidence".into(),
20283 config,
20284 );
20285 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20286 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20287 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20288 let mut names = Vec::new();
20289 for _ in 0..2 {
20290 let (outcomes, _) = run_commands(
20291 &mut state,
20292 "gate",
20293 "gate",
20294 0,
20295 &["sh".into(), "-c".into()],
20296 &[script.into()],
20297 tmp.path(),
20298 Duration::from_secs(60),
20299 )
20300 .await;
20301 let tail = &outcomes[0].output_tail;
20302 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20303 assert!(tail.contains("boom assertion"));
20304 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20305 let line = tail.lines().next().unwrap();
20306 names.push(line.to_owned());
20307 }
20308 assert_ne!(names[0], names[1], "the second run must not overwrite");
20309 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20310 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20311 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20312
20313 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20315 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20316 let (outcomes, _) = run_commands(
20317 &mut state,
20318 "gate",
20319 "gate",
20320 0,
20321 &["sh".into(), "-c".into()],
20322 &["echo failure-output; exit 7".into()],
20323 tmp.path(),
20324 Duration::from_secs(10),
20325 )
20326 .await;
20327 assert_eq!(outcomes[0].code, Some(7));
20328 assert!(state.events.iter().any(|e| {
20329 e.node == "gate" && e.message.contains("could not save the full gate output:")
20330 }));
20331 }
20332
20333 #[cfg(unix)]
20334 #[tokio::test]
20335 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20336 crate::run::pin_test_home();
20337 let tmp = tempfile::tempdir().unwrap();
20338 let mut config = Config::default();
20339 config.disk.min_free_bytes = 0;
20340 let mut state = RunState::new(
20341 tmp.path().to_owned(),
20342 "main".into(),
20343 "base".into(),
20344 "verification evidence".into(),
20345 config,
20346 );
20347 let (outcomes, pids) = run_commands(
20348 &mut state,
20349 "gate",
20350 "gate",
20351 1,
20352 &["sh".into(), "-c".into()],
20353 &[
20354 "printf before-timeout; exec sleep 30".into(),
20355 "kill -TERM $$".into(),
20356 "exit 7".into(),
20357 "printf success".into(),
20358 ],
20359 tmp.path(),
20360 Duration::from_secs(1),
20361 )
20362 .await;
20363 wait_for_timed_out_children_to_die(&pids).await;
20364 assert_eq!(
20365 pids.len(),
20366 1,
20367 "unleased timeout retains its death-check PID"
20368 );
20369 assert_eq!(outcomes.len(), 4);
20370 assert_eq!(outcomes[0].code, None);
20371 assert!(outcomes[0].output_tail.contains("before-timeout"));
20372 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20373 assert_eq!(outcomes[1].code, None);
20374 assert!(outcomes[1].output_tail.contains("command terminated:"));
20375 assert!(!outcomes[1].output_tail.contains("timed out"));
20376 assert_eq!(outcomes[2].code, Some(7));
20377 assert!(!outcomes[2].output_tail.contains("timed out"));
20378 assert_eq!(outcomes[3].code, Some(0));
20379 assert_eq!(outcomes[3].output_tail, "success");
20380 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20381 }
20382
20383 #[tokio::test]
20393 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20394 crate::run::pin_test_home();
20395
20396 let tmp = tempfile::tempdir().expect("tempdir");
20397 let repo = tmp.path().join("repo");
20398 std::fs::create_dir_all(&repo).unwrap();
20399 init_repo(&repo);
20400
20401 let mut config = Config::default();
20402 config.disk.min_free_bytes = 0;
20403 config.verify.gate = vec![
20404 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20405 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20406 .to_owned(),
20407 ];
20408
20409 let mut state = RunState::new(
20410 repo.clone(),
20411 "main".to_owned(),
20412 "deadbeef".to_owned(),
20413 "task".to_owned(),
20414 config,
20415 );
20416 let run_id = state.id.clone();
20417 state.candidates = vec![Candidate {
20418 index: 0,
20419 label: 'A',
20420 agent: "alpha".to_owned(),
20421 branch: "does-not-exist".to_owned(),
20422 worktree: repo.clone(),
20423 summary: String::new(),
20424 stat: String::new(),
20425 files: 0,
20426 commits: 0,
20427 empty: false,
20428 failed: None,
20429 verified_noop: None,
20430 duration_ms: 0,
20431 folded: false,
20432 }];
20433 state.tally = Some(Tally {
20434 first_choice: BTreeMap::from([('A', 1)]),
20435 borda: BTreeMap::new(),
20436 winner: 'A',
20437 rankings: 1,
20438 unanimous_initial: true,
20439 deliberated: false,
20440 changed_votes: 0,
20441 unanimous_final: true,
20442 tie_break: None,
20443 judges: 0,
20444 present: 0,
20445 quorum: 0,
20446 met_quorum: true,
20447 uncontested: Some("only candidate A produced a change".to_owned()),
20448 });
20449 state.reviews = vec![ReviewRound {
20450 round: 1,
20451 head: "deadbeef".to_owned(),
20452 verified_head: None,
20453 verified_at: None,
20454 reviews: Vec::new(),
20455 e2e: Vec::new(),
20456 fix: None,
20457 blocking: 0,
20458 answered: 0,
20459 expected: 0,
20460 clean: true,
20461 verify_retried: false,
20462 e2e_deferred: false,
20463 e2e_defer_reason: None,
20464 progressed: false,
20465 vote_split: false,
20466 reconsideration: Vec::new(),
20467 verdict: None,
20468 }];
20469
20470 let mut runner = Runner {
20471 state,
20472 roles: ResolvedRoles {
20473 implementers: Vec::new(),
20474 judges: Vec::new(),
20475 reviewers: Vec::new(),
20476 fixer: None,
20477 conductor: conductor(),
20478 implementer_roster: Vec::new(),
20479 judge_roster: Vec::new(),
20480 reviewer_roster: Vec::new(),
20481 },
20482 sem: Arc::new(Semaphore::new(1)),
20483 pause: Pause::new(),
20484 interrupt: Pause::new(),
20485 };
20486
20487 let started_marker = repo.join("started.marker");
20488 let release_marker = repo.join("release.marker");
20489 let poller = tokio::spawn(async move {
20490 for _ in 0..100 {
20495 if started_marker.exists()
20496 && let Ok(s) = crate::run::RunState::load(&run_id)
20497 && let Some(a) = s.active.get("gate")
20498 {
20499 std::fs::write(&release_marker, b"go").expect("release marker");
20500 return Some(a.clone());
20501 }
20502 tokio::time::sleep(Duration::from_millis(50)).await;
20503 }
20504 None
20505 });
20506
20507 runner.gate().await.expect("gate");
20508 let captured = poller.await.expect("poller task");
20509 let captured = captured.expect(
20510 "the poller never saw a `gate` task entry in run.json while the command was \
20511 still blocked on its own release marker",
20512 );
20513
20514 assert_eq!(captured.task.as_deref(), Some("gate"));
20515 assert_eq!(captured.node, "gate");
20516 assert_eq!(captured.index, Some(1));
20517 assert_eq!(captured.total, Some(1));
20518 assert!(
20519 captured
20520 .command
20521 .as_deref()
20522 .is_some_and(|c| c.contains("started.marker")),
20523 "{captured:?}"
20524 );
20525
20526 assert!(
20527 runner.state.active.is_empty(),
20528 "the entry must be cleared once the command actually finished: {:?}",
20529 runner.state.active
20530 );
20531 assert!(runner.state.gate_ran);
20532 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20533 }
20534
20535 #[tokio::test]
20538 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20539 use crate::verdict::{Finding, ReviewVote, Severity};
20540 crate::run::pin_test_home();
20541 let tmp = tempfile::tempdir().expect("tempdir");
20542 let repo = tmp.path().join("repo");
20543 std::fs::create_dir_all(&repo).unwrap();
20544 init_repo(&repo);
20545
20546 for (severity, vote, expect) in [
20547 (Severity::Major, ReviewVote::Reject, true),
20548 (Severity::Minor, ReviewVote::Reject, false),
20549 (Severity::Major, ReviewVote::Approve, false),
20550 ] {
20551 let mut round = review_round(false, 1, 1, 1, false, true);
20552 round.reviews = vec![ReviewRecord {
20553 reviewer: 1,
20554 agent: "alpha".to_owned(),
20555 summary: String::new(),
20556 findings: vec![Finding {
20557 id: "R1-1-1".to_owned(),
20558 severity,
20559 file: None,
20560 line: None,
20561 title: "t".to_owned(),
20562 detail: String::new(),
20563 }],
20564 vote: Some(vote),
20565 failed: None,
20566 duration_ms: 0,
20567 attempts: 0,
20568 }];
20569 let mut state = RunState::new(
20570 repo.clone(),
20571 "main".to_owned(),
20572 "deadbeef".to_owned(),
20573 "task".to_owned(),
20574 Config::default(),
20575 );
20576 state.reviews = vec![round];
20577 let mut runner = Runner {
20578 state,
20579 roles: ResolvedRoles {
20580 implementers: Vec::new(),
20581 judges: Vec::new(),
20582 reviewers: Vec::new(),
20583 fixer: None,
20584 conductor: conductor(),
20585 implementer_roster: Vec::new(),
20586 judge_roster: Vec::new(),
20587 reviewer_roster: Vec::new(),
20588 },
20589 sem: Arc::new(Semaphore::new(1)),
20590 pause: Pause::new(),
20591 interrupt: Pause::new(),
20592 };
20593 let shell = runner.state.config.shell();
20594 runner
20595 .stop_reviewing("round budget spent", &shell, &repo)
20596 .await
20597 .expect("stop_reviewing");
20598 assert_eq!(runner.state.status, RunStatus::Gating);
20599 assert_eq!(
20600 runner.state.contested_handoff.is_some(),
20601 expect,
20602 "{severity:?} + {vote:?}"
20603 );
20604 }
20605 }
20606
20607 #[tokio::test]
20620 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20621 crate::run::pin_test_home();
20622 let home = crate::run::home();
20623
20624 let tmp = tempfile::tempdir().expect("tempdir");
20625 let repo = tmp.path().join("repo");
20626 std::fs::create_dir_all(&repo).unwrap();
20627 init_repo(&repo);
20628 let head = crate::git::rev_parse(&repo, "HEAD")
20629 .await
20630 .expect("rev-parse");
20631 let cache_dir = tmp.path().join("target");
20634
20635 let mut config = Config::default();
20636 config.verify.e2e = vec![format!(
20637 "CARGO_TARGET_DIR='{}' test -f README.md",
20638 cache_dir.display()
20639 )];
20640 config.graph.review_rounds = 1;
20641 config.graph.timeout_verify = Some(2);
20644 config.graph.timeout_resource_wait = 2;
20645
20646 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20647 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20648 .expect("no io error acquiring directly")
20649 {
20650 crate::cache::AcquireOutcome::Acquired(g) => g,
20651 crate::cache::AcquireOutcome::Busy(b) => {
20652 panic!("expected the direct acquire to win the lease first: {b:?}")
20653 }
20654 };
20655
20656 let mut state = RunState::new(
20657 repo.clone(),
20658 "main".to_owned(),
20659 head.clone(),
20660 "task".to_owned(),
20661 config,
20662 );
20663 state.candidates = vec![Candidate {
20664 index: 0,
20665 label: 'A',
20666 agent: "alpha".to_owned(),
20667 branch: "does-not-exist".to_owned(),
20668 worktree: repo.clone(),
20669 summary: String::new(),
20670 stat: String::new(),
20671 files: 0,
20672 commits: 0,
20673 empty: false,
20674 failed: None,
20675 verified_noop: None,
20676 duration_ms: 0,
20677 folded: false,
20678 }];
20679 state.tally = Some(Tally {
20680 first_choice: BTreeMap::from([('A', 1)]),
20681 borda: BTreeMap::new(),
20682 winner: 'A',
20683 rankings: 1,
20684 unanimous_initial: true,
20685 deliberated: false,
20686 changed_votes: 0,
20687 unanimous_final: true,
20688 tie_break: None,
20689 judges: 0,
20690 present: 0,
20691 quorum: 0,
20692 met_quorum: true,
20693 uncontested: Some("only candidate A produced a change".to_owned()),
20694 });
20695 state.reviews = vec![ReviewRound {
20699 round: 1,
20700 head: head.clone(),
20701 verified_head: None,
20702 verified_at: None,
20703 reviews: Vec::new(),
20704 e2e: Vec::new(),
20705 fix: None,
20706 blocking: 1,
20707 answered: 1,
20708 expected: 1,
20709 clean: false,
20710 verify_retried: false,
20711 e2e_deferred: true,
20712 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20713 progressed: false,
20714 vote_split: false,
20715 reconsideration: Vec::new(),
20716 verdict: None,
20717 }];
20718
20719 let mut runner = Runner {
20720 state,
20721 roles: ResolvedRoles {
20722 implementers: Vec::new(),
20723 judges: Vec::new(),
20724 reviewers: Vec::new(),
20725 fixer: None,
20726 conductor: conductor(),
20727 implementer_roster: Vec::new(),
20728 judge_roster: Vec::new(),
20729 reviewer_roster: Vec::new(),
20730 },
20731 sem: Arc::new(Semaphore::new(1)),
20732 pause: Pause::new(),
20733 interrupt: Pause::new(),
20734 };
20735
20736 let shell = runner.state.config.shell();
20737 runner
20738 .stop_reviewing("round budget spent", &shell, &repo)
20739 .await
20740 .expect("stop_reviewing");
20741
20742 let last = runner.state.reviews.last().expect("round record");
20743 assert_eq!(
20744 last.e2e_status(),
20745 E2eStatus::ResourceBlocked,
20746 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20747 failed: {last:?}"
20748 );
20749 assert_eq!(
20750 last.verified_head.as_deref(),
20751 Some(head.as_str()),
20752 "which commit this attempt targeted is known even though nothing finished checking \
20753 it"
20754 );
20755 let first_attempt_at = last
20756 .verified_at
20757 .expect("when this attempt ran is known too");
20758 assert_ne!(
20759 runner.state.status,
20760 RunStatus::Blocked,
20761 "contention is evidence about the machine, not the patch — it must not settle the \
20762 run as blocked: {:?}",
20763 runner.state.status
20764 );
20765 assert!(
20766 !runner
20767 .state
20768 .events
20769 .iter()
20770 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20771 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20772 runner.state.events
20773 );
20774
20775 runner
20780 .stop_reviewing("round budget spent", &shell, &repo)
20781 .await
20782 .expect("stop_reviewing retry");
20783 assert_eq!(
20784 runner.state.reviews.len(),
20785 1,
20786 "no new round was started: {:?}",
20787 runner.state.reviews
20788 );
20789 let last = runner.state.reviews.last().expect("round record");
20790 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20791 assert!(
20792 last.verified_at.expect("still known") > first_attempt_at,
20793 "a second reentry must be a fresh attempt, not a stale copy of the first"
20794 );
20795 assert_ne!(runner.state.status, RunStatus::Blocked);
20796
20797 held.release();
20798 }
20799
20800 #[cfg(unix)]
20801 #[tokio::test]
20802 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20803 ask_test_home();
20804 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20805 let tmp = tempfile::tempdir().unwrap();
20806 let repo = tmp.path().join("repo");
20807 std::fs::create_dir(&repo).unwrap();
20808 init_repo(&repo);
20809 let run_git = |args: &[&str]| {
20810 let out = std::process::Command::new("git")
20811 .args(args)
20812 .current_dir(&repo)
20813 .output()
20814 .unwrap();
20815 assert!(
20816 out.status.success(),
20817 "{args:?}: {}",
20818 String::from_utf8_lossy(&out.stderr)
20819 );
20820 };
20821 let origin = tmp.path().join("origin.git");
20822 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20823 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20824 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20825 run_git(&["add", "spec.rs"]);
20826 run_git(&["commit", "-m", "initial spec"]);
20827 run_git(&["push", "origin", "main"]);
20828 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20829 run_git(&["checkout", "-b", "fix/review"]);
20830 std::fs::write(
20831 repo.join("check.rs"),
20832 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20833 )
20834 .unwrap();
20835 run_git(&["add", "check.rs"]);
20836 run_git(&["commit", "-m", "use spec"]);
20837 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20838 run_git(&["checkout", "main"]);
20839 std::fs::write(
20840 repo.join("spec.rs"),
20841 "struct AgentSpec { max_turn_secs: u64 }\n",
20842 )
20843 .unwrap();
20844 run_git(&["add", "spec.rs"]);
20845 run_git(&["commit", "-m", "extend spec"]);
20846 run_git(&["push", "origin", "main"]);
20847 run_git(&["checkout", "fix/review"]);
20848
20849 let mut runner = runner_at(RunStatus::Reviewing);
20850 runner.state.repo = repo.clone();
20851 runner.state.base_commit = base;
20852 runner.state.pr_head = Some("fix/review".into());
20853 runner.state.config.graph.review_rounds = max_rounds;
20854 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20855 runner.state.config.graph.sessions = false;
20856 runner.state.config.disk.min_free_bytes = 0;
20857 runner.state.config.verify.e2e =
20858 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20859 runner.state.candidates = vec![Candidate {
20860 index: 0,
20861 label: 'A',
20862 agent: "fixture".into(),
20863 branch: "fix/review".into(),
20864 worktree: repo.clone(),
20865 summary: String::new(),
20866 stat: String::new(),
20867 files: 1,
20868 commits: 1,
20869 empty: false,
20870 failed: None,
20871 verified_noop: None,
20872 duration_ms: 0,
20873 folded: false,
20874 }];
20875 runner.state.tally = Some(Tally {
20876 first_choice: BTreeMap::new(),
20877 borda: BTreeMap::new(),
20878 winner: 'A',
20879 rankings: 0,
20880 unanimous_initial: true,
20881 deliberated: false,
20882 changed_votes: 0,
20883 unanimous_final: true,
20884 tie_break: None,
20885 judges: 0,
20886 present: 0,
20887 quorum: 0,
20888 met_quorum: true,
20889 uncontested: Some("review-only".into()),
20890 });
20891 let mut last = review_round(false, 0, 1, 1, false, false);
20892 last.head = reviewed.clone();
20893 last.e2e_deferred = true;
20894 last.e2e[0].resource_blocked = true;
20895 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20896 if max_rounds == 1 {
20897 last.e2e.clear();
20898 }
20899 runner.state.reviews = vec![last];
20900 let calls = tmp.path().join("fix-calls");
20901 let mut fixer = spec("repair");
20902 fixer
20903 .env
20904 .insert("FIX_CALLS".into(), calls.display().to_string());
20905 fixer.command = vec!["sh".into(), "-c".into(), r#"
20906 echo fix >> "$FIX_CALLS"
20907 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20908 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20909 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20910 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20911 "#.into()];
20912 let mut reviewer = spec("reviewer");
20913 reviewer.command = vec![
20914 "sh".into(),
20915 "-c".into(),
20916 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20917 .into(),
20918 ];
20919 runner.roles.fixer = Some(vec![fixer]);
20920 runner.roles.reviewers = vec![reviewer];
20921
20922 runner.sync_to_base().await.unwrap();
20923 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20924 assert_ne!(rebased, reviewed);
20925 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20926 if reentry {
20927 let shell = runner.state.config.shell();
20930 runner
20931 .stop_unresolved_review("deferred verification", &shell, &repo)
20932 .await
20933 .unwrap();
20934 assert!(runner.catchup_repairable());
20935 assert!(!calls.exists());
20936 runner.state = RunState::load(&runner.state.id).unwrap();
20937 assert!(runner.catchup_repairable());
20938 }
20939 runner.review_loop().await.unwrap();
20940 let first = &runner.state.reviews[0];
20941 assert_eq!(first.head, reviewed);
20942 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20943 assert!(
20944 first.verify_retried,
20945 "a reproduced build failure gets exactly one retry: {:?}",
20946 first.e2e
20947 );
20948 assert!(first.e2e[0].output_tail.contains("E0063"));
20949 assert!(!runner.catchup_repairable());
20950 if max_rounds == 2 {
20951 assert_eq!(
20952 runner.state.status,
20953 RunStatus::Gating,
20954 "{}",
20955 crate::report::run(&runner.state)
20956 );
20957 assert_eq!(runner.state.reviews.len(), 2);
20958 assert!(first.fix.as_ref().unwrap().committed);
20959 assert!(runner.state.reviews[1].clean);
20960 assert_eq!(
20961 runner
20962 .state
20963 .reviews
20964 .iter()
20965 .filter(|r| r.fix.is_some())
20966 .count(),
20967 1
20968 );
20969 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20970 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20971 } else {
20972 assert_eq!(runner.state.status, RunStatus::Blocked);
20973 assert!(first.fix.is_none());
20974 assert!(!calls.exists());
20975 let recovery = crate::recovery::assess(
20976 "prior",
20977 Some(&runner.state),
20978 &runner.landing_base(),
20979 crate::recovery::Relation::Diverged,
20980 );
20981 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20982 let reasons = recovery.reasons.join("\n");
20983 assert!(
20984 reasons.contains(&reviewed) && reasons.contains(&rebased),
20985 "{reasons}"
20986 );
20987 assert!(
20988 reasons.contains("base sync to")
20989 && reasons.contains("additional review is required"),
20990 "{reasons}"
20991 );
20992 }
20993 }
20994 }
20995
20996 #[tokio::test]
21008 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21009 crate::run::pin_test_home();
21010 let home = crate::run::home();
21011
21012 let tmp = tempfile::tempdir().expect("tempdir");
21013 let repo = tmp.path().join("repo");
21014 std::fs::create_dir_all(&repo).unwrap();
21015 init_repo(&repo);
21016 let head = crate::git::rev_parse(&repo, "HEAD")
21017 .await
21018 .expect("rev-parse");
21019 let cache_dir = tmp.path().join("target");
21020
21021 let mut config = Config::default();
21022 config.verify.e2e = vec![format!(
21023 "CARGO_TARGET_DIR='{}' test -f README.md",
21024 cache_dir.display()
21025 )];
21026 config.graph.review_rounds = 1;
21027 config.graph.timeout_verify = Some(2);
21028 config.graph.timeout_resource_wait = 2;
21029
21030 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21031 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21032 .expect("no io error acquiring directly")
21033 {
21034 crate::cache::AcquireOutcome::Acquired(g) => g,
21035 crate::cache::AcquireOutcome::Busy(b) => {
21036 panic!("expected the direct acquire to win the lease first: {b:?}")
21037 }
21038 };
21039
21040 let mut state = RunState::new(
21041 repo.clone(),
21042 "main".to_owned(),
21043 head.clone(),
21044 "task".to_owned(),
21045 config,
21046 );
21047 state.candidates = vec![Candidate {
21048 index: 0,
21049 label: 'A',
21050 agent: "alpha".to_owned(),
21051 branch: "does-not-exist".to_owned(),
21052 worktree: repo.clone(),
21053 summary: String::new(),
21054 stat: String::new(),
21055 files: 0,
21056 commits: 0,
21057 empty: false,
21058 failed: None,
21059 verified_noop: None,
21060 duration_ms: 0,
21061 folded: false,
21062 }];
21063 state.tally = Some(Tally {
21064 first_choice: BTreeMap::from([('A', 1)]),
21065 borda: BTreeMap::new(),
21066 winner: 'A',
21067 rankings: 1,
21068 unanimous_initial: true,
21069 deliberated: false,
21070 changed_votes: 0,
21071 unanimous_final: true,
21072 tie_break: None,
21073 judges: 0,
21074 present: 0,
21075 quorum: 0,
21076 met_quorum: true,
21077 uncontested: Some("only candidate A produced a change".to_owned()),
21078 });
21079 state.reviews = vec![ReviewRound {
21083 round: 1,
21084 head: head.clone(),
21085 verified_head: Some(head.clone()),
21086 verified_at: Some(jiff::Timestamp::now()),
21087 reviews: Vec::new(),
21088 e2e: vec![CommandOutcome {
21089 command: format!(
21090 "CARGO_TARGET_DIR='{}' test -f README.md",
21091 cache_dir.display()
21092 ),
21093 code: None,
21094 output_tail: "waiting for the shared build cache".to_owned(),
21095 duration_ms: 0,
21096 resource_blocked: true,
21097 }],
21098 fix: None,
21099 blocking: 1,
21100 answered: 1,
21101 expected: 1,
21102 clean: false,
21103 verify_retried: false,
21104 e2e_deferred: false,
21105 e2e_defer_reason: None,
21106 progressed: false,
21107 vote_split: false,
21108 reconsideration: Vec::new(),
21109 verdict: None,
21110 }];
21111
21112 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21113 let mut runner = Runner {
21114 state,
21115 roles: ResolvedRoles {
21116 implementers: Vec::new(),
21117 judges: Vec::new(),
21118 reviewers: Vec::new(),
21119 fixer: None,
21120 conductor: conductor(),
21121 implementer_roster: Vec::new(),
21122 judge_roster: Vec::new(),
21123 reviewer_roster: Vec::new(),
21124 },
21125 sem: Arc::new(Semaphore::new(1)),
21126 pause: Pause::new(),
21127 interrupt: Pause::new(),
21128 };
21129
21130 runner.review_loop().await.expect("review_loop");
21135
21136 assert_eq!(
21137 runner.state.reviews.len(),
21138 1,
21139 "no new round was started on top of the unresolved one: {:?}",
21140 runner.state.reviews
21141 );
21142 let last = &runner.state.reviews[0];
21143 assert_eq!(
21144 last.e2e_status(),
21145 E2eStatus::ResourceBlocked,
21146 "still contended: {last:?}"
21147 );
21148 assert!(
21149 last.verified_at.expect("still known") > first_attempt_at,
21150 "review_loop must have actually retried the check, not left it exactly as found"
21151 );
21152 assert_ne!(
21153 runner.state.status,
21154 RunStatus::Blocked,
21155 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21156 runner.state.status
21157 );
21158
21159 held.release();
21160 }
21161
21162 #[tokio::test]
21163 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21164 crate::run::pin_test_home();
21165 for status in [RunStatus::Blocked, RunStatus::Gating] {
21166 let mut runner = runner_at(status);
21167 runner.state.config.graph.land = true;
21168 runner.state.gate_ran = true;
21169 runner.state.merge = Some(MergeOutcome {
21170 mode: MergeMode::Pr,
21171 ok: false,
21172 detail: "the fixer produced no commit".into(),
21173 empty: false,
21174 });
21175 let url = "https://example.invalid/o/r/pull/1";
21176 runner.state.pr = Some(crate::run::PrRecord {
21177 url: url.into(),
21178 number: 1,
21179 state: "open".into(),
21180 checks: "red".into(),
21181 round: 1,
21182 rounds: 1,
21183 red_at_merge: Vec::new(),
21184 });
21185 runner.state.save().unwrap();
21186 runner.execute_graph().await.unwrap();
21190 assert_eq!(runner.state.status, RunStatus::Blocked);
21191 assert!(runner.state.land_sync_pending);
21192 assert!(runner.state.events.iter().any(|event|
21193 event.node == "land" && event.message == format!("watching {url}")));
21194 assert_eq!(
21195 runner.state.merge.as_ref().unwrap().detail,
21196 "the fixer produced no commit"
21197 );
21198 }
21199 }
21200
21201 #[tokio::test]
21202 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21203 crate::run::pin_test_home();
21204 let tmp = tempfile::tempdir().expect("tempdir");
21205 let repo = tmp.path().join("repo");
21206 std::fs::create_dir_all(&repo).unwrap();
21207 init_repo(&repo);
21208
21209 let mut config = Config::default();
21210 config.merge.mode = MergeMode::Pr;
21211 config.graph.land = true;
21212 config.graph.land_approval = false;
21213
21214 let mut state = RunState::new(
21215 repo.clone(),
21216 "main".to_owned(),
21217 "deadbeef".to_owned(),
21218 "task".to_owned(),
21219 config,
21220 );
21221 state.candidates = vec![Candidate {
21222 index: 0,
21223 label: 'A',
21224 agent: "alpha".to_owned(),
21225 branch: "does-not-exist".to_owned(),
21226 worktree: repo.clone(),
21227 summary: String::new(),
21228 stat: String::new(),
21229 files: 0,
21230 commits: 0,
21231 empty: false,
21232 failed: None,
21233 verified_noop: None,
21234 duration_ms: 0,
21235 folded: false,
21236 }];
21237 state.tally = Some(Tally {
21238 first_choice: BTreeMap::from([('A', 1)]),
21239 borda: BTreeMap::new(),
21240 winner: 'A',
21241 rankings: 1,
21242 unanimous_initial: true,
21243 deliberated: false,
21244 changed_votes: 0,
21245 unanimous_final: true,
21246 tie_break: None,
21247 judges: 0,
21248 present: 0,
21249 quorum: 0,
21250 met_quorum: true,
21251 uncontested: Some("only candidate A produced a change".to_owned()),
21252 });
21253 state.reviews = vec![ReviewRound {
21254 round: 1,
21255 head: "deadbeef".to_owned(),
21256 verified_head: None,
21257 verified_at: None,
21258 reviews: Vec::new(),
21259 e2e: Vec::new(),
21260 fix: None,
21261 blocking: 0,
21262 answered: 0,
21263 expected: 0,
21264 clean: true,
21265 verify_retried: false,
21266 e2e_deferred: false,
21267 e2e_defer_reason: None,
21268 progressed: false,
21269 vote_split: false,
21270 reconsideration: Vec::new(),
21271 verdict: None,
21272 }];
21273 state.gate = vec![CommandOutcome {
21274 command: "test".to_owned(),
21275 code: Some(0),
21276 output_tail: String::new(),
21277 duration_ms: 0,
21278 resource_blocked: false,
21279 }];
21280 state.gate_ran = true;
21281 state.status = RunStatus::Landing;
21285 state.merge = Some(MergeOutcome {
21286 mode: MergeMode::Pr,
21287 ok: true,
21288 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21289 empty: false,
21290 });
21291
21292 ask_test_home();
21296 state.save().expect("save landing run");
21299 let store = ask::Questions::open();
21300 let q = ask_open_question(&store, &state.id);
21301
21302 let mut runner = Runner {
21303 state,
21304 roles: ResolvedRoles {
21305 implementers: Vec::new(),
21306 judges: Vec::new(),
21307 reviewers: Vec::new(),
21308 fixer: None,
21309 conductor: conductor(),
21310 implementer_roster: Vec::new(),
21311 judge_roster: Vec::new(),
21312 reviewer_roster: Vec::new(),
21313 },
21314 sem: Arc::new(Semaphore::new(1)),
21315 pause: Pause::new(),
21316 interrupt: Pause::new(),
21317 };
21318
21319 runner.execute().await.expect("execute");
21324
21325 assert_eq!(
21326 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21327 Some("https://example.invalid/x/y/pull/1"),
21328 "reentry must not push again or open a second pull request over the \
21329 one `land` is already watching"
21330 );
21331 assert_ne!(
21332 runner.state.status,
21333 RunStatus::Landing,
21334 "land could not actually reach the fake pull request, so it must \
21335 have given up rather than left the run silently parked forever"
21336 );
21337 assert_eq!(runner.state.status, RunStatus::Blocked);
21341 assert!(
21342 store.get(&q.id).unwrap().status.open(),
21343 "Blocked is still alive; settle_questions must have been a no-op here"
21344 );
21345 }
21346
21347 fn state_with_round(round: ReviewRound) -> RunState {
21348 let mut s = RunState::new(
21349 PathBuf::from("/repo"),
21350 "main".to_owned(),
21351 "abc1234".to_owned(),
21352 "add retries".to_owned(),
21353 Config::default(),
21354 );
21355 s.reviews = vec![round];
21356 s
21357 }
21358
21359 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21360 crate::verdict::Finding {
21361 id: id.to_owned(),
21362 severity,
21363 file: None,
21364 line: None,
21365 title: title.to_owned(),
21366 detail: String::new(),
21367 }
21368 }
21369
21370 #[test]
21371 fn pr_body_names_open_findings_and_declined_ones() {
21372 let round = ReviewRound {
21373 round: 2,
21374 head: "deadbee".to_owned(),
21375 verified_head: None,
21376 verified_at: None,
21377 reviews: vec![ReviewRecord {
21378 attempts: 0,
21379 reviewer: 1,
21380 agent: "alpha".to_owned(),
21381 summary: String::new(),
21382 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21383 vote: None,
21384 failed: None,
21385 duration_ms: 0,
21386 }],
21387 e2e: vec![CommandOutcome {
21388 command: "cargo test".to_owned(),
21389 code: Some(0),
21390 output_tail: String::new(),
21391 duration_ms: 0,
21392 resource_blocked: false,
21393 }],
21394 verify_retried: false,
21395 e2e_deferred: false,
21396 e2e_defer_reason: None,
21397 fix: Some(FixRecord {
21398 agent: "alpha".to_owned(),
21399 addressed: Vec::new(),
21400 rejected: vec![crate::verdict::Rejection {
21401 id: "R1-1-1".to_owned(),
21402 why: "not reachable from any caller".to_owned(),
21403 }],
21404 notes: String::new(),
21405 committed: true,
21406 failed: None,
21407 duration_ms: 0,
21408 continuation: None,
21409 }),
21410 blocking: 0,
21411 answered: 1,
21412 expected: 1,
21413 clean: false,
21414 progressed: true,
21415 vote_split: false,
21416 reconsideration: Vec::new(),
21417 verdict: None,
21418 };
21419 let state = state_with_round(round);
21420 let body = pr_message(&state, 'A').body;
21421
21422 assert!(body.contains("add retries"), "the task must still be there");
21423 assert!(body.contains("R2-1-1"), "{body}");
21424 assert!(!body.contains("unused import"), "{body}");
21425 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21426 assert!(
21427 !body.contains("not reachable from any caller"),
21428 "the reason it was declined: {body}"
21429 );
21430 }
21431
21432 #[test]
21433 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21434 let round = ReviewRound {
21435 round: 1,
21436 head: "deadbee".to_owned(),
21437 verified_head: None,
21438 verified_at: None,
21439 reviews: vec![ReviewRecord {
21440 attempts: 0,
21441 reviewer: 1,
21442 agent: "alpha".to_owned(),
21443 summary: String::new(),
21444 findings: Vec::new(),
21445 vote: None,
21446 failed: None,
21447 duration_ms: 0,
21448 }],
21449 e2e: Vec::new(),
21450 verify_retried: false,
21451 e2e_deferred: false,
21452 e2e_defer_reason: None,
21453 fix: None,
21454 blocking: 0,
21455 answered: 1,
21456 expected: 1,
21457 clean: true,
21458 progressed: false,
21459 vote_split: false,
21460 reconsideration: Vec::new(),
21461 verdict: None,
21462 };
21463 let state = state_with_round(round);
21464 let body = pr_message(&state, 'A').body;
21465 assert!(!body.contains("Open review findings"), "{body}");
21466 assert!(!body.contains("Declined"), "{body}");
21467 }
21468
21469 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21470 let mut state = RunState::new(
21471 PathBuf::from("/repo"),
21472 "main".to_owned(),
21473 "abc1234".to_owned(),
21474 instruction.to_owned(),
21475 Config::default(),
21476 );
21477 state.config.disk.min_free_bytes = 0;
21479 state.candidates.push(Candidate {
21480 index: 0,
21481 label: 'A',
21482 agent: "alpha".to_owned(),
21483 branch: "magi/x/A".to_owned(),
21484 worktree: PathBuf::from("/wt"),
21485 summary: summary.to_owned(),
21486 stat: String::new(),
21487 files: 1,
21488 commits: 1,
21489 empty: false,
21490 failed: None,
21491 verified_noop: None,
21492 folded: false,
21493 duration_ms: 0,
21494 });
21495 state
21496 }
21497
21498 fn review_state(subjects: &[&str]) -> RunState {
21499 let mut state = state_with_summary(
21500 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21501 "",
21502 );
21503 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21504 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21505 state
21506 }
21507
21508 #[tokio::test]
21511 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21512 ask_test_home();
21513 let tmp = tempfile::tempdir().unwrap();
21514 let repo = tmp.path().join("repo");
21515 std::fs::create_dir_all(&repo).unwrap();
21516 init_repo(&repo);
21517 let origin = tmp.path().join("origin.git");
21518 let g = |dir: &Path, args: &[&str]| {
21519 let out = std::process::Command::new("git")
21520 .args(args)
21521 .current_dir(dir)
21522 .quiet()
21523 .output()
21524 .expect("spawn git");
21525 assert!(
21526 out.status.success(),
21527 "git {args:?}: {}",
21528 String::from_utf8_lossy(&out.stderr)
21529 );
21530 };
21531 g(
21532 tmp.path(),
21533 &[
21534 "clone",
21535 "--bare",
21536 "-q",
21537 repo.to_str().unwrap(),
21538 origin.to_str().unwrap(),
21539 ],
21540 );
21541 g(
21542 &repo,
21543 &["remote", "add", "origin", origin.to_str().unwrap()],
21544 );
21545 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21546
21547 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21549 g(&repo, &["add", "-A"]);
21550 g(
21551 &repo,
21552 &["commit", "-q", "-m", "chore(deps): update a crate"],
21553 );
21554 g(&repo, &["push", "-q", "origin", "main"]);
21555 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21556 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21557 g(&repo, &["add", "-A"]);
21558 g(
21559 &repo,
21560 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21561 );
21562 g(&repo, &["checkout", "-q", "main"]);
21563
21564 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21565 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21566 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21568 assert_eq!(fallback, c1);
21569
21570 let mut state = review_state(&[
21571 "chore(deps): update a crate",
21572 "fix(daemon): apply a chosen action",
21573 ]);
21574 state.repo = repo.clone();
21575 state.base_branch = "main".to_owned();
21576 state.base_commit = c1;
21577 refresh_reviewed_commits(&mut state, "feat/own").await;
21578 assert_eq!(
21579 state.reviewed_commits,
21580 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21581 );
21582 assert_eq!(
21583 review_title(&state).as_deref(),
21584 Some("fix(daemon): apply a chosen action")
21585 );
21586 assert_eq!(
21587 leaked_subjects(&state, "feat/own").await,
21588 Some(vec!["chore(deps): update a crate".to_owned()])
21589 );
21590 state.config.merge.remote = "nowhere".to_owned();
21593 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21594 }
21595
21596 struct SummaryFixture {
21600 _tmp: tempfile::TempDir,
21601 repo: PathBuf,
21602 reply: PathBuf,
21603 runner: Runner,
21604 }
21605
21606 fn summary_fixture() -> SummaryFixture {
21607 ask_test_home();
21608 let tmp = tempfile::tempdir().unwrap();
21609 let repo = tmp.path().join("repo");
21610 std::fs::create_dir_all(&repo).unwrap();
21611 init_repo(&repo);
21612 let origin = tmp.path().join("origin.git");
21613 let g = |args: &[&str]| {
21614 let out = std::process::Command::new("git")
21615 .args(args)
21616 .current_dir(&repo)
21617 .quiet()
21618 .output()
21619 .expect("spawn git");
21620 assert!(
21621 out.status.success(),
21622 "git {args:?}: {}",
21623 String::from_utf8_lossy(&out.stderr)
21624 );
21625 };
21626 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21627 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21628 g(&["push", "-q", "origin", "main"]);
21629 g(&["checkout", "-q", "-b", "feat/own"]);
21630 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21631 g(&["add", "-A"]);
21632 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21633 let reply = tmp.path().join("reply.txt");
21634 std::fs::write(
21635 &reply,
21636 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21637 )
21638 .unwrap();
21639 let mut runner = runner_at(RunStatus::Gating);
21640 let mut state = review_state(&["feat(own): add the own file"]);
21641 state.repo = repo.clone();
21642 state.base_branch = "main".to_owned();
21643 state.candidates[0].branch = "feat/own".to_owned();
21644 state.candidates[0].worktree = repo.clone();
21645 let mut author = spec("alpha");
21646 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21647 author
21648 .env
21649 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21650 state.config.agents = vec![author];
21651 runner.state = state;
21652 SummaryFixture {
21653 _tmp: tmp,
21654 repo,
21655 reply,
21656 runner,
21657 }
21658 }
21659
21660 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21661 let winner = f.runner.state.candidates[0].clone();
21662 f.runner
21663 .guarded_pr_message(&winner, None, true)
21664 .await
21665 .unwrap()
21666 .expect("an English description never parks")
21667 }
21668
21669 fn clean_round_with_minor_findings() -> ReviewRound {
21670 let mut round = passing_round(2, None);
21671 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21672 round.reviews = vec![ReviewRecord {
21673 attempts: 0,
21674 reviewer: 1,
21675 agent: "alpha".into(),
21676 summary: "内部レビュー".into(),
21677 findings: vec![
21678 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21679 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21680 ],
21681 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21682 failed: None,
21683 duration_ms: 0,
21684 }];
21685 round
21686 }
21687
21688 #[tokio::test]
21689 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21690 for review_only in [false, true] {
21691 let mut f = summary_fixture();
21692 if !review_only {
21693 f.runner.state.candidates[0].agent = "alpha".into();
21694 }
21695 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21696 assert!(f.runner.state.reviews[0].clean);
21697 assert!(f.runner.state.reviews[0].fix.is_none());
21698 assert!(f.runner.state.open_findings().is_empty());
21699 assert_eq!(
21700 crate::pr_outcome::required_finding_ids(&f.runner.state),
21701 ["R2-1-1", "R2-1-2"]
21702 );
21703 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21704 let evidence = crate::pr_outcome::summary_evidence(
21705 &f.runner.state,
21706 crate::pr_outcome::FinalHead::At(&head),
21707 );
21708 assert!(
21709 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21710 );
21711 let omitted = posted(&mut f).await;
21713 assert!(omitted.body.contains("No description"));
21714 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21715 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21716 assert!(f.runner.state.change_summary.is_none());
21717 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21718 let described = posted(&mut f).await;
21719 assert!(described.body.contains("improve documentation for callers"));
21720 assert!(described.body.contains("remaining edge case"));
21721 assert!(described.body.contains("## Open review findings"));
21722 assert!(!described.body.contains("未修正"));
21723 }
21724 }
21725
21726 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21727 let mut f = summary_fixture();
21728 if !review_only {
21729 f.runner.state.candidates[0].agent = "alpha".into();
21730 }
21731 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21732 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21733 fixed
21734 .fix
21735 .as_mut()
21736 .unwrap()
21737 .rejected
21738 .push(crate::verdict::Rejection {
21739 id: "R1-1-1".into(),
21740 why: "呼び出されないコードのため辞退".into(),
21741 });
21742 let mut clean = clean_round_with_minor_findings();
21743 clean.verified_head = Some(head.clone());
21744 f.runner.state.reviews = vec![fixed, clean];
21745 ok_gate(&mut f.runner.state, Some(&head));
21746 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21747 f.runner.state.config.agents.push(scripted("beta", reply));
21748 f.runner.state.config.roles.language_judge =
21749 Some(crate::config::AgentChoice::One("beta".into()));
21750 f
21751 }
21752
21753 #[tokio::test]
21754 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21755 for review_only in [false, true] {
21756 let mut f = rewrite_fixture_with_findings(review_only,
21757 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21758 let m = posted(&mut f).await;
21759 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21760 let expected =
21761 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21762 assert!(m.body.contains(&expected));
21763 assert!(m.body.contains("## Review rounds"));
21764 assert!(
21765 m.body
21766 .contains("Gate (`verify.gate`): passed on the final commit")
21767 );
21768 assert!(
21769 m.body
21770 .contains("Tests (`verify.e2e`): passed on the final commit")
21771 );
21772 let c = f.runner.state.change_summary.as_ref().unwrap();
21773 assert!(
21774 m.body
21775 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21776 );
21777 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21778 assert!(crate::pr_outcome::covers_required_findings(
21779 &f.runner.state,
21780 &m.body
21781 ));
21782 assert!(m.body.contains("documentation remains incomplete"));
21783 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21784 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21785 assert!(f.runner.state.github_text.is_none());
21786 }
21787 }
21788
21789 #[tokio::test]
21790 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21791 for review_only in [false, true] {
21792 for reply in [
21793 "TITLE: fix: public description\nA short English description.",
21794 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21795 ] {
21796 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21797 let winner = f.runner.state.candidates[0].clone();
21798 let posted = f
21799 .runner
21800 .guarded_pr_message(&winner, None, true)
21801 .await
21802 .unwrap();
21803 assert!(posted.is_none());
21804 assert!(f.runner.state.parked);
21805 let withheld = f.runner.state.github_text.as_ref().unwrap();
21806 assert!(
21807 withheld.rewrite.is_none(),
21808 "an incomplete rewrite must not be selectable"
21809 );
21810 let q = ask::Questions::open()
21811 .get(withheld.question.as_ref().unwrap())
21812 .unwrap();
21813 assert!(
21814 !q.choices
21815 .contains(&crate::github_text::USE_REWRITE.to_owned())
21816 );
21817 assert!(
21818 q.detail.contains("R1-1-1")
21819 && q.detail.contains("R2-1-1")
21820 && q.detail.contains("R2-1-2")
21821 );
21822 }
21823 }
21824 }
21825
21826 #[tokio::test]
21827 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21828 for review_only in [false, true] {
21829 let mut f = summary_fixture();
21830 if !review_only {
21831 f.runner.state.candidates[0].agent = "alpha".into();
21832 }
21833 f.runner.state.instruction = "内部タスクの説明".into();
21834 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21835 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21836 f.runner.state.reviews = vec![
21837 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21838 round_with_notes(2, Some("二回目の内部修正報告です。")),
21839 passing_round(3, Some(&head)),
21840 ];
21841 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21842 ok_gate(&mut f.runner.state, Some(&head));
21843 let captured = f.repo.join("public-prompt.txt");
21844 let author = &mut f.runner.state.config.agents[0];
21845 author.command = vec![
21846 "sh".into(),
21847 "-c".into(),
21848 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21849 ];
21850 author.env.insert(
21851 "PROMPT_FILE".into(),
21852 captured.to_string_lossy().into_owned(),
21853 );
21854 let m = posted(&mut f).await;
21855 assert!(m.body.contains("Adds `own.txt`"));
21856 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21857 assert!(m.body.contains("R1-1-1"));
21858 assert!(
21859 m.body
21860 .contains("Gate (`verify.gate`): passed on the final commit")
21861 );
21862 assert!(
21863 m.body
21864 .contains("Tests (`verify.e2e`): passed on the final commit")
21865 );
21866 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21867 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21868 let prompt = std::fs::read_to_string(captured).unwrap();
21869 assert!(prompt.contains(&format!(
21870 "git diff {} {head}",
21871 f.runner.state.change_summary.as_ref().unwrap().base
21872 )));
21873 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21874 assert_eq!(
21875 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21876 "二回目の内部修正報告です。"
21877 );
21878 }
21879 }
21880
21881 #[tokio::test]
21882 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21883 for review_only in [false, true] {
21884 let mut f = summary_fixture();
21885 if !review_only {
21886 f.runner.state.candidates[0].agent = "alpha".into();
21887 }
21888 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21889 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21890 std::fs::write(&f.reply, "no usable description").unwrap();
21891 let failed = posted(&mut f).await;
21892 assert!(
21893 failed
21894 .body
21895 .contains("No description of what this change does could be generated")
21896 );
21897 assert!(failed.body.is_ascii());
21898 assert!(f.runner.state.github_text.is_none());
21899 assert!(f.runner.state.change_summary.is_none());
21900 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21901 let evidence = crate::pr_outcome::summary_evidence(
21902 &f.runner.state,
21903 crate::pr_outcome::FinalHead::At(&head),
21904 );
21905 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21906 assert!(
21907 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21908 .exists()
21909 );
21910 f.runner.state =
21911 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21912 std::fs::write(
21913 &f.reply,
21914 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21915 )
21916 .unwrap();
21917 let retried = posted(&mut f).await;
21918 assert!(retried.body.contains("Adds the committed own file."));
21919 assert_eq!(
21920 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21921 "日本語の修正報告"
21922 );
21923 }
21924 }
21925
21926 #[tokio::test]
21927 async fn same_head_verification_and_findings_invalidate_public_prose() {
21928 let mut f = summary_fixture();
21929 posted(&mut f).await;
21930 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21931 let original = f
21932 .runner
21933 .state
21934 .change_summary
21935 .as_ref()
21936 .unwrap()
21937 .evidence
21938 .clone();
21939 ok_gate(&mut f.runner.state, Some(&head));
21940 std::fs::write(
21941 &f.reply,
21942 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21943 )
21944 .unwrap();
21945 let verified = posted(&mut f).await;
21946 assert!(verified.body.contains("now verified"));
21947 assert_ne!(
21948 f.runner.state.change_summary.as_ref().unwrap().evidence,
21949 original
21950 );
21951 let mut round = round_with_notes(1, Some("内部報告"));
21952 round.reviews.push(ReviewRecord {
21953 attempts: 0,
21954 reviewer: 1,
21955 agent: "alpha".into(),
21956 summary: "内部レビュー".into(),
21957 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21958 vote: None,
21959 failed: None,
21960 duration_ms: 0,
21961 });
21962 round
21963 .fix
21964 .as_mut()
21965 .unwrap()
21966 .rejected
21967 .push(crate::verdict::Rejection {
21968 id: "R1-1-1".into(),
21969 why: "呼び出し元から到達できない".into(),
21970 });
21971 f.runner.state.reviews = vec![round];
21972 let omitted = posted(&mut f).await;
21974 assert!(omitted.body.contains("No description"));
21975 assert!(omitted.body.contains("R1-1-1"));
21976 assert!(!omitted.body.contains("到達"));
21977 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21978 let described = posted(&mut f).await;
21979 assert!(described.body.contains("unreachable from callers"));
21980 assert!(described.body.contains("R1-1-1"));
21981 assert!(!described.body.contains("到達"));
21982 let evidence = f
21983 .runner
21984 .state
21985 .change_summary
21986 .as_ref()
21987 .unwrap()
21988 .evidence
21989 .clone();
21990 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21991 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21992 assert!(posted(&mut f).await.body.contains("rationale updated"));
21993 assert_ne!(
21994 f.runner.state.change_summary.as_ref().unwrap().evidence,
21995 evidence
21996 );
21997 f.runner
21999 .state
22000 .change_summary
22001 .as_mut()
22002 .unwrap()
22003 .evidence
22004 .clear();
22005 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22006 assert!(posted(&mut f).await.body.contains("current specification"));
22007 }
22008
22009 #[tokio::test]
22010 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22011 let mut f = summary_fixture();
22012 let m = posted(&mut f).await;
22013 assert_eq!(m.title, "feat(own): add the own file");
22014 assert!(
22015 m.body
22016 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22017 "{}",
22018 m.body
22019 );
22020 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22021 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22022 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22023 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22024 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22025 let kept = f.runner.state.change_summary.as_ref().unwrap();
22026 assert_eq!(
22027 (kept.head.as_str(), kept.base.as_str()),
22028 (head.as_str(), base.as_str())
22029 );
22030 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22031 }
22032
22033 #[tokio::test]
22034 async fn review_only_summary_receives_the_configured_github_language() {
22035 let mut f = summary_fixture();
22036 f.runner.state.config.graph.github_language = "ja".into();
22037 let captured = f.repo.join("summary-prompt.txt");
22038 let author = &mut f.runner.state.config.agents[0];
22039 author.command = vec![
22040 "sh".into(),
22041 "-c".into(),
22042 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22043 ];
22044 author.env.insert(
22045 "PROMPT_FILE".into(),
22046 captured.to_string_lossy().into_owned(),
22047 );
22048 std::fs::write(
22049 &f.reply,
22050 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22051 )
22052 .unwrap();
22053 let m = posted(&mut f).await;
22054 let prompt = std::fs::read_to_string(captured).unwrap();
22055 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22056 assert!(!prompt.contains("in English"));
22057 assert_eq!(m.title, "feat(own): ファイルを追加");
22058 assert!(m.body.contains("own.txt を追加します。"));
22059 }
22060
22061 #[tokio::test]
22062 async fn review_fixes_come_after_the_whole_change_summary() {
22063 let mut f = summary_fixture();
22064 f.runner.state.reviews = vec![
22065 round_with_notes(1, Some("handled the PATH case")),
22066 round_with_notes(2, None),
22067 ];
22068 let m = posted(&mut f).await;
22069 let summary = m.body.find("## Summary").expect("summary");
22070 let fixes = m.body.find("## Review rounds").expect("review rounds");
22071 assert!(summary < fixes, "{}", m.body);
22072 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22073 assert!(m.body.contains("## Final verification"), "{}", m.body);
22074 }
22075
22076 #[tokio::test]
22077 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22078 let mut f = summary_fixture();
22079 let first = posted(&mut f).await;
22080 let first_head = f.runner.state.change_summary.clone().unwrap().head;
22081 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22083 let again = posted(&mut f).await;
22084 assert_eq!((again.title, again.body), (first.title, first.body));
22085
22086 let g = |args: &[&str]| {
22087 let out = std::process::Command::new("git")
22088 .args(args)
22089 .current_dir(&f.repo)
22090 .quiet()
22091 .output()
22092 .expect("spawn git");
22093 assert!(out.status.success(), "git {args:?}");
22094 };
22095 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22096 g(&["add", "-A"]);
22097 g(&["commit", "-q", "-m", "fix(own): handle more"]);
22098 let second = posted(&mut f).await;
22099 let kept = f.runner.state.change_summary.clone().unwrap();
22100 assert_ne!(kept.head, first_head);
22101 assert!(second.body.contains("Not read yet."), "{}", second.body);
22102 assert!(!second.body.contains("so the first summary is read"));
22103 assert_eq!(second.title, "feat(own): other");
22104 }
22105
22106 #[tokio::test]
22107 async fn an_unavailable_summary_is_stated_and_never_invented() {
22108 let mut f = summary_fixture();
22109 std::fs::write(&f.reply, "no title line here").unwrap();
22110 let m = posted(&mut f).await;
22111 assert!(
22112 m.body
22113 .contains("No description of what this change does could be generated"),
22114 "{}",
22115 m.body
22116 );
22117 assert!(m.body.contains("internal reports remain in local records"));
22118 assert!(f.runner.state.change_summary.is_none());
22119 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22120 let winner = f.runner.state.candidates[0].clone();
22122 let preview = f
22123 .runner
22124 .guarded_pr_message(&winner, None, false)
22125 .await
22126 .unwrap()
22127 .unwrap();
22128 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22129 }
22130
22131 #[test]
22132 fn pr_message_review_single_commit_uses_its_subject() {
22133 let state = review_state(&["feat(nats): per-role user"]);
22134 let m = pr_message(&state, 'A');
22135 assert_eq!(m.title, "feat(nats): per-role user");
22136 assert!(!m.body.contains("Review the work already"), "{}", m.body);
22137 assert!(m.body.contains("## Commits under review"), "{}", m.body);
22138 }
22139
22140 #[test]
22141 fn pr_message_review_multi_commit_takes_the_oldest() {
22142 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22143 let m = pr_message(&state, 'A');
22144 assert_eq!(m.title, "feat: the change");
22145 for s in ["feat: the change", "fix: typo", "fix: again"] {
22146 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22147 }
22148 }
22149
22150 #[test]
22151 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22152 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22153 let mut state = review_state(&[first, "fix: later fixup"]);
22154 state.candidates[0].branch = "機能/ログイン".to_owned();
22155 let m = pr_message(&state, 'A');
22156 assert!(
22157 m.title.starts_with("chore: land candidate A of run"),
22158 "{}",
22159 m.title
22160 );
22161 }
22162 }
22163
22164 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22165 BranchFacts {
22166 commits: commits
22167 .iter()
22168 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22169 .collect(),
22170 stat: stat.to_owned(),
22171 }
22172 }
22173
22174 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22175 let mut r = review_round(true, 0, 1, 1, true, true);
22176 r.round = round;
22177 r.fix = notes.map(|n| FixRecord {
22178 agent: "fixer".to_owned(),
22179 addressed: Vec::new(),
22180 rejected: Vec::new(),
22181 notes: n.to_owned(),
22182 committed: true,
22183 failed: None,
22184 duration_ms: 0,
22185 continuation: None,
22186 });
22187 r
22188 }
22189
22190 #[test]
22191 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22192 let state = review_state(&["ignored"]);
22193 let f = facts(
22194 &[
22195 (
22196 "fix(login): resolve PATH on macOS",
22197 "Login shells skip rc files.",
22198 ),
22199 ("fix: address review", ""),
22200 ],
22201 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22202 );
22203 let m = pr_message_with(&state, 'A', Some(&f));
22204 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22205 assert!(
22206 m.body
22207 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22208 "{}",
22209 m.body
22210 );
22211 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22212 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22213 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22214 for banned in [
22215 "Review the work already",
22216 "no task statement",
22217 "Original task",
22218 ] {
22219 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22220 }
22221 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22222 }
22223
22224 #[test]
22225 fn pr_message_review_truncates_a_huge_first_commit_body() {
22226 let state = review_state(&["ignored"]);
22227 let f = facts(
22228 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22229 "s",
22230 );
22231 let m = pr_message_with(&state, 'A', Some(&f));
22232 assert!(m.body.len() < 30_000, "{}", m.body.len());
22233 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22234 assert!(
22235 m.body.contains("- ... 1 more commit(s)"),
22236 "{}",
22237 m.body.len()
22238 );
22239 assert!(m.body.ends_with("magi:candidate-a\n"));
22240 }
22241
22242 #[test]
22243 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22244 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22245 assert_eq!(m.title, "feat: x");
22246 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22247 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22248 assert!(!m.body.contains("no task statement"), "{}", m.body);
22249 }
22250
22251 #[test]
22252 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22253 let mut state = review_state(&["日本語の件名"]);
22254 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22255 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22256 state.candidates[0].branch = "機能/ログイン".to_owned();
22257 assert!(
22258 pr_message(&state, 'A')
22259 .title
22260 .starts_with("chore: land candidate A")
22261 );
22262 }
22263
22264 #[test]
22265 fn pr_message_review_fixes_survive_a_clean_final_round() {
22266 let mut state = review_state(&["feat: x"]);
22267 state.reviews = vec![
22268 round_with_notes(1, Some("handled the PATH case")),
22269 round_with_notes(2, None),
22270 ];
22271 let body = pr_message(&state, 'A').body;
22272 assert!(
22273 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22274 "{body}"
22275 );
22276 assert!(body.contains("### Round 1"), "{body}");
22277
22278 state.reviews = vec![
22279 round_with_notes(1, Some("first fix")),
22280 round_with_notes(2, Some("")),
22281 round_with_notes(3, Some("second fix")),
22282 round_with_notes(4, None),
22283 ];
22284 let body = pr_message(&state, 'A').body;
22285 assert!(
22286 body.contains("### Round 1") && !body.contains("first fix"),
22287 "{body}"
22288 );
22289 assert!(
22290 body.contains("### Round 3") && !body.contains("second fix"),
22291 "{body}"
22292 );
22293 assert!(body.contains("### Round 2"), "{body}");
22294 }
22295
22296 const FINAL: &str = "abcdef1234567890";
22297
22298 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22299 let mut r = round_with_notes(round, None);
22300 r.verified_head = verified.map(str::to_owned);
22301 r
22302 }
22303
22304 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22305 state.gate = vec![CommandOutcome {
22306 command: "gate".to_owned(),
22307 code: Some(0),
22308 output_tail: String::new(),
22309 duration_ms: 0,
22310 resource_blocked: false,
22311 }];
22312 state.gate_ran = true;
22313 state.gate_head = head.map(str::to_owned);
22314 }
22315
22316 fn body_at(state: &RunState) -> String {
22317 pr_message_raw(
22318 state,
22319 'A',
22320 None,
22321 crate::pr_outcome::FinalHead::At(FINAL),
22322 ChangeNote::NotAsked,
22323 )
22324 .body
22325 }
22326
22327 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22328
22329 #[test]
22330 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22331 let mut state = review_state(&["feat: x"]);
22332 state.reviews = vec![
22333 round_with_notes(1, Some(SANDBOXED)),
22334 passing_round(2, Some(FINAL)),
22335 ];
22336 ok_gate(&mut state, Some(FINAL));
22337 let body = body_at(&state);
22338 assert!(!body.contains("Handled the PATH case."), "{body}");
22339 assert!(!body.contains("could not commit"), "{body}");
22340 assert!(!body.contains("could not run"), "{body}");
22341 assert!(
22342 body.contains("The fix is committed on the branch"),
22343 "{body}"
22344 );
22345 assert!(!body.contains("sandbox"), "{body}");
22346 assert!(
22347 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22348 "{body}"
22349 );
22350 assert!(
22351 body.contains("Gate (`verify.gate`): passed on the final commit"),
22352 "{body}"
22353 );
22354 assert!(
22356 state.reviews[0]
22357 .fix
22358 .as_ref()
22359 .unwrap()
22360 .notes
22361 .contains("could not commit")
22362 );
22363 }
22364
22365 #[test]
22366 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22367 let mut state = review_state(&["feat: x"]);
22368 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22369 r1.fix.as_mut().unwrap().committed = false;
22370 let mut r2 = passing_round(2, Some(FINAL));
22371 r2.e2e[0].code = Some(1);
22372 state.reviews = vec![r1, r2];
22373 let body = body_at(&state);
22374 assert!(!body.contains("could not commit"), "{body}");
22375 assert!(!body.contains("could not run"), "{body}");
22376 assert!(body.contains("This round produced no commit"), "{body}");
22377 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22378 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22379 assert!(!body.contains("passed"), "{body}");
22380 }
22381
22382 #[test]
22383 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22384 let mut state = review_state(&["feat: x"]);
22385 state.reviews = vec![
22386 round_with_notes(1, Some(SANDBOXED)),
22387 passing_round(2, Some("0123456789abcdef")),
22388 ];
22389 ok_gate(&mut state, Some("0123456789abcdef"));
22390 let body = body_at(&state);
22391 assert!(!body.contains("HTTP tests could not run"), "{body}");
22392 assert!(!body.contains("could not commit"), "{body}");
22394 assert!(
22395 body.contains("passed on an earlier commit (`0123456`)"),
22396 "{body}"
22397 );
22398 assert!(!body.contains("passed on the final commit"), "{body}");
22399
22400 ok_gate(&mut state, None);
22402 let body = body_at(&state);
22403 assert!(
22404 body.contains("passed on a commit that was not recorded"),
22405 "{body}"
22406 );
22407 }
22408
22409 #[test]
22410 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22411 let state = state_with_summary(
22412 "add retries to the client",
22413 "TITLE: feat: retries\n\nDid it.",
22414 );
22415 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22416 assert_eq!(m.title, "feat: retries");
22417 assert!(
22418 m.body.contains("<summary>Original task</summary>"),
22419 "{}",
22420 m.body
22421 );
22422 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22423 assert!(
22424 m.body
22425 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22426 "{}",
22427 m.body
22428 );
22429 }
22430
22431 #[test]
22432 fn pr_message_review_bounds_a_long_english_subject() {
22433 let long = format!("feat: {}", "word ".repeat(100));
22434 let m = pr_message(&review_state(&[&long]), 'A');
22435 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22436 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22437 }
22438
22439 #[test]
22440 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22441 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22442 let m = pr_message(&state, 'A');
22443 assert_eq!(m.title, "add retries");
22444 assert!(m.body.contains("<summary>Original task</summary>"));
22445 assert!(!m.body.contains("Commits under review"));
22446 assert_eq!(landing_subject_source(&state), state.instruction);
22447 }
22448
22449 #[test]
22450 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22451 let state = review_state(&["feat: the change", "fix: typo"]);
22452 let source = landing_subject_source(&state);
22453 assert_eq!(land::merge_subject("", &source), "feat: the change");
22454 assert_eq!(
22455 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22456 "feat: the change"
22457 );
22458 assert_eq!(
22460 land::merge_subject("feat: renamed by hand", &source),
22461 "feat: renamed by hand"
22462 );
22463 let blank = review_state(&["日本語"]);
22464 assert!(
22465 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22466 );
22467 }
22468
22469 #[test]
22470 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22471 let state = review_state(&["feat: the change"]);
22472 let source = landing_subject_source(&state);
22473 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22474 assert_eq!(
22475 land::merge_subject(landing_title(&state, old), &source),
22476 "feat: the change"
22477 );
22478 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22479 let task = state_with_summary("add retries", "");
22480 assert_eq!(landing_title(&task, old), old);
22481 }
22482
22483 #[test]
22484 fn pr_message_describes_the_change_not_the_task() {
22485 let state = state_with_summary(
22486 "今回やってほしいこと: results projector を直す",
22487 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22488 );
22489 let m = pr_message(&state, 'A');
22490 assert_eq!(m.title, "fix(web): batch the runs list reads");
22491 assert!(
22492 m.body.starts_with("## Summary\n\n- reads run.json once"),
22493 "{}",
22494 m.body
22495 );
22496 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22497 let task_at = m.body.find("今回やってほしいこと").unwrap();
22498 let details_at = m.body.find("<details>").unwrap();
22499 assert!(
22500 details_at < task_at,
22501 "the task lives inside <details>: {}",
22502 m.body
22503 );
22504 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22505 assert!(m.body.contains("magi:candidate-a"));
22506 }
22507
22508 #[test]
22509 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22510 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22511 let m = pr_message(&state, 'A');
22512 assert_eq!(m.title, "add retries");
22513 assert!(
22514 m.body.contains("## Summary\n\n- did some things"),
22515 "{}",
22516 m.body
22517 );
22518
22519 let none = RunState::new(
22520 PathBuf::from("/repo"),
22521 "main".to_owned(),
22522 "abc1234".to_owned(),
22523 "add retries".to_owned(),
22524 Config::default(),
22525 );
22526 let m = pr_message(&none, 'A');
22527 assert_eq!(m.title, "add retries");
22528 assert!(!m.body.contains("## Summary"), "{}", m.body);
22529 }
22530
22531 #[test]
22532 fn pr_message_refuses_the_candidate_commit_subject() {
22533 for bad in [
22534 "TITLE: magi: candidate A (uncommitted work)",
22535 "TITLE: chore: stuff (uncommitted work)",
22536 "TITLE: ",
22537 ] {
22538 let state = state_with_summary("add retries", bad);
22539 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22540 }
22541 }
22542
22543 #[test]
22544 fn pr_message_bounds_a_very_long_task_and_title() {
22545 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22546 let state = state_with_summary(&long, "- nothing");
22547 let m = pr_message(&state, 'A');
22548 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22549 assert!(!m.title.contains('\n'));
22550
22551 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22552 let m = pr_message(&state, 'A');
22553 assert!(m.title.starts_with("feat: "));
22554 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22555 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22556 }
22557
22558 fn long_title_of(instruction: &str) -> String {
22559 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22560 }
22561
22562 #[test]
22563 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22564 let first = "Make the landing path keep a readable title for long tasks";
22565 let line = format!(
22566 "{first}. {}",
22567 "And then keep going with more words ".repeat(20)
22568 );
22569 let t = long_title_of(&line);
22570 assert_eq!(t, first);
22571 assert!(!t.starts_with("chore: land"));
22572 }
22573
22574 #[test]
22575 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22576 let line = "word ".repeat(200);
22577 let t = long_title_of(&line);
22578 assert!(t.ends_with("word..."), "{t}");
22579 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22580 }
22581
22582 #[test]
22583 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22584 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22585 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22586 }
22587 }
22588
22589 #[test]
22590 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22591 let mut state = state_with_summary("再試行を追加する", "- nothing");
22592 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22593 state.config.graph.github_language = "ja".into();
22594 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22595 }
22596
22597 #[test]
22598 fn pr_message_title_limit_is_exact() {
22599 let at = "a".repeat(PR_TITLE_MAX);
22600 assert_eq!(long_title_of(&at), at);
22601 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22602 assert!(over.ends_with("..."), "{over}");
22603 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22604 }
22605
22606 #[test]
22607 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22608 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22609 let t = long_title_of(&line);
22610 assert!(t.ends_with("..."), "{t}");
22611 assert!(t.is_ascii(), "{t}");
22612 }
22613
22614 #[test]
22615 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22616 let line = format!(
22617 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22618 "plus more filler words ".repeat(20)
22619 );
22620 let t = long_title_of(&line);
22621 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22622 }
22623
22624 #[test]
22625 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22626 let line = format!("\n\n# {}", "title words ".repeat(40));
22627 let state = state_with_summary(&line, "- nothing");
22628 let m = pr_message(&state, 'A');
22629 assert!(m.title.starts_with("title words"), "{}", m.title);
22630 assert_eq!(
22631 land::merge_subject(&m.title, &landing_subject_source(&state)),
22632 m.title
22633 );
22634 assert_eq!(
22636 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22637 "feat: renamed by hand"
22638 );
22639 }
22640
22641 #[tokio::test]
22642 async fn github_text_rewrite_is_bounded_and_falls_back() {
22643 crate::run::pin_test_home();
22644 for (reply, accepted) in [
22645 (
22646 "TITLE: fix: retries\nAdd retries for failed requests.",
22647 true,
22648 ),
22649 (
22650 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22651 false,
22652 ),
22653 (
22654 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22655 true,
22656 ),
22657 (
22658 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22659 true,
22660 ),
22661 (
22662 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22663 true,
22664 ),
22665 (
22666 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22667 false,
22668 ),
22669 ("TITLE: fix: retries\n", false),
22670 ] {
22671 let dir = tempfile::tempdir().unwrap();
22672 let mut runner = runner_at(RunStatus::Gating);
22673 runner.state = state_with_summary(
22674 "add retries",
22675 "TITLE: fix: retries\n日本語の説明を書きます。",
22676 );
22677 runner.state.repo = dir.path().to_owned();
22678 runner.state.candidates[0].worktree = dir.path().to_owned();
22679 let mut author = spec("alpha");
22680 author.command = vec![
22681 "sh".into(),
22682 "-c".into(),
22683 "printf '%s' \"$REWRITE_REPLY\"".into(),
22684 ];
22685 author.env.insert("REWRITE_REPLY".into(), reply.into());
22686 runner.state.config.agents = vec![author];
22687 seed_public_summary(&mut runner);
22688 let winner = runner.state.candidates[0].clone();
22689 let first = runner
22690 .guarded_pr_message(&winner, None, true)
22691 .await
22692 .unwrap();
22693 let message = if accepted {
22694 first.expect("a passing rewrite never asks")
22695 } else {
22696 assert!(first.is_none());
22698 assert!(runner.state.parked);
22699 let store = ask::Questions::open();
22700 let asked: Vec<_> = store
22701 .list()
22702 .into_iter()
22703 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22704 .collect();
22705 assert_eq!(asked.len(), 1);
22706 assert!(!asked[0].detail.contains("secret"));
22707 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22708 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22709 assert_eq!(
22710 asked[0]
22711 .choices
22712 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22713 rewrite.is_some()
22714 );
22715 store
22718 .update(&asked[0].id, |q| {
22719 q.abandon("test");
22720 Ok(())
22721 })
22722 .unwrap();
22723 let again = runner
22724 .guarded_pr_message(&winner, None, true)
22725 .await
22726 .unwrap()
22727 .expect("an abandoned question falls back");
22728 let n = store
22729 .list()
22730 .into_iter()
22731 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22732 .count();
22733 assert_eq!(n, 1, "asked once per rejected text");
22734 again
22735 };
22736 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22737 assert!(!message.body.contains("secret"));
22738 assert!(!message.body.contains("prefix"));
22739 assert!(!message.title.contains("secret"));
22740 assert_eq!(
22741 message.body.contains("Add retries for failed requests."),
22742 accepted
22743 );
22744 assert_eq!(
22745 runner.state.seats["github-text-rewrite"].turns, 1,
22746 "only one rewrite invocation"
22747 );
22748 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22749 if accepted {
22750 assert!(runner.state.github_text.is_none());
22751 assert!(!runner.state.parked);
22752 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22753 assert!(message.body.contains("## Final verification"));
22754 assert!(message.body.contains("Evidence: final diff"));
22755 assert!(
22756 message
22757 .body
22758 .contains(&format!("magi:run/{}", runner.state.id))
22759 );
22760 }
22761 if !accepted {
22762 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22763 assert!(
22764 message
22765 .body
22766 .contains(&format!("magi:run/{}", runner.state.id))
22767 );
22768 }
22769 }
22770 }
22771
22772 fn scripted(id: &str, reply: &str) -> AgentSpec {
22773 let mut a = spec(id);
22774 a.command = vec![
22775 "sh".into(),
22776 "-c".into(),
22777 "printf '%s' \"$REWRITE_REPLY\"".into(),
22778 ];
22779 a.env.insert("REWRITE_REPLY".into(), reply.into());
22780 a
22781 }
22782
22783 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22784 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22785
22786 fn seed_public_summary(runner: &mut Runner) {
22788 let repo = &runner.state.repo;
22789 let g = |args: &[&str]| {
22790 let out = std::process::Command::new("git")
22791 .args(args)
22792 .current_dir(repo)
22793 .quiet()
22794 .output()
22795 .unwrap();
22796 assert!(out.status.success(), "git {args:?}");
22797 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22798 };
22799 g(&["init", "-q", "-b", "main"]);
22800 g(&[
22801 "-c",
22802 "user.name=Test",
22803 "-c",
22804 "user.email=test@example.invalid",
22805 "commit",
22806 "-q",
22807 "--allow-empty",
22808 "-m",
22809 "feat: fixture",
22810 ]);
22811 let head = g(&["rev-parse", "HEAD"]);
22812 g(&["update-ref", "refs/remotes/origin/main", &head]);
22813 let evidence = crate::pr_outcome::summary_evidence(
22814 &runner.state,
22815 crate::pr_outcome::FinalHead::At(&head),
22816 );
22817 let generated = &runner.state.candidates[0].summary;
22818 runner.state.change_summary = Some(crate::run::ChangeSummary {
22819 base: head.clone(),
22820 head,
22821 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22822 title: summary_title(generated).unwrap(),
22823 body: summary_without_title(generated),
22824 });
22825 }
22826
22827 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22828 let mut runner = runner_at(RunStatus::Gating);
22829 runner.state = state_with_summary("add retries", JA_SUMMARY);
22830 if review_only {
22831 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22832 }
22833 runner.state.repo = dir.to_owned();
22834 runner.state.candidates[0].worktree = dir.to_owned();
22835 runner.state.config.agents = agents;
22836 seed_public_summary(&mut runner);
22837 runner
22838 }
22839
22840 fn gate_questions(runner: &Runner) -> usize {
22841 ask::Questions::open()
22842 .list()
22843 .into_iter()
22844 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22845 .count()
22846 }
22847
22848 #[tokio::test]
22849 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22850 crate::run::pin_test_home();
22851 let dir = tempfile::tempdir().unwrap();
22852 let mut runner = rewrite_runner(
22853 dir.path(),
22854 true,
22855 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22856 );
22857 runner.state.config.roles.language_judge =
22858 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22859 let winner = runner.state.candidates[0].clone();
22860 let msg = runner
22861 .guarded_pr_message(&winner, None, true)
22862 .await
22863 .unwrap()
22864 .expect("an English rewrite posts without asking");
22865 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22866 assert!(msg.body.contains("<details>kept</details>"));
22867 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22868 assert_eq!(gate_questions(&runner), 0);
22869 assert!(runner.state.github_text.is_none());
22870 assert!(!runner.state.seats.contains_key("impl-A"));
22871 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22872 }
22873
22874 #[tokio::test]
22875 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22876 crate::run::pin_test_home();
22877 let dir = tempfile::tempdir().unwrap();
22878 let mut runner = rewrite_runner(
22879 dir.path(),
22880 false,
22881 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22882 );
22883 runner.state.config.roles.language_judge =
22884 Some(crate::config::AgentChoice::One("beta".into()));
22885 runner.state.seats.insert(
22886 "impl-A".into(),
22887 SeatState::new("impl-A", "alpha", runner.state.seed),
22888 );
22889 let winner = runner.state.candidates[0].clone();
22890 let msg = runner
22891 .guarded_pr_message(&winner, None, true)
22892 .await
22893 .unwrap()
22894 .expect("rewritten");
22895 assert!(msg.body.contains("<details>kept</details>"));
22896 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22897 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22898 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22899 }
22900
22901 #[tokio::test]
22902 async fn rewrite_reuses_a_resumable_implementer_session() {
22903 crate::run::pin_test_home();
22904 let dir = tempfile::tempdir().unwrap();
22905 let mut runner = rewrite_runner(
22906 dir.path(),
22907 false,
22908 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22909 );
22910 runner.state.config.roles.language_judge =
22911 Some(crate::config::AgentChoice::One("beta".into()));
22912 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22913 seat.turns = 1;
22914 runner.state.seats.insert("impl-A".into(), seat);
22915 let winner = runner.state.candidates[0].clone();
22916 runner
22917 .guarded_pr_message(&winner, None, true)
22918 .await
22919 .unwrap()
22920 .expect("rewritten");
22921 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22922 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22923 }
22924
22925 #[tokio::test]
22926 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22927 crate::run::pin_test_home();
22930 let dir = tempfile::tempdir().unwrap();
22931 let mut exhausted = spec("alpha");
22932 exhausted.command = vec!["false".into()];
22933 let mut runner = rewrite_runner(
22934 dir.path(),
22935 false,
22936 vec![exhausted, scripted("beta", EN_REWRITE)],
22937 );
22938 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22940 "alpha".into(),
22941 "beta".into(),
22942 ]));
22943 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22944 seat.turns = 1;
22945 runner.state.seats.insert("impl-A".into(), seat);
22946 let winner = runner.state.candidates[0].clone();
22947 let msg = runner
22948 .guarded_pr_message(&winner, None, true)
22949 .await
22950 .unwrap()
22951 .expect("the role rewrites after the implementer fails");
22952 assert!(msg.body.contains("`src/net.rs`"));
22953 assert_eq!(
22954 runner.state.seats["impl-A"].turns, 2,
22955 "the implementer was tried once"
22956 );
22957 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22958 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22959 }
22960
22961 #[tokio::test]
22962 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22963 crate::run::pin_test_home();
22964 let dir = tempfile::tempdir().unwrap();
22965 let mut broken = spec("bad");
22966 broken.command = vec!["false".into()];
22967 let mut runner = rewrite_runner(
22968 dir.path(),
22969 true,
22970 vec![
22971 broken,
22972 scripted("noformat", "Add retries."),
22973 scripted("good", EN_REWRITE),
22974 ],
22975 );
22976 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22977 "bad".into(),
22978 "noformat".into(),
22979 "good".into(),
22980 ]));
22981 let winner = runner.state.candidates[0].clone();
22982 let msg = runner
22983 .guarded_pr_message(&winner, None, true)
22984 .await
22985 .unwrap()
22986 .expect("the third agent rewrites");
22987 assert!(msg.body.contains("`src/net.rs`"));
22988 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22989 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22990 }
22991
22992 #[tokio::test]
22993 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22994 crate::run::pin_test_home();
22995 for (agents, role, saved) in [
22997 (
22998 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22999 Some("beta"),
23000 true,
23001 ),
23002 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23003 (Vec::new(), None, false),
23004 ] {
23005 let dir = tempfile::tempdir().unwrap();
23006 let mut runner = rewrite_runner(dir.path(), true, agents);
23007 runner.state.config.roles.language_judge =
23008 role.map(|r| crate::config::AgentChoice::One(r.into()));
23009 let winner = runner.state.candidates[0].clone();
23010 let first = runner
23011 .guarded_pr_message(&winner, None, true)
23012 .await
23013 .unwrap();
23014 assert!(first.is_none(), "withheld text parks the run");
23015 assert!(runner.state.parked);
23016 assert_eq!(gate_questions(&runner), 1);
23017 let g = runner.state.github_text.as_ref().unwrap();
23018 assert_eq!(g.rewrite.is_some(), saved);
23019 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23020 }
23021 }
23022
23023 #[tokio::test]
23024 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23025 crate::run::pin_test_home();
23026 let dir = tempfile::tempdir().unwrap();
23027 let mut runner = runner_at(RunStatus::Gating);
23028 runner.state =
23029 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23030 runner.state.repo = dir.path().to_owned();
23031 runner.state.candidates[0].worktree = dir.path().to_owned();
23032 let mut author = spec("alpha");
23033 author.command = vec![
23034 "sh".into(),
23035 "-c".into(),
23036 "printf '%s' \"$REWRITE_REPLY\"".into(),
23037 ];
23038 author.env.insert(
23039 "REWRITE_REPLY".into(),
23040 format!(
23041 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23042 "English sentence. ".repeat(600)
23043 ),
23044 );
23045 runner.state.config.agents = vec![author];
23046 seed_public_summary(&mut runner);
23047 let winner = runner.state.candidates[0].clone();
23048 assert!(
23049 runner
23050 .guarded_pr_message(&winner, None, true)
23051 .await
23052 .unwrap()
23053 .is_none()
23054 );
23055 let record = runner.state.github_text.clone().unwrap();
23056 let rewrite = record.rewrite.clone().unwrap();
23057 assert_eq!(rewrite.failed, ["body-language"]);
23058 assert!(
23059 rewrite
23060 .body
23061 .ends_with(&format!("magi:run/{}", runner.state.id))
23062 );
23063 let store = ask::Questions::open();
23064 let q = store.get(record.question.as_ref().unwrap()).unwrap();
23065 assert!(
23066 q.choices
23067 .contains(&crate::github_text::USE_REWRITE.to_owned())
23068 );
23069 for label in [
23070 "ORIGINAL",
23071 "FALLBACK",
23072 "REWRITE",
23073 "Rules still failed: body-language",
23074 "truncated",
23075 ] {
23076 assert!(q.detail.contains(label), "{label}");
23077 }
23078 let artifact = std::fs::read_to_string(crate::run::artifact_path(
23079 &runner.state,
23080 &crate::github_text::artifact_name(&record.fingerprint),
23081 ))
23082 .unwrap();
23083 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23084 runner.state =
23086 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23087 store
23088 .update(&q.id, |q| {
23089 q.say("まだ日本語")?;
23090 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23091 })
23092 .unwrap();
23093 assert!(
23094 runner
23095 .guarded_pr_message(&winner, None, true)
23096 .await
23097 .unwrap()
23098 .is_none()
23099 );
23100 let retry = runner.state.github_text.as_ref().unwrap();
23101 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23102 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23103 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23104 assert!(
23105 q.posting_comparison
23106 .as_ref()
23107 .unwrap()
23108 .matches_detail(&q.detail)
23109 );
23110 store
23111 .update(&q.id, |q| {
23112 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23113 })
23114 .unwrap();
23115 let message = runner
23116 .guarded_pr_message(&winner, None, true)
23117 .await
23118 .unwrap()
23119 .unwrap();
23120 assert_eq!(message.title, rewrite.title);
23121 assert_eq!(message.body, rewrite.body);
23122 assert_eq!(
23123 runner.state.github_text.as_ref().unwrap().choice,
23124 Some(GithubTextChoice::Rewrite)
23125 );
23126 let again = runner
23127 .guarded_pr_message(&winner, None, true)
23128 .await
23129 .unwrap()
23130 .unwrap();
23131 assert_eq!(again.title, message.title);
23132 assert_eq!(again.body, message.body);
23133 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23134 assert!(
23135 !runner
23136 .state
23137 .events
23138 .iter()
23139 .any(|e| e.message.contains(&rewrite.title)
23140 || e.message.contains("English sentence."))
23141 );
23142 }
23143
23144 #[tokio::test]
23145 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23146 crate::run::pin_test_home();
23147 let dir = tempfile::tempdir().unwrap();
23148 let mut runner = runner_at(RunStatus::Gating);
23149 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23150 runner.state.repo = dir.path().to_owned();
23151 runner.state.candidates[0].worktree = dir.path().to_owned();
23152 seed_public_summary(&mut runner);
23153 let winner = runner.state.candidates[0].clone();
23154 assert!(
23155 runner
23156 .guarded_pr_message(&winner, None, true)
23157 .await
23158 .unwrap()
23159 .is_none()
23160 );
23161 let record = runner.state.github_text.as_ref().unwrap();
23162 assert!(record.rewrite.is_none());
23163 let mut old = serde_json::to_value(record).unwrap();
23165 old.as_object_mut().unwrap().remove("rewrite");
23166 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23167 assert!(old.rewrite.is_none());
23168 let id = record.question.clone().unwrap();
23169 ask::Questions::open()
23170 .update(&id, |q| {
23171 q.choices.push(crate::github_text::USE_REWRITE.into());
23172 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23173 })
23174 .unwrap();
23175 let message = runner
23176 .guarded_pr_message(&winner, None, true)
23177 .await
23178 .unwrap()
23179 .unwrap();
23180 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23181 assert_eq!(
23182 runner.state.github_text.as_ref().unwrap().choice,
23183 Some(GithubTextChoice::Fallback)
23184 );
23185 }
23186
23187 #[tokio::test]
23188 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23189 crate::run::pin_test_home();
23190 let dir = tempfile::tempdir().unwrap();
23191 let mut runner = runner_at(RunStatus::Gating);
23192 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23193 runner.state.repo = dir.path().to_owned();
23194 runner.state.candidates[0].worktree = dir.path().to_owned();
23195 seed_public_summary(&mut runner);
23196 let winner = runner.state.candidates[0].clone();
23197 assert!(
23198 runner
23199 .guarded_pr_message(&winner, None, true)
23200 .await
23201 .unwrap()
23202 .is_none()
23203 );
23204 let store = ask::Questions::open();
23205 let find = |run: &str| {
23206 store
23207 .list()
23208 .into_iter()
23209 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23210 .unwrap()
23211 };
23212 let run = runner.state.id.clone();
23213 let q = find(&run);
23214 let fp = runner
23215 .state
23216 .github_text
23217 .as_ref()
23218 .unwrap()
23219 .fingerprint
23220 .clone();
23221 let name = crate::github_text::artifact_name(&fp);
23222 let path = crate::run::artifact_path(&runner.state, &name);
23223 assert!(q.detail.contains(&path.display().to_string()));
23224 assert!(q.detail.contains("title-language: title"));
23225 let saved = std::fs::read_to_string(&path).expect("artifact written");
23226 assert!(saved.contains("日本語のタイトル"));
23227 assert!(q.cwd.is_none());
23228 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23229 let say = |id: &str, text: &str, choice: &str| {
23231 store
23232 .update(id, |q| {
23233 q.thread.push(ask::Turn {
23234 who: ask::Who::Operator,
23235 body: text.to_owned(),
23236 at: jiff::Timestamp::now(),
23237 note: None,
23238 });
23239 q.answer(ask::Answer::Choice(choice.to_owned()))
23240 })
23241 .unwrap();
23242 };
23243 say(&q.id, "まだ日本語", "use my text");
23244 assert!(
23245 runner
23246 .guarded_pr_message(&winner, None, true)
23247 .await
23248 .unwrap()
23249 .is_none()
23250 );
23251 let second = find(&run);
23252 assert_ne!(second.id, q.id);
23253 assert!(second.detail.contains("did not pass"));
23254 assert!(second.detail.contains("日本語のタイトル"));
23257 assert!(!second.detail.contains("まだ日本語"));
23258 assert!(second.detail.contains(&path.display().to_string()));
23259 say(&second.id, "fix: retry failed requests", "use my text");
23260 let message = runner
23261 .guarded_pr_message(&winner, None, true)
23262 .await
23263 .unwrap()
23264 .expect("a vetted replacement is posted");
23265 assert_eq!(message.title, "fix: retry failed requests");
23266 assert_eq!(
23267 runner
23268 .state
23269 .github_text
23270 .as_ref()
23271 .unwrap()
23272 .chosen_title
23273 .as_deref(),
23274 Some("fix: retry failed requests")
23275 );
23276 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23277 let again = runner
23279 .guarded_pr_message(&winner, None, true)
23280 .await
23281 .unwrap()
23282 .unwrap();
23283 assert_eq!(again.title, message.title);
23284 assert_eq!(
23285 store
23286 .list()
23287 .into_iter()
23288 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23289 .count(),
23290 2
23291 );
23292 }
23293
23294 #[test]
23295 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23296 let mut state = state_with_summary(
23300 "add retries",
23301 "TITLE: fix(web): batch reads\n- reads run.json once",
23302 );
23303 state.config.graph.language = "ja".to_owned();
23304 let m = pr_message(&state, 'A');
23305 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23306 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23307
23308 let task = "今回やってほしいこと: results projector を直す";
23311 let mut state = state_with_summary(task, "- no title line");
23312 state.config.graph.language = "ja".to_owned();
23313 let m = pr_message(&state, 'A');
23314 assert_eq!(
23315 m.title,
23316 format!("chore: land candidate A of run {}", state.id)
23317 );
23318 assert!(
23319 m.body.contains(&format!(
23320 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23321 )),
23322 "{}",
23323 m.body
23324 );
23325 }
23326
23327 #[test]
23328 fn pr_message_scrubs_home_paths_and_addresses() {
23329 let state = state_with_summary(
23330 "fix it in /Users/someone/src/x",
23331 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23332 );
23333 let m = pr_message(&state, 'A');
23334 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23335 assert!(!m.body.contains(leak), "{}", m.body);
23336 }
23337 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23338 }
23339
23340 #[test]
23341 fn pr_message_survives_a_task_that_closes_details() {
23342 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23343 let m = pr_message(&state, 'A');
23344 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23345 }
23346
23347 #[test]
23348 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23349 let cmd = manual_merge_command(
23350 MergeStyle::Squash,
23351 Path::new("/repo"),
23352 "b",
23353 "fix: \"quoted\" $(x) `y`\n\nbody",
23354 );
23355 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23356 }
23357
23358 #[test]
23359 fn manual_merge_command_matches_the_configured_style() {
23360 let repo = Path::new("/repo");
23361 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23362
23363 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23364 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23365
23366 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23367 assert_eq!(
23368 squash,
23369 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23370 \"Merge magi run 0832 (candidate A)\""
23371 );
23372
23373 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23374 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23375 }
23376
23377 #[test]
23378 fn a_nudge_gets_a_quarter_of_the_budget() {
23379 assert_eq!(retry_budget(secs(1200), true), secs(300));
23381 assert_eq!(retry_budget(secs(3600), true), secs(900));
23382 }
23383
23384 #[test]
23385 fn a_resent_prompt_keeps_the_whole_budget() {
23386 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23389 assert_eq!(retry_budget(secs(60), false), secs(60));
23390 }
23391
23392 #[test]
23393 fn the_floor_never_exceeds_the_original_budget() {
23394 assert_eq!(retry_budget(secs(60), true), secs(60));
23398 assert_eq!(retry_budget(secs(480), true), secs(120));
23399 assert_eq!(retry_budget(secs(0), true), secs(0));
23400 }
23401
23402 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23403 agent::CommandEvidence {
23404 id: "item1".to_owned(),
23405 description: "cargo test".to_owned(),
23406 exit_code,
23407 result_summary: String::new(),
23408 source: "codex".to_owned(),
23409 }
23410 }
23411
23412 #[test]
23413 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23414 assert!(!has_unconfirmed_command(&[]));
23418 }
23419
23420 #[test]
23421 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23422 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23426 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23427 assert!(!has_unconfirmed_command(&[
23428 evidence(Some(0)),
23429 evidence(Some(101))
23430 ]));
23431 }
23432
23433 #[test]
23434 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23435 assert!(has_unconfirmed_command(&[
23436 evidence(Some(0)),
23437 evidence(None)
23438 ]));
23439 }
23440
23441 #[test]
23442 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23443 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23444 assert_eq!(
23445 verified_noop_claim(true, &[], text).as_deref(),
23446 Some("already fixed by b32cfc4, on main.")
23447 );
23448 }
23449
23450 #[test]
23451 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23452 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23455 assert!(verified_noop_claim(false, &[], text).is_none());
23456 }
23457
23458 #[test]
23459 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23460 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23461 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23462 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23464 }
23465
23466 #[test]
23467 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23468 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23469 }
23470
23471 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23474 runner.state.candidates = shape
23475 .iter()
23476 .enumerate()
23477 .map(|(i, &(empty, verified))| Candidate {
23478 index: i,
23479 label: (b'A' + i as u8) as char,
23480 agent: "sonnet".to_owned(),
23481 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23482 worktree: PathBuf::from(format!("/wt/{i}")),
23483 summary: String::new(),
23484 stat: String::new(),
23485 files: 0,
23486 commits: 0,
23487 empty,
23488 failed: None,
23489 verified_noop: verified.map(str::to_owned),
23490 duration_ms: 0,
23491 folded: false,
23492 })
23493 .collect();
23494 }
23495
23496 #[test]
23497 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23498 ask_test_home();
23499 let mut runner = runner_at(RunStatus::Implementing);
23500 set_candidates(
23501 &mut runner,
23502 &[
23503 (true, Some("already on main at b32cfc4")),
23504 (true, Some("same fix, see the existing test")),
23505 ],
23506 );
23507
23508 runner
23509 .after_implement()
23510 .expect("a verified no-op is not an error");
23511
23512 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23513 }
23514
23515 #[test]
23516 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23517 ask_test_home();
23518 let mut runner = runner_at(RunStatus::Implementing);
23519 set_candidates(
23523 &mut runner,
23524 &[(true, Some("already on main at b32cfc4")), (true, None)],
23525 );
23526
23527 let err = runner
23528 .after_implement()
23529 .expect_err("an unverified empty candidate must still fail the run");
23530
23531 assert!(
23532 err.to_string().contains("no candidate produced a change"),
23533 "{err}"
23534 );
23535 assert_eq!(runner.state.status, RunStatus::Failed);
23536 }
23537
23538 #[test]
23539 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23540 ask_test_home();
23541 let mut runner = runner_at(RunStatus::Implementing);
23542 set_candidates(&mut runner, &[(true, None), (true, None)]);
23543
23544 let err = runner
23545 .after_implement()
23546 .expect_err("no candidate declared anything; this is an ordinary failure");
23547
23548 assert!(
23549 err.to_string().contains("no candidate produced a change"),
23550 "{err}"
23551 );
23552 assert_eq!(runner.state.status, RunStatus::Failed);
23553 }
23554}
23555
23556#[derive(Debug)]
23559struct GraftRestoreFailed(String);
23560
23561impl std::fmt::Display for GraftRestoreFailed {
23562 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23563 write!(
23564 f,
23565 "could not restore the winner to its pre-graft commit: {}",
23566 self.0
23567 )
23568 }
23569}
23570
23571impl std::error::Error for GraftRestoreFailed {}
23572
23573#[cfg(test)]
23577mod resource_wait_tests {
23578 use super::*;
23579 use std::collections::BTreeMap;
23580
23581 fn sh(id: &str, script: &str) -> AgentSpec {
23582 AgentSpec {
23583 id: id.to_owned(),
23584 kind: crate::config::AgentKind::Command,
23585 model: None,
23586 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23587 extra_args: Vec::new(),
23588 env: BTreeMap::new(),
23589 prompt_delivery: None,
23590 max_turn_secs: None,
23591 }
23592 }
23593
23594 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23595 SeatJob {
23596 seat: SeatState::new("impl-A", &spec.id, 1),
23597 spec,
23598 cwd: dir.to_path_buf(),
23599 prompt: "do it".to_owned(),
23600 timeout,
23601 allow_write: true,
23602 sessions: false,
23603 artifacts: dir.join("art"),
23604 stem: "impl-A".to_owned(),
23605 handover: None,
23606 }
23607 }
23608
23609 fn state(resource_wait_secs: u64) -> RunState {
23610 let mut config = Config::default();
23611 config.graph.timeout_resource_wait = resource_wait_secs;
23612 RunState::new(
23613 PathBuf::from("/repo"),
23614 "main".to_owned(),
23615 "abc1234".to_owned(),
23616 "task".to_owned(),
23617 config,
23618 )
23619 }
23620
23621 fn hold(cache: &Path) -> crate::cache::Guard {
23622 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23623 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23624 crate::cache::AcquireOutcome::Acquired(g) => g,
23625 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23626 }
23627 }
23628
23629 async fn run(
23630 jobs: Vec<SeatJob>,
23631 sem: Arc<Semaphore>,
23632 cache: Option<&Path>,
23633 st: &mut RunState,
23634 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23635 let prompts = Prompts::default();
23636 let ctx = WaveCtx {
23637 run: "r",
23638 node: "implement",
23639 prompts: &prompts,
23640 cache,
23641 round: None,
23642 carry_seats: false,
23643 pauses: None,
23644 };
23645 wave(jobs, sem, &ctx, st, 0).await
23646 }
23647
23648 #[tokio::test]
23649 async fn verification_cache_wait_preserves_the_command_budget() {
23650 crate::run::pin_test_home();
23651 let tmp = tempfile::tempdir().unwrap();
23652 let cache = tmp.path().join("target");
23653 let guard = hold(&cache);
23654 let release = tokio::spawn(async move {
23655 tokio::time::sleep(Duration::from_millis(100)).await;
23656 guard.release();
23657 });
23658 let mut st = state(10);
23659 let budget = Duration::from_millis(50);
23660 let wait = repository_wait(&st, None);
23661 let (outcomes, _) = with_cache_lease(
23662 &mut st,
23663 Some(&cache),
23664 "gate",
23665 "gate",
23666 tmp.path(),
23667 "abc1234",
23668 budget,
23669 "test gate",
23670 &wait,
23671 |_, available| async move {
23672 assert_eq!(
23673 available, budget,
23674 "contention must not shorten verification"
23675 );
23676 (Vec::new(), false, Vec::new())
23677 },
23678 )
23679 .await
23680 .unwrap();
23681 release.await.unwrap();
23682 assert!(outcomes.is_empty(), "{outcomes:?}");
23683 assert!(
23684 st.events
23685 .iter()
23686 .any(|event| event.message.contains("waiting for"))
23687 );
23688 assert!(st.resource_wait.is_none());
23689 hold(&cache).release();
23691 }
23692
23693 #[tokio::test]
23694 async fn verification_cache_wait_expires_without_starting_commands() {
23695 crate::run::pin_test_home();
23696 let tmp = tempfile::tempdir().unwrap();
23697 let cache = tmp.path().join("target");
23698 let _held = hold(&cache);
23699 let mut st = state(0);
23700 let wait = repository_wait(&st, None);
23701 let (outcomes, _) = with_cache_lease(
23702 &mut st,
23703 Some(&cache),
23704 "gate",
23705 "gate",
23706 tmp.path(),
23707 "abc1234",
23708 Duration::from_secs(60),
23709 "test gate",
23710 &wait,
23711 |_, _| async { panic!("commands must not start without the cache lease") },
23712 )
23713 .await
23714 .unwrap();
23715 assert_eq!(outcomes.len(), 1);
23716 assert!(outcomes[0].resource_blocked);
23717 assert!(st.resource_wait.is_none());
23718 assert!(matches!(
23719 st.resource_block.unwrap().cause,
23720 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23721 ));
23722 }
23723
23724 #[tokio::test]
23725 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23726 crate::run::pin_test_home();
23727 let tmp = tempfile::tempdir().unwrap();
23728 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23729 let cache = tmp.path().join("cache");
23730 let mut state = state(21600);
23731 let wait = repository_wait(&state, None);
23732 let (outcomes, _) = with_cache_lease(
23733 &mut state,
23734 Some(&cache),
23735 "gate",
23736 "gate",
23737 tmp.path(),
23738 "head",
23739 Duration::from_secs(60),
23740 "gate",
23741 &wait,
23742 |_, _| async { panic!("freshness failure must not start verification") },
23743 )
23744 .await
23745 .unwrap();
23746 assert_eq!(outcomes.len(), 1);
23747 assert!(outcomes[0].resource_blocked);
23748 let block = state.resource_block.unwrap();
23749 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23750 assert!(!block.description().contains("timed out"));
23751 }
23752
23753 #[tokio::test]
23754 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23755 crate::run::pin_test_home();
23756 let tmp = tempfile::tempdir().unwrap();
23757 let cache = tmp.path().join("cache");
23758 let _held = hold(&cache);
23759 let mut state = state(21600);
23760 let pause = Pause::new();
23761 let interrupt = Pause::new();
23762 let cause = ParkCause::Operator;
23763 pause.park_for(cause.clone());
23764 let prompts = Prompts::default();
23765 let ctx = WaveCtx {
23766 run: "test",
23767 node: "fix",
23768 prompts: &prompts,
23769 cache: Some(&cache),
23770 round: None,
23771 carry_seats: false,
23772 pauses: Some((&pause, &interrupt)),
23773 };
23774 let result = wave(
23775 vec![job(
23776 sh("writer", "exit 99"),
23777 tmp.path(),
23778 Duration::from_secs(1),
23779 )],
23780 Arc::new(Semaphore::new(1)),
23781 &ctx,
23782 &mut state,
23783 0,
23784 )
23785 .await;
23786 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23787 assert_eq!(result[0].1.turns, 0);
23788 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23789 assert_eq!(
23790 state.resource_block.unwrap().cause,
23791 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23792 );
23793 assert!(state.active.is_empty());
23794 }
23795
23796 #[tokio::test]
23797 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23798 crate::run::pin_test_home();
23799 let tmp = tempfile::tempdir().unwrap();
23800 let cache = tmp.path().join("target-a");
23801 let guard = hold(&cache);
23802 tokio::spawn(async move {
23804 tokio::time::sleep(Duration::from_millis(1500)).await;
23805 guard.release();
23806 });
23807 let mut st = state(60);
23808 let results = run(
23809 vec![job(
23810 sh("alpha", "echo ok"),
23811 tmp.path(),
23812 Duration::from_secs(1),
23813 )],
23814 Arc::new(Semaphore::new(1)),
23815 Some(&cache),
23816 &mut st,
23817 )
23818 .await;
23819 assert!(
23820 matches!(results[0].2, AgentOutcome::Ok(_)),
23821 "the wait must not be charged to the agent"
23822 );
23823 assert!(st.resource_block.is_none());
23824 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23825 assert!(st.resource_wait.is_none() && st.active.is_empty());
23826 }
23827
23828 #[tokio::test]
23829 #[cfg(unix)]
23830 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23831 use std::os::fd::AsRawFd as _;
23832 crate::run::pin_test_home();
23833 let tmp = tempfile::tempdir().unwrap();
23834 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23835 let common = tmp.path().join(".git");
23836 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23837 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23838 assert_eq!(
23839 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23840 0
23841 );
23842 let mut st = state(1);
23843 st.config.coordination = crate::config::test_coordination();
23844 let mut reader = job(
23845 sh("reader", "echo must-not-run"),
23846 tmp.path(),
23847 Duration::from_secs(30),
23848 );
23849 reader.allow_write = false;
23850 let results = run(
23851 vec![reader.clone()],
23852 Arc::new(Semaphore::new(1)),
23853 None,
23854 &mut st,
23855 )
23856 .await;
23857 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23858 assert_eq!(results[0].1.turns, 0);
23859 assert!(FailClass::of(&results[0].2).is_none());
23860 assert_eq!(
23861 st.resource_block.as_ref().unwrap().kind,
23862 "repository coordination"
23863 );
23864 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23865 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23866 drop(external);
23867 let editor = crate::repo_lock::SeatLease::for_checkout(
23868 tmp.path(),
23869 true,
23870 crate::config::test_coordination().as_ref(),
23871 )
23872 .await
23873 .unwrap();
23874 let pause = Pause::new();
23875 let interrupt = Pause::new();
23876 let prompts = Prompts::default();
23877 let ctx = WaveCtx {
23878 run: "r",
23879 node: "review",
23880 prompts: &prompts,
23881 cache: None,
23882 round: None,
23883 carry_seats: false,
23884 pauses: Some((&pause, &interrupt)),
23885 };
23886 let (results, ()) = tokio::join!(
23887 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23888 async {
23889 tokio::time::sleep(Duration::from_millis(50)).await;
23890 pause.park();
23891 }
23892 );
23893 assert!(
23894 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23895 );
23896 assert_eq!(results[0].1.turns, 0);
23897 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23898 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23899 drop(editor);
23900 }
23901
23902 #[tokio::test]
23903 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23904 crate::run::pin_test_home();
23905 let tmp = tempfile::tempdir().unwrap();
23906 let cache = tmp.path().join("target-b");
23907 let _held = hold(&cache);
23908 let mut st = state(1);
23909 let results = run(
23910 vec![job(
23911 sh("alpha", "echo ok"),
23912 tmp.path(),
23913 Duration::from_secs(1),
23914 )],
23915 Arc::new(Semaphore::new(1)),
23916 Some(&cache),
23917 &mut st,
23918 )
23919 .await;
23920 let (_, seat, out) = &results[0];
23921 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23922 assert!(wave_blocked(&results));
23923 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23924 assert_eq!(seat.turns, 0, "nothing was asked");
23925 assert!(
23926 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23927 "no agent was spawned"
23928 );
23929 assert!(FailClass::of(out).is_none(), "not a failure class");
23930 let block = st.resource_block.as_ref().expect("recorded for the resume");
23931 assert_eq!(block.kind, "shared build cache");
23932 assert!(matches!(
23933 block.cause,
23934 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23935 ));
23936 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23937 assert!(st.active.is_empty());
23938 }
23939
23940 #[tokio::test]
23941 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23942 crate::run::pin_test_home();
23943 let tmp = tempfile::tempdir().unwrap();
23944 let mut st = state(1);
23945 let results = run(
23946 vec![job(
23947 sh("alpha", "echo ok"),
23948 tmp.path(),
23949 Duration::from_secs(1),
23950 )],
23951 Arc::new(Semaphore::new(0)),
23952 None,
23953 &mut st,
23954 )
23955 .await;
23956 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23957 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23958 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23959 }
23960
23961 #[tokio::test]
23962 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23963 crate::run::pin_test_home();
23964 let tmp = tempfile::tempdir().unwrap();
23965 let mut st = state(60);
23966 let results = run(
23967 vec![job(
23968 sh("alpha", "sleep 30"),
23969 tmp.path(),
23970 Duration::from_secs(1),
23971 )],
23972 Arc::new(Semaphore::new(1)),
23973 None,
23974 &mut st,
23975 )
23976 .await;
23977 let out = &results[0].2;
23978 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23979 let class = FailClass::of(out).expect("a failure");
23980 assert_eq!(class, FailClass::Timeout);
23981 assert!(should_hand_over(None, &class), "existing handover policy");
23982 assert!(st.resource_block.is_none());
23983 }
23984}