1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false, None).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true, None).await
1357 }
1358
1359 pub(crate) async fn authorize_base_recovery_control(
1360 home: &Path,
1361 task: &str,
1362 control: &mut crate::base_recovery_control::Transaction,
1363 ) -> Result<()> {
1364 let id = control.request.run.clone();
1365 Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366 }
1367
1368 async fn authorize_base_recovery_request(
1369 home: &Path,
1370 id: &str,
1371 task: &str,
1372 resumption: bool,
1373 mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374 ) -> Result<()> {
1375 let _owner = crate::pr_head::owner_lock(home, id)?;
1376 let mut state = RunState::load_under(id, home)?;
1377 crate::pr_head::ensure_owner(&state)?;
1378 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379 anyhow::ensure!(
1380 queued.execution_runs().last().is_some_and(|run| run == id)
1381 && queued.status != crate::queue::TaskStatus::Done
1382 && !queued.fresh_start
1383 && queued.review_branch.is_none(),
1384 "run task ownership changed"
1385 );
1386 anyhow::ensure!(
1387 state
1388 .origin
1389 .as_ref()
1390 .and_then(|o| o.task.as_deref())
1391 .is_none_or(|owner| owner == task),
1392 "run belongs to another task"
1393 );
1394 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395 anyhow::ensure!(
1396 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397 "run is live or its ownership is unknown"
1398 );
1399 let _execution = crate::driver::maintenance(home, id)?;
1402 anyhow::ensure!(
1403 state.status.resumable()
1404 && state.released_to.is_none()
1405 && state.retired_to.is_none()
1406 && state.pr_taken_over_by.is_none(),
1407 "run ownership is final"
1408 );
1409 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410 anyhow::ensure!(
1411 !state.pr_publication_pending
1412 && !crate::handover::landing_checkpoint(&state)
1413 && state.merge_decision.is_none(),
1414 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415 );
1416 let winner = state.winner().context("run has no retained winner")?;
1417 anyhow::ensure!(
1418 git::is_clean(&winner.worktree).await?,
1419 "retained worktree has uncommitted changes"
1420 );
1421 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422 anyhow::ensure!(
1423 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424 "retained worktree does not match the winner branch"
1425 );
1426 if let Some(control) = control.as_ref() {
1427 anyhow::ensure!(
1428 head.starts_with(&control.request.head),
1429 "retained HEAD changed since the owner request"
1430 );
1431 anyhow::ensure!(
1432 state
1433 .base_sync_recovery
1434 .stop
1435 .as_ref()
1436 .is_none_or(|stop| stop.head == head),
1437 "retained HEAD differs from the recorded stop head; inspect it first"
1438 );
1439 }
1440 state.ensure_base_episode();
1441 state.record_base_sync_stop(&head);
1442 state.save_under(home)?;
1443 let fetched =
1444 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1445 anyhow::ensure!(
1446 fetched.ok(),
1447 "cannot authorize base recovery without a fetched base"
1448 );
1449 let approved_tip = git::rev_parse(
1450 &state.repo,
1451 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1452 )
1453 .await?;
1454 if let Some(control) = control.as_deref_mut() {
1455 let queue = crate::queue::Queue::at(home.join("queue"));
1456 control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1457 if let Some(prior) = state
1458 .base_sync_recovery
1459 .authorizations
1460 .iter()
1461 .find(|a| a.request.as_deref() == Some(&control.key()))
1462 {
1463 anyhow::ensure!(
1464 !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1465 "recovery request was consumed or invalidated; it cannot queue again"
1466 );
1467 return control.enqueue(&queue);
1468 }
1469 }
1470 if resumption {
1471 anyhow::ensure!(
1472 crate::daemon::compatible_run_instruction(&state, &queued),
1473 "resumption instruction does not match the retained run"
1474 );
1475 let approval = queued
1476 .resumption
1477 .as_ref()
1478 .context("no operator resumption")?;
1479 anyhow::ensure!(
1480 approval.instruction == queued.instruction,
1481 "resumption approval does not match the current task instruction"
1482 );
1483 state.authorize_base_sync_resumption(task, approval)?;
1484 } else {
1485 state.authorize_base_sync(task)?;
1486 }
1487 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1488 state
1489 .base_sync_recovery
1490 .authorizations
1491 .last_mut()
1492 .unwrap()
1493 .scope = Some(crate::base_sync_recovery::Scope {
1494 tip: approved_tip,
1495 instruction: queued.instruction.clone(),
1496 episode,
1497 });
1498 if let Some(control) = control.as_deref_mut() {
1499 let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1500 authorization.request = Some(control.key());
1501 authorization.generation = control.generation();
1502 authorization.decision = control.decision();
1503 authorization.question = control
1504 .request
1505 .decision
1506 .as_ref()
1507 .map(|_| control.request.request.clone());
1508 }
1509 state.event(
1510 "base-recovery",
1511 "owner authorized a bounded base synchronization interval; fresh verification required",
1512 );
1513 state.save_under(home)?;
1514 if let Some(control) = control {
1515 control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1516 }
1517 Ok(())
1518 }
1519
1520 pub fn resume(id: &str) -> Result<Self> {
1522 let state = RunState::load(id)?;
1523 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1524 if state.startup.as_ref().is_some_and(|s| !s.complete)
1525 || (state.status == RunStatus::Prep
1526 && state.instruction.trim().is_empty()
1527 && state.candidates.is_empty()
1528 && state.prep_complete == Some(false))
1529 {
1530 bail!(
1531 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1532 );
1533 }
1534 crate::pr_head::ensure_owner(&state)?;
1535 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1536 bail!(
1537 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1538 );
1539 }
1540 if let Some(to) = &state.released_to {
1541 bail!(
1542 "run {} cannot be resumed: its worktree was released to run {}",
1543 state.short(),
1544 crate::run::short_of(to)
1545 );
1546 }
1547 let roles = state.config.resolve_roles()?;
1548 if let Some(id) = &state.config.graph.selected_implementer
1549 && state.candidates.iter().any(|c| c.agent != *id)
1550 {
1551 bail!(
1552 "replacement implementer pin does not match the saved candidates; task must stay held"
1553 );
1554 }
1555 let max_parallel = state.config.graph.max_parallel.max(1);
1556 Ok(Self {
1557 state,
1558 roles,
1559 sem: Arc::new(Semaphore::new(max_parallel)),
1560 pause: Pause::new(),
1561 interrupt: Pause::new(),
1562 })
1563 }
1564
1565 pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1572 Box::pin(self.execute_inner())
1575 }
1576
1577 async fn execute_inner(&mut self) -> Result<()> {
1578 anyhow::ensure!(
1579 self.state.startup.as_ref().is_none_or(|s| s.complete),
1580 "startup incomplete; inspect retained evidence before explicit recovery"
1581 );
1582 if let Some(selection) = &self.state.pr_recovery
1583 && (selection.source_proof.is_some() || selection.source_run.is_some())
1584 {
1585 Box::pin(selection.verify_source(&self.state.repo)).await?;
1586 }
1587 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1590 || self.state.land_approval.is_some()
1591 || self.state.merge_decision.is_some()
1592 || !self.state.merge_repairs.is_empty()
1593 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1594 {
1595 let claim = FixClaim::acquire(&self.state.dir())?;
1596 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1597 let disk = RunState::load(&self.state.id)?;
1598 let checkpoint = |s: &RunState| {
1599 (
1600 s.merge_repairs.len(),
1601 s.merge_repairs
1602 .last()
1603 .map(|r| (r.question.clone(), r.phase)),
1604 s.land_approval
1605 .as_ref()
1606 .map(|a| (a.question.clone(), a.head.clone())),
1607 )
1608 };
1609 if checkpoint(&disk) != checkpoint(&self.state) {
1610 bail!(
1611 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1612 );
1613 }
1614 Some(claim)
1615 } else {
1616 None
1617 };
1618 let mut execution = self.claim_driver().await?;
1619 let runner = &mut *execution.runner;
1620 let wait = runner.repository_wait();
1621 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1624 let result = match result {
1625 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1626 record_repository_block(&mut runner.state, &e);
1627 runner.park_here()?;
1628 Ok(())
1629 }
1630 other => other,
1631 };
1632 execution.driver.finish(&mut runner.state).await?;
1633 let ended = if result.is_err() {
1634 Some({
1635 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1636 if let Err(error) = &result {
1637 notice.message = format!(
1638 "Run {} stopped with an error: {}",
1639 runner.state.id,
1640 error.to_string().lines().next().unwrap_or("unknown error")
1641 );
1642 }
1643 notice
1644 })
1645 } else {
1646 crate::notices::run_ended(&runner.state)
1647 };
1648 if let Some(notice) = ended {
1649 crate::notices::raise(notice);
1650 }
1651 result
1652 }
1653
1654 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1655 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1656 }
1657
1658 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1659 let wait = self.repository_wait();
1660 coordinate_operation(&mut self.state, cwd, &wait).await
1661 }
1662
1663 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1665 let owner = match crate::pr_head::owner_lock_wait(
1666 &crate::run::home(),
1667 &self.state.id,
1668 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1669 )
1670 .await
1671 {
1672 Ok(owner) => owner,
1673 Err(error) => {
1674 let detail = format!("worktree maintenance or ownership update: {error:#}");
1675 self.state.resource_block = Some(crate::run::ResourceBlock {
1679 cause: crate::run::ResourceBlockCause::Other,
1680 node: "start".into(),
1681 kind: "worktree maintenance".into(),
1682 detail: detail.clone(),
1683 at: Timestamp::now(),
1684 });
1685 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1686 }
1687 };
1688 let disk = RunState::load(&self.state.id)?;
1689 crate::pr_head::ensure_owner(&disk)?;
1690 let claimed =
1691 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1692 anyhow::ensure!(
1697 disk.liveness(claimed) != crate::run::Liveness::Live,
1698 "run already has a live driver; wait for its execution to end"
1699 );
1700 if let Some(to) = &disk.released_to {
1701 bail!(
1702 "run {} cannot continue: its worktree was released to run {}",
1703 self.state.short(),
1704 crate::run::short_of(to)
1705 );
1706 }
1707 anyhow::ensure!(
1708 serde_json::to_value(&disk.base_sync_recovery)?
1709 == serde_json::to_value(&self.state.base_sync_recovery)?,
1710 "base recovery changed since this run was loaded; reload the saved run"
1711 );
1712 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1713 let mut execution = DriverExecution {
1714 runner: self,
1715 driver,
1716 };
1717 let _owner = owner;
1720 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1721 .await?;
1722 let state = &mut execution.runner.state;
1723 state.parked = false;
1728 state.resource_wait = None;
1729 state.resource_block = None;
1730 state.clear_active();
1737 let pid = std::process::id();
1752 state.driver_pid = Some(pid);
1753 state.driver_started_at = crate::proc::process_started_at(pid);
1754 state.driver_exited = false;
1755 state.driver_claimed = Some(true);
1756 state.begin_pass();
1757 execution.driver.bind(state);
1758 state.save()?;
1759 execution.driver.record(state)?;
1760 Ok(execution)
1761 }
1762
1763 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1766 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1767 if let Err(error) = &result
1768 && !error.is::<crate::repo_lock::ResourceBlocked>()
1769 && self
1770 .state
1771 .repair_grants
1772 .last()
1773 .is_some_and(|g| g.conflict.is_none())
1774 {
1775 self.state.repair_grants.last_mut().unwrap().phase =
1776 crate::repair_budget::Phase::Complete;
1777 self.state.status = RunStatus::Blocked;
1778 self.state.save()?;
1779 bail!(
1780 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1781 self.state.passes.last().map_or(0, |p| p.index)
1782 );
1783 }
1784 result
1785 }
1786
1787 async fn execute_granted_repair(&mut self) -> Result<()> {
1790 use crate::repair_budget::{Kind, Phase};
1791 let index = self.state.repair_grants.len() - 1;
1792 let winner = self
1793 .state
1794 .winner()
1795 .cloned()
1796 .context("repair winner missing")?;
1797 loop {
1798 let grant = self.state.repair_grants[index].clone();
1799 match grant.phase {
1800 Phase::Pending => {
1801 if grant.spent >= grant.receipt.request.rounds {
1802 self.state.repair_grants[index].phase = Phase::Complete;
1803 self.state.status = RunStatus::Blocked;
1804 self.state.save()?;
1805 return Ok(());
1806 }
1807 self.state.repair_grants[index].spent += 1;
1810 self.state.repair_grants[index].phase = Phase::Fixing;
1811 self.state.save()?;
1812 match grant.receipt.request.kind {
1813 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1814 Kind::Gate => {
1815 let failed = self.state.gate.clone();
1816 let _ = self.gate_fix_round(&winner, &failed).await?;
1817 }
1818 Kind::Review => {
1819 let mut record = self
1820 .state
1821 .reviews
1822 .last()
1823 .cloned()
1824 .context("review failure missing")?;
1825 let round = record.round;
1826 let step = self.run_fix_round(&winner, &mut record).await?;
1827 self.state.reviews[round - 1] = record;
1828 if matches!(step, FixStep::Recorded { report_lost: true }) {
1829 self.state.repair_grants[index].phase = Phase::Complete;
1830 self.state.status = RunStatus::Blocked;
1831 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1832 self.state.save()?;
1833 return Ok(());
1834 }
1835 if matches!(step, FixStep::Resource) {
1836 self.stop_for_resource()?;
1837 return Ok(());
1838 }
1839 self.state.save()?;
1840 }
1841 }
1842 if self.resource_halt()? {
1843 return Ok(());
1844 }
1845 }
1846 Phase::Fixing => {
1847 if grant.receipt.request.kind == Kind::BaseConflict {
1848 self.repair_base_conflict(false).await?;
1849 }
1850 let _coordination = self.coordinate(&winner.worktree).await?;
1853 let rescued = git::rescue_commit(
1854 &winner.worktree,
1855 "magi: preserve interrupted granted repair",
1856 )
1857 .await?;
1858 self.state.note_withheld("repair-grant", &rescued.withheld);
1859 self.state
1862 .verification_history
1863 .push(crate::run::VerificationEvidence {
1864 superseded_by: None,
1865 archived_at: Some(Timestamp::now()),
1866 fixes_spent: Some(self.state.gate_fixes.len()),
1867 gate_head: self.state.gate_head.take(),
1868 gate_ran: std::mem::take(&mut self.state.gate_ran),
1869 gate: std::mem::take(&mut self.state.gate),
1870 e2e: self.state.head_e2e.take(),
1871 });
1872 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1873 self.state
1874 .verification_history
1875 .push(crate::run::VerificationEvidence {
1876 superseded_by: None,
1877 archived_at: Some(Timestamp::now()),
1878 fixes_spent: Some(self.state.gate_fixes.len()),
1879 gate_head: None,
1880 gate_ran: false,
1881 gate: Vec::new(),
1882 e2e: Some(e2e),
1883 });
1884 }
1885 self.state.repair_grants[index].review_start = self.state.reviews.len();
1886 self.state.repair_grants[index].head =
1887 git::rev_parse(&winner.worktree, "HEAD").await?;
1888 self.state.repair_grants[index].phase = Phase::Reviewing;
1889 if grant.receipt.request.kind == Kind::BaseConflict {
1890 self.state.base_sync_recovery.stop = None;
1891 if let Some(sync) = &mut self.state.base_sync {
1892 sync.tip = grant
1893 .receipt
1894 .request
1895 .tip
1896 .clone()
1897 .context("repair base tip missing")?;
1898 sync.conflict = None;
1899 sync.behind = 0;
1900 }
1901 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1902 }
1903 self.state.save()?;
1904 }
1905 Phase::Reviewing => {
1906 self.review_loop_pass().await?;
1907 if self.resource_halt()? {
1908 return Ok(());
1909 }
1910 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1911 let clean = self.state.reviews.len() > grant.review_start
1912 && self.state.reviews.last().is_some_and(|r| {
1913 r.head == head
1914 && r.clean
1915 && r.expected > 0
1916 && r.answered == r.expected
1917 && matches!(
1918 r.e2e_status(),
1919 E2eStatus::Passed | E2eStatus::NotConfigured
1920 )
1921 });
1922 if !clean {
1923 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1925 == Kind::Review
1926 && grant.spent < grant.receipt.request.rounds
1927 && self.state.reviews.last().is_some_and(|r| {
1928 r.fix.is_none()
1929 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1930 }) {
1931 Phase::Pending
1932 } else {
1933 Phase::Complete
1934 };
1935 self.state.status = RunStatus::Blocked;
1936 self.state.event(
1937 "repair-grant",
1938 "fresh review did not clear the repaired head; no publication",
1939 );
1940 self.state.save()?;
1941 if self.state.repair_grants[index].phase == Phase::Pending {
1942 continue;
1943 }
1944 return Ok(());
1945 }
1946 self.state.repair_grants[index].phase = Phase::Gating;
1947 self.state.save()?;
1948 }
1949 Phase::Gating => {
1950 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1951 self.state.reconcile_verification_head(&head);
1952 let wait = self.repository_wait();
1953 if self
1954 .state
1955 .reviews
1956 .last()
1957 .is_none_or(|r| r.head != head || !r.clean)
1958 {
1959 self.state.status = RunStatus::Blocked;
1960 self.state.repair_grants[index].phase = Phase::Complete;
1961 self.state.event(
1962 "repair-grant",
1963 "head changed since fresh review; repair held",
1964 );
1965 self.state.save()?;
1966 return Ok(());
1967 }
1968 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1969 return Ok(());
1970 }
1971 let outcomes = self.run_gate(&winner).await?;
1972 if verify_inconclusive(&outcomes) {
1973 return Ok(());
1974 }
1975 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1976 if after != head {
1977 self.state.repair_grants[index].review_start = self.state.reviews.len();
1978 self.state.repair_grants[index].head = after;
1979 self.state.repair_grants[index].phase = Phase::Reviewing;
1980 self.state.status = RunStatus::Blocked;
1981 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1982 self.state.save()?;
1983 return Ok(());
1984 }
1985 let passed = outcomes.iter().all(CommandOutcome::ok);
1986 if passed
1989 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1990 {
1991 return Ok(());
1992 }
1993 self.state.repair_grants[index].succeeded = passed;
1994 self.state.gate = outcomes;
1995 self.state.gate_ran = true;
1996 self.state.status = if passed {
1997 RunStatus::Gating
1998 } else {
1999 RunStatus::Blocked
2000 };
2001 self.state.repair_grants[index].phase = if !passed
2002 && grant.receipt.request.kind == Kind::Gate
2003 && grant.spent < grant.receipt.request.rounds
2004 {
2005 Phase::Pending
2006 } else {
2007 Phase::Complete
2008 };
2009 if passed && self.state.config.merge.mode == MergeMode::Pr {
2010 self.state.pr_publication_pending = true;
2012 } else if passed {
2013 self.state.status = RunStatus::Ready;
2014 }
2015 self.state.save()?;
2016 if !passed {
2017 continue;
2018 }
2019 if self.state.config.merge.mode != MergeMode::Pr {
2023 self.state.status = RunStatus::Ready;
2024 self.state.event(
2025 "repair-grant",
2026 "repair verified locally; grant carries no merge authorization",
2027 );
2028 self.state.save()?;
2029 return Ok(());
2030 }
2031 return self.merge().await;
2032 }
2033 Phase::Complete => return Ok(()),
2034 }
2035 }
2036 }
2037
2038 async fn execute_graph(&mut self) -> Result<()> {
2039 let archive_home = crate::run::home().canonicalize()?;
2043 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2044 let repo = self.state.repo.clone();
2045 let _coordination = self.coordinate(&repo).await?;
2046 ensure_setup_config(&mut self.state, &repo).await?;
2047 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2048 }
2049 self.state.configure_recovery_resume()?;
2050 if self
2051 .state
2052 .repair_grants
2053 .last()
2054 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2055 {
2056 return self.execute_granted_repair().await;
2057 }
2058 if self
2059 .state
2060 .repair_grants
2061 .last()
2062 .is_some_and(|g| !g.succeeded)
2063 {
2064 self.state.status = RunStatus::Blocked;
2065 self.state.save()?;
2066 return Ok(());
2067 }
2068 self.persist_candidates().await?;
2071 if self.state.base_sync_blocked() {
2074 self.sync_to_base().await?;
2075 if !self.state.base_sync_blocked()
2076 && self.state.status != RunStatus::AlreadyInBase
2077 && !self.resource_halt()?
2078 {
2079 self.finish_after_tally().await?;
2080 }
2081 return Ok(());
2082 }
2083 if self
2084 .state
2085 .merge_decision
2086 .as_ref()
2087 .is_some_and(|decision| !decision.closed)
2088 && self.state.status != RunStatus::Landing
2089 {
2090 if crate::merge_decision::resumable_hold(&self.state) {
2091 self.state.merge = Some(MergeOutcome {
2092 mode: MergeMode::Pr,
2093 ok: false,
2094 detail: self
2095 .state
2096 .merge_decision
2097 .as_ref()
2098 .unwrap()
2099 .target
2100 .pr
2101 .clone(),
2102 empty: false,
2103 });
2104 self.run_land().await?;
2105 self.settle_questions();
2106 return Ok(());
2107 }
2108 bail!(
2109 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2110 );
2111 }
2112 if self.state.config.graph.selected_implementer.is_some()
2113 && self.state.status == RunStatus::Blocked
2114 && self.state.tally.is_none()
2115 && self.state.candidate_preservation.is_empty()
2116 && self.state.interrupted_implementations.is_empty()
2117 {
2118 bail!(
2119 "selected implementation is held; explicitly release the owning task to retry it"
2120 );
2121 }
2122
2123 if self.state.merge_repairs.last().is_some_and(|r| {
2124 matches!(
2125 r.phase,
2126 crate::run::MergeRepairPhase::Requested
2127 | crate::run::MergeRepairPhase::Fixing
2128 | crate::run::MergeRepairPhase::Reviewing
2129 )
2130 }) {
2131 if self.repair_merge_review().await? {
2132 self.run_land().await?;
2133 }
2134 self.settle_questions();
2135 return Ok(());
2136 }
2137 if self.state.pr_publication_pending {
2140 self.merge().await?;
2141 if self.resource_halt()? {
2142 return Ok(());
2143 }
2144 self.state.save()?;
2145 self.settle_questions();
2146 return Ok(());
2147 }
2148 if self.state.status == RunStatus::Stalled {
2161 if self.recover_stall().await? {
2162 self.finish_after_tally().await?;
2163 } else {
2164 self.state.save()?;
2166 }
2167 return Ok(());
2168 }
2169 if self.state.status == RunStatus::Landing
2179 || crate::handover::landing_checkpoint(&self.state)
2180 {
2181 self.state.land_sync_pending = true;
2182 self.state.save()?;
2183 self.run_land().await?;
2184 self.settle_questions();
2189 return Ok(());
2190 }
2191 if self.state.github_text.is_some()
2196 && self.state.merge.is_none()
2197 && self.state.status == RunStatus::Gating
2198 {
2199 self.merge().await?;
2200 if self.resource_halt()? {
2201 return Ok(());
2202 }
2203 self.state.save()?;
2204 return Ok(());
2205 }
2206 if let Err(e) = self.prep().await {
2207 if e.is::<crate::repo_lock::ResourceBlocked>() {
2208 return Err(e);
2209 }
2210 self.state.status = RunStatus::Blocked;
2211 self.state
2212 .prep_failure
2213 .get_or_insert_with(|| format!("{e:#}"));
2214 self.state
2215 .event("prep", format!("preparation stopped: {e:#}"));
2216 self.state.save()?;
2217 return Err(e);
2218 }
2219 if self.park_here()? {
2220 return Ok(());
2221 }
2222 self.advise().await?;
2223 if self.park_here()? {
2224 return Ok(());
2225 }
2226 self.implement().await?;
2227 if self.park_here()? {
2228 return Ok(());
2229 }
2230 if self.state.status == RunStatus::VerifiedNoop {
2234 return Ok(());
2235 }
2236 self.judge().await?;
2237 if self.park_here()? {
2238 return Ok(());
2239 }
2240 self.deliberate().await?;
2241 if self.park_here()? {
2242 return Ok(());
2243 }
2244 self.vote().await?;
2245 if self.park_here()? {
2246 return Ok(());
2247 }
2248 self.tally()?;
2249 if self.state.status == RunStatus::Stalled {
2254 self.state.save()?;
2258 return Ok(());
2259 }
2260 self.finish_after_tally().await?;
2261 Ok(())
2262 }
2263
2264 fn stop_for_resource(&mut self) -> Result<()> {
2269 if self.pause.parked() || self.interrupt.parked() {
2272 self.park_here()?;
2273 return Ok(());
2274 }
2275 if self.state.resource_block.is_some() {
2276 self.state.status = RunStatus::Blocked;
2277 }
2278 self.state.save()
2279 }
2280
2281 fn resource_halt(&mut self) -> Result<bool> {
2283 if self.state.resource_block.is_none() {
2284 return Ok(false);
2285 }
2286 self.stop_for_resource()?;
2287 Ok(true)
2288 }
2289
2290 fn park_here(&mut self) -> Result<bool> {
2297 let parking = self.pause.parked() || self.interrupt.parked();
2298 if !parking && self.state.resource_block.is_some() {
2299 self.stop_for_resource()?;
2300 return Ok(true);
2301 }
2302 if !self.pause.parked() && !self.interrupt.parked() {
2307 return Ok(false);
2308 }
2309 if self.state.parked {
2310 return Ok(true);
2311 }
2312 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2313 if let Some(block) = &mut self.state.resource_block
2314 && matches!(
2315 block.cause,
2316 crate::run::ResourceBlockCause::Cancelled { .. }
2317 )
2318 {
2319 block.cause = crate::run::ResourceBlockCause::Cancelled {
2320 park: cause.clone(),
2321 };
2322 }
2323 let why = match cause.as_ref().map(ParkCause::label) {
2324 Some(reason) => format!(
2325 "parked after `{}` ({reason}) — resume to carry on from here",
2326 self.state.status.as_str()
2327 ),
2328 None => format!(
2329 "parked after `{}` — resume to carry on from here",
2330 self.state.status.as_str()
2331 ),
2332 };
2333 self.state.event("park", why);
2334 self.state
2335 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2336 self.state.parked = true;
2337 self.state.save()?;
2338 Ok(true)
2339 }
2340
2341 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2343 if self.state.prep_failure.is_none() {
2344 return Ok(());
2345 }
2346 let failed_at = self
2347 .state
2348 .events
2349 .iter()
2350 .rev()
2351 .find(|event| event.node == "prep")
2352 .map_or(self.state.updated_at, |event| event.at);
2353 anyhow::ensure!(
2354 approval_at > failed_at,
2355 "preparation retry requires a new explicit release after the failure"
2356 );
2357 self.state.prep_failure = None;
2358 self.state.prep_complete = Some(false);
2359 self.state
2360 .event("prep-retry", "operator approved retry of preparation");
2361 Ok(())
2362 }
2363
2364 pub fn on_pause(&mut self, pause: Pause) {
2366 self.pause = pause;
2367 }
2368
2369 pub fn watch_interrupt(&mut self, pause: Pause) {
2375 self.interrupt = pause;
2376 }
2377
2378 fn settle_questions(&mut self) {
2398 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2399 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2400 }
2401 }
2402
2403 fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2406 Box::pin(self.finish_after_tally_inner())
2407 }
2408
2409 async fn finish_after_tally_inner(&mut self) -> Result<()> {
2410 self.graft().await?;
2413 if self.resource_halt()? {
2414 return Ok(());
2415 }
2416 self.fold_losers().await?;
2417 self.sync_to_base().await?;
2422 if self.hold_base_sync().await? || self.resource_halt()? {
2423 return Ok(());
2424 }
2425 if self.state.status == RunStatus::AlreadyInBase {
2426 return Ok(());
2427 }
2428 self.review_loop().await?;
2429 if self.resource_halt()? {
2430 return Ok(());
2431 }
2432 self.sync_to_base().await?;
2433 if self.hold_base_sync().await? || self.resource_halt()? {
2434 return Ok(());
2435 }
2436 if self.state.status == RunStatus::AlreadyInBase {
2437 return Ok(());
2438 }
2439 self.gate().await?;
2440 self.merge().await?;
2441 if self.resource_halt()? {
2442 return Ok(());
2443 }
2444 self.state.save()?;
2445 Ok(())
2446 }
2447
2448 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2455 let result = match ensure_setup_config(&mut self.state, repo).await {
2456 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2457 Err(e) => Err(e),
2458 };
2459 if let Err(e) = result {
2460 git::worktree_remove_clean(repo, wt).await.ok();
2461 if in_prep {
2462 self.rollback_prep(repo).await;
2463 }
2464 self.state.status = RunStatus::Blocked;
2465 self.state.event(
2466 "setup",
2467 format!("worktree setup failed in {}: {e:#}", wt.display()),
2468 );
2469 self.state.save()?;
2470 return Err(e);
2471 }
2472 Ok(())
2473 }
2474
2475 async fn rollback_prep(&mut self, repo: &Path) {
2477 let root = self.state.worktree_root();
2478 for c in std::mem::take(&mut self.state.candidates) {
2479 if matches!(
2480 git::worktree_remove_clean(repo, &c.worktree).await,
2481 Ok(true)
2482 ) {
2483 git::branch_delete(repo, &c.branch).await.ok();
2484 } else {
2485 self.state.candidates.push(c);
2486 }
2487 }
2488 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2489 for j in 1..=seats {
2490 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2491 .await
2492 .ok();
2493 }
2494 for k in 1..=self.state.config.graph.advisors {
2495 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2496 .await
2497 .ok();
2498 }
2499 }
2500
2501 async fn prep(&mut self) -> Result<()> {
2502 if let Some(reason) = &self.state.prep_failure {
2503 bail!(
2504 "candidate preparation previously failed: {reason}; correct the source before retrying"
2505 );
2506 }
2507 if self
2508 .state
2509 .prep_complete
2510 .unwrap_or(!self.state.candidates.is_empty())
2511 {
2512 return Ok(());
2513 }
2514 let repo = self.state.repo.clone();
2515 let _coordination = self.coordinate(&repo).await?;
2518 let base = self.state.base_commit.clone();
2519 let plan = self.state.sources.plan(&repo, &base).await?;
2520 self.rollback_prep(&repo).await;
2521 if !self.state.candidates.is_empty() {
2522 bail!(
2523 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2524 );
2525 }
2526 self.state.prep_complete = Some(false);
2527 self.state.status = RunStatus::Prep;
2528 self.state.save()?;
2529 let repo = self.state.repo.clone();
2530 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2531 let root = self.state.worktree_root();
2532 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2533
2534 let hooks_dir = self.state.dir().join("hooks");
2537 if self.state.config.blind.commit_msg_hook {
2538 std::fs::create_dir_all(&hooks_dir)
2539 .with_context(|| format!("create {}", hooks_dir.display()))?;
2540 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2541 let path = hooks_dir.join("commit-msg");
2542 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2543 make_executable(&path)?;
2544 git::acquire_worktree_config(&repo).await?;
2552 self.state.enabled_worktree_config = true;
2553 }
2554
2555 ensure_setup_config(&mut self.state, &repo).await?;
2558
2559 for (index, (spec, label)) in self
2560 .roles
2561 .implementers
2562 .clone()
2563 .into_iter()
2564 .zip(labels)
2565 .enumerate()
2566 {
2567 let branch = self.state.branch_for(label);
2568 let worktree = root.join(format!("cand-{label}"));
2569 self.state.event(
2573 "prep-intent",
2574 serde_json::json!({
2575 "branch": branch,
2576 "worktree": worktree,
2577 "start": start,
2578 "picks": plan.picks,
2579 })
2580 .to_string(),
2581 );
2582 self.state.save()?;
2583 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2584 if self.state.config.blind.commit_msg_hook {
2585 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2586 }
2587 git::local_exclude(&worktree, "/.magi/").await?;
2588 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2589 git::worktree_remove_clean(&repo, &worktree).await.ok();
2592 git::branch_delete(&repo, &branch).await.ok();
2593 self.rollback_prep(&repo).await;
2594 self.state.status = RunStatus::Blocked;
2595 self.state
2596 .event("prep", format!("worktree setup failed: {e:#}"));
2597 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2598 self.state.save()?;
2599 return Err(e);
2600 }
2601 self.state.candidates.push(Candidate {
2602 index,
2603 label,
2604 agent: spec.id.clone(),
2605 branch,
2606 worktree,
2607 summary: String::new(),
2608 stat: String::new(),
2609 files: 0,
2610 commits: 0,
2611 empty: false,
2612 failed: None,
2613 verified_noop: None,
2614 duration_ms: 0,
2615 folded: false,
2616 });
2617 self.state.save()?;
2618 for pick in &plan.picks {
2619 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2620 {
2621 self.state.status = RunStatus::Blocked;
2622 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2623 self.state
2624 .event("prep", format!("cannot apply referenced commit: {e}"));
2625 self.state.save()?;
2626 return Err(e);
2627 }
2628 }
2629 }
2630
2631 for j in 1..=self.roles.judges.len() {
2632 let wt = root.join(format!("judge-{j}"));
2633 if !wt.exists() {
2634 git::worktree_add_detached(&repo, &wt, &base).await?;
2635 self.setup_seat_worktree(&repo, &wt, true).await?;
2636 }
2637 }
2638
2639 if self.state.config.graph.advise {
2647 for k in 1..=self.state.config.graph.advisors {
2648 let wt = root.join(format!("advisor-{k}"));
2649 if !wt.exists() {
2650 git::worktree_add_detached(&repo, &wt, &base).await?;
2651 self.setup_seat_worktree(&repo, &wt, true).await?;
2652 }
2653 }
2654 }
2655
2656 let authors: Vec<&str> = self
2661 .roles
2662 .implementers
2663 .iter()
2664 .map(|a| a.id.as_str())
2665 .collect();
2666 let overlap: Vec<String> = self
2667 .roles
2668 .judges
2669 .iter()
2670 .enumerate()
2671 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2672 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2673 .collect();
2674 if !overlap.is_empty() {
2675 let note = format!(
2676 "{} also authored a candidate; blind, but the panel is less \
2677 independent than {} distinct agents would be",
2678 overlap.join(", "),
2679 self.roles.judges.len()
2680 );
2681 self.state.event("prep", note);
2682 }
2683
2684 self.state.event(
2685 "prep",
2686 format!(
2687 "{} candidates, {} judges, base {} ({})",
2688 self.state.candidates.len(),
2689 self.roles.judges.len(),
2690 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2691 self.state.base_branch
2692 ),
2693 );
2694 self.state.prep_complete = Some(true);
2695 self.state.status = RunStatus::Implementing;
2696 self.state.save()?;
2697 Ok(())
2698 }
2699
2700 async fn advise(&mut self) -> Result<()> {
2733 let implement_untouched = self
2734 .state
2735 .candidates
2736 .iter()
2737 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2738 if !self.state.config.graph.advise || self.state.advise_attempted {
2739 return Ok(());
2740 }
2741 if !implement_untouched {
2742 self.state.event(
2743 "advise",
2744 "skipping the design-deliberation stage: at least one \
2745 candidate already shows implementation progress, so this \
2746 run is past the point the stage exists to run before"
2747 .to_owned(),
2748 );
2749 self.state.advise_attempted = true;
2750 self.state.save()?;
2751 return Ok(());
2752 }
2753 let run_id = self.state.id.clone();
2754 let prompts = self.state.config.prompts.clone();
2755 let instruction = self.state.instruction.clone();
2756 let language = self.state.config.graph.language.clone();
2757 let root = self.state.worktree_root();
2758 let n = self.state.config.graph.advisors;
2759 let where_recorded = self.state.dir().join("run.json");
2760
2761 let seats = match self.state.config.advisors() {
2762 Ok(seats) if !seats.is_empty() => seats,
2763 Ok(_) => {
2764 self.state.event(
2765 "advise",
2766 format!(
2767 "[graph] advisors is 0; skipping the design-deliberation \
2768 stage and continuing without a synthesis brief (see {})",
2769 where_recorded.display()
2770 ),
2771 );
2772 self.state.advise_attempted = true;
2773 self.state.save()?;
2774 return Ok(());
2775 }
2776 Err(e) => {
2777 self.state.event(
2778 "advise",
2779 format!(
2780 "could not resolve advisor seats ({e:#}); continuing \
2781 without a design-deliberation brief (see {})",
2782 where_recorded.display()
2783 ),
2784 );
2785 self.state.advise_attempted = true;
2786 self.state.save()?;
2787 return Ok(());
2788 }
2789 };
2790
2791 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2792 let artifacts = agent::artifacts_dir(&self.state.dir());
2793 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2794
2795 let mut jobs = Vec::new();
2796 for (i, spec) in seats.iter().cloned().enumerate() {
2797 let seat_key = format!("advisor-{}", i + 1);
2798 let seat = self.seat(&seat_key, &spec.id);
2799 jobs.push(SeatJob {
2800 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2801 spec,
2802 seat,
2803 cwd: worktrees[i % worktrees.len()].clone(),
2804 timeout,
2805 allow_write: false,
2806 sessions: false,
2807 artifacts: artifacts.clone(),
2808 stem: seat_key,
2809 handover: None,
2810 });
2811 }
2812
2813 self.state.event(
2814 "advise",
2815 format!(
2816 "{} advisor seat(s) sketching a design in parallel",
2817 jobs.len()
2818 ),
2819 );
2820 let mut quota_losses = Vec::new();
2821 let cache = self.state.config.cache_dir();
2822 let ctx = WaveCtx {
2823 carry_seats: false,
2824 pauses: Some((&self.pause, &self.interrupt)),
2825 run: &run_id,
2826 node: "advise",
2827 prompts: &prompts,
2828 cache: cache.as_deref(),
2829 round: None,
2830 };
2831 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2832 let results = ask_json_wave::<Proposal>(
2833 jobs,
2834 Arc::clone(&self.sem),
2835 self.state.config.graph.retries,
2836 &advisor_roster,
2837 &ctx,
2838 &mut quota_losses,
2839 &mut self.state,
2840 &|p: &Proposal| p.validate(),
2841 )
2842 .await;
2843 if self.state.resource_block.is_some() {
2844 return self.stop_for_resource();
2845 }
2846 self.state.quota.extend(quota_losses);
2847
2848 let mut records = Vec::with_capacity(results.len());
2849 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2850 let agent_id = seat.agent.clone();
2851 self.state.seats.insert(seat.key.clone(), seat);
2852 match res {
2853 Ok((proposal, out)) => {
2854 self.state
2855 .event("advise", format!("advisor-{} proposed a design", i + 1));
2856 records.push(advise::AdvisorRecord::proposed(
2857 i + 1,
2858 agent_id,
2859 proposal,
2860 out.duration_ms,
2861 ));
2862 }
2863 Err(e) => {
2864 self.state.event(
2865 "advise",
2866 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2867 );
2868 records.push(advise::AdvisorRecord::failed(
2869 i + 1,
2870 agent_id,
2871 e.to_string(),
2872 ));
2873 }
2874 }
2875 }
2876
2877 let mut advice = advise::Advice {
2878 records,
2879 synthesis: None,
2880 };
2881 if advice.proposals().is_empty() {
2882 self.state.event(
2883 "advise",
2884 "no advisor produced a usable proposal; continuing without a \
2885 synthesis brief"
2886 .to_owned(),
2887 );
2888 } else {
2889 match self
2890 .synthesize_brief(
2891 &advice,
2892 &instruction,
2893 &language,
2894 &worktrees[0],
2895 &artifacts,
2896 &run_id,
2897 &prompts,
2898 cache.as_deref(),
2899 )
2900 .await
2901 {
2902 Ok(Some(text)) => {
2903 self.state.event(
2904 "advise",
2905 "synthesized a design brief for the implementer".to_owned(),
2906 );
2907 advice.synthesis = Some(text);
2908 }
2909 Ok(None) => {
2910 self.state.event(
2911 "advise",
2912 "the synthesis seat produced nothing usable; continuing \
2913 without a design brief"
2914 .to_owned(),
2915 );
2916 }
2917 Err(e) => {
2918 if e.is::<crate::repo_lock::ResourceBlocked>() {
2919 return Err(e);
2920 }
2921 self.state.event(
2922 "advise",
2923 format!("could not synthesize a design brief: {e:#}"),
2924 );
2925 }
2926 }
2927 }
2928 advise::apply_reflection(&mut advice);
2929
2930 self.state.advice = Some(advice);
2931 self.state.advise_attempted = true;
2932 self.state.save()?;
2933 Ok(())
2934 }
2935
2936 #[allow(clippy::too_many_arguments)]
2948 async fn synthesize_brief(
2949 &mut self,
2950 advice: &advise::Advice,
2951 instruction: &str,
2952 language: &str,
2953 cwd: &Path,
2954 artifacts: &Path,
2955 run_id: &str,
2956 prompts: &Prompts,
2957 cache: Option<&Path>,
2958 ) -> Result<Option<String>> {
2959 let chain = agent::pick_chain(
2960 &self.state.config.agents,
2961 self.state.config.roles.synthesizer.as_ref(),
2962 &agent::installed,
2963 "synthesizer",
2964 )?;
2965 let proposals = advice.proposals();
2966 let mut prompt = prompt::with_overlay(
2967 prompt::synthesize_brief(instruction, &proposals, language),
2968 prompts.overlay("advise"),
2969 );
2970 if cache.is_some() {
2971 prompt.push('\n');
2976 prompt.push_str(&prompt::build_cache_note("advise", false));
2977 }
2978 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2979 let mut last = None;
2984 for (n, spec) in chain.iter().enumerate() {
2985 if n > 0 {
2986 self.state
2987 .event("advise", format!("synthesis falling back to {}", spec.id));
2988 }
2989 let mut seat = self.seat("advise-synthesis", &spec.id);
2990 let wait = self.repository_wait();
2991 let outcome = wait
2992 .scope(agent::invoke(
2993 spec,
2994 &mut seat,
2995 &Invocation {
2996 coordination: self.state.config.coordination.as_ref(),
2997 fetched_base: None,
2998 cwd,
2999 prompt: &prompt,
3000 timeout,
3001 allow_write: false,
3002 sessions: false,
3003 artifacts,
3004 stem: &if n == 0 {
3005 "advise-synthesis".to_owned()
3006 } else {
3007 format!("advise-synthesis-{}", spec.id)
3008 },
3009 run: run_id,
3010 node: "advise",
3011 cache_dir: None,
3012 attachments: &[],
3013 writable: &[],
3014 },
3015 ))
3016 .await;
3017 if outcome.is_ok() {
3018 self.state.seats.insert(seat.key.clone(), seat);
3019 }
3020 let advance = agent::chain_advances(&outcome);
3021 last = Some(outcome);
3022 if !advance {
3023 break;
3024 }
3025 }
3026 let out = last.expect("a chain holds at least one agent")?;
3029 if !out.usable() {
3030 return Ok(None);
3031 }
3032 let text =
3033 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3034 Ok((!text.trim().is_empty()).then_some(text))
3035 }
3036
3037 async fn implement(&mut self) -> Result<()> {
3040 self.persist_candidates().await?;
3041 let run_id = self.state.id.clone();
3046 let prompts = self.state.config.prompts.clone();
3047 let todo: Vec<usize> = self
3048 .state
3049 .candidates
3050 .iter()
3051 .enumerate()
3052 .filter(|(_, c)| {
3053 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3054 && c.failed.is_none()
3055 && !c.empty
3056 })
3057 .map(|(i, _)| i)
3058 .collect();
3059 if todo.is_empty() {
3060 return self.after_implement();
3061 }
3062 self.state.status = RunStatus::Implementing;
3063
3064 let language = self.state.config.graph.language.clone();
3065 let github_language = self.state.config.graph.github_language.clone();
3066 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3067 let sessions = self.state.config.graph.sessions;
3068 let artifacts = agent::artifacts_dir(&self.state.dir());
3069 let brief = self
3073 .state
3074 .advice
3075 .as_ref()
3076 .and_then(|a| a.synthesis.as_deref())
3077 .map(str::to_owned);
3078 let attachments = self.state.attachments.clone();
3079
3080 let mut jobs = Vec::new();
3081 for &i in &todo {
3082 let (index, label, worktree) = {
3083 let c = &self.state.candidates[i];
3084 (c.index, c.label, c.worktree.clone())
3085 };
3086 let spec = self.roles.implementers[index].clone();
3087 let seat_key = format!("impl-{label}");
3088 let seat = self.seat(&seat_key, &spec.id);
3089 let instruction = seeded_instruction(&self.state);
3090 jobs.push(SeatJob {
3091 spec,
3092 seat,
3093 prompt: prompt::implement_in(
3094 &instruction,
3095 &worktree.to_string_lossy(),
3096 &language,
3097 &github_language,
3098 brief.as_deref(),
3099 &attachments,
3100 ),
3101 cwd: worktree,
3102 timeout,
3103 allow_write: true,
3104 sessions,
3105 artifacts: artifacts.clone(),
3106 stem: format!("impl-{label}"),
3107 handover: None,
3108 });
3109 }
3110
3111 self.state.event(
3112 "implement",
3113 format!("{} candidates in parallel", jobs.len()),
3114 );
3115 let mut sent = jobs.clone();
3121 let cache = self.state.config.cache_dir();
3122 let ctx = WaveCtx {
3123 carry_seats: false,
3124 pauses: Some((&self.pause, &self.interrupt)),
3125 run: &run_id,
3126 node: "implement",
3127 prompts: &prompts,
3128 cache: cache.as_deref(),
3129 round: None,
3130 };
3131 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3132 if !wave_blocked(&results) {
3133 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3134 .await;
3135 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3136 .await?;
3137 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3138 .await;
3139 }
3140 if !results.is_empty()
3143 && results
3144 .iter()
3145 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3146 {
3147 let losses: Vec<_> = results
3148 .iter()
3149 .map(|(_, seat, out)| {
3150 let AgentOutcome::Quota(o) = out else {
3151 unreachable!()
3152 };
3153 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3154 })
3155 .collect();
3156 self.state.quota.extend(losses.clone());
3157 for (_, seat, _) in results {
3158 self.state.seats.insert(seat.key.clone(), seat);
3159 }
3160 for &i in &todo {
3161 let label = self.state.candidates[i].label;
3162 self.state.interrupted_implementations.insert(label);
3163 self.state
3164 .candidate_preservation
3165 .insert(label, "machine outage awaiting persistence".into());
3166 }
3167 block_machine_outage(&mut self.state, "implement", &losses);
3170 self.state.save()?;
3171 self.persist_candidates().await?;
3172 for &i in &todo {
3173 self.state
3174 .interrupted_implementations
3175 .insert(self.state.candidates[i].label);
3176 }
3177 return self.stop_for_resource();
3178 }
3179 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3180
3181 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3182 && results
3183 .iter()
3184 .any(|(_, _, out)| FailClass::of(out).is_some());
3185 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3186 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3187 if self
3191 .state
3192 .candidate_preservation
3193 .contains_key(&self.state.candidates[i].label)
3194 {
3195 self.state.seats.insert(seat.key.clone(), seat);
3196 }
3197 continue;
3198 }
3199 let seat_key = seat.key.clone();
3200 let agent = seat.agent.clone();
3209 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3210 self.state.seats.insert(seat.key.clone(), seat);
3211 let label = self.state.candidates[i].label;
3212 self.state.interrupted_implementations.remove(&label);
3213
3214 let (summary, duration, failed, verified_claim) = match out {
3215 AgentOutcome::Ok(o) => {
3216 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3217 let failed = (!o.usable()).then(|| {
3218 if o.timed_out {
3219 "agent timed out".to_owned()
3220 } else {
3221 format!(
3222 "agent exited with {}",
3223 crate::notices::exit_status(o.exit_code)
3224 )
3225 }
3226 });
3227 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3228 (text, o.duration_ms, failed, verified_claim)
3229 }
3230 AgentOutcome::Dropped(o) => {
3236 let why = o
3237 .dropped
3238 .as_ref()
3239 .map(|d| d.why.as_str())
3240 .unwrap_or("the CLI ended the stream without delivering its answer");
3241 (
3242 String::new(),
3243 o.duration_ms,
3244 Some(format!("the CLI dropped the stream ({why})")),
3245 None,
3246 )
3247 }
3248 AgentOutcome::Quota(o) => {
3249 self.state.quota.push(QuotaLoss::from_report(
3250 seat_key,
3251 "implement",
3252 o.quota.as_ref(),
3253 ));
3254 (
3255 String::new(),
3256 o.duration_ms,
3257 Some("rate limited (quota); produced no change".to_owned()),
3258 None,
3259 )
3260 }
3261 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3262 (String::new(), 0, Some(e), None)
3263 }
3264 };
3265
3266 let c = &mut self.state.candidates[i];
3269 if !exhausted_the_fallback_chain {
3270 c.agent = agent;
3271 }
3272 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3273 c.duration_ms = duration;
3274 c.failed = failed;
3275 c.verified_noop = verified_claim;
3276 self.state
3277 .candidate_preservation
3278 .insert(label, "awaiting persistence".to_owned());
3279 }
3280 self.state.save()?;
3281 self.persist_candidates().await?;
3282
3283 if resource_blocked {
3284 return self.stop_for_resource();
3285 }
3286 if selected_failed {
3287 self.state.status = RunStatus::Blocked;
3288 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3289 self.state.save()?;
3290 bail!(
3291 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3292 );
3293 }
3294 self.after_implement()
3295 }
3296
3297 async fn persist_candidates(&mut self) -> Result<()> {
3299 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3300 for label in labels {
3301 let i = self
3302 .state
3303 .candidates
3304 .iter()
3305 .position(|c| c.label == label)
3306 .context("pending candidate is missing")?;
3307 let c = self.state.candidates[i].clone();
3308 let interrupted = self.state.interrupted_implementations.contains(&label);
3309 let result = async {
3310 let wait = crate::repo_lock::WaitPolicy::new(
3311 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3312 || false,
3313 );
3314 let _coordination =
3315 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3316 if let Err(e) = git::snapshot_uncommitted(
3319 &c.worktree,
3320 &self
3321 .state
3322 .dir()
3323 .join("artifacts")
3324 .join(format!("candidate-{label}-recovery")),
3325 )
3326 .await
3327 {
3328 self.state.event(
3329 "preserve",
3330 format!("candidate {label} snapshot incomplete: {e:#}"),
3331 );
3332 }
3333 let rescued = git::rescue_commit(
3334 &c.worktree,
3335 &format!("magi: candidate {label} (uncommitted work)"),
3336 )
3337 .await?;
3338 self.state.note_withheld("implement", &rescued.withheld);
3339 let base = &self.state.base_commit;
3340 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3341 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3342 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3343 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3344 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3345 if c.empty && commits > 0 && !patch.trim().is_empty()
3346 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3347 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3348 || self.state.tally.is_some())
3349 {
3350 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3354 }
3355 let c = &mut self.state.candidates[i];
3356 c.commits = commits;
3357 c.stat = stat;
3358 c.files = files;
3359 let no_changes = commits == 0 || patch.trim().is_empty();
3360 c.empty = no_changes && !interrupted;
3361 if interrupted && no_changes {
3362 c.failed = None;
3363 c.verified_noop = None;
3364 }
3365 if !c.empty {
3366 c.failed = None;
3367 c.verified_noop = None;
3368 }
3369 self.state.event(
3370 "preserve",
3371 format!("candidate {label}: {files} files, {commits} commits"),
3372 );
3373 Ok::<_, anyhow::Error>(())
3374 }
3375 .await;
3376 match result {
3377 Ok(()) => {
3378 self.state.candidate_preservation.remove(&label);
3379 if !interrupted || self.state.candidates[i].files > 0 {
3380 self.state.interrupted_implementations.remove(&label);
3381 }
3382 }
3383 Err(e) => {
3384 let reason = format!("candidate {label} preservation failed: {e:#}");
3385 self.state.event("preserve", &reason);
3386 self.state.candidate_preservation.insert(label, reason);
3387 }
3388 }
3389 if self.state.candidate_preservation.is_empty()
3390 && self.state.status == RunStatus::Blocked
3391 && self.state.candidates.iter().all(|c| c.failed.is_none())
3392 {
3393 self.state.status = RunStatus::Implementing;
3397 self.state.event(
3398 "preserve",
3399 "candidate persistence recovered; implementation can continue",
3400 );
3401 }
3402 self.state.save()?;
3403 }
3404 if !self.state.candidate_preservation.is_empty() {
3405 self.state.status = RunStatus::Blocked;
3406 self.state.save()?;
3407 bail!(
3408 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3409 );
3410 }
3411 Ok(())
3412 }
3413
3414 async fn resume_undelivered(
3442 &mut self,
3443 results: &mut [(usize, SeatState, AgentOutcome)],
3444 sent: &[SeatJob],
3445 prompts: &Prompts,
3446 run_id: &str,
3447 ) {
3448 for (wi, seat, out) in results.iter_mut() {
3449 let Some(dropped) = (match &*out {
3450 AgentOutcome::Dropped(o) => o.dropped.clone(),
3451 _ => None,
3452 }) else {
3453 continue;
3454 };
3455 let Some(job) = sent.get(*wi) else { continue };
3456 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3458 self.state.event(
3459 "implement",
3460 format!(
3461 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3462 work is in the tree",
3463 seat.key, dropped.output_tokens, dropped.why
3464 ),
3465 );
3466 continue;
3467 }
3468 if !has_context(&job.spec, seat, job.sessions) {
3476 self.state.event(
3477 "implement",
3478 format!(
3479 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3480 is no session left to resume",
3481 seat.key, dropped.output_tokens, dropped.why
3482 ),
3483 );
3484 continue;
3485 }
3486 self.state.event(
3487 "implement",
3488 format!(
3489 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3490 conversation",
3491 seat.key, dropped.output_tokens, dropped.why
3492 ),
3493 );
3494 let mut retry = job.clone();
3495 retry.seat = seat.clone();
3496 retry.prompt = prompt::resume_after_drop(&dropped.why);
3497 retry.timeout = retry_budget(job.timeout, true);
3498 retry.stem = format!("{}-resume", job.stem);
3499 let cache = self.state.config.cache_dir();
3500 let ctx = WaveCtx {
3501 carry_seats: false,
3502 pauses: Some((&self.pause, &self.interrupt)),
3503 run: run_id,
3504 node: "implement",
3505 prompts,
3506 cache: cache.as_deref(),
3507 round: None,
3508 };
3509 let (resumed_seat, resumed) =
3510 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3511 *seat = resumed_seat;
3512 *out = resumed;
3513 }
3514 }
3515
3516 async fn resume_seat_handovers(
3581 &mut self,
3582 results: &mut [(usize, SeatState, AgentOutcome)],
3583 sent: &mut [SeatJob],
3584 prompts: &Prompts,
3585 run_id: &str,
3586 ) -> Result<()> {
3587 let instruction = seeded_instruction(&self.state);
3588 let language = self.state.config.graph.language.clone();
3589 let github_language = self.state.config.graph.github_language.clone();
3590 let brief = self
3591 .state
3592 .advice
3593 .as_ref()
3594 .and_then(|a| a.synthesis.as_deref())
3595 .map(str::to_owned);
3596 let attachments = self.state.attachments.clone();
3597 for (wi, seat, out) in results.iter_mut() {
3598 let others: BTreeSet<String> = sent
3601 .iter()
3602 .enumerate()
3603 .filter(|(j, _)| j != wi)
3604 .map(|(_, j)| j.spec.id.clone())
3605 .collect();
3606 let Some(job) = sent.get_mut(*wi) else {
3607 continue;
3608 };
3609 let start = self
3614 .roles
3615 .implementer_roster
3616 .iter()
3617 .position(|s| s.id == job.spec.id)
3618 .unwrap_or(0);
3619 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3620 let mut fallback_attempt = 0usize;
3621 let mut prev: Option<FailClass> = None;
3622 while let Some(cur) = FailClass::of(&*out) {
3623 if !should_hand_over(prev.as_ref(), &cur) {
3624 break;
3625 }
3626 let Some(next) =
3627 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3628 .cloned()
3629 else {
3630 break;
3631 };
3632 tried.insert(next.id.clone());
3633 fallback_attempt += 1;
3634
3635 let coordination = self.coordinate(&job.cwd).await?;
3636 if let Ok(r) = git::rescue_commit(
3637 &job.cwd,
3638 &format!(
3639 "magi: candidate {} (uncommitted work before {} fallback)",
3640 seat.key,
3641 if cur == FailClass::Quota {
3642 "quota"
3643 } else {
3644 "handover"
3645 }
3646 ),
3647 )
3648 .await
3649 {
3650 self.state.note_withheld("implement", &r.withheld);
3651 }
3652 drop(coordination);
3653
3654 record_handover(
3655 &mut self.state,
3656 "implement",
3657 &seat.key,
3658 &seat.agent,
3659 &next.id,
3660 &cur,
3661 &fail_reason(&*out),
3662 );
3663 prev = Some(cur.clone());
3664
3665 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3666 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3667 job.spec = next.clone();
3675 let mut retry = job.clone();
3676 retry.seat = new_seat;
3677 retry.prompt = prompt::implement_in(
3678 &instruction,
3679 &job.cwd.to_string_lossy(),
3680 &language,
3681 &github_language,
3682 brief.as_deref(),
3683 &attachments,
3684 );
3685 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3686 let cache = self.state.config.cache_dir();
3687 let ctx = WaveCtx {
3688 carry_seats: false,
3689 pauses: Some((&self.pause, &self.interrupt)),
3690 run: run_id,
3691 node: "implement",
3692 prompts,
3693 cache: cache.as_deref(),
3694 round: None,
3695 };
3696 let (fallback_seat, fallback_out) = run_one(
3697 retry,
3698 Arc::clone(&self.sem),
3699 &ctx,
3700 &mut self.state,
3701 fallback_attempt,
3702 )
3703 .await;
3704 *seat = fallback_seat;
3705 *out = fallback_out;
3706 }
3707 }
3708 Ok(())
3709 }
3710
3711 async fn resume_unconfirmed_commands(
3735 &mut self,
3736 results: &mut [(usize, SeatState, AgentOutcome)],
3737 sent: &[SeatJob],
3738 prompts: &Prompts,
3739 run_id: &str,
3740 ) {
3741 for (wi, seat, out) in results.iter_mut() {
3742 let AgentOutcome::Ok(o) = &*out else {
3743 continue;
3744 };
3745 if !has_unconfirmed_command(&o.commands) {
3746 continue;
3747 }
3748 let Some(job) = sent.get(*wi) else { continue };
3749 if !has_context(&job.spec, seat, job.sessions) {
3750 self.state.event(
3751 "implement",
3752 format!(
3753 "{}: the reply named a command whose own CLI never confirmed the exit \
3754 status of, but there is no session left to resume",
3755 seat.key
3756 ),
3757 );
3758 continue;
3759 }
3760 self.state.event(
3761 "implement",
3762 format!(
3763 "{}: the reply named a command whose own CLI never confirmed the exit \
3764 status of; resuming the conversation",
3765 seat.key
3766 ),
3767 );
3768 let mut retry = job.clone();
3769 retry.seat = seat.clone();
3770 retry.prompt = prompt::resume_incomplete(
3771 "a command in your last reply had no confirmed exit status",
3772 );
3773 retry.timeout = retry_budget(job.timeout, true);
3774 retry.stem = format!("{}-confirm", job.stem);
3775 let cache = self.state.config.cache_dir();
3776 let ctx = WaveCtx {
3777 carry_seats: false,
3778 pauses: Some((&self.pause, &self.interrupt)),
3779 run: run_id,
3780 node: "implement",
3781 prompts,
3782 cache: cache.as_deref(),
3783 round: None,
3784 };
3785 let (resumed_seat, resumed) =
3786 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3787 *seat = resumed_seat;
3788 *out = resumed;
3789 }
3790 }
3791
3792 async fn continue_fix_report(
3813 &mut self,
3814 mut seat: SeatState,
3815 parse_err: String,
3816 job: &SeatJob,
3817 prompts: &Prompts,
3818 run_id: &str,
3819 round: usize,
3820 ) -> (
3821 SeatState,
3822 Option<FixReport>,
3823 Option<String>,
3824 ContinuationRecord,
3825 ) {
3826 let mut last_err = parse_err;
3827 let mut cumulative_wait_ms = 0u64;
3828 let mut attempts = 0usize;
3829 loop {
3830 if !has_context(&job.spec, &seat, job.sessions) {
3831 self.state.event(
3832 "fix",
3833 format!(
3834 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3835 session left to resume into"
3836 ),
3837 );
3838 let outcome = if attempts == 0 {
3839 ContinuationOutcome::NoSession
3840 } else {
3841 ContinuationOutcome::Exhausted
3842 };
3843 return (
3844 seat,
3845 None,
3846 Some(format!("unparsable fix report: {last_err}")),
3847 ContinuationRecord {
3848 attempts,
3849 cumulative_wait_ms,
3850 outcome,
3851 },
3852 );
3853 }
3854 if attempts >= MAX_FIX_CONTINUATIONS {
3855 self.state.event(
3856 "fix",
3857 format!(
3858 "round {round}: fixer's reply still had no adoption report after \
3859 {attempts} continuation(s) ({last_err}); giving up"
3860 ),
3861 );
3862 return (
3863 seat,
3864 None,
3865 Some(format!(
3866 "unparsable fix report after {attempts} continuation(s): {last_err}"
3867 )),
3868 ContinuationRecord {
3869 attempts,
3870 cumulative_wait_ms,
3871 outcome: ContinuationOutcome::Exhausted,
3872 },
3873 );
3874 }
3875 attempts += 1;
3876 self.state.event(
3877 "fix",
3878 format!(
3879 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3880 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3881 ),
3882 );
3883 let mut retry = job.clone();
3884 retry.seat = seat.clone();
3885 retry.prompt = prompt::resume_incomplete(&last_err);
3886 retry.timeout = retry_budget(job.timeout, true);
3887 retry.stem = format!("{}-continue{attempts}", job.stem);
3888 let cache = self.state.config.cache_dir();
3889 let ctx = WaveCtx {
3890 carry_seats: false,
3891 pauses: Some((&self.pause, &self.interrupt)),
3892 run: run_id,
3893 node: "fix",
3894 prompts,
3895 cache: cache.as_deref(),
3896 round: Some(round),
3897 };
3898 let (resumed_seat, resumed_out) = run_one(
3899 retry,
3900 Arc::clone(&self.sem),
3901 &ctx,
3902 &mut self.state,
3903 attempts,
3904 )
3905 .await;
3906 seat = resumed_seat;
3907 match resumed_out {
3908 AgentOutcome::Ok(o) => {
3909 cumulative_wait_ms += o.duration_ms;
3910 match verdict::extract_json::<FixReport>(&o.text) {
3911 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3912 self.state.event(
3913 "fix",
3914 format!(
3915 "round {round}: fixer's adoption report recovered after \
3916 {attempts} continuation(s)"
3917 ),
3918 );
3919 return (
3920 seat,
3921 Some(report),
3922 None,
3923 ContinuationRecord {
3924 attempts,
3925 cumulative_wait_ms,
3926 outcome: ContinuationOutcome::Resumed,
3927 },
3928 );
3929 }
3930 Ok(_) => {
3938 last_err = "the reply parsed, but it reported a command whose own CLI \
3939 never confirmed an exit status"
3940 .to_owned();
3941 }
3942 Err(e) => last_err = e.to_string(),
3943 }
3944 }
3945 AgentOutcome::Quota(o) => {
3946 cumulative_wait_ms += o.duration_ms;
3947 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3948 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3949 self.state.quota.push(loss);
3950 self.state.event(
3951 "fix",
3952 format!(
3953 "round {round}: continuation rate limited (quota); not retrying now"
3954 ),
3955 );
3956 return (
3957 seat,
3958 None,
3959 Some("rate limited (quota) while recovering the fix report".to_owned()),
3960 ContinuationRecord {
3961 attempts,
3962 cumulative_wait_ms,
3963 outcome: ContinuationOutcome::QuotaLost,
3964 },
3965 );
3966 }
3967 AgentOutcome::Dropped(o) => {
3968 cumulative_wait_ms += o.duration_ms;
3969 let why = o
3970 .dropped
3971 .as_ref()
3972 .map(|d| d.why.as_str())
3973 .unwrap_or("the CLI ended the stream without delivering its answer");
3974 last_err = format!("the CLI dropped the stream ({why})");
3975 }
3976 AgentOutcome::Failed(e) => last_err = e,
3977 AgentOutcome::ResourceBlocked(e) => {
3980 return (
3981 seat,
3982 None,
3983 Some(format!("the continuation was not started: {e}")),
3984 ContinuationRecord {
3985 attempts: attempts.saturating_sub(1),
3986 cumulative_wait_ms,
3987 outcome: ContinuationOutcome::NoSession,
3988 },
3989 );
3990 }
3991 }
3992 }
3993 }
3994
3995 fn after_implement(&mut self) -> Result<()> {
3996 if !self.state.candidate_preservation.is_empty() {
3997 bail!("candidate preservation incomplete; judging is unsafe");
3998 }
3999 if self.state.leaks.is_empty() {
4001 let cfg = self.state.config.blind.clone();
4002 let mut leaks = Vec::new();
4003 for c in &self.state.candidates {
4004 let Some(patch) =
4005 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4006 else {
4007 continue;
4008 };
4009 leaks.extend(blind::scan(
4010 &format!("candidate {} patch", c.label),
4011 &patch,
4012 &cfg.vendor_tokens,
4013 ));
4014 }
4015 if !leaks.is_empty() {
4016 let summary = leaks
4017 .iter()
4018 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4019 .collect::<Vec<_>>()
4020 .join(", ");
4021 match cfg.on_leak {
4022 LeakPolicy::Fail => {
4023 self.state.status = RunStatus::Failed;
4024 self.state
4025 .event("blind", format!("vendor text in a patch: {summary}"));
4026 self.state.leaks = leaks;
4027 self.state.save()?;
4028 self.settle_questions();
4029 bail!(
4030 "blind.on_leak = \"fail\" and vendor text reached a \
4031 judged patch: {summary}"
4032 );
4033 }
4034 LeakPolicy::Redact => self.state.event(
4035 "blind",
4036 format!("redacting vendor text for judging: {summary}"),
4037 ),
4038 LeakPolicy::Warn => self.state.event(
4039 "blind",
4040 format!("vendor text present in a judged patch (shown as-is): {summary}"),
4041 ),
4042 }
4043 self.state.leaks = leaks;
4044 }
4045 }
4046
4047 if self.state.viable().is_empty() {
4048 if self.state.all_candidates_verified_noop() {
4049 self.state.status = RunStatus::VerifiedNoop;
4060 self.state.save()?;
4061 self.settle_questions();
4062 return Ok(());
4063 }
4064 self.state.status = RunStatus::Failed;
4065 self.state.save()?;
4066 self.settle_questions();
4067 bail!("no candidate produced a change; nothing to judge");
4068 }
4069 self.state.status = RunStatus::Judging;
4070 self.state.save()?;
4071 Ok(())
4072 }
4073
4074 async fn judge(&mut self) -> Result<()> {
4077 let run_id = self.state.id.clone();
4082 let prompts = self.state.config.prompts.clone();
4083 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4084 return Ok(());
4085 }
4086 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4087 if viable.len() == 1 {
4088 self.state.judge_skipped = true;
4095 self.state.event(
4096 "judge",
4097 format!(
4098 "only candidate {} produced a change; judging skipped",
4099 viable[0].label
4100 ),
4101 );
4102 self.state.save()?;
4103 return Ok(());
4104 }
4105 self.state.status = RunStatus::Judging;
4106
4107 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4108 let language = self.state.config.graph.language.clone();
4109 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4110 let sessions = self.state.config.graph.sessions;
4111 let artifacts = agent::artifacts_dir(&self.state.dir());
4112 let root = self.state.worktree_root();
4113 let base_short = short(&self.state.base_commit);
4114
4115 let mut jobs = Vec::new();
4116 let mut orders = Vec::new();
4117 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4118 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4119 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4120 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4121 let seat_key = format!("judge-{}", j + 1);
4122 let seat = self.seat(&seat_key, &spec.id);
4123 jobs.push(SeatJob {
4124 prompt: prompt::judge(
4125 &self.state.instruction,
4126 &views,
4127 self.roles.judges.len(),
4128 &base_short,
4129 &language,
4130 ),
4131 spec,
4132 seat,
4133 cwd: root.join(format!("judge-{}", j + 1)),
4134 timeout,
4135 allow_write: false,
4136 sessions,
4137 artifacts: artifacts.clone(),
4138 stem: format!("judge-{}", j + 1),
4139 handover: None,
4140 });
4141 }
4142
4143 self.state.event(
4144 "judge",
4145 format!(
4146 "{} judges ranking {} candidates blind",
4147 jobs.len(),
4148 viable.len()
4149 ),
4150 );
4151 let labels_for_check = labels.clone();
4152 let mut quota_losses = Vec::new();
4153 let cache = self.state.config.cache_dir();
4154 let ctx = WaveCtx {
4155 carry_seats: false,
4156 pauses: Some((&self.pause, &self.interrupt)),
4157 run: &run_id,
4158 node: "judge",
4159 prompts: &prompts,
4160 cache: cache.as_deref(),
4161 round: None,
4162 };
4163 let results = ask_json_wave::<Ranking>(
4164 jobs,
4165 Arc::clone(&self.sem),
4166 self.state.config.graph.retries,
4167 &self.roles.judge_roster,
4168 &ctx,
4169 &mut quota_losses,
4170 &mut self.state,
4171 &move |r: &Ranking| r.validate(&labels_for_check),
4172 )
4173 .await;
4174 if self.state.resource_block.is_some() {
4175 return self.stop_for_resource();
4176 }
4177 self.state.quota.extend(quota_losses);
4178
4179 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4180 let agent_id = seat.agent.clone();
4181 self.state.seats.insert(seat.key.clone(), seat);
4182 let mut record = Judgement {
4183 judge: j + 1,
4184 seat: format!("judge-{}", j + 1),
4185 agent: agent_id,
4186 ranking: Vec::new(),
4187 reasons: BTreeMap::new(),
4188 confidence: None,
4189 strengths: BTreeMap::new(),
4190 order: orders[j].clone(),
4191 failed: None,
4192 duration_ms: 0,
4193 };
4194 match res {
4195 Ok((ranking, out)) => {
4196 record.ranking = ranking.normalized();
4197 record.reasons = ranking.reasons;
4198 record.strengths = ranking.strengths;
4199 record.confidence = ranking.confidence;
4200 record.duration_ms = out.duration_ms;
4201 self.state.event(
4202 "judge",
4203 format!(
4204 "judge {} ranked {}",
4205 j + 1,
4206 record.ranking.iter().collect::<String>()
4207 ),
4208 );
4209 }
4210 Err(e) => {
4211 record.failed = Some(e.to_string());
4212 self.state
4213 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4214 }
4215 }
4216 self.state.judgements.push(record);
4217 self.state.save()?;
4218 }
4219 Ok(())
4220 }
4221
4222 async fn deliberate(&mut self) -> Result<()> {
4225 if self.hold_base_sync().await? {
4226 return Ok(());
4227 }
4228 let run_id = self.state.id.clone();
4233 let prompts = self.state.config.prompts.clone();
4234 let rounds = self.state.config.graph.deliberate_rounds;
4235 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4237 return Ok(());
4238 }
4239 let tops: Vec<char> = self
4240 .state
4241 .judgements
4242 .iter()
4243 .filter_map(|j| j.ranking.first().copied())
4244 .collect();
4245 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4246 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4247 self.state.event(
4248 "deliberate",
4249 format!("judges agreed on {} outright; no deliberation", tops[0]),
4250 );
4251 }
4252 self.state.status = RunStatus::Voting;
4253 self.state.save()?;
4254 return Ok(());
4255 }
4256
4257 self.state.status = RunStatus::Deliberating;
4258 self.state.event(
4259 "deliberate",
4260 format!(
4261 "split: first choices were {} — opening {rounds} round(s)",
4262 tops.iter().collect::<String>()
4263 ),
4264 );
4265
4266 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4267 let language = self.state.config.graph.language.clone();
4268 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4269 let sessions = self.state.config.graph.sessions;
4270 let artifacts = agent::artifacts_dir(&self.state.dir());
4271 let root = self.state.worktree_root();
4272 let base_short = short(&self.state.base_commit);
4273
4274 for round in self.state.deliberation.len() + 1..=rounds {
4278 let mut turns: Vec<DeliberationTurn> = Vec::new();
4279 let mut round_losses = Vec::new();
4280 let mut asked = 0;
4281 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4282 if self.state.judgements[j].failed.is_some() {
4283 continue;
4284 }
4285 let seat_key = format!("judge-{}", j + 1);
4286 let spec = self.occupant(&seat_key, spec);
4287 let mut seat = self.seat(&seat_key, &spec.id);
4288 let transcript = self.transcript(&turns, j);
4289 let build = |context: Option<&str>| {
4290 prompt::deliberate(
4291 &self.state.instruction,
4292 context,
4293 &transcript,
4294 round,
4295 rounds,
4296 &language,
4297 )
4298 };
4299 let block = self.candidate_block(&viable, &base_short);
4300 let full = build(Some(&block));
4301 let text = if has_context(&spec, &seat, sessions) {
4302 build(None)
4303 } else {
4304 full.clone()
4305 };
4306 let job = SeatJob {
4307 spec,
4308 seat: seat.clone(),
4309 prompt: text,
4310 cwd: root.join(format!("judge-{}", j + 1)),
4311 timeout,
4312 allow_write: false,
4313 sessions,
4314 artifacts: artifacts.clone(),
4315 stem: format!("delib-{round}-judge-{}", j + 1),
4316 handover: Some(full),
4317 };
4318 let cache = self.state.config.cache_dir();
4319 let ctx = WaveCtx {
4320 carry_seats: false,
4321 pauses: Some((&self.pause, &self.interrupt)),
4322 run: &run_id,
4323 node: "deliberate",
4324 prompts: &prompts,
4325 cache: cache.as_deref(),
4326 round: None,
4327 };
4328 let mut losses = Vec::new();
4332 asked += 1;
4333 let mut results = ask_wave_with::<String>(
4334 vec![job],
4335 Arc::clone(&self.sem),
4336 0,
4337 &self.roles.judge_roster,
4338 &ctx,
4339 &mut losses,
4340 &mut self.state,
4341 &|text: &str| {
4342 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4343 },
4344 )
4345 .await;
4346 if self.state.resource_block.is_some() {
4347 return self.stop_for_resource();
4348 }
4349 round_losses.extend(losses.iter().cloned());
4350 self.state.quota.extend(losses);
4351 let (updated, res, _) = results.pop().expect("one job in, one result out");
4352 seat = updated;
4353 let agent_id = seat.agent.clone();
4354 self.state.seats.insert(seat.key.clone(), seat);
4355 let body = match res {
4356 Ok((body, _)) => body,
4357 Err(e) => {
4360 self.state
4361 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4362 continue;
4363 }
4364 };
4365 let tentative = verdict::extract_json::<Position>(&body)
4366 .ok()
4367 .and_then(|p| p.tentative)
4368 .and_then(|s| s.trim().chars().next())
4369 .map(|c| c.to_ascii_uppercase());
4370 self.state.event(
4371 "deliberate",
4372 format!(
4373 "round {round}: judge {} now favours {}",
4374 j + 1,
4375 tentative.map_or("—".to_owned(), |c| c.to_string())
4376 ),
4377 );
4378 turns.push(DeliberationTurn {
4379 judge: j + 1,
4380 agent: agent_id,
4381 body: blind::sanitize_prose(&body, &self.state.config.blind),
4382 tentative,
4383 });
4384 }
4385 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4386 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4387 return self.stop_for_resource();
4388 }
4389 self.state
4390 .deliberation
4391 .push(DeliberationRound { round, turns });
4392 self.state.save()?;
4393 }
4394
4395 self.state.status = RunStatus::Voting;
4396 self.state.save()?;
4397 Ok(())
4398 }
4399
4400 async fn vote(&mut self) -> Result<()> {
4403 if self.hold_base_sync().await? {
4404 return Ok(());
4405 }
4406 let run_id = self.state.id.clone();
4411 let prompts = self.state.config.prompts.clone();
4412 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4413 return Ok(());
4414 }
4415 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4416 if viable.len() == 1 {
4417 return Ok(());
4418 }
4419 self.state.status = RunStatus::Voting;
4420
4421 let language = self.state.config.graph.language.clone();
4422 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4423 let sessions = self.state.config.graph.sessions;
4424 let artifacts = agent::artifacts_dir(&self.state.dir());
4425 let root = self.state.worktree_root();
4426 let base_short = short(&self.state.base_commit);
4427 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4428
4429 let mut jobs = Vec::new();
4430 let mut seats_at = Vec::new();
4431 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4432 if self
4433 .state
4434 .judgements
4435 .get(j)
4436 .is_some_and(|r| r.failed.is_some())
4437 {
4438 continue;
4439 }
4440 let seat_key = format!("judge-{}", j + 1);
4441 let spec = self.occupant(&seat_key, spec);
4442 let seat = self.seat(&seat_key, &spec.id);
4443 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4444 let text = if has_context(&spec, &seat, sessions) {
4445 prompt::final_vote(&viable, &language)
4446 } else {
4447 full.clone()
4448 };
4449 jobs.push(SeatJob {
4450 spec,
4451 seat,
4452 prompt: text,
4453 cwd: root.join(format!("judge-{}", j + 1)),
4454 timeout,
4455 allow_write: false,
4456 sessions,
4457 artifacts: artifacts.clone(),
4458 stem: format!("vote-judge-{}", j + 1),
4459 handover: Some(full),
4460 });
4461 seats_at.push(j);
4462 }
4463
4464 self.state.event(
4465 "vote",
4466 format!(
4467 "collecting {} final votes one by one, privately",
4468 jobs.len()
4469 ),
4470 );
4471 let allowed = viable.clone();
4472 let mut quota_losses = Vec::new();
4473 let cache = self.state.config.cache_dir();
4474 let ctx = WaveCtx {
4475 carry_seats: false,
4476 pauses: Some((&self.pause, &self.interrupt)),
4477 run: &run_id,
4478 node: "vote",
4479 prompts: &prompts,
4480 cache: cache.as_deref(),
4481 round: None,
4482 };
4483 let results = ask_json_wave::<FinalVote>(
4484 jobs,
4485 Arc::clone(&self.sem),
4486 self.state.config.graph.retries,
4487 &self.roles.judge_roster,
4488 &ctx,
4489 &mut quota_losses,
4490 &mut self.state,
4491 &move |v: &FinalVote| match v.label() {
4492 Some(c) if allowed.contains(&c) => Ok(()),
4493 other => bail!("vote {other:?} is not one of {allowed:?}"),
4494 },
4495 )
4496 .await;
4497 if self.state.resource_block.is_some() {
4498 return self.stop_for_resource();
4499 }
4500 self.state.quota.extend(quota_losses);
4501
4502 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4503 let agent_id = seat.agent.clone();
4504 self.state.seats.insert(seat.key.clone(), seat);
4505 let initial = self
4506 .state
4507 .judgements
4508 .get(j)
4509 .and_then(|r| r.ranking.first().copied());
4510 let mut record = VoteRecord {
4511 judge: j + 1,
4512 agent: agent_id,
4513 vote: None,
4514 reason: String::new(),
4515 changed: false,
4516 };
4517 match res {
4518 Ok((v, _)) => {
4519 record.vote = v.label();
4520 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4521 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4522 self.state.event(
4523 "vote",
4524 format!(
4525 "judge {} voted {}{}",
4526 j + 1,
4527 record.vote.unwrap_or('?'),
4528 if record.changed { " (changed)" } else { "" }
4529 ),
4530 );
4531 }
4532 Err(e) => {
4533 self.state
4534 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4535 }
4536 }
4537 self.state.votes.push(record);
4538 self.state.save()?;
4539 }
4540 Ok(())
4541 }
4542
4543 fn tally(&mut self) -> Result<()> {
4546 if self.state.tally.is_some() {
4547 return Ok(());
4548 }
4549 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4550 let tops: Vec<char> = self
4551 .state
4552 .judgements
4553 .iter()
4554 .filter_map(|j| j.ranking.first().copied())
4555 .collect();
4556 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4557
4558 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4561 let mut cast: Vec<char> = Vec::new();
4562 for (i, j) in self.state.judgements.iter().enumerate() {
4563 let vote = self
4564 .state
4565 .votes
4566 .iter()
4567 .find(|v| v.judge == i + 1)
4568 .and_then(|v| v.vote)
4569 .or_else(|| j.ranking.first().copied());
4570 if let Some(v) = vote {
4571 *first_choice.entry(v).or_insert(0) += 1;
4572 cast.push(v);
4573 }
4574 }
4575
4576 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4577 for j in &self.state.judgements {
4578 let n = j.ranking.len();
4579 for (pos, label) in j.ranking.iter().enumerate() {
4580 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4581 }
4582 }
4583
4584 let best = first_choice.values().copied().max().unwrap_or(0);
4585 let mut leaders: Vec<char> = first_choice
4586 .iter()
4587 .filter(|(_, v)| **v == best)
4588 .map(|(k, _)| *k)
4589 .collect();
4590 let mut tie_break = None;
4591 if leaders.len() > 1 {
4592 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4593 let borda_leaders: Vec<char> = leaders
4594 .iter()
4595 .copied()
4596 .filter(|l| borda[l] == top_borda)
4597 .collect();
4598 tie_break = Some(if borda_leaders.len() == 1 {
4599 format!(
4600 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4601 leaders.len()
4602 )
4603 } else {
4604 format!(
4605 "{} way tie on both first-choice votes and Borda points, broken by label order",
4606 leaders.len()
4607 )
4608 });
4609 leaders = borda_leaders;
4610 leaders.sort_unstable();
4611 }
4612 let winner = *leaders
4613 .first()
4614 .or(viable.first())
4615 .context("no candidate to declare a winner from")?;
4616
4617 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4618 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4619 let deliberated = !self.state.deliberation.is_empty();
4620
4621 let quota_seats: std::collections::BTreeSet<&str> =
4625 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4626 let mut present = 0usize;
4627 for (i, j) in self.state.judgements.iter().enumerate() {
4628 if quota_seats.contains(j.seat.as_str()) {
4629 continue;
4630 }
4631 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4632 let voted = self
4633 .state
4634 .votes
4635 .iter()
4636 .any(|v| v.judge == i + 1 && v.vote.is_some());
4637 if ranked || voted {
4638 present += 1;
4639 }
4640 }
4641 let needs_quorum = viable.len() > 1;
4647 let judges_total = if needs_quorum {
4648 self.roles.judges.len()
4649 } else {
4650 0
4651 };
4652 let quorum = if needs_quorum {
4653 judges_total / 2 + 1
4654 } else {
4655 0
4656 };
4657 let met_quorum = !needs_quorum || present >= quorum;
4658 let uncontested = (!needs_quorum).then(|| {
4659 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4660 });
4661
4662 self.state.event(
4663 "tally",
4664 match &uncontested {
4665 Some(reason) => format!("winner {winner} — {reason}"),
4666 None => format!(
4667 "winner {winner} — votes {} | initial {} | {} changed | \
4668 {present}/{judges_total} judges{}",
4669 first_choice
4670 .iter()
4671 .map(|(k, v)| format!("{k}:{v}"))
4672 .collect::<Vec<_>>()
4673 .join(" "),
4674 if unanimous_initial {
4675 "unanimous"
4676 } else {
4677 "split"
4678 },
4679 changed_votes,
4680 if met_quorum {
4681 String::new()
4682 } else {
4683 format!(" — below quorum ({quorum} required)")
4684 },
4685 ),
4686 },
4687 );
4688 if !met_quorum {
4689 self.state.event(
4690 "stall",
4691 format!(
4692 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4693 the run stops here, resumable"
4694 ),
4695 );
4696 }
4697 self.state.tally = Some(Tally {
4698 first_choice,
4699 borda,
4700 winner,
4701 rankings: tops.len(),
4702 unanimous_initial,
4703 deliberated,
4704 changed_votes,
4705 unanimous_final,
4706 tie_break,
4707 judges: judges_total,
4708 present,
4709 quorum,
4710 met_quorum,
4711 uncontested,
4712 });
4713 self.state.status = if met_quorum {
4714 RunStatus::Reviewing
4715 } else {
4716 RunStatus::Stalled
4717 };
4718 self.state.save()?;
4719 Ok(())
4720 }
4721
4722 #[allow(clippy::too_many_lines)]
4743 async fn recover_stall(&mut self) -> Result<bool> {
4744 let run_id = self.state.id.clone();
4749 let prompts = self.state.config.prompts.clone();
4750 let quota_seats: BTreeSet<&str> =
4755 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4756 let absent: Vec<String> = self
4757 .state
4758 .judgements
4759 .iter()
4760 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4761 .map(|j| j.seat.clone())
4762 .collect();
4763 if absent.is_empty() {
4764 return Ok(false);
4765 }
4766 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4767 if viable.len() <= 1 {
4768 return Ok(false);
4769 }
4770 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4771 let language = self.state.config.graph.language.clone();
4772 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4773 let sessions = self.state.config.graph.sessions;
4774 let artifacts = agent::artifacts_dir(&self.state.dir());
4775 let root = self.state.worktree_root();
4776 let base_short = short(&self.state.base_commit);
4777 let candidates: Vec<Candidate> = viable.clone();
4778
4779 let mut positions: Vec<usize> = absent
4781 .iter()
4782 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4783 .collect();
4784 if positions.is_empty() {
4785 return Ok(false);
4786 }
4787 positions.sort_unstable();
4788 positions.dedup();
4789
4790 let mut judge_jobs = Vec::new();
4792 for &j in &positions {
4793 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4794 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4795 let seat_key = format!("judge-{}", j + 1);
4796 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4797 let seat = self.seat(&seat_key, &spec.id);
4798 judge_jobs.push(SeatJob {
4799 spec,
4800 seat,
4801 prompt: prompt::judge(
4802 &self.state.instruction,
4803 &views,
4804 self.roles.judges.len(),
4805 &base_short,
4806 &language,
4807 ),
4808 cwd: root.join(seat_key),
4809 timeout,
4810 allow_write: false,
4811 sessions,
4812 artifacts: artifacts.clone(),
4813 stem: format!("judge-{}-recover", j + 1),
4814 handover: None,
4815 });
4816 }
4817
4818 let labels_for_check = labels.clone();
4819 let mut judge_losses = Vec::new();
4820 let retries = self.state.config.graph.retries;
4821 let cache = self.state.config.cache_dir();
4822 let ctx = WaveCtx {
4823 carry_seats: false,
4824 pauses: Some((&self.pause, &self.interrupt)),
4825 run: &run_id,
4826 node: "judge",
4827 prompts: &prompts,
4828 cache: cache.as_deref(),
4829 round: None,
4830 };
4831 let results = ask_json_wave::<Ranking>(
4832 judge_jobs,
4833 Arc::clone(&self.sem),
4834 retries,
4835 &self.roles.judge_roster,
4836 &ctx,
4837 &mut judge_losses,
4838 &mut self.state,
4839 &move |r: &Ranking| r.validate(&labels_for_check),
4840 )
4841 .await;
4842 if self.state.resource_block.is_some() {
4843 self.stop_for_resource()?;
4844 return Ok(false);
4845 }
4846
4847 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4849 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4850 let agent_id = seat.agent.clone();
4851 self.state.seats.insert(seat.key.clone(), seat);
4852 let record = &mut self.state.judgements[j];
4853 match res {
4854 Ok((ranking, out)) => {
4855 record.agent = agent_id;
4856 record.ranking = ranking.normalized();
4857 record.reasons = ranking.reasons;
4858 record.strengths = ranking.strengths;
4859 record.confidence = ranking.confidence;
4860 record.failed = None;
4861 record.duration_ms = out.duration_ms;
4862 recovered.insert(j);
4863 self.state.event(
4864 "recover",
4865 format!("judge {} ranked again after the limit", j + 1),
4866 );
4867 }
4868 Err(e) => {
4869 self.state
4870 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4871 }
4872 }
4873 }
4874
4875 let mut vote_jobs = Vec::new();
4877 let mut vote_pos: Vec<usize> = Vec::new();
4878 for &j in &recovered {
4879 let seat_key = format!("judge-{}", j + 1);
4880 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4881 let seat = self.seat(&seat_key, &spec.id);
4882 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4883 let text = if has_context(&spec, &seat, sessions) {
4884 prompt::final_vote(&labels, &language)
4885 } else {
4886 full.clone()
4887 };
4888 vote_jobs.push(SeatJob {
4889 spec,
4890 seat,
4891 prompt: text,
4892 cwd: root.join(seat_key),
4893 timeout,
4894 allow_write: false,
4895 sessions,
4896 artifacts: artifacts.clone(),
4897 stem: format!("vote-judge-{}-recover", j + 1),
4898 handover: Some(full),
4899 });
4900 vote_pos.push(j);
4901 }
4902 let allowed = labels.clone();
4903 let mut vote_losses = Vec::new();
4904 let vote_retries = self.state.config.graph.retries;
4905 let vote_cache = self.state.config.cache_dir();
4906 let ctx = WaveCtx {
4907 carry_seats: false,
4908 pauses: Some((&self.pause, &self.interrupt)),
4909 run: &run_id,
4910 node: "vote",
4911 prompts: &prompts,
4912 cache: vote_cache.as_deref(),
4913 round: None,
4914 };
4915 let votes = ask_json_wave::<FinalVote>(
4916 vote_jobs,
4917 Arc::clone(&self.sem),
4918 vote_retries,
4919 &self.roles.judge_roster,
4920 &ctx,
4921 &mut vote_losses,
4922 &mut self.state,
4923 &move |v: &FinalVote| match v.label() {
4924 Some(c) if allowed.contains(&c) => Ok(()),
4925 other => bail!("vote {other:?} is not one of {allowed:?}"),
4926 },
4927 )
4928 .await;
4929 if self.state.resource_block.is_some() {
4930 self.stop_for_resource()?;
4931 return Ok(false);
4932 }
4933 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4934 let agent_id = seat.agent.clone();
4935 self.state.seats.insert(seat.key.clone(), seat);
4936 match res {
4937 Ok((v, _)) => {
4938 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4939 rec.vote = v.label();
4940 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4941 } else {
4942 self.state.votes.push(VoteRecord {
4943 judge: j + 1,
4944 agent: agent_id,
4945 vote: v.label(),
4946 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4947 changed: false,
4948 });
4949 }
4950 self.state.event(
4951 "recover",
4952 format!("judge {} voted again after the limit", j + 1),
4953 );
4954 }
4955 Err(e) => {
4956 self.state
4957 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4958 }
4959 }
4960 }
4961
4962 let recovered_keys: BTreeSet<String> = recovered
4966 .iter()
4967 .map(|&j| format!("judge-{}", j + 1))
4968 .collect();
4969 self.state
4970 .quota
4971 .retain(|q| !recovered_keys.contains(&q.seat));
4972 for loss in judge_losses.into_iter().chain(vote_losses) {
4976 if recovered_keys.contains(&loss.seat) {
4977 continue;
4978 }
4979 self.state.quota.retain(|q| q.seat != loss.seat);
4980 self.state.quota.push(loss);
4981 }
4982
4983 self.state.tally = None;
4985 self.tally()?;
4986 Ok(self
4987 .state
4988 .tally
4989 .as_ref()
4990 .map(|t| t.met_quorum)
4991 .unwrap_or(false))
4992 }
4993
4994 async fn graft(&mut self) -> Result<()> {
5010 let Err(e) = self.graft_inner().await else {
5011 return Ok(());
5012 };
5013 if e.is::<crate::repo_lock::ResourceBlocked>() {
5014 return Err(e);
5015 }
5016 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5017 self.state.status = RunStatus::Blocked;
5018 self.state.event("graft", format!("blocked: {e:#}"));
5019 self.state.save()?;
5020 return Err(e);
5021 }
5022 if let Some(rec) = self.state.graft.clone()
5027 && !rec.outcome.settled()
5028 {
5029 if let Some(winner) = self.state.winner().cloned() {
5030 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5031 if re.is::<crate::repo_lock::ResourceBlocked>() {
5032 return Err(re);
5033 }
5034 self.state.status = RunStatus::Blocked;
5035 self.state.event("graft", format!("blocked: {re:#}"));
5036 self.state.save()?;
5037 return Err(re);
5038 }
5039 }
5040 if let Some(g) = self.state.graft.as_mut() {
5041 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5042 }
5043 }
5044 self.state.event("graft", format!("skipped: {e:#}"));
5045 if let Err(e) = self.state.save() {
5046 tracing::warn!("save after graft: {e:#}");
5047 }
5048 Ok(())
5049 }
5050
5051 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5054 let _coordination = self.coordinate(wt).await?;
5055 crate::recovery::restore_repair(wt, pre)
5056 .await
5057 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5058 }
5059
5060 fn graft_skipped(&mut self, why: &str) {
5062 let message = format!("skipped: {why}");
5063 if !self
5064 .state
5065 .events
5066 .iter()
5067 .any(|e| e.node == "graft" && e.message == message)
5068 {
5069 self.state.event("graft", message);
5070 }
5071 }
5072
5073 async fn graft_inner(&mut self) -> Result<()> {
5074 if let Some(rec) = self.state.graft.clone()
5077 && !rec.outcome.settled()
5078 {
5079 if self.state.reviews.is_empty()
5083 && let Some(winner) = self.state.winner().cloned()
5084 {
5085 self.graft_restore(&winner.worktree, &rec.pre).await?;
5086 }
5087 if rec.outcome == GraftOutcome::WaitingForRestore {
5088 self.state.graft = None;
5089 self.state.save()?;
5090 } else {
5091 if let Some(g) = self.state.graft.as_mut() {
5092 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5093 }
5094 self.state.event(
5095 "graft",
5096 "rolled back: the previous round was interrupted; the winner is untouched",
5097 );
5098 self.state.save()?;
5099 return Ok(());
5100 }
5101 }
5102 let plan = match graft::plan(&self.state) {
5103 Ok(plan) => plan,
5104 Err(skip) => {
5105 if skip != graft::Skip::AlreadyDone {
5106 self.graft_skipped(skip.reason());
5107 self.state.save()?;
5108 }
5109 return Ok(());
5110 }
5111 };
5112 let Some(winner) = self.state.winner().cloned() else {
5113 return Ok(());
5114 };
5115 let wt = winner.worktree.clone();
5116 if !git::is_clean(&wt).await? {
5117 self.graft_skipped("the winner's worktree has uncommitted changes");
5118 self.state.save()?;
5119 return Ok(());
5120 }
5121 let pre = git::rev_parse(&wt, "HEAD").await?;
5122 self.state.graft = Some(GraftRecord {
5125 pre: pre.clone(),
5126 post: None,
5127 lines: None,
5128 limit: None,
5129 agent: None,
5130 outcome: GraftOutcome::Running,
5131 requested: graft::requested(&plan),
5132 adoption: None,
5133 adoption_error: None,
5134 });
5135 self.state.event(
5136 "graft",
5137 format!(
5138 "porting strengths of candidate(s) {} onto {}",
5139 plan.strengths
5140 .iter()
5141 .map(|(l, _, _)| l.to_string())
5142 .collect::<Vec<_>>()
5143 .join(", "),
5144 plan.winner
5145 ),
5146 );
5147 self.state.save()?;
5148
5149 let fix_prompt = prompt::graft(
5150 &self.state.instruction,
5151 plan.winner,
5152 &plan.strengths,
5153 &self.state.config.graph.language,
5154 );
5155 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5156 let sessions = self.state.config.graph.sessions;
5157 let artifacts = agent::artifacts_dir(&self.state.dir());
5158 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5159 let (_, seat, out) = self
5160 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5161 SeatJob {
5162 prompt: fix_prompt.clone(),
5163 spec,
5164 seat,
5165 cwd: wt.clone(),
5166 timeout,
5167 allow_write: true,
5168 sessions,
5169 artifacts: artifacts.clone(),
5170 stem: "graft".to_owned(),
5171 handover: None,
5172 }
5173 })
5174 .await;
5175 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5176 self.state.graft.as_mut().expect("graft record").outcome =
5179 GraftOutcome::WaitingForRestore;
5180 return self.stop_for_resource();
5181 }
5182 let mut why: Option<String> = None;
5183 match &out {
5184 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5185 why = Some("the fixer produced nothing usable".to_owned());
5186 }
5187 AgentOutcome::Ok(_) => {}
5188 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5189 AgentOutcome::Quota(o) => {
5190 self.state.quota.push(QuotaLoss::from_report(
5191 seat.key.clone(),
5192 "graft",
5193 o.quota.as_ref(),
5194 ));
5195 why = Some("rate limited (quota)".to_owned());
5196 }
5197 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5198 why = Some(format!(
5199 "the fixer failed: {}",
5200 e.lines()
5201 .next()
5202 .unwrap_or("")
5203 .chars()
5204 .take(160)
5205 .collect::<String>()
5206 ));
5207 }
5208 }
5209 let agent_id = seat.agent.clone();
5210 self.state.seats.insert(seat.key.clone(), seat);
5211
5212 let mut post = None;
5213 let mut lines = None;
5214 let _coordination = self.coordinate(&wt).await?;
5215 if why.is_none() {
5216 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5217 if let Ok(r) = &rescued {
5218 self.state.note_withheld("graft", &r.withheld);
5219 }
5220 let head = git::rev_parse(&wt, "HEAD").await?;
5221 let dirty = match &rescued {
5225 Ok(r) => git::status_porcelain(&wt)
5226 .await
5227 .map(|st| {
5228 st.lines().any(|l| {
5229 !r.withheld
5230 .iter()
5231 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5232 })
5233 })
5234 .unwrap_or(true),
5235 Err(_) => true,
5236 };
5237 if dirty {
5238 why = Some("the fixer left changes that could not be committed".to_owned());
5239 } else if head == pre {
5240 why = Some("the fixer changed nothing".to_owned());
5241 } else {
5242 lines = git::changed_lines(&wt, &pre, &head).await;
5243 post = Some(head);
5244 }
5245 }
5246 let (adoption, adoption_error) = match &out {
5247 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5248 _ => (None, None),
5249 };
5250 let requested = graft::requested(&plan);
5251 let outcome = match why {
5252 None => {
5253 self.state.event(
5254 "graft",
5255 format!(
5256 "applied: {}",
5257 lines.map_or_else(
5258 || "changed lines unavailable".to_owned(),
5259 |n| format!("{n} changed lines")
5260 )
5261 ),
5262 );
5263 GraftOutcome::Applied
5264 }
5265 Some(why) => {
5266 crate::recovery::restore_repair(&wt, &pre)
5269 .await
5270 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5271 self.state.event(
5272 "graft",
5273 format!("rolled back: {why}; the winner is untouched"),
5274 );
5275 GraftOutcome::RolledBack(why)
5276 }
5277 };
5278 self.state.graft = Some(GraftRecord {
5279 pre,
5280 post,
5281 lines,
5282 limit: None,
5283 agent: Some(agent_id),
5284 outcome,
5285 requested,
5286 adoption,
5287 adoption_error,
5288 });
5289 self.state.save()?;
5290 Ok(())
5291 }
5292
5293 async fn fold_losers(&mut self) -> Result<()> {
5296 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5297 return Ok(());
5298 };
5299 if !self.state.candidate_preservation.is_empty() {
5300 bail!("candidate preservation incomplete; refusing cleanup");
5301 }
5302 let repo = self.state.repo.clone();
5303 let _coordination = self.coordinate(&repo).await?;
5304 let mut folded = Vec::new();
5305 for i in 0..self.state.candidates.len() {
5306 let c = &self.state.candidates[i];
5307 if c.label == winner || c.folded {
5308 continue;
5309 }
5310 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5311 remove_preserved_worktree(&repo, &wt).await?;
5312 if git::branch_exists(&repo, &branch).await?
5313 && !git::branch_delete(&repo, &branch).await?
5314 {
5315 bail!("candidate branch removal refused");
5316 }
5317 self.state.candidates[i].folded = true;
5318 self.state.save()?;
5319 folded.push(label.to_string());
5320 }
5321 let root = self.state.worktree_root();
5323 for j in 1..=self.roles.judges.len() {
5324 let wt = root.join(format!("judge-{j}"));
5325 if wt.exists() {
5326 remove_preserved_worktree(&repo, &wt).await?;
5327 }
5328 }
5329 if self.state.config.graph.advise {
5332 for k in 1..=self.state.config.graph.advisors {
5333 let wt = root.join(format!("advisor-{k}"));
5334 if wt.exists() {
5335 remove_preserved_worktree(&repo, &wt).await?;
5336 }
5337 }
5338 }
5339 if !folded.is_empty() {
5340 self.state
5341 .event("fold", format!("folded candidates {}", folded.join(", ")));
5342 self.state.save()?;
5343 }
5344 Ok(())
5345 }
5346
5347 async fn hold_base_sync(&mut self) -> Result<bool> {
5350 self.hold_base_sync_as(None).await
5351 }
5352
5353 async fn hold_base_sync_kind(
5354 &mut self,
5355 kind: crate::base_sync_recovery::StopKind,
5356 ) -> Result<()> {
5357 self.hold_base_sync_as(Some(kind)).await?;
5358 Ok(())
5359 }
5360
5361 async fn hold_base_sync_as(
5362 &mut self,
5363 kind: Option<crate::base_sync_recovery::StopKind>,
5364 ) -> Result<bool> {
5365 if !self.state.base_sync_blocked() {
5366 self.state.base_sync_recovery.stop = None;
5367 return Ok(false);
5368 }
5369 let head = match self.state.winner() {
5370 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5371 .await
5372 .unwrap_or_default(),
5373 None => String::new(),
5374 };
5375 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5376 && !authorization.consumed
5377 && authorization.stop.head != head
5378 {
5379 authorization.refuse(format!(
5380 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5381 authorization.stop.head
5382 ));
5383 }
5384 self.state.record_base_sync_stop(&head);
5385 if let Some(kind) = kind {
5386 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5387 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5388 stop.kind = kind;
5389 }
5390 }
5391 self.state.save()?;
5392 Ok(true)
5393 }
5394
5395 async fn sync_to_base(&mut self) -> Result<()> {
5437 if self.state.status == RunStatus::AlreadyInBase {
5438 return Ok(());
5439 }
5440 self.hold_base_sync().await?;
5441 let conflicted = self
5442 .state
5443 .base_sync
5444 .as_ref()
5445 .is_some_and(|s| s.conflict.is_some());
5446 let Some(winner) = self.state.winner().cloned() else {
5447 return Ok(());
5448 };
5449
5450 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5451
5452 let repo = self.state.repo.clone();
5453 let remote = self.state.config.merge.remote.clone();
5454 let base_branch = self.state.base_branch.clone();
5455 let tracking = format!("{remote}/{base_branch}");
5456
5457 let fetched_base =
5458 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5459 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5464 return Ok(());
5465 };
5466
5467 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5468 self.state.ensure_base_episode();
5469 let mut pending = self
5470 .state
5471 .base_sync_recovery
5472 .operations
5473 .last()
5474 .filter(|o| !o.finished)
5475 .cloned();
5476 let scratch = self.state.dir().join("base-sync");
5477 if git::rebase_in_progress(&scratch).await {
5478 if pending.is_none()
5481 && self.state.base_sync_recovery.operations.is_empty()
5482 && !conflicted
5483 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5484 .await
5485 .ok()
5486 .as_deref()
5487 == Some(head.as_str())
5488 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5489 && git::is_clean(&winner.worktree).await?
5490 {
5491 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5492 if self.state.base_sync.is_none() {
5493 self.state.base_sync = Some(BaseSync {
5494 tip: saved_tip.clone(),
5495 behind: 0,
5496 attempts: 0,
5497 conflict: None,
5498 already_in: None,
5499 });
5500 }
5501 self.state.reserve_base_operation(&head, &saved_tip);
5502 self.state.event(
5503 "base-recovery",
5504 "imported standing legacy rebase as a spent synchronization reservation",
5505 );
5506 self.state.save()?;
5507 pending = self.state.base_sync_recovery.operations.last().cloned();
5508 }
5509 let expected = pending.as_ref().map(|op| op.tip.as_str());
5510 if expected.is_none()
5511 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5512 {
5513 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5514 if self.state.base_sync.is_none() {
5515 self.state.base_sync = Some(BaseSync {
5516 tip: tip.clone(),
5517 behind: 0,
5518 attempts: 0,
5519 conflict: None,
5520 already_in: None,
5521 });
5522 }
5523 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5524 self.state.event("land", why);
5525 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5526 .await?;
5527 return Ok(());
5528 }
5529 }
5530 if let Some(op) = &pending {
5531 let mut replay_proven =
5533 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5534 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5535 let wait = repository_wait(&self.state, None);
5536 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5537 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5538 Ok(true) => {
5539 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5540 self.state.save()?;
5541 git::worktree_remove(&repo, &scratch).await.ok();
5542 replay_proven = true;
5543 }
5544 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5545 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5546 }
5547 }
5548 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5549 if self.state.foreign_checkout.is_some() {
5550 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5551 .await
5552 .ok()
5553 .as_deref()
5554 == Some(op.head.as_str())
5555 && git::rev_parse(&repo, &format!("{head}^2"))
5556 .await
5557 .ok()
5558 .as_deref()
5559 == Some(op.tip.as_str());
5560 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5561 .await
5562 .is_ok_and(|m| m.is_empty())
5563 && let Ok(added) =
5564 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5565 {
5566 replay_proven = true;
5567 for commit in added {
5568 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5569 replay_proven = false;
5570 break;
5571 }
5572 }
5573 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5574 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5575 }
5576 }
5577 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5581 || (head != op.head
5582 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5583 .await?
5584 .is_empty())
5585 || (!git::is_clean(&winner.worktree).await?
5586 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5587 .await
5588 .is_ok_and(|o| o.ok()))
5589 || !replay_proven
5590 {
5591 let why = format!(
5592 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5593 op.id
5594 );
5595 let sync = self
5596 .state
5597 .base_sync
5598 .as_mut()
5599 .context("reserved synchronization has no lifetime accounting")?;
5600 sync.conflict = Some(why.clone());
5601 self.state.event("land", why);
5602 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5603 .await?;
5604 return Ok(());
5605 }
5606 if head != op.head {
5607 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5608 }
5609 tip = op.tip.clone();
5610 }
5611 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5612 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5613 if pending.is_none()
5614 && !conflicted
5615 && fetched_base
5616 && self
5617 .state
5618 .base_sync_recovery
5619 .episodes
5620 .last()
5621 .is_some_and(|e| e.boundary.is_some())
5622 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5623 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5624 && owner.execution_runs().last() == Some(&self.state.id)
5625 && owner.status != crate::queue::TaskStatus::Done
5626 && !owner.fresh_start
5627 && owner.review_branch.is_none()
5628 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5629 && let Some(approval) = owner.resumption.as_ref()
5630 && approval.instruction == owner.instruction
5631 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5632 && git::is_clean(&winner.worktree).await?
5633 && self.state.renew_base_episode(&task, approval, &head, &tip)
5634 {
5635 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5636 self.state.save()?;
5637 }
5638
5639 if (behind > 0 || conflicted || head == tip)
5648 && self
5649 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5650 .await?
5651 {
5652 return Ok(());
5653 }
5654 if conflicted && self.state.base_recovery_pending() {
5655 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5656 let owner =
5657 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5658 let owner_valid = owner.as_ref().is_ok_and(|t| {
5659 t.execution_runs().last() == Some(&self.state.id)
5660 && t.status != crate::queue::TaskStatus::Done
5661 && !t.fresh_start
5662 && t.review_branch.is_none()
5663 });
5664 let invalidation = if !owner_valid {
5665 Some("latest task ownership changed")
5666 } else if !owner.as_ref().is_ok_and(|t| {
5667 authorization
5668 .generation
5669 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5670 }) {
5671 Some("owner approval generation changed")
5672 } else if authorization.question.as_ref().is_some_and(|question| {
5673 !crate::base_recovery_control::decision_current(
5674 &crate::queue::Queue::at(crate::run::home().join("queue")),
5675 &self.state,
5676 &authorization.task,
5677 question,
5678 )
5679 }) {
5680 Some("recovery decision ownership, routing or stop evidence changed")
5681 } else if authorization.request.is_some()
5682 && !owner
5683 .as_ref()
5684 .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5685 {
5686 Some("owner decision changed")
5687 } else if let Some(scope) = &authorization.scope {
5688 if scope.tip != tip {
5689 Some("fetched base changed")
5690 } else if !self
5691 .state
5692 .base_sync_recovery
5693 .episodes
5694 .last()
5695 .is_some_and(|e| e.id == scope.episode)
5696 {
5697 Some("active synchronization interval changed")
5698 } else if !owner
5699 .as_ref()
5700 .is_ok_and(|t| t.instruction == scope.instruction)
5701 {
5702 Some("task instruction changed")
5703 } else {
5704 None
5705 }
5706 } else {
5707 None
5708 };
5709 if let Some(reason) = invalidation {
5710 let authorization = self
5711 .state
5712 .base_sync_recovery
5713 .authorizations
5714 .last_mut()
5715 .unwrap();
5716 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5717 self.state.save()?;
5718 return Ok(());
5719 }
5720 if !fetched_base
5721 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5722 || !git::is_clean(&winner.worktree).await?
5723 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5724 {
5725 return Ok(());
5726 }
5727 }
5728 let recovery = conflicted && self.state.consume_base_recovery(&head);
5729 if conflicted {
5730 self.state.save()?;
5732 if !recovery {
5733 return Ok(());
5734 }
5735 }
5736
5737 if behind == 0 {
5738 if let Some(from) = self
5747 .state
5748 .rebase_fixes
5749 .iter()
5750 .rev()
5751 .find_map(|r| r.from.clone())
5752 && from != head
5753 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5754 .await
5755 .is_ok_and(|o| o.ok())
5756 {
5757 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5758 }
5759 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5765 if let Some(why) = &conflict {
5766 self.state.status = RunStatus::Blocked;
5767 self.state.event("land", why.clone());
5768 }
5769 self.state.base_sync = Some(BaseSync {
5770 tip,
5771 behind: 0,
5772 attempts,
5773 conflict,
5774 already_in: None,
5775 });
5776 if pending.is_some() {
5777 self.state.finish_base_operation(Some(&head));
5778 }
5779 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5780 .await?;
5781 self.state.save()?;
5782 return Ok(());
5783 }
5784
5785 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5786 let why = format!(
5787 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5788 synchronization(s); synchronizing again would only race it",
5789 winner.branch
5790 );
5791 self.state.status = RunStatus::Blocked;
5792 self.state.base_sync = Some(BaseSync {
5793 tip,
5794 behind,
5795 attempts,
5796 conflict: Some(why.clone()),
5797 already_in: None,
5798 });
5799 self.state.event("land", why);
5800 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5801 .await?;
5802 self.state.save()?;
5803 return Ok(());
5804 }
5805
5806 if self.state.foreign_checkout.is_some() {
5807 let wait = repository_wait(&self.state, None);
5810 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5811 if self.state.base_sync.is_none() {
5812 self.state.base_sync = Some(BaseSync {
5813 tip: tip.clone(),
5814 behind,
5815 attempts: 0,
5816 conflict: None,
5817 already_in: None,
5818 });
5819 }
5820 self.state.reserve_base_operation(&head, &tip);
5821 self.state.save()?;
5822 let merged = git::git_raw(
5823 &winner.worktree,
5824 &[
5825 "-c",
5826 "core.hooksPath=/dev/null",
5827 "merge",
5828 "--no-ff",
5829 "--no-autostash",
5830 "--no-edit",
5831 "-m",
5832 &format!("Merge {tracking} before verifying"),
5833 &tip,
5834 ],
5835 )
5836 .await?;
5837 let conflict = if merged.ok() {
5838 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5839 self.state.event(
5840 "land",
5841 format!("merged {tracking} into {} before verifying", winner.branch),
5842 );
5843 None
5844 } else {
5845 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5846 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5847 }
5848 let why = format!(
5849 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5850 winner.branch,
5851 merged.stderr.chars().take(600).collect::<String>()
5852 );
5853 self.state.status = RunStatus::Blocked;
5854 self.state.event("land", &why);
5855 Some(why)
5856 };
5857 let synced = conflict.is_none();
5858 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5859 self.state
5860 .finish_base_operation(synced.then_some(fresh.as_str()));
5861 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5862 self.state.base_sync = Some(BaseSync {
5863 tip,
5864 behind: if synced { 0 } else { behind },
5865 attempts,
5866 conflict,
5867 already_in: None,
5868 });
5869 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5870 .await?;
5871 self.state.save()?;
5872 return Ok(());
5873 }
5874
5875 self.state.event(
5876 "land",
5877 format!(
5878 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5879 winner.branch
5880 ),
5881 );
5882 self.state.save()?;
5883
5884 let branch_tracking = format!("{remote}/{}", winner.branch);
5890 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5891 let remote_tip = if self.state.task_pr_branch().is_none()
5892 && matches!(&fetched_branch, Ok(o) if o.ok())
5893 {
5894 git::rev_parse(&repo, &branch_tracking).await.ok()
5895 } else {
5896 None
5897 };
5898 if let Some(theirs) = &remote_tip
5902 && !git::is_ancestor(&repo, theirs, &head).await
5903 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5904 .await
5905 .is_ok_and(|missing| missing.is_empty())
5906 {
5907 let why = format!(
5908 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5909 them",
5910 short(theirs),
5911 winner.branch
5912 );
5913 self.state.status = RunStatus::Blocked;
5914 self.state.base_sync = Some(BaseSync {
5915 tip,
5916 behind,
5917 attempts,
5918 conflict: Some(why.clone()),
5919 already_in: None,
5920 });
5921 self.state.event("land", why);
5922 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5923 .await?;
5924 self.state.save()?;
5925 return Ok(());
5926 }
5927
5928 if self.state.base_sync.is_none() {
5929 self.state.base_sync = Some(BaseSync {
5930 tip: tip.clone(),
5931 behind,
5932 attempts: 0,
5933 conflict: None,
5934 already_in: None,
5935 });
5936 }
5937 self.state.reserve_base_operation(&head, &tip);
5938 self.state.save()?;
5939 let scratch = self.state.dir().join("base-sync");
5940 let rebased =
5941 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5942 .await
5943 {
5944 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5945 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5946 Err(e) => Err(e),
5947 };
5948 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5949 let stop_kind = match &rebased {
5950 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5951 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5952 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5953 };
5954 match rebased {
5955 Ok(None) => {
5956 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5960 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5961 self.state.finish_base_operation(Some(&fresh));
5962 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5963 let mut conflict = None;
5964 if let Some(pinned) = &remote_tip {
5965 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5966 match pushed {
5967 Ok(o) if o.ok() => self.state.event(
5968 "land",
5969 format!("pushed rebased {} to {remote}", winner.branch),
5970 ),
5971 Ok(o) => {
5972 conflict = Some(format!(
5973 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5974 winner.branch,
5975 short(pinned),
5976 o.stderr.chars().take(600).collect::<String>()
5977 ));
5978 }
5979 Err(e) => {
5980 conflict = Some(format!(
5981 "rebased {} locally but could not push it: {e:#}",
5982 winner.branch
5983 ));
5984 }
5985 }
5986 }
5987 if let Some(why) = &conflict {
5988 self.state.status = RunStatus::Blocked;
5989 self.state.event("land", why.clone());
5990 }
5991 self.state.base_sync = Some(BaseSync {
5992 tip: tip.clone(),
5993 behind: 0,
5994 attempts,
5995 conflict,
5996 already_in: None,
5997 });
5998 self.state
5999 .event("land", format!("rebased {} onto {tracking}", winner.branch));
6000 }
6001 Ok(Some(conflict)) => {
6002 self.state.finish_base_operation(None);
6003 let why = format!(
6004 "{} conflicts with {tracking} and did not rebase: {}",
6005 winner.branch,
6006 conflict.chars().take(600).collect::<String>()
6007 );
6008 self.state.status = RunStatus::Blocked;
6009 self.state.base_sync = Some(BaseSync {
6010 tip,
6011 behind,
6012 attempts,
6013 conflict: Some(why.clone()),
6014 already_in: None,
6015 });
6016 self.state.event("land", why);
6017 }
6018 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6022 self.state.save()?;
6023 return Ok(());
6024 }
6025 Err(e) => {
6026 self.state.finish_base_operation(None);
6027 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6028 self.state.status = RunStatus::Blocked;
6029 self.state.base_sync = Some(BaseSync {
6030 tip,
6031 behind,
6032 attempts,
6033 conflict: Some(why.clone()),
6034 already_in: None,
6035 });
6036 self.state.event("land", why);
6037 }
6038 }
6039 if !self.state.base_sync_blocked() {
6040 self.state.base_sync_recovery.stop = None;
6041 }
6042 self.hold_base_sync_kind(stop_kind).await?;
6043 self.state.save()?;
6044 Ok(())
6045 }
6046
6047 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6053 if self.state.task_pr_branch().is_some() {
6056 return None;
6057 }
6058 let repo = self.state.repo.clone();
6059 let remote = self.state.config.merge.remote.clone();
6060 let fetched = git::fetch(&repo, &remote, branch).await;
6061 if !matches!(&fetched, Ok(o) if o.ok()) {
6062 return None;
6063 }
6064 let branch_tracking = format!("{remote}/{branch}");
6065 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6066 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6067 return None;
6068 }
6069 if !crate::reconcile::origin_missing(&repo, head, &theirs)
6070 .await
6071 .is_ok_and(|missing| missing.is_empty())
6072 {
6073 return Some(format!(
6074 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6075 them",
6076 short(&theirs)
6077 ));
6078 }
6079 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6080 Ok(o) if o.ok() => {
6081 self.state
6082 .event("land", format!("pushed rebased {branch} to {remote}"));
6083 None
6084 }
6085 Ok(o) => Some(format!(
6086 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6087 someone may have pushed): {}",
6088 short(&theirs),
6089 o.stderr.chars().take(600).collect::<String>()
6090 )),
6091 Err(e) => Some(format!(
6092 "{branch} is rebased locally but could not be pushed: {e:#}"
6093 )),
6094 }
6095 }
6096
6097 async fn settle_already_in(
6114 &mut self,
6115 branch: &str,
6116 tip: &str,
6117 head: &str,
6118 attempts: usize,
6119 behind: usize,
6120 ) -> Result<bool> {
6121 let repo = self.state.repo.clone();
6122 let remote = self.state.config.merge.remote.clone();
6123 let start = self.state.base_commit.clone();
6124 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6125 Ok(Some(e)) => e,
6126 Ok(None) => return Ok(false),
6127 Err(e) => {
6128 tracing::warn!("already-in-base check for {branch}: {e:#}");
6129 return Ok(false);
6130 }
6131 };
6132 let mut verified = vec![head.to_owned()];
6133 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6134 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6135 if matches!(&fetched, Ok(o) if o.ok())
6136 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6137 && theirs != head
6138 {
6139 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6140 Ok(Some(_)) => verified.push(theirs),
6141 _ => return Ok(false),
6142 }
6143 }
6144 let base_branch = self.state.base_branch.clone();
6145 let message = format!(
6146 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6147 land",
6148 evidence.names(),
6149 evidence.proof.as_str()
6150 );
6151 let closed =
6152 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6153 .await;
6154 match closed {
6155 Ok(Ok(url)) => self
6156 .state
6157 .event("land", format!("closed {url}: superseded on {base_branch}")),
6158 Ok(Err(why)) => self
6159 .state
6160 .event("land", format!("did not close a pull request: {why}")),
6161 Err(e) => {
6162 let why = format!(
6163 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6164 not be closed ({e:#}); resume to retry"
6165 );
6166 self.state.status = RunStatus::Blocked;
6167 self.state.base_sync = Some(BaseSync {
6168 tip: tip.to_owned(),
6169 behind,
6170 attempts,
6171 conflict: Some(why.clone()),
6172 already_in: None,
6173 });
6174 self.state.event("land", why);
6175 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6176 .await?;
6177 self.state.save()?;
6178 return Ok(true);
6179 }
6180 }
6181 self.state.status = RunStatus::AlreadyInBase;
6182 self.state.base_sync_recovery.stop = None;
6183 self.state.base_sync = Some(BaseSync {
6184 tip: tip.to_owned(),
6185 behind,
6186 attempts,
6187 conflict: None,
6188 already_in: Some(evidence),
6189 });
6190 self.state.event("land", message);
6191 self.state.save()?;
6192 self.settle_questions();
6193 Ok(true)
6194 }
6195
6196 fn landing_base(&self) -> String {
6206 self.state
6207 .base_sync
6208 .as_ref()
6209 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6210 }
6211
6212 pub async fn fix_selected(
6245 &mut self,
6246 ids: &[String],
6247 reason: &str,
6248 allow_stale: bool,
6249 ) -> Result<()> {
6250 let reason = reason.trim();
6251 if reason.is_empty() {
6252 bail!("a fix request needs a reason — that is the operator's own record of why");
6253 }
6254 if ids.is_empty() {
6255 bail!("no finding id given");
6256 }
6257 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6258 bail!(
6259 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6260 has already concluded — can be given a targeted fix. A run still \
6261 in progress should simply be resumed; a `merged` run's branch has \
6262 already landed, so its answer is a fresh `magi review <branch>`, \
6263 not reopening this run's own record",
6264 self.state.id,
6265 self.state.status.as_str()
6266 );
6267 }
6268 let _claim = FixClaim::acquire(&self.state.dir())?;
6271 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6272 let Some(winner) = self.state.winner().cloned() else {
6273 bail!("run {} has no winning candidate to fix", self.state.id);
6274 };
6275 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6276 bail!(
6277 "branch `{}` no longer exists; this run cannot be extended",
6278 winner.branch
6279 );
6280 }
6281 let home = crate::run::home();
6282 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6283 bail!(
6284 "run {} is currently being worked on by another magi process",
6285 self.state.id
6286 );
6287 }
6288 let mut seen = BTreeSet::new();
6292 let mut findings = Vec::new();
6293 let mut missing = Vec::new();
6294 for id in ids {
6295 if !seen.insert(id.clone()) {
6296 continue;
6297 }
6298 match self.state.finding(id) {
6299 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6300 id: f.id.clone(),
6301 severity: f.severity,
6302 reviewer_vote: rec.vote,
6303 round: round.round,
6304 round_head: round.head.clone(),
6305 reviewer: rec.reviewer,
6306 agent: rec.agent.clone(),
6307 file: f.file.clone(),
6308 line: f.line,
6309 title: f.title.clone(),
6310 detail: f.detail.clone(),
6311 outcome: OperatorFixOutcome::Pending,
6312 }),
6313 None => missing.push(id.clone()),
6314 }
6315 }
6316 if !missing.is_empty() {
6317 bail!(
6318 "unknown finding id(s): {}; nothing was changed",
6319 missing.join(", ")
6320 );
6321 }
6322
6323 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6324 let stale_details: Vec<(String, String)> = findings
6325 .iter()
6326 .filter(|f| f.round_head != head_at_request)
6327 .map(|f| (f.id.clone(), f.round_head.clone()))
6328 .collect();
6329 let stale = !stale_details.is_empty();
6330 if stale && !allow_stale {
6331 bail!(
6332 "the branch has moved since some finding(s) were raised — {} — now \
6333 at {}; pass --allow-stale to fix anyway, or re-run review first",
6334 stale_details
6335 .iter()
6336 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6337 .collect::<Vec<_>>()
6338 .join(", "),
6339 short(&head_at_request)
6340 );
6341 }
6342
6343 let request = OperatorFixRequest {
6344 requested_at: Timestamp::now(),
6345 reason: reason.to_owned(),
6346 findings,
6347 head_at_request: head_at_request.clone(),
6348 allow_stale,
6349 stale,
6350 fix: None,
6351 result_head: None,
6352 follow_up_review_run: None,
6353 };
6354 self.state.event(
6355 "fix",
6356 format!(
6357 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6358 request.findings.len(),
6359 request
6360 .findings
6361 .iter()
6362 .map(|f| f.id.as_str())
6363 .collect::<Vec<_>>()
6364 .join(", "),
6365 ),
6366 );
6367 self.state.operator_fixes.push(request);
6374 self.state.save()?;
6375 let request_index = self.state.operator_fixes.len() - 1;
6376
6377 let repo = self.state.repo.clone();
6386 let coordination = self.coordinate(&repo).await?;
6387 if winner.worktree.exists() {
6388 let dirty = git::git(
6391 &winner.worktree,
6392 &["status", "--porcelain", "--untracked-files=all"],
6393 )
6394 .await?;
6395 let only_withheld = dirty.lines().all(|l| {
6396 l.strip_prefix("?? ")
6397 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6398 });
6399 if !only_withheld {
6400 bail!(
6401 "`{}` has uncommitted changes; refusing to touch it — commit or \
6402 discard them first",
6403 winner.worktree.display()
6404 );
6405 }
6406 git::worktree_remove(&self.state.repo, &winner.worktree)
6407 .await
6408 .ok();
6409 }
6410 let fix_worktree = self.state.worktree_root().join("operator-fix");
6411 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6412 git::git(
6413 &self.state.repo,
6414 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6415 )
6416 .await
6417 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6418 if !git::is_clean(&fix_worktree).await? {
6419 git::worktree_remove(&self.state.repo, &fix_worktree)
6420 .await
6421 .ok();
6422 bail!(
6423 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6424 winner.branch
6425 );
6426 }
6427 let repo_for_setup = self.state.repo.clone();
6428 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6429 Ok(()) => {
6430 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6431 }
6432 Err(e) => Err(e),
6433 };
6434 if let Err(e) = setup {
6435 git::worktree_remove(&self.state.repo, &fix_worktree)
6436 .await
6437 .ok();
6438 return Err(e);
6439 }
6440
6441 self.state.resource_block = None;
6444
6445 let run_id = self.state.id.clone();
6446 let prompts = self.state.config.prompts.clone();
6447 let language = self.state.config.graph.language.clone();
6448 let sessions = self.state.config.graph.sessions;
6449 let artifacts = agent::artifacts_dir(&self.state.dir());
6450 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6451 .findings
6452 .iter()
6453 .map(|f| Finding {
6454 id: f.id.clone(),
6455 severity: f.severity,
6456 file: f.file.clone(),
6457 line: f.line,
6458 title: f.title.clone(),
6459 detail: f.detail.clone(),
6460 })
6461 .collect();
6462 let fix_prompt = prompt::operator_fix_in(
6463 &self.state.instruction,
6464 &finding_list,
6465 reason,
6466 &stale_details,
6467 &head_at_request,
6468 &language,
6469 &self.state.config.graph.github_language,
6470 );
6471 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6472 drop(coordination);
6473 let (job, seat, out) = self
6474 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6475 prompt: fix_prompt.clone(),
6476 spec,
6477 seat,
6478 cwd: fix_worktree.clone(),
6479 timeout,
6480 allow_write: true,
6481 sessions,
6482 artifacts: artifacts.clone(),
6483 stem: "operator-fix".to_owned(),
6484 handover: None,
6485 })
6486 .await;
6487 let agent_id = seat.agent.clone();
6488
6489 let mut fix = FixRecord {
6490 agent: agent_id,
6491 addressed: Vec::new(),
6492 rejected: Vec::new(),
6493 notes: String::new(),
6494 committed: false,
6495 failed: None,
6496 duration_ms: 0,
6497 continuation: None,
6498 };
6499 let mut final_seat = seat.clone();
6500 match out {
6501 AgentOutcome::Ok(o) => {
6502 fix.duration_ms = o.duration_ms;
6503 let parsed = verdict::extract_json::<FixReport>(&o.text);
6504 let incomplete_reason = match &parsed {
6505 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6506 "the reply parsed, but it reported a command whose own CLI \
6507 never confirmed an exit status"
6508 .to_owned(),
6509 ),
6510 Ok(_) => None,
6511 Err(e) => Some(e.to_string()),
6512 };
6513 match incomplete_reason {
6514 None => {
6515 let report = parsed.expect("checked Ok above");
6516 fix.addressed = report.addressed;
6517 fix.rejected = report.rejected;
6518 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6519 }
6520 Some(reason) => {
6521 let (resumed_seat, resolved, failure, cont) = self
6522 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6523 .await;
6524 fix.duration_ms += cont.cumulative_wait_ms;
6525 fix.continuation = Some(cont);
6526 final_seat = resumed_seat;
6527 match resolved {
6528 Some(report) => {
6529 fix.addressed = report.addressed;
6530 fix.rejected = report.rejected;
6531 fix.notes =
6532 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6533 }
6534 None => fix.failed = failure,
6535 }
6536 }
6537 }
6538 }
6539 AgentOutcome::Dropped(o) => {
6540 fix.duration_ms = o.duration_ms;
6541 let why = o
6542 .dropped
6543 .as_ref()
6544 .map(|d| d.why.as_str())
6545 .unwrap_or("the CLI ended the stream without delivering its answer");
6546 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6547 }
6548 AgentOutcome::Quota(o) => {
6549 self.state.quota.push(QuotaLoss::from_report(
6550 final_seat.key.clone(),
6551 "fix",
6552 o.quota.as_ref(),
6553 ));
6554 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6555 }
6556 AgentOutcome::Failed(e) => fix.failed = Some(e),
6557 AgentOutcome::ResourceBlocked(e) => {
6558 fix.failed = Some(format!("the fixer was not started: {e}"));
6559 }
6560 }
6561 if fix.continuation.is_none() {
6564 fix.continuation = Some(ContinuationRecord::not_needed());
6565 }
6566 self.state.seats.insert(final_seat.key.clone(), final_seat);
6567
6568 let rescue_message = format!(
6569 "magi: operator-selected fix ({}) (uncommitted work)",
6570 self.state.operator_fixes[request_index]
6571 .findings
6572 .iter()
6573 .map(|f| f.id.as_str())
6574 .collect::<Vec<_>>()
6575 .join(", ")
6576 );
6577 let repo = self.state.repo.clone();
6578 let coordination = self.coordinate(&repo).await?;
6579 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6580 self.state.note_withheld("fix", &r.withheld);
6581 }
6582 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6583 fix.committed = after != head_at_request;
6584 git::worktree_remove(&self.state.repo, &fix_worktree)
6585 .await
6586 .ok();
6587 drop(coordination);
6588
6589 self.state.event(
6590 "fix",
6591 match &fix.failed {
6592 Some(reason) => format!(
6593 "operator fix: adoption report was lost ({reason}); {}",
6594 if fix.committed {
6595 "committed"
6596 } else {
6597 "NO new commit"
6598 }
6599 ),
6600 None => format!(
6601 "operator fix: {} addressed, {} rejected, {}",
6602 fix.addressed.len(),
6603 fix.rejected.len(),
6604 if fix.committed {
6605 "committed"
6606 } else {
6607 "NO new commit"
6608 }
6609 ),
6610 },
6611 );
6612
6613 for f in &mut self.state.operator_fixes[request_index].findings {
6620 f.outcome = if fix.failed.is_some() {
6621 OperatorFixOutcome::Unreported
6622 } else if fix.addressed.contains(&f.id) {
6623 OperatorFixOutcome::Addressed
6624 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6625 OperatorFixOutcome::Rejected { why: r.why.clone() }
6626 } else {
6627 OperatorFixOutcome::Unreported
6628 };
6629 }
6630
6631 let committed = fix.committed;
6632 if committed {
6633 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6634 }
6635 self.state.operator_fixes[request_index].fix = Some(fix);
6636 self.state.save()?;
6639 if self.state.resource_block.is_some() {
6640 return self.stop_for_resource();
6641 }
6642
6643 if committed {
6644 self.state.event(
6645 "fix",
6646 format!(
6647 "operator fix committed {}; opening a follow-up review-only run",
6648 short(&after)
6649 ),
6650 );
6651 let origin =
6654 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6655 match Self::review(
6656 &self.state.repo,
6657 &winner.branch,
6658 self.state.config.clone(),
6659 origin,
6660 )
6661 .await
6662 {
6663 Ok(mut follow_up) => {
6664 follow_up.state.event(
6665 "start",
6666 format!(
6667 "requested by an operator fix on run {} for finding(s) {}",
6668 self.state.id,
6669 self.state.operator_fixes[request_index]
6670 .findings
6671 .iter()
6672 .map(|f| f.id.as_str())
6673 .collect::<Vec<_>>()
6674 .join(", "),
6675 ),
6676 );
6677 follow_up.state.save()?;
6678 let follow_up_id = follow_up.state.id.clone();
6679 let adopted = match crate::direct::adopt(
6682 &follow_up.state,
6683 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6684 ) {
6685 Ok(a) => a,
6686 Err(e) => {
6687 self.state.event(
6690 "fix",
6691 format!(
6692 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6693 ),
6694 );
6695 self.state.save()?;
6696 return Ok(());
6697 }
6698 };
6699 let executed = follow_up.execute().await;
6700 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6701 if let Some(a) = adopted {
6702 a.finish(&follow_up.state, executed);
6703 }
6704 if let Some(e) = failure {
6705 self.state.event(
6706 "fix",
6707 format!(
6708 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6709 ),
6710 );
6711 }
6712 self.state.operator_fixes[request_index].follow_up_review_run =
6713 Some(follow_up_id);
6714 }
6715 Err(e) => {
6716 self.state.event(
6717 "fix",
6718 format!("committed the fix but could not open a follow-up review: {e:#}"),
6719 );
6720 }
6721 }
6722 self.state.save()?;
6723 }
6724
6725 Ok(())
6726 }
6727
6728 async fn ask_fixer(
6748 &mut self,
6749 winner: &Candidate,
6750 node: &'static str,
6751 round: Option<usize>,
6752 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6753 ) -> (SeatJob, SeatState, AgentOutcome) {
6754 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6755 self.ask_fixer_among(attempts, node, round, None, build)
6756 .await
6757 }
6758
6759 async fn ask_fixer_among(
6764 &mut self,
6765 attempts: Vec<(AgentSpec, String)>,
6766 node: &'static str,
6767 round: Option<usize>,
6768 restore: Option<(&Path, &str)>,
6769 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6770 ) -> (SeatJob, SeatState, AgentOutcome) {
6771 let run_id = self.state.id.clone();
6772 let prompts = self.state.config.prompts.clone();
6773 let cache = self.state.config.cache_dir();
6774 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6775 let mut last = None;
6776 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6777 let seat = self.seat(&key, &spec.id);
6778 let mut job = build(spec.clone(), seat);
6779 if i > 0 {
6780 job.stem = format!("{}-{}", job.stem, spec.id);
6781 }
6782 let ctx = WaveCtx {
6783 carry_seats: false,
6784 pauses: Some((&self.pause, &self.interrupt)),
6785 run: &run_id,
6786 node,
6787 prompts: &prompts,
6788 cache: cache.as_deref(),
6789 round,
6790 };
6791 let (seat, out) =
6792 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6793 let advances = match &out {
6796 AgentOutcome::Ok(o) => agent::output_advances(o),
6797 AgentOutcome::ResourceBlocked(_) => false,
6798 _ => true,
6799 };
6800 if let Some(next) = ids.get(i + 1)
6801 && advances
6802 {
6803 self.state.seats.insert(seat.key.clone(), seat.clone());
6804 let class =
6805 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6806 let reason = match &out {
6807 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6808 other => fail_reason(other),
6809 };
6810 if let Some((wt, rev)) = restore
6814 && let Err(e) = self.graft_restore(wt, rev).await
6815 {
6816 let why =
6817 format!("could not restore the worktree before the next fixer: {e:#}");
6818 tracing::warn!("{why}");
6819 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6820 AgentOutcome::ResourceBlocked(why)
6821 } else {
6822 AgentOutcome::Failed(why)
6823 };
6824 last = Some((job, seat, out));
6825 break;
6826 }
6827 record_handover(
6828 &mut self.state,
6829 node,
6830 &seat.key,
6831 &spec.id,
6832 next,
6833 &class,
6834 &reason,
6835 );
6836 continue;
6837 }
6838 last = Some((job, seat, out));
6839 break;
6840 }
6841 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6842 if let AgentOutcome::Quota(output) = &out
6843 && node != "graft"
6844 {
6845 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6850 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6851 self.state.quota.push(loss);
6852 self.state.seats.insert(seat.key.clone(), seat.clone());
6853 self.state.status = RunStatus::Blocked;
6854 return (
6855 job,
6856 seat,
6857 AgentOutcome::ResourceBlocked(
6858 "fixer unavailable; restore agent availability and resume".into(),
6859 ),
6860 );
6861 }
6862 (job, seat, out)
6863 }
6864
6865 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6876 use crate::recovery::{Mode, Repair};
6877 if !self.state.reviews.is_empty() {
6878 return Ok(());
6879 }
6880 let Some(mut rec) = self.state.recovery.clone() else {
6881 return Ok(());
6882 };
6883 if rec.mode != Mode::VerifyRepair {
6884 return Ok(());
6885 }
6886 let wt = winner.worktree.clone();
6887 let mut head = git::rev_parse(&wt, "HEAD").await?;
6888 match rec.repair.clone() {
6889 Repair::Done { .. } => return Ok(()),
6890 Repair::WaitingForRestore { before } => {
6891 self.graft_restore(&wt, &before).await?;
6892 head = before;
6893 rec.repair = Repair::NotStarted;
6894 self.state.recovery = Some(rec.clone());
6895 self.state.save()?;
6896 }
6897 Repair::Started { before } => {
6898 if let Err(e) = self.graft_restore(&wt, &before).await {
6902 if e.is::<crate::repo_lock::ResourceBlocked>() {
6903 return Err(e);
6904 }
6905 return self
6906 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6907 .await;
6908 }
6909 rec.repair = Repair::Done {
6910 before: before.clone(),
6911 after: before,
6912 committed: false,
6913 failed: Some("interrupted; rolled back".to_owned()),
6914 };
6915 self.state
6916 .event("recovery", "the repair was interrupted and rolled back");
6917 self.state.recovery = Some(rec);
6918 self.state.save()?;
6919 return Ok(());
6920 }
6921 Repair::NotStarted => {}
6922 }
6923 if crate::recovery::recheck(&mut rec, &head) {
6924 self.state.event("recovery", rec.summary());
6925 self.state.recovery = Some(rec);
6926 self.state.save()?;
6927 return Ok(());
6928 }
6929 rec.repair = Repair::Started {
6930 before: head.clone(),
6931 };
6932 self.state.recovery = Some(rec.clone());
6933 self.state.save()?;
6934
6935 let language = self.state.config.graph.language.clone();
6936 let sessions = self.state.config.graph.sessions;
6937 let artifacts = agent::artifacts_dir(&self.state.dir());
6938 let fix_prompt = prompt::recovery_fix(
6939 &self.state.instruction,
6940 &rec,
6941 self.state.config.graph.review_rounds,
6942 &language,
6943 );
6944 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6945 self.state.event(
6946 "recovery",
6947 format!(
6948 "repair: the fixer starts from the failed verification on {}",
6949 short(&head)
6950 ),
6951 );
6952 let (_job, seat, out) = self
6953 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6954 prompt: fix_prompt.clone(),
6955 spec,
6956 seat,
6957 cwd: wt.clone(),
6958 timeout,
6959 allow_write: true,
6960 sessions,
6961 artifacts: artifacts.clone(),
6962 stem: "fix-recovery".to_owned(),
6963 handover: None,
6964 })
6965 .await;
6966 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6967 rec.repair = Repair::WaitingForRestore { before: head };
6968 self.state.recovery = Some(rec);
6969 return self.stop_for_resource();
6970 }
6971 let failed = match &out {
6972 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6973 AgentOutcome::Quota(o) => {
6974 self.state.quota.push(QuotaLoss::from_report(
6975 seat.key.clone(),
6976 "fix",
6977 o.quota.as_ref(),
6978 ));
6979 Some("rate limited (quota); the fixer could not run".to_owned())
6980 }
6981 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6982 AgentOutcome::Failed(e) => Some(e.clone()),
6983 };
6984 self.state.seats.insert(seat.key.clone(), seat);
6985 let coordination = self.coordinate(&wt).await?;
6986 let failed = match failed {
6987 Some(why) => {
6989 if let Err(e) = git::restore_to(&wt, &head).await {
6990 return self
6991 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6992 .await;
6993 }
6994 Some(why)
6995 }
6996 None => {
6997 if let Ok(r) = git::rescue_commit(
6998 &wt,
6999 "magi: recovery repair of the failed verification (uncommitted work)",
7000 )
7001 .await
7002 {
7003 self.state.note_withheld("fix", &r.withheld);
7004 }
7005 None
7006 }
7007 };
7008 let after = git::rev_parse(&wt, "HEAD").await?;
7009 drop(coordination);
7010 self.state.event(
7011 "recovery",
7012 match &failed {
7013 Some(why) => format!("repair: nothing kept ({why})"),
7014 None if after != head => {
7015 format!("repair: committed {}", short(&after))
7016 }
7017 None => "repair: the fixer made no commit".to_owned(),
7018 },
7019 );
7020 rec.repair = Repair::Done {
7021 before: head.clone(),
7022 committed: after != head,
7023 after,
7024 failed,
7025 };
7026 self.state.recovery = Some(rec);
7027 self.state.save()?;
7028 Ok(())
7029 }
7030
7031 async fn stop_recovery(
7034 &mut self,
7035 mut rec: crate::recovery::RecoveryRecord,
7036 head: &str,
7037 why: String,
7038 ) -> Result<()> {
7039 rec.repair = crate::recovery::Repair::Started {
7041 before: head.to_owned(),
7042 };
7043 self.state.recovery = Some(rec);
7044 self.state
7045 .event("recovery", format!("repair could not be undone: {why}"));
7046 self.state.status = RunStatus::Blocked;
7047 self.state.save()?;
7048 bail!("recovery repair could not be rolled back: {why}")
7049 }
7050
7051 async fn review_loop(&mut self) -> Result<()> {
7052 if self.hold_base_sync().await? {
7053 return Ok(());
7054 }
7055 loop {
7056 if self
7057 .state
7058 .base_sync
7059 .as_ref()
7060 .is_some_and(|s| s.conflict.is_some())
7061 {
7062 return Ok(());
7063 }
7064 if self.catchup_repairable() {
7065 let record = self.state.reviews.last().expect("repairable round").clone();
7066 let winner = self
7067 .state
7068 .winner()
7069 .cloned()
7070 .context("catch-up repair has no winner")?;
7071 if !self.fix_review_round(&winner, record).await? {
7072 return Ok(());
7073 }
7074 }
7075 self.review_loop_pass().await?;
7076 if !self.catchup_repairable() {
7077 return Ok(());
7078 }
7079 }
7080 }
7081
7082 fn catchup_repairable(&self) -> bool {
7085 self.state.reviews.len() < self.state.config.graph.review_rounds
7086 && self.state.reviews.last().is_some_and(|r| {
7087 !r.clean
7088 && r.blocking == 0
7089 && r.fix.is_none()
7090 && r.e2e_status() == E2eStatus::Failed
7091 })
7092 }
7093
7094 async fn review_loop_pass(&mut self) -> Result<()> {
7095 if self
7100 .state
7101 .base_sync
7102 .as_ref()
7103 .is_some_and(|s| s.conflict.is_some())
7104 {
7105 return Ok(());
7106 }
7107 let run_id = self.state.id.clone();
7112 let prompts = self.state.config.prompts.clone();
7113 let Some(winner) = self.state.winner().cloned() else {
7114 return Ok(());
7115 };
7116 let granted_review = self
7117 .state
7118 .repair_grants
7119 .last()
7120 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7121 .cloned();
7122 let fresh_review = granted_review
7123 .as_ref()
7124 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7125 let max_rounds = granted_review
7126 .as_ref()
7127 .map_or(self.state.config.graph.review_rounds, |g| {
7128 g.review_start.saturating_add(1)
7129 });
7130 if let Some(status) =
7140 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7141 {
7142 if status == RunStatus::Gating {
7145 self.record_contested_handoff();
7146 }
7147 self.state.status = status;
7148 self.state.save()?;
7149 return Ok(());
7150 }
7151 self.state.status = RunStatus::Reviewing;
7152 if !fresh_review
7162 && self.state.reviews.last().is_some_and(|r| {
7163 r.e2e_status() == E2eStatus::ResourceBlocked
7164 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7165 })
7166 {
7167 let shell = self.state.config.shell();
7168 return self
7169 .stop_unresolved_review(
7170 "the last round's own verification never resolved",
7171 &shell,
7172 &winner.worktree,
7173 )
7174 .await;
7175 }
7176
7177 if granted_review.is_none() {
7178 self.recovery_repair(&winner).await?;
7179 }
7180 if self.resource_halt()? {
7181 return Ok(());
7182 }
7183
7184 let repo = self.state.repo.clone();
7185 let root = self.state.worktree_root();
7186 let language = self.state.config.graph.language.clone();
7187 let sessions = self.state.config.graph.sessions;
7188 let artifacts = self.merge_repair_artifacts();
7189 let base = self.landing_base();
7190 let base_short = short(&base);
7191 let reviewers = self.roles.reviewers.clone();
7192 let shell = self.state.config.shell();
7193
7194 for round in (self.state.reviews.len() + 1)..=max_rounds {
7195 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7196 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7197 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7198 let prev_verification = self
7207 .state
7208 .reviews
7209 .last()
7210 .and_then(|r| r.verification_summary(&head));
7211 let prior_section = match (&self.state.recovery, round) {
7214 (Some(rec), 1) if self.state.reviews.is_empty() => {
7215 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7216 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7217 Some(crate::recovery::head_of(
7218 &git::diff(&winner.worktree, reviewed, "HEAD")
7219 .await
7220 .unwrap_or_default(),
7221 crate::recovery::SINCE_DIFF_MAX,
7222 ))
7223 }
7224 _ => None,
7225 };
7226 Some(prompt::recovery_review(rec, since.as_deref()))
7227 }
7228 _ => None,
7229 };
7230
7231 let mut jobs = Vec::new();
7235 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7236 self.coordinate(&repo).await?
7237 } else {
7238 None
7239 };
7240 for (r, spec) in reviewers.iter().cloned().enumerate() {
7241 let wt = root.join(format!("review-{}", r + 1));
7242 if wt.exists() {
7245 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7246 } else {
7247 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7248 None
7249 } else {
7250 self.coordinate(&repo).await?
7251 };
7252 git::worktree_add_detached(&repo, &wt, &head).await?;
7253 }
7254 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7255 None
7256 } else {
7257 self.coordinate(&repo).await?
7258 };
7259 self.setup_seat_worktree(&repo, &wt, false).await?;
7260 let seat_key = format!("review-{}", r + 1);
7261 let spec = pick_start_spec(
7264 &self.roles.reviewer_roster,
7265 spec,
7266 self.state.seat_history.get(&seat_key),
7267 );
7268 let seat = self.seat(&seat_key, &spec.id);
7269 jobs.push(SeatJob {
7270 prompt: prompt::review(&prompt::ReviewCtx {
7271 instruction: &self.state.instruction,
7272 branch: &winner.branch,
7273 base_short: &base_short,
7274 stat: &stat,
7275 patch: &patch,
7276 verification: prev_verification.as_ref(),
7277 prior: prior_section.as_deref(),
7278 reviewers: reviewers.len(),
7279 round,
7280 rounds: max_rounds,
7281 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7284 lens: Lens::for_seat(r),
7285 language: &language,
7286 github_language: &self.state.config.graph.github_language,
7287 }),
7288 spec,
7289 seat,
7290 cwd: wt,
7291 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7292 allow_write: false,
7293 sessions,
7294 artifacts: artifacts.clone(),
7295 stem: format!("review-{round}-{}", r + 1),
7296 handover: None,
7297 });
7298 }
7299
7300 drop(coordination);
7301 self.state.event(
7302 "review",
7303 format!(
7304 "round {round}: {} reviewers on {}",
7305 jobs.len(),
7306 short(&head)
7307 ),
7308 );
7309 let mut quota_losses = Vec::new();
7310 let review_retries = self.state.config.graph.retries;
7311 let review_cache = self.state.config.cache_dir();
7312 let ctx = WaveCtx {
7313 carry_seats: true,
7314 pauses: Some((&self.pause, &self.interrupt)),
7315 run: &run_id,
7316 node: "review",
7317 prompts: &prompts,
7318 cache: review_cache.as_deref(),
7319 round: Some(round),
7320 };
7321 let results = ask_json_wave::<Review>(
7322 jobs,
7323 Arc::clone(&self.sem),
7324 review_retries,
7325 &self.roles.reviewer_roster,
7326 &ctx,
7327 &mut quota_losses,
7328 &mut self.state,
7329 &|_: &Review| Ok(()),
7330 )
7331 .await;
7332 if self.state.resource_block.is_some() {
7333 return self.stop_for_resource();
7334 }
7335 let round_quota_missing = quota_losses.len();
7339 self.state.quota.extend(quota_losses);
7340
7341 let mut records = Vec::new();
7342 let mut all_findings = Vec::new();
7343 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7344 let agent_id = seat.agent.clone();
7345 self.state.seats.insert(seat.key.clone(), seat);
7346 let mut record = ReviewRecord {
7347 reviewer: r + 1,
7348 agent: agent_id,
7349 summary: String::new(),
7350 findings: Vec::new(),
7351 vote: None,
7352 failed: None,
7353 duration_ms: 0,
7354 attempts,
7360 };
7361 match res {
7362 Ok((review, out)) => {
7363 record.summary =
7371 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7372 record.vote = Some(review.vote);
7373 record.duration_ms = out.duration_ms;
7374 for (n, mut f) in review.findings.into_iter().enumerate() {
7375 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7378 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7379 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7380 f.file = f
7386 .file
7387 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7388 all_findings.push(f.clone());
7389 record.findings.push(f);
7390 }
7391 self.state.event(
7392 "review",
7393 format!(
7394 "round {round}: reviewer {} voted {} with {} finding(s)",
7395 r + 1,
7396 review.vote.label(),
7397 record.findings.len()
7398 ),
7399 );
7400 }
7401 Err(e) => {
7402 record.failed = Some(e.to_string());
7403 self.state.event(
7404 "review",
7405 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7406 );
7407 }
7408 }
7409 records.push(record);
7410 }
7411
7412 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7419 let vote_split =
7420 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7421 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7422 if vote_split {
7423 self.state.event(
7424 "review",
7425 format!(
7426 "round {round}: votes split ({}) — one round of reconsideration",
7427 initial_votes
7428 .iter()
7429 .map(|v| v.label())
7430 .collect::<Vec<_>>()
7431 .join(", ")
7432 ),
7433 );
7434 let panel: Vec<ReviewSeatReport<'_>> = records
7437 .iter()
7438 .filter_map(|r| {
7439 r.vote.map(|vote| ReviewSeatReport {
7440 reviewer: r.reviewer,
7441 vote,
7442 summary: &r.summary,
7443 findings: &r.findings,
7444 })
7445 })
7446 .collect();
7447
7448 let mut jobs = Vec::new();
7449 let mut seats_at = Vec::new();
7450 for (r, spec) in reviewers.iter().cloned().enumerate() {
7451 if records[r].vote.is_none() {
7455 continue;
7456 }
7457 let wt = root.join(format!("review-{}", r + 1));
7458 let seat_key = format!("review-{}", r + 1);
7459 let spec = self.occupant(&seat_key, spec);
7460 let seat = self.seat(&seat_key, &spec.id);
7461 let build = |with_patch: bool| {
7469 prompt::review_reconsider(&ReviewReconsiderCtx {
7470 instruction: &self.state.instruction,
7471 reviewer: r + 1,
7472 lens: Lens::for_seat(r),
7473 panel: &panel,
7474 patch: with_patch.then_some(ReviewPatch {
7475 branch: &winner.branch,
7476 base_short: &base_short,
7477 stat: &stat,
7478 patch: &patch,
7479 }),
7480 round,
7481 rounds: max_rounds,
7482 language: &language,
7483 })
7484 };
7485 let full = build(true);
7486 let prompt = if has_context(&spec, &seat, sessions) {
7487 build(false)
7488 } else {
7489 full.clone()
7490 };
7491 jobs.push(SeatJob {
7492 prompt,
7493 spec,
7494 seat,
7495 cwd: wt,
7496 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7497 allow_write: false,
7498 sessions,
7499 artifacts: artifacts.clone(),
7500 stem: format!("review-{round}-reconsider-{}", r + 1),
7501 handover: Some(full),
7502 });
7503 seats_at.push(r);
7504 }
7505
7506 let mut recon_quota_losses = Vec::new();
7507 let recon_cache = self.state.config.cache_dir();
7508 let recon_ctx = WaveCtx {
7509 carry_seats: true,
7510 pauses: Some((&self.pause, &self.interrupt)),
7511 run: &run_id,
7512 node: "review",
7513 prompts: &prompts,
7514 cache: recon_cache.as_deref(),
7515 round: Some(round),
7516 };
7517 let recon_results = ask_json_wave::<ReviewRevote>(
7518 jobs,
7519 Arc::clone(&self.sem),
7520 review_retries,
7521 &self.roles.reviewer_roster,
7522 &recon_ctx,
7523 &mut recon_quota_losses,
7524 &mut self.state,
7525 &|_: &ReviewRevote| Ok(()),
7526 )
7527 .await;
7528 if self.state.resource_block.is_some() {
7529 return self.stop_for_resource();
7530 }
7531 self.state.quota.extend(recon_quota_losses);
7532
7533 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7534 let agent_id = seat.agent.clone();
7535 self.state.seats.insert(seat.key.clone(), seat);
7536 let mut rec = ReviewRevoteRecord {
7537 reviewer: r + 1,
7538 agent: agent_id,
7539 vote: None,
7540 reason: String::new(),
7541 failed: None,
7542 };
7543 match res {
7544 Ok((rv, _)) => {
7545 rec.vote = Some(rv.vote);
7546 rec.reason =
7547 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7548 self.state.event(
7549 "review",
7550 format!(
7551 "round {round}: reviewer {} revoted {}",
7552 r + 1,
7553 rv.vote.label()
7554 ),
7555 );
7556 }
7557 Err(e) => {
7558 rec.failed = Some(e.to_string());
7559 self.state.event(
7560 "review",
7561 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7562 );
7563 }
7564 }
7565 reconsideration.push(rec);
7566 }
7567 } else if initial_votes.len() > 1 {
7568 self.state.event(
7569 "review",
7570 format!(
7571 "round {round}: votes agreed ({}) — no reconsideration",
7572 initial_votes[0].label()
7573 ),
7574 );
7575 }
7576
7577 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7578 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7579 let defer_e2e =
7590 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7591 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7592 let reason =
7593 format!("{blocking} blocking finding(s) already required a fix this round");
7594 self.state.event(
7595 "verify",
7596 format!(
7597 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7598 {}); it will run once a round has none left",
7599 short(&head)
7600 ),
7601 );
7602 (Vec::new(), false, true, Some(reason))
7603 } else {
7604 let e2e_commands = self.state.config.verify.e2e.clone();
7605 let cache_dir = self.state.config.cache_dir();
7606 let context = format!("round {round}");
7607 let wait = self.repository_wait();
7608 let (e2e, verify_retried) = with_cache_lease(
7609 &mut self.state,
7610 cache_dir.as_deref(),
7611 "e2e",
7612 "e2e",
7613 &winner.worktree,
7614 &head,
7615 verify_timeout,
7616 &context,
7617 &wait,
7618 |state, budget| {
7619 let shell = shell.clone();
7620 let e2e_commands = e2e_commands.clone();
7621 let worktree = winner.worktree.clone();
7622 let context = context.clone();
7623 async move {
7624 run_e2e_with_retry(
7625 state,
7626 &shell,
7627 &e2e_commands,
7628 &worktree,
7629 budget,
7630 &context,
7631 )
7632 .await
7633 }
7634 },
7635 )
7636 .await?;
7637 (e2e, verify_retried, false, None)
7638 };
7639
7640 let expected = records.len();
7641 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7642 let incomplete = answered < expected;
7643 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7644 let policy = self.state.config.graph.incomplete_review;
7645 let clean = round_is_clean(
7646 blocking,
7647 e2e_ok,
7648 answered,
7649 expected,
7650 round_quota_missing,
7651 policy,
7652 );
7653
7654 let mut round_record = ReviewRound {
7655 round,
7656 head: head.clone(),
7657 verified_head: None,
7658 verified_at: None,
7659 reviews: records,
7660 e2e,
7661 verify_retried,
7662 e2e_deferred,
7663 e2e_defer_reason,
7664 fix: None,
7665 blocking,
7666 answered,
7667 expected,
7668 clean,
7669 progressed: false,
7670 vote_split,
7671 reconsideration,
7672 verdict: None,
7673 };
7674 round_record.verdict = ReviewVote::worst(
7678 round_record
7679 .final_votes()
7680 .into_iter()
7681 .map(|(_, _, vote)| vote),
7682 );
7683 if !matches!(
7694 round_record.e2e_status(),
7695 E2eStatus::Deferred | E2eStatus::NotConfigured
7696 ) {
7697 round_record.verified_head = Some(head.clone());
7698 round_record.verified_at = Some(Timestamp::now());
7699 }
7700
7701 if incomplete {
7702 let missing: Vec<String> = round_record
7703 .reviews
7704 .iter()
7705 .filter(|r| r.failed.is_some())
7706 .map(|r| format!("review-{}", r.reviewer))
7707 .collect();
7708 self.state.event(
7709 "review",
7710 format!(
7711 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7712 missing.join(", ")
7713 ),
7714 );
7715 }
7716
7717 if clean {
7718 self.state.event(
7719 "review",
7720 if incomplete && policy == IncompleteReviewPolicy::Warn {
7721 format!(
7722 "round {round}: clean (warn policy, incomplete panel) — no \
7723 blocking findings from the seats that answered, verification green"
7724 )
7725 } else if incomplete {
7726 format!(
7727 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7728 quorum) — no blocking findings from the seats that answered, \
7729 verification green",
7730 expected - answered
7731 )
7732 } else {
7733 format!("round {round}: clean — no blocking findings, verification green")
7734 },
7735 );
7736 self.state.reviews.push(round_record);
7737 self.state.status = RunStatus::Gating;
7738 self.state.save()?;
7739 return Ok(());
7740 }
7741
7742 if incomplete && blocking == 0 && e2e_ok {
7750 self.state.reviews.push(round_record);
7751 self.state.save()?;
7752 if round == max_rounds {
7753 self.state.status = RunStatus::Blocked;
7754 self.state.event(
7755 "review",
7756 format!(
7757 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7758 refusing to call it clean",
7759 expected - answered
7760 ),
7761 );
7762 return Ok(());
7763 }
7764 continue;
7765 }
7766
7767 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7779 self.state.reviews.push(round_record);
7780 return self
7781 .stop_unresolved_review(
7782 "the round's own verification could not run",
7783 &shell,
7784 &winner.worktree,
7785 )
7786 .await;
7787 }
7788
7789 if round == max_rounds {
7790 self.state.reviews.push(round_record);
7791 return self
7792 .stop_reviewing(
7793 &format!(
7794 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7795 ),
7796 &shell,
7797 &winner.worktree,
7798 )
7799 .await;
7800 }
7801
7802 if !self.fix_review_round(&winner, round_record).await? {
7803 return Ok(());
7804 }
7805 }
7806 Ok(())
7807 }
7808
7809 async fn fix_review_round(
7812 &mut self,
7813 winner: &Candidate,
7814 mut round_record: ReviewRound,
7815 ) -> Result<bool> {
7816 let round = round_record.round;
7817 let shell = self.state.config.shell();
7818 let step = self.run_fix_round(winner, &mut round_record).await?;
7819 if matches!(step, FixStep::Resource) {
7820 self.stop_for_resource()?;
7821 return Ok(false);
7822 }
7823 if self.state.reviews.len() < round {
7824 self.state.reviews.push(round_record);
7825 } else {
7826 self.state.reviews[round - 1] = round_record;
7827 }
7828 self.state.save()?;
7829
7830 if matches!(step, FixStep::Recorded { report_lost: true }) {
7843 self.stop_reviewing(
7844 "the fixer's adoption report never came back, even after resuming its \
7845 own seat; refusing to start another round against the same worktree \
7846 while that is unresolved",
7847 &shell,
7848 &winner.worktree,
7849 )
7850 .await?;
7851 return Ok(false);
7852 }
7853
7854 let streak = self
7855 .state
7856 .reviews
7857 .iter()
7858 .rev()
7859 .take_while(|r| !r.progressed)
7860 .count();
7861 if streak >= STAGNANT_LIMIT {
7862 self.stop_reviewing(
7863 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7864 &shell,
7865 &winner.worktree,
7866 )
7867 .await?;
7868 return Ok(false);
7869 }
7870 Ok(true)
7871 }
7872
7873 async fn run_fix_round(
7875 &mut self,
7876 winner: &Candidate,
7877 round_record: &mut ReviewRound,
7878 ) -> Result<FixStep> {
7879 let round = round_record.round;
7880 let max_rounds = self.state.config.graph.review_rounds;
7881 let run_id = self.state.id.clone();
7882 let prompts = self.state.config.prompts.clone();
7883 let language = self.state.config.graph.language.clone();
7884 let sessions = self.state.config.graph.sessions;
7885 let artifacts = self.merge_repair_artifacts();
7886 let base = self.landing_base();
7887 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7888 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7889 let this_round_verification = round_record.verification_summary(&head);
7890 let all_findings: Vec<_> = round_record
7891 .reviews
7892 .iter()
7893 .flat_map(|r| r.findings.iter().cloned())
7894 .collect();
7895 let blocking_findings: Vec<_> = all_findings
7898 .iter()
7899 .filter(|f| f.severity.blocks())
7900 .cloned()
7901 .collect();
7902 let fix_prompt = prompt::fix_in(
7903 &self.state.instruction,
7904 &blocking_findings,
7905 this_round_verification.as_ref(),
7906 round,
7907 max_rounds,
7908 &language,
7909 &self.state.config.graph.github_language,
7910 );
7911 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7912 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7913 let (job, seat, out) = self
7914 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7915 prompt: fix_prompt.clone(),
7916 spec,
7917 seat,
7918 cwd: winner.worktree.clone(),
7919 timeout,
7920 allow_write: true,
7921 sessions,
7922 artifacts: artifacts.clone(),
7923 stem: format!("fix-{round}"),
7924 handover: None,
7925 })
7926 .await;
7927 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7930 return Ok(FixStep::Resource);
7931 }
7932 let agent_id = seat.agent.clone();
7933
7934 let mut fix = FixRecord {
7935 agent: agent_id,
7936 addressed: Vec::new(),
7937 rejected: Vec::new(),
7938 notes: String::new(),
7939 committed: false,
7940 failed: None,
7941 duration_ms: 0,
7942 continuation: None,
7943 };
7944 let mut continuation = ContinuationRecord::not_needed();
7945 let mut final_seat = seat.clone();
7946 match out {
7947 AgentOutcome::Ok(o) => {
7948 fix.duration_ms = o.duration_ms;
7949 let parsed = verdict::extract_json::<FixReport>(&o.text);
7950 let incomplete_reason = match &parsed {
7957 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7958 "the reply parsed, but it reported a command whose own CLI never \
7959 confirmed an exit status"
7960 .to_owned(),
7961 ),
7962 Ok(_) => None,
7963 Err(e) => Some(e.to_string()),
7964 };
7965 match incomplete_reason {
7966 None => {
7967 let report = parsed.expect("checked Ok above");
7968 fix.addressed = report.addressed;
7969 fix.rejected = report.rejected;
7970 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7971 }
7972 Some(reason) => {
7973 let (resumed_seat, resolved, failure, cont) = self
7974 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7975 .await;
7976 fix.duration_ms += cont.cumulative_wait_ms;
7977 continuation = cont;
7978 final_seat = resumed_seat;
7979 match resolved {
7980 Some(report) => {
7981 fix.addressed = report.addressed;
7982 fix.rejected = report.rejected;
7983 fix.notes =
7984 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7985 }
7986 None => fix.failed = failure,
7987 }
7988 }
7989 }
7990 }
7991 AgentOutcome::Dropped(o) => {
7993 fix.duration_ms = o.duration_ms;
7994 let why = o
7995 .dropped
7996 .as_ref()
7997 .map(|d| d.why.as_str())
7998 .unwrap_or("the CLI ended the stream without delivering its answer");
7999 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8000 }
8001 AgentOutcome::Quota(o) => {
8002 self.state.quota.push(QuotaLoss::from_report(
8003 final_seat.key.clone(),
8004 "fix",
8005 o.quota.as_ref(),
8006 ));
8007 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8008 }
8009 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8010 }
8011 if self.state.resource_block.is_some() {
8012 self.state.seats.insert(final_seat.key.clone(), final_seat);
8013 return Ok(FixStep::Resource);
8014 }
8015 fix.continuation = Some(continuation);
8016 self.state.seats.insert(final_seat.key.clone(), final_seat);
8017 let coordination = self.coordinate(&winner.worktree).await?;
8018 if let Ok(r) = git::rescue_commit(
8019 &winner.worktree,
8020 &format!("magi: review round {round} fixes (uncommitted work)"),
8021 )
8022 .await
8023 {
8024 self.state.note_withheld("fix", &r.withheld);
8025 }
8026 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8027 fix.committed = after != before;
8028 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8036 drop(coordination);
8037 let progressed = diff_after != patch;
8038 let commit_note = if fix.committed {
8039 "committed"
8040 } else {
8041 "NO new commit"
8042 };
8043 let tree_note = if progressed {
8044 "changed vs base"
8045 } else {
8046 "unchanged vs base"
8047 };
8048 self.state.event(
8049 "fix",
8050 match &fix.failed {
8051 Some(reason) => {
8057 format!(
8058 "round {round}: fixer's adoption report was lost ({reason}); \
8059 {commit_note}, tree {tree_note}"
8060 )
8061 }
8062 None => format!(
8063 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8064 {tree_note}{}",
8065 fix.addressed.len(),
8066 fix.rejected.len(),
8067 if continuation.outcome == ContinuationOutcome::Resumed {
8068 format!(
8069 " (adoption report recovered after {} continuation(s))",
8070 continuation.attempts
8071 )
8072 } else {
8073 String::new()
8074 },
8075 ),
8076 },
8077 );
8078 round_record.fix = Some(fix);
8079 round_record.progressed = progressed;
8080 Ok(FixStep::Recorded {
8081 report_lost: matches!(
8082 continuation.outcome,
8083 ContinuationOutcome::Exhausted
8084 | ContinuationOutcome::QuotaLost
8085 | ContinuationOutcome::NoSession
8086 ),
8087 })
8088 }
8089
8090 fn record_contested_handoff(&mut self) {
8124 if self.state.contested_handoff.is_some() {
8125 return;
8126 }
8127 let Some(contested) = self
8128 .state
8129 .reviews
8130 .last()
8131 .and_then(ReviewRound::contested_handoff)
8132 else {
8133 return;
8134 };
8135 self.state.event(
8136 "review",
8137 format!(
8138 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8139 wait for the owner's approval",
8140 contested.findings.len(),
8141 contested.rejecters.len()
8142 ),
8143 );
8144 self.state.contested_handoff = Some(contested);
8145 }
8146
8147 async fn stop_unresolved_review(
8148 &mut self,
8149 why: &str,
8150 shell: &[String],
8151 worktree: &Path,
8152 ) -> Result<()> {
8153 self.stop_reviewing_inner(why, shell, worktree, true).await
8154 }
8155
8156 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8157 self.stop_reviewing_inner(why, shell, worktree, false).await
8158 }
8159
8160 async fn stop_reviewing_inner(
8161 &mut self,
8162 why: &str,
8163 shell: &[String],
8164 worktree: &Path,
8165 repairable: bool,
8166 ) -> Result<()> {
8167 let round_idx = self.state.reviews.len() - 1;
8168 let needs_catchup_run = matches!(
8176 self.state.reviews[round_idx].e2e_status(),
8177 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8178 );
8179 if needs_catchup_run {
8180 let round = self.state.reviews[round_idx].round;
8181 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8182 let commands = self.state.config.verify.e2e.clone();
8183 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8184 let cache_dir = self.state.config.cache_dir();
8185 let context = format!(
8186 "round {round}: verification unresolved, catching up before the final decision"
8187 );
8188 let wait = self.repository_wait();
8189 let (outcomes, verify_retried) = with_cache_lease(
8190 &mut self.state,
8191 cache_dir.as_deref(),
8192 "e2e",
8193 "e2e",
8194 worktree,
8195 &attempted_head,
8196 timeout,
8197 &context,
8198 &wait,
8199 |state, budget| {
8200 let shell = shell.to_vec();
8201 let commands = commands.clone();
8202 let context = context.clone();
8203 async move {
8204 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8205 .await
8206 }
8207 },
8208 )
8209 .await?;
8210 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8211 self.state.record_base_progress(&attempted_head);
8212 }
8213 let last = &mut self.state.reviews[round_idx];
8214 last.e2e = outcomes;
8215 last.verify_retried = verify_retried;
8216 last.verified_head = Some(attempted_head);
8223 last.verified_at = Some(Timestamp::now());
8224 if verify_inconclusive(&last.e2e) {
8225 self.state.save()?;
8232 return Ok(());
8233 }
8234 last.e2e_deferred = false;
8235 if last.answered == last.expected {
8240 last.clean = round_is_clean(
8241 last.blocking,
8242 last.e2e.iter().all(CommandOutcome::ok),
8243 last.answered,
8244 last.expected,
8245 0,
8246 self.state.config.graph.incomplete_review,
8247 );
8248 }
8249 }
8250 let last = &self.state.reviews[round_idx];
8251 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8252
8253 match last.e2e_status() {
8254 E2eStatus::Failed => {
8255 let red: Vec<String> = last
8256 .e2e
8257 .iter()
8258 .filter(|o| !o.ok())
8259 .map(|o| {
8260 format!(
8261 "`{}` -> {:?}\n{}",
8262 o.command,
8263 o.code,
8264 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8265 )
8266 })
8267 .collect();
8268 let can_repair = last.blocking == 0 && last.fix.is_none();
8269 self.state.reviews[round_idx].clean = false;
8270 self.state
8271 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8272 self.state.status = RunStatus::Blocked;
8273 if repairable
8274 && can_repair
8275 && self.state.reviews.len() < self.state.config.graph.review_rounds
8276 {
8277 self.state.status = RunStatus::Reviewing;
8278 }
8279 }
8280 E2eStatus::ResourceBlocked => {
8285 self.state.event(
8286 "review",
8287 format!(
8288 "{why}; e2e could not run (resource unavailable); not \
8289 deciding yet"
8290 ),
8291 );
8292 }
8293 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8294 self.state.event(
8295 "review",
8296 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8297 );
8298 self.record_contested_handoff();
8299 self.state.status = RunStatus::Gating;
8300 }
8301 }
8302 self.state.save()?;
8303 Ok(())
8304 }
8305
8306 async fn gate(&mut self) -> Result<()> {
8309 if self.hold_base_sync().await? {
8310 return Ok(());
8311 }
8312 if self.state.status == RunStatus::Failed
8324 || self
8325 .state
8326 .base_sync
8327 .as_ref()
8328 .is_some_and(|s| s.conflict.is_some())
8329 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8330 != Some(RunStatus::Gating)
8331 {
8332 return Ok(());
8333 }
8334 let Some(winner) = self.state.winner().cloned() else {
8335 return Ok(());
8336 };
8337 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8338 self.state.reconcile_verification_head(&head);
8339 self.state.save()?;
8340 let wait = self.repository_wait();
8341 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8342 return Ok(());
8343 }
8344 if self.state.gate_ran {
8345 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8356 self.state.status = RunStatus::Blocked;
8357 self.state.save()?;
8358 }
8359 return Ok(());
8360 }
8361 self.state.status = RunStatus::Gating;
8362 let mut outcomes = self.run_gate(&winner).await?;
8363 let mut attempted_head = self.state.gate_head.clone();
8364 loop {
8365 if verify_inconclusive(&outcomes) {
8376 self.state.save()?;
8377 return Ok(());
8378 }
8379 if outcomes.iter().all(CommandOutcome::ok) {
8380 break;
8381 }
8382 let fixes_before = self.state.gate_fixes.len();
8383 match self.gate_fix_round(&winner, &outcomes).await? {
8384 GateFix::Retry => {
8385 outcomes = self.run_gate(&winner).await?;
8386 attempted_head = self.state.gate_head.clone();
8387 }
8388 GateFix::Stop => {
8389 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8390 if self.state.gate_fixes.len() > fixes_before
8391 && attempted_head.as_deref() != Some(current.as_str())
8392 && verify_current_head(&mut self.state, &winner, &wait).await?
8393 {
8394 outcomes = self.run_gate(&winner).await?;
8397 attempted_head = self.state.gate_head.clone();
8398 } else {
8399 break;
8400 }
8401 }
8402 GateFix::Defer => {
8403 self.state.save()?;
8404 return Ok(());
8405 }
8406 }
8407 }
8408 let passed = outcomes.iter().all(CommandOutcome::ok);
8409 self.state.gate = outcomes;
8410 self.state.gate_ran = true;
8411 self.state.gate_head = attempted_head;
8412 if !self.state.gate.is_empty()
8413 && !verify_inconclusive(&self.state.gate)
8414 && let Some(head) = self.state.gate_head.clone()
8415 {
8416 self.state.record_base_progress(&head);
8417 }
8418 self.state.save()?;
8419 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8420 return Ok(());
8421 }
8422 if !passed {
8423 self.state.status = RunStatus::Blocked;
8424 let spent = self.state.gate_fixes.len();
8425 self.state.event(
8426 "gate",
8427 if spent == 0 {
8428 "gate failed; not merging".to_owned()
8429 } else {
8430 format!("gate failed after {spent} gate-fix round(s); not merging")
8431 },
8432 );
8433 }
8434 self.state.save()?;
8435 Ok(())
8436 }
8437
8438 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8448 let commands = state.config.verify.pre_gate.clone();
8449 if commands.is_empty() {
8450 return;
8451 }
8452 let shell = state.config.shell();
8453 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8454 let (outcomes, _) = run_commands(
8455 state,
8456 "pre_gate",
8457 "pre_gate",
8458 0,
8459 &shell,
8460 &commands,
8461 &winner.worktree,
8462 timeout,
8463 )
8464 .await;
8465 for o in &outcomes {
8466 if !o.ok() {
8467 tracing::warn!(
8468 "pre_gate `{}` failed ({:?}); the gate decides",
8469 o.command,
8470 o.code
8471 );
8472 }
8473 state.event(
8474 "pre_gate",
8475 format!(
8476 "`{}` -> {}",
8477 o.command,
8478 if o.ok() {
8479 "pass".to_owned()
8480 } else {
8481 format!(
8482 "FAIL ({:?})\n{}",
8483 o.code,
8484 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8485 )
8486 }
8487 ),
8488 );
8489 }
8490 state.pre_gate = outcomes;
8491 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8492 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8493 Ok(head) => {
8494 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8495 state.pre_gate_commit = Some(head);
8496 }
8497 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8498 },
8499 Ok(false) => {}
8500 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8501 }
8502 if let Err(e) = state.save() {
8503 tracing::warn!("could not persist the pre_gate record: {e:#}");
8504 }
8505 }
8506
8507 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8510 let wait = self.repository_wait();
8511 run_landing_gate(&mut self.state, winner, &wait).await
8512 }
8513
8514 async fn gate_fix_round(
8527 &mut self,
8528 winner: &Candidate,
8529 outcomes: &[CommandOutcome],
8530 ) -> Result<GateFix> {
8531 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8532 g.phase == crate::repair_budget::Phase::Fixing
8533 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8534 }) {
8535 self.state.gate_fixes.len().saturating_add(1)
8536 } else {
8537 self.state.config.graph.gate_fix_rounds
8538 };
8539 let spent = self.state.gate_fixes.len();
8540 if spent >= cap {
8541 if cap > 0 {
8542 self.state.event(
8543 "gate",
8544 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8545 );
8546 }
8547 return Ok(GateFix::Stop);
8548 }
8549 if !gate_fixable(outcomes) {
8550 self.state.event(
8551 "gate",
8552 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8553 command or similar); not spending a fix round on it",
8554 );
8555 return Ok(GateFix::Stop);
8556 }
8557 let min_free = self.state.config.disk.min_free_bytes;
8558 if min_free > 0 {
8559 match crate::disk::free_bytes(&winner.worktree) {
8560 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8561 Ok(free) => {
8562 self.state.event(
8563 "gate",
8564 format!(
8565 "only {free} bytes free ({min_free} required by `[disk] \
8566 min_free_bytes`); not spending a fix round on a failure the disk \
8567 may explain"
8568 ),
8569 );
8570 return Ok(GateFix::Stop);
8571 }
8572 Err(e) => {
8573 self.state.event(
8574 "gate",
8575 format!("free disk space could not be measured ({e:#}); no fix round"),
8576 );
8577 return Ok(GateFix::Stop);
8578 }
8579 }
8580 }
8581
8582 let attempt = spent + 1;
8583 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8584 let base = self.landing_base();
8585 let fix_prompt = prompt::gate_fix_in(
8586 &self.state.instruction,
8587 &failed,
8588 attempt,
8589 cap,
8590 &self.state.config.graph.language,
8591 &self.state.config.graph.github_language,
8592 );
8593 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8594 let sessions = self.state.config.graph.sessions;
8595 let artifacts = self.merge_repair_artifacts();
8596 self.state.event(
8597 "gate",
8598 format!("gate failed; gate-fix round {attempt} of {cap}"),
8599 );
8600 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8601 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8602 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8605 head: before.clone(),
8606 passed: false,
8607 outcomes: Vec::new(),
8608 });
8609 self.state.save()?;
8610 let (_, seat, out) = self
8611 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8612 prompt: fix_prompt.clone(),
8613 spec,
8614 seat,
8615 cwd: winner.worktree.clone(),
8616 timeout,
8617 allow_write: true,
8618 sessions,
8619 artifacts: artifacts.clone(),
8620 stem: format!("gate-fix-{attempt}"),
8621 handover: None,
8622 })
8623 .await;
8624 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8627 return Ok(GateFix::Defer);
8628 }
8629 let mut record = GateFixRecord {
8630 agent: seat.agent.clone(),
8631 failed,
8632 notes: String::new(),
8633 committed: false,
8634 error: None,
8635 };
8636 match out {
8637 AgentOutcome::Ok(o) => {
8638 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8641 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8642 }
8643 }
8644 AgentOutcome::Dropped(_) => {
8645 record.error = Some("the CLI dropped the stream".to_owned());
8646 }
8647 AgentOutcome::Quota(o) => {
8648 self.state.quota.push(QuotaLoss::from_report(
8649 seat.key.clone(),
8650 "gate-fix",
8651 o.quota.as_ref(),
8652 ));
8653 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8654 }
8655 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8656 }
8657 self.state.seats.insert(seat.key.clone(), seat);
8658 let coordination = self.coordinate(&winner.worktree).await?;
8659 if let Ok(r) = git::rescue_commit(
8660 &winner.worktree,
8661 &format!("magi: gate fix {attempt} (uncommitted work)"),
8662 )
8663 .await
8664 {
8665 self.state.note_withheld("gate-fix", &r.withheld);
8666 }
8667 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8668 record.committed = after != before;
8669 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8670 let note = record.error.clone();
8671 self.state.gate_fixes.push(record);
8672 self.state.save()?;
8673 if !changed {
8674 self.state.event(
8675 "gate",
8676 match note {
8677 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8678 None => format!("gate-fix round {attempt}: the tree did not change"),
8679 },
8680 );
8681 return Ok(GateFix::Stop);
8682 }
8683 self.state.event(
8684 "gate",
8685 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8686 );
8687
8688 drop(coordination);
8689 let wait = self.repository_wait();
8690 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8691 }
8692
8693 async fn guarded_pr_message(
8700 &mut self,
8701 winner: &Candidate,
8702 facts: Option<&BranchFacts>,
8703 posting: bool,
8704 ) -> Result<Option<PrMessage>> {
8705 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8706 let head = match final_head.as_deref() {
8707 Some(h) => crate::pr_outcome::FinalHead::At(h),
8708 None => crate::pr_outcome::FinalHead::Unknown,
8709 };
8710 let summarized = if posting {
8712 Some(self.summarize_change(winner, final_head.as_deref()).await)
8713 } else {
8714 None
8715 };
8716 if self.state.resource_block.is_some() {
8721 self.state.event(
8722 "pr-summary",
8723 "a shared resource was busy; nothing was published, resume to summarize again",
8724 );
8725 return Ok(None);
8726 }
8727 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8728 anyhow::bail!(
8729 "the final commit moved during public description generation; retry before publishing"
8730 );
8731 }
8732 let note = match (&summarized, &self.state.change_summary) {
8733 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8734 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8735 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8736 (None, _) => ChangeNote::NotAsked,
8737 };
8738 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8739 if !posting {
8740 let identity = crate::scrub::Identity::current();
8741 return Ok(Some(PrMessage {
8742 title: crate::scrub::scrub(&pr.title, &identity),
8743 body: crate::scrub::scrub(&pr.body, &identity),
8744 }));
8745 }
8746 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8749 if let Some(known) = self
8750 .state
8751 .github_text
8752 .clone()
8753 .filter(|g| g.fingerprint == raw_fp)
8754 {
8755 return self.settle_withheld(winner, &pr, known).await;
8756 }
8757 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8758 let mut violations =
8759 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8760 let identity = crate::scrub::Identity::current();
8761 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8762 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8763 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8764 {
8765 violations.push(crate::github_text::Violation::SensitiveData);
8766 }
8767 let has_language = violations
8771 .iter()
8772 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8773 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8774 self.state.event(
8775 "github-text",
8776 if has_language {
8777 format!("description rejected: {violations:?}; requesting one rewrite")
8778 } else {
8779 format!("description rejected: {violations:?}; asking the owner")
8780 },
8781 );
8782 let mut rewritten = false;
8783 let mut rejected_rewrite = None;
8784 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8785 {
8786 let output = text;
8787 if let Some(title) = summary_title(&output) {
8788 let raw_body = summary_without_title(&output);
8791 let clean_title = crate::scrub::scrub(&title, &identity);
8792 let mut body = crate::scrub::scrub(&raw_body, &identity);
8793 let redacted = clean_title != title || body != raw_body;
8794 let title = clean_title;
8795 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8796 if !complete {
8797 self.state.event(
8798 "github-text",
8799 "description rewrite omitted required findings",
8800 );
8801 }
8802 if !body.trim().is_empty() {
8803 body.push_str("\n\n");
8804 body.push_str(&public_pr_evidence(&self.state, head));
8805 }
8806 let marker = format!("magi:run/{}", self.state.id);
8807 if !body.trim().is_empty() && !body.contains(&marker) {
8808 body.push_str(&format!("\n\n{marker}"));
8809 }
8810 let rewrite = PrMessage {
8811 title: title.clone(),
8812 body: body.clone(),
8813 };
8814 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8815 let mut failed = crate::github_text::check_for(
8816 &self.state.config.graph,
8817 &title,
8818 &body,
8819 redecision,
8820 );
8821 if (!crate::github_text::shareable(&title)
8822 || !crate::github_text::shareable(&body))
8823 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8824 {
8825 failed.push(crate::github_text::Violation::SensitiveData);
8826 }
8827 if complete
8828 && !title.trim().is_empty()
8829 && !body.trim().is_empty()
8830 && !failed.is_empty()
8831 {
8832 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8833 title: title.clone(),
8834 body: body.clone(),
8835 failed: failed
8836 .iter()
8837 .map(|v| crate::github_text::category(*v).to_owned())
8838 .collect(),
8839 });
8840 }
8841 if complete
8842 && !title.trim().is_empty()
8843 && !raw_body.trim().is_empty()
8844 && failed.is_empty()
8845 {
8846 decision = redecision;
8847 pr = PrMessage { title, body };
8848 rewritten = true;
8849 self.state
8850 .event("github-text", "description rewrite passed");
8851 if redacted {
8852 self.state.event(
8853 "github-text",
8854 "sensitive data removed from description rewrite",
8855 );
8856 }
8857 }
8858 }
8859 }
8860 if !rewritten {
8861 self.state.event(
8862 "github-text",
8863 "description rewrite unavailable or rejected; asking the owner",
8864 );
8865 if let Some(w) =
8866 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8867 {
8868 self.retire_github_text_question("the withheld text changed");
8871 let record = crate::run::GithubTextAsk {
8872 rewrite: rejected_rewrite,
8873 fingerprint: raw_fp,
8874 title: w.title,
8875 body: w.body,
8876 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8877 question: None,
8878 asks: 0,
8879 resolved: false,
8880 chosen_title: None,
8881 choice: None,
8882 publication: None,
8883 };
8884 return self.settle_withheld(winner, &pr, record).await;
8885 }
8886 }
8887 }
8888 let (title, body) =
8889 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8890 Ok(Some(PrMessage { title, body }))
8891 }
8892
8893 async fn summarize_change(
8904 &mut self,
8905 winner: &Candidate,
8906 head: Option<&str>,
8907 ) -> std::result::Result<(), String> {
8908 let Some(head) = head else {
8909 return Err("the final commit could not be read".to_owned());
8910 };
8911 let remote = self.state.config.merge.remote.clone();
8912 let tracking = format!("{remote}/{}", self.state.base_branch);
8913 if !git::rev_exists(&winner.worktree, &tracking).await {
8914 return Err(format!("the remote base `{tracking}` could not be read"));
8915 }
8916 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8917 Ok(b) if !b.is_empty() => b,
8918 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8919 };
8920 let evidence = crate::pr_outcome::summary_evidence(
8921 &self.state,
8922 crate::pr_outcome::FinalHead::At(head),
8923 );
8924 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8925 if self
8926 .state
8927 .change_summary
8928 .as_ref()
8929 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8930 {
8931 return Ok(());
8932 }
8933 if self.state.change_summary.take().is_some() {
8934 self.state.event(
8935 "pr-summary",
8936 "the final diff or evidence changed; the earlier change summary was discarded",
8937 );
8938 }
8939 if let Err(e) = write_artifact(
8941 &self.state,
8942 &format!("public-description-{fingerprint}.md"),
8943 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8944 ) {
8945 tracing::warn!("could not save public description evidence: {e:#}");
8946 }
8947 let chain = match agent::pick_chain(
8948 &self.state.config.agents,
8949 self.state.config.roles.synthesizer.as_ref(),
8950 &agent::installed,
8951 "synthesizer",
8952 ) {
8953 Ok(chain) => chain,
8954 Err(e) => {
8955 tracing::warn!("no agent for the change summary: {e:#}");
8956 self.state.event("pr-summary", "no summary agent available");
8957 return Err("no agent was available to summarize it".to_owned());
8958 }
8959 };
8960 let subjects = git::subjects(&winner.worktree, &base, head)
8961 .await
8962 .unwrap_or_default();
8963 let stat = git::diff_stat(&winner.worktree, &base, head)
8964 .await
8965 .unwrap_or_default();
8966 let stat: String = stat.chars().take(6_000).collect();
8967 let mut prompt = crate::prompt::change_summary_in(
8968 &base,
8969 head,
8970 &subjects,
8971 stat.trim(),
8972 &self.state.config.graph.github_language,
8973 );
8974 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8975 let sessions = self.state.config.graph.sessions;
8976 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8977 let prompts = self.state.config.prompts.clone();
8978 let cache = self.state.config.cache_dir();
8979 let run = self.state.id.clone();
8980 for (i, spec) in chain.into_iter().enumerate() {
8981 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8982 let github_spec = match agent::for_github_text(
8983 &spec,
8984 &self.state.config.graph.github_language,
8985 &winner.worktree,
8986 ) {
8987 Ok(spec) => spec,
8988 Err(e) => {
8989 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8990 continue;
8991 }
8992 };
8993 let job = SeatJob {
8994 spec: github_spec,
8995 seat,
8996 cwd: winner.worktree.clone(),
8997 prompt: prompt.clone(),
8998 timeout,
8999 allow_write: false,
9000 sessions,
9001 artifacts: agent::artifacts_dir(&self.state.dir()),
9002 stem: if i == 0 {
9003 "change-summary".to_owned()
9004 } else {
9005 format!("change-summary-{}", spec.id)
9006 },
9007 handover: None,
9008 };
9009 let ctx = WaveCtx {
9010 run: &run,
9011 node: "pr-summary",
9012 prompts: &prompts,
9013 cache: cache.as_deref(),
9014 round: None,
9015 carry_seats: false,
9016 pauses: Some((&self.pause, &self.interrupt)),
9017 };
9018 let (seat, outcome) =
9019 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9020 self.state.seats.insert(seat.key.clone(), seat);
9021 let text = match outcome {
9022 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9023 AgentOutcome::ResourceBlocked(why) => {
9024 if self.state.resource_block.is_none() {
9027 self.state.resource_block = Some(crate::run::ResourceBlock {
9028 cause: crate::run::ResourceBlockCause::Other,
9029 node: "pr-summary".to_owned(),
9030 kind: "shared resource".to_owned(),
9031 detail: why,
9032 at: Timestamp::now(),
9033 });
9034 }
9035 return Err("a shared resource was busy".to_owned());
9036 }
9037 _ => continue,
9038 };
9039 let blind = &self.state.config.blind;
9040 let Some(title) = summary_title(&text) else {
9041 continue;
9042 };
9043 let body: String = summary_without_title(&text)
9044 .lines()
9045 .filter(|l| !l.contains("magi:run/"))
9046 .collect::<Vec<_>>()
9047 .join("\n");
9048 let body = blind::sanitize_prose(body.trim(), blind);
9049 let title = blind::sanitize_prose(&title, blind);
9050 if body.trim().is_empty() || title.trim().is_empty() {
9051 continue;
9052 }
9053 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9054 continue;
9055 }
9056 if git::rev_parse(&winner.worktree, "HEAD")
9058 .await
9059 .ok()
9060 .as_deref()
9061 != Some(head)
9062 {
9063 return Err("the branch moved while the summary was written".to_owned());
9064 }
9065 if git::merge_base(&winner.worktree, &tracking, head)
9066 .await
9067 .ok()
9068 .as_deref()
9069 != Some(base.as_str())
9070 || crate::pr_outcome::summary_evidence(
9071 &self.state,
9072 crate::pr_outcome::FinalHead::At(head),
9073 ) != evidence
9074 {
9075 return Err("the summary evidence changed while it was written".to_owned());
9076 }
9077 self.state.change_summary = Some(crate::run::ChangeSummary {
9078 head: head.to_owned(),
9079 base,
9080 evidence: fingerprint,
9081 title,
9082 body,
9083 });
9084 self.state.event(
9085 "pr-summary",
9086 format!("change summary written by {} for the final diff", spec.id),
9087 );
9088 if let Err(e) = self.state.save() {
9089 tracing::warn!("could not save the change summary: {e:#}");
9090 }
9091 return Ok(());
9092 }
9093 self.state
9094 .event("pr-summary", "no agent produced a change summary");
9095 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9096 }
9097
9098 async fn rewrite_pr_text(
9108 &mut self,
9109 winner: &Candidate,
9110 pr: &PrMessage,
9111 violations: &[crate::github_text::Violation],
9112 ) -> Option<String> {
9113 let prompt = format!(
9114 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9115 pr.title, pr.body
9116 );
9117 let sessions = self.state.config.graph.sessions;
9118 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9119 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9122 let key = format!("impl-{}", winner.label);
9123 if let Some(spec) = self
9124 .state
9125 .config
9126 .agents
9127 .iter()
9128 .find(|a| a.id == winner.agent)
9129 .cloned()
9130 && let Some(existing) = self.state.seats.get(&key)
9131 && existing.agent == spec.id
9132 && agent::has_session(spec.kind, existing, sessions)
9133 {
9134 attempts.push((spec, Some(existing.clone())));
9135 }
9136 match agent::pick_chain(
9137 &self.state.config.agents,
9138 self.state.config.roles.language_judge.as_ref(),
9139 &agent::installed,
9140 "language_judge",
9141 ) {
9142 Ok(chain) => {
9143 for spec in chain {
9144 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9145 continue;
9146 }
9147 attempts.push((spec, None));
9148 }
9149 }
9150 Err(e) if attempts.is_empty() => {
9151 tracing::warn!("no agent for the description rewrite: {e:#}");
9152 self.state
9153 .event("github-text", "no rewrite agent available");
9154 return None;
9155 }
9156 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9157 }
9158 let prompts = self.state.config.prompts.clone();
9159 let cache = self.state.config.cache_dir();
9160 let run = self.state.id.clone();
9161 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9162 let resumed = resumed_seat.is_some();
9163 let seat = resumed_seat.unwrap_or_else(|| {
9164 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9165 });
9166 let github_spec = match agent::for_github_text(
9167 &spec,
9168 &self.state.config.graph.github_language,
9169 &winner.worktree,
9170 ) {
9171 Ok(spec) => spec,
9172 Err(e) => {
9173 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9174 continue;
9175 }
9176 };
9177 let job = SeatJob {
9178 spec: github_spec,
9179 seat,
9180 cwd: winner.worktree.clone(),
9181 prompt: prompt.clone(),
9182 timeout: retry_budget(timeout_full, resumed),
9183 allow_write: false,
9184 sessions,
9185 artifacts: agent::artifacts_dir(&self.state.dir()),
9186 stem: if i == 0 {
9187 "github-text-rewrite".to_owned()
9188 } else {
9189 format!("github-text-rewrite-{}", spec.id)
9190 },
9191 handover: None,
9192 };
9193 let ctx = WaveCtx {
9194 run: &run,
9195 node: "github-text",
9196 prompts: &prompts,
9197 cache: cache.as_deref(),
9198 round: None,
9199 carry_seats: false,
9200 pauses: Some((&self.pause, &self.interrupt)),
9201 };
9202 let (seat, outcome) =
9203 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9204 self.state.seats.insert(seat.key.clone(), seat);
9205 match outcome {
9206 AgentOutcome::Ok(o)
9207 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9208 {
9209 return Some(o.text);
9210 }
9211 AgentOutcome::ResourceBlocked(_) => return None,
9212 _ => {}
9213 }
9214 }
9215 None
9216 }
9217
9218 fn retire_github_text_question(&mut self, why: &str) {
9220 let Some(id) = self
9221 .state
9222 .github_text
9223 .as_ref()
9224 .and_then(|g| g.question.clone())
9225 else {
9226 return;
9227 };
9228 let _ = ask::Questions::open().update(&id, |q| {
9229 q.abandon(why);
9230 Ok(())
9231 });
9232 }
9233
9234 async fn settle_withheld(
9238 &mut self,
9239 winner: &Candidate,
9240 pr: &PrMessage,
9241 mut g: crate::run::GithubTextAsk,
9242 ) -> Result<Option<PrMessage>> {
9243 use crate::github_text as gt;
9244 let store = ask::Questions::open();
9245 let timeout = self.state.config.graph.answer_timeout;
9246 if !g.resolved {
9247 let mut standing = g
9248 .question
9249 .as_ref()
9250 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9251 if let Some(q) = standing.as_ref()
9252 && q.status.open()
9253 && gt::expired(q, timeout)
9254 {
9255 let why = format!("no answer within {}s of asking", timeout.max(1));
9256 standing = store
9257 .update(&q.id, |q| {
9258 q.abandon(&why);
9259 Ok(())
9260 })
9261 .ok()
9262 .map(|(q, ())| q);
9263 }
9264 let mut ask_again: Option<Option<Vec<String>>> = None;
9265 match standing {
9266 None if g.asks <= 1 => {
9269 g.asks = 0;
9270 ask_again = Some(None);
9271 }
9272 None => {
9273 g.resolved = true;
9274 g.chosen_title = None;
9275 g.choice = Some(GithubTextChoice::Fallback);
9276 }
9277 Some(q) if q.status.open() => {
9278 self.park_on_github_text(&g)?;
9279 return Ok(None);
9280 }
9281 Some(q) => match gt::read_reply(&q) {
9282 gt::Reply::Fallback => {
9283 g.resolved = true;
9284 g.chosen_title = None;
9285 g.choice = Some(GithubTextChoice::Fallback);
9286 }
9287 gt::Reply::Original => {
9288 g.resolved = true;
9289 g.chosen_title = None;
9290 g.choice = Some(GithubTextChoice::Original);
9291 }
9292 gt::Reply::Rewrite => {
9293 g.resolved = true;
9294 g.chosen_title = None;
9295 g.choice = Some(if g.rewrite.is_some() {
9296 GithubTextChoice::Rewrite
9297 } else {
9298 GithubTextChoice::Fallback
9299 });
9300 }
9301 gt::Reply::Title(text) => {
9302 let first = text
9303 .lines()
9304 .map(str::trim)
9305 .find(|l| !l.is_empty())
9306 .unwrap_or("");
9307 let decision = if first.is_empty() {
9308 None
9309 } else {
9310 let candidate = PrMessage {
9311 title: first.to_owned(),
9312 body: String::new(),
9313 };
9314 self.judge_pr_language(&winner.worktree, &candidate).await
9315 };
9316 match gt::vet_title(
9317 &text,
9318 decision,
9319 self.state.config.graph.github_english(),
9320 ) {
9321 Ok(title) => {
9322 g.resolved = true;
9323 g.chosen_title = Some(title);
9324 g.choice = Some(GithubTextChoice::MyTitle);
9325 }
9326 Err(cats) if g.asks < 2 => {
9327 ask_again = Some(Some(
9328 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9329 ))
9330 }
9331 Err(_) => {
9332 g.resolved = true;
9333 g.chosen_title = None;
9334 g.choice = Some(GithubTextChoice::Fallback);
9335 }
9336 }
9337 }
9338 },
9339 }
9340 if let Some(rejected) = ask_again {
9341 let w = gt::Withheld {
9344 title: g.title,
9345 body: g.body,
9346 categories: g.categories.clone(),
9347 };
9348 let artifact = match crate::run::write_artifact(
9351 &self.state,
9352 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9353 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9354 ) {
9355 Ok(p) => Some(p),
9356 Err(e) => {
9357 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9358 None
9359 }
9360 };
9361 let (detail, comparison) = gt::question_content(
9362 &w,
9363 &pr.title,
9364 &pr.body,
9365 &self.state.id,
9366 rejected.as_deref(),
9367 artifact.as_deref(),
9368 g.rewrite.as_ref(),
9369 );
9370 let mut q = ask::Question::new(
9371 self.state.id.clone(),
9372 gt::ASK_NODE.to_owned(),
9373 gt::ASK_SEAT.to_owned(),
9374 gt::question_summary(&self.state.config.graph.language, &w),
9375 detail,
9376 gt::question_choices(&w, g.rewrite.as_ref()),
9377 );
9378 q.posting_comparison = Some(comparison);
9379 q.answer_timeout = timeout;
9380 g.question = Some(q.id.clone());
9383 g.asks += 1;
9384 self.state.github_text = Some(g.clone());
9385 self.state.event(
9386 "github-text",
9387 format!("asking the owner about the withheld text ({})", q.short()),
9388 );
9389 self.state.save()?;
9390 store
9391 .put_executable(&mut q)
9392 .context("file the posting-gate question")?;
9393 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9394 tracing::warn!(
9395 "could not notify about posting-gate question {}: {e:#}",
9396 q.short()
9397 );
9398 }
9399 self.park_on_github_text(&g)?;
9400 return Ok(None);
9401 }
9402 }
9403 let resolved = g.chosen_title.clone();
9404 let original = g.choice == Some(GithubTextChoice::Original);
9405 self.state.event(
9407 "github-text",
9408 if original {
9409 format!(
9410 "posting the original text for: {}",
9411 match (g.title, g.body) {
9412 (true, true) => "title, description",
9413 (true, false) => "title",
9414 _ => "description",
9415 }
9416 )
9417 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9418 "posting the saved rewrite title and description".to_owned()
9419 } else if resolved.is_some() {
9420 "posting the owner's replacement title".to_owned()
9421 } else {
9422 "posting the neutral text for the withheld fields".to_owned()
9423 },
9424 );
9425 self.state.github_text = Some(g.clone());
9426 self.state.save()?;
9427 if g.choice == Some(GithubTextChoice::Rewrite)
9428 && let Some(r) = g.rewrite.as_ref()
9429 {
9430 return Ok(Some(PrMessage {
9431 title: r.title.clone(),
9432 body: r.body.clone(),
9433 }));
9434 }
9435 let w = gt::Withheld {
9436 title: g.title,
9437 body: g.body,
9438 categories: g.categories.clone(),
9439 };
9440 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9441 if let Some(t) = resolved {
9442 title = t;
9443 }
9444 let english = gt::LanguageDecision {
9446 title_english: true,
9447 body_english: true,
9448 body_complete: true,
9449 };
9450 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9451 if original {
9452 if g.title {
9455 title = pr.title.clone();
9456 }
9457 if g.body {
9458 body = pr.body.clone();
9459 }
9460 }
9461 Ok(Some(PrMessage { title, body }))
9462 }
9463
9464 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9465 self.state.github_text = Some(g.clone());
9466 self.state.end_pass_parked(ParkCause::Question);
9467 self.state.parked = true;
9468 self.state.event(
9469 "github-text",
9470 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9471 );
9472 self.state.save()
9473 }
9474
9475 async fn judge_pr_language(
9478 &mut self,
9479 cwd: &Path,
9480 pr: &PrMessage,
9481 ) -> Option<crate::github_text::LanguageDecision> {
9482 self.state.config.roles.language_judge.as_ref()?;
9483 if !self.state.config.graph.language_gate() {
9484 return None;
9485 }
9486 let decision =
9487 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9488 self.state.event(
9489 "github-text",
9490 if decision.is_some() {
9491 "language decided by the language judge"
9492 } else {
9493 "language judge unavailable; using built-in heuristics"
9494 },
9495 );
9496 decision
9497 }
9498
9499 fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9500 Box::pin(self.merge_inner())
9501 }
9502
9503 async fn merge_inner(&mut self) -> Result<()> {
9504 if self.resource_halt()? || self.hold_base_sync().await? {
9505 return Ok(());
9506 }
9507 if self
9522 .state
9523 .base_sync
9524 .as_ref()
9525 .is_some_and(|s| s.conflict.is_some())
9526 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9527 != Some(RunStatus::Gating)
9528 || !self.state.gate_allows_progress()
9536 {
9537 return Ok(());
9538 }
9539 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9548 return Ok(());
9549 }
9550 let Some(winner) = self.state.winner().cloned() else {
9551 return Ok(());
9552 };
9553 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9554 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9555 self.state.status = RunStatus::Blocked;
9556 self.state.event(
9557 "gate",
9558 "winner changed after gate; fresh verification required",
9559 );
9560 self.state.save()?;
9561 return Ok(());
9562 }
9563 let wait = self.repository_wait();
9564 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9565 return Ok(());
9566 }
9567 let repo = self.state.repo.clone();
9568 let base = self.state.base_branch.clone();
9569 let mode = self.state.config.merge.mode;
9570 let style = self.state.config.merge.style;
9571 let facts = if is_review_run(&self.state) {
9572 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9573 let start = review_base(
9574 &repo,
9575 &self.state.config.merge.remote,
9576 &base,
9577 &self.state.base_commit,
9578 &winner.branch,
9579 )
9580 .await;
9581 branch_facts(&repo, &start, &winner.branch).await
9582 } else {
9583 None
9584 };
9585 let leaked = if is_review_run(&self.state) {
9588 leaked_subjects(&self.state, &winner.branch).await
9589 } else {
9590 Some(Vec::new())
9591 };
9592 let Some(pr) = self
9596 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9597 .await?
9598 else {
9599 return Ok(());
9600 };
9601 let message = pr.commit_message();
9602
9603 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9604 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9605 return Ok(());
9606 }
9607 let outcome = match mode {
9608 MergeMode::None => MergeOutcome {
9609 mode,
9610 ok: true,
9611 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9612 empty: false,
9613 },
9614 MergeMode::Pr | MergeMode::Local
9615 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9616 {
9617 MergeOutcome {
9618 mode,
9619 ok: false,
9620 detail: empty_candidate_detail(&self.state, &base),
9621 empty: true,
9622 }
9623 }
9624 MergeMode::Local => {
9625 let on = git::current_branch(&repo).await?;
9626 if on.as_deref() != Some(base.as_str()) {
9627 MergeOutcome {
9628 mode,
9629 ok: false,
9630 detail: format!(
9631 "{} has {} checked out, not the base branch {base}",
9632 repo.display(),
9633 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9634 ),
9635 empty: false,
9636 }
9637 } else if !git::is_clean(&repo).await? {
9638 MergeOutcome {
9639 mode,
9640 ok: false,
9641 detail: format!("{} is dirty; refusing to merge", repo.display()),
9642 empty: false,
9643 }
9644 } else {
9645 let out = match style {
9646 MergeStyle::Merge => {
9647 git::merge_no_ff(&repo, &winner.branch, &message).await?
9648 }
9649 MergeStyle::Squash => {
9650 git::merge_squash(&repo, &winner.branch, &message).await?
9651 }
9652 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9653 };
9654 MergeOutcome {
9655 mode,
9656 ok: out.ok(),
9657 detail: if out.ok() { out.stdout } else { out.stderr },
9658 empty: false,
9659 }
9660 }
9661 }
9662 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9663 self.state.pr_publication_pending = true;
9664 self.state.save()?;
9665 let out =
9666 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9667 self.state.pr_publication_pending = out.is_err();
9668 MergeOutcome {
9669 mode,
9670 ok: out.is_ok(),
9671 detail: match out {
9672 Ok(url) => url,
9673 Err(e) => format!("{e:#}"),
9674 },
9675 empty: false,
9676 }
9677 }
9678 MergeMode::Pr => {
9679 let remote = self.state.config.merge.remote.clone();
9680 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9681 let pushed = if self.state.foreign_checkout.is_some() {
9682 let refspec = format!("{delivery_head}:refs/heads/{public}");
9683 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9684 } else {
9685 git::push(&winner.worktree, &remote, &winner.branch).await?
9686 };
9687 if !pushed.ok() {
9688 MergeOutcome {
9689 mode,
9690 ok: false,
9691 detail: pushed.stderr,
9692 empty: false,
9693 }
9694 } else {
9695 if self.state.foreign_checkout.is_some() {
9696 self.state.pr_pushed_head = Some(delivery_head.clone());
9697 }
9698 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9704 let out = match pr_merge_plan(found) {
9705 PrPlan::Create => {
9706 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9707 .await
9708 }
9709 PrPlan::Adopt { url, title } => {
9710 self.state
9711 .event("merge", format!("Pr: adopted open pull request {url}"));
9712 if self.state.explicit_github_text_selection() {
9713 land::set_pr_text_verbatim(
9716 &winner.worktree,
9717 &url,
9718 &pr.title,
9719 &pr.body,
9720 )
9721 .await
9722 .map(|()| url)
9723 } else {
9724 if title != pr.title
9725 && (!is_review_run(&self.state)
9726 || leaked
9727 .as_deref()
9728 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9729 && let Err(e) =
9730 if self.state.github_text.as_ref().is_some_and(|g| {
9731 g.resolved
9732 && g.title
9733 && g.choice == Some(GithubTextChoice::Original)
9734 }) {
9735 land::set_pr_title_verbatim(
9737 &winner.worktree,
9738 &url,
9739 &pr.title,
9740 )
9741 .await
9742 } else {
9743 land::set_pr_title(
9744 &mut self.state,
9745 &winner.worktree,
9746 &url,
9747 &pr.title,
9748 )
9749 .await
9750 }
9751 {
9752 tracing::warn!("could not refresh title of {url}: {e:#}");
9753 self.state
9754 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9755 }
9756 Ok(url)
9757 }
9758 }
9759 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9760 };
9761 match out {
9762 Ok(url) => MergeOutcome {
9763 mode,
9764 ok: true,
9765 detail: url,
9766 empty: false,
9767 },
9768 Err(e) => MergeOutcome {
9769 mode,
9770 ok: false,
9771 detail: e.to_string(),
9772 empty: false,
9773 },
9774 }
9775 }
9776 }
9777 };
9778
9779 self.state.status = match (mode, outcome.ok) {
9780 (MergeMode::None, _) => RunStatus::Ready,
9781 (_, true) if outcome.empty => RunStatus::Ready,
9782 (MergeMode::Pr, true) => RunStatus::Ready,
9783 (_, true) => RunStatus::Merged,
9784 (_, false) => RunStatus::Blocked,
9785 };
9786 self.state.event(
9787 "merge",
9788 format!(
9789 "{:?}: {}",
9790 mode,
9791 outcome.detail.lines().next().unwrap_or("")
9792 ),
9793 );
9794 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9795 self.state.record_github_text_publication(&outcome.detail);
9796 }
9797 self.state.merge = Some(outcome);
9798 self.state.save()?;
9799
9800 if self.state.config.graph.land
9806 && mode == MergeMode::Pr
9807 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9808 {
9809 self.run_land().await?;
9810 }
9811 self.settle_questions();
9816 Ok(())
9817 }
9818
9819 fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9830 Box::pin(self.run_land_inner())
9831 }
9832
9833 async fn run_land_inner(&mut self) -> Result<()> {
9834 crate::pr_head::ensure_owner(&self.state)?;
9835 let url = self
9836 .state
9837 .pr
9838 .as_ref()
9839 .map(|pr| pr.url.clone())
9840 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9841 .unwrap_or_default();
9842 let url = url.lines().next().unwrap_or("").trim().to_owned();
9843 if !url.starts_with("http") {
9844 return Ok(());
9845 }
9846 loop {
9849 match land::land(&mut self.state, &url).await {
9850 Ok(_)
9851 if self
9852 .state
9853 .merge_repairs
9854 .last()
9855 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9856 {
9857 if self.repair_merge_review().await? {
9858 continue;
9859 }
9860 }
9861 Ok(_) if self.state.resource_block.is_some() => {
9862 return self.stop_for_resource();
9863 }
9864 Ok(pr) if self.state.parked => {
9865 let _ = pr;
9869 }
9870 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9871 self.state.status = RunStatus::Superseded;
9872 self.state.save()?;
9873 }
9874 Ok(pr) => {
9875 self.state.status = match pr.state {
9876 land::PrLifecycle::Merged => RunStatus::Merged,
9877 _ => RunStatus::Blocked,
9878 };
9879 if bump::should_release_bump(self.state.status)
9886 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9887 {
9888 self.state
9894 .event("bump", format!("release bump skipped: {e:#}"));
9895 }
9896 if self.state.status == RunStatus::Merged {
9899 crate::followup::after_merge(&mut self.state, &pr.url).await;
9900 }
9901 self.state.save()?;
9902 }
9903 Err(e) => {
9904 if e.is::<crate::repo_lock::ResourceBlocked>() {
9905 return Err(e);
9906 }
9907 self.state.status = RunStatus::Blocked;
9908 self.state.event("land", format!("gave up: {e}"));
9909 self.state.save()?;
9910 }
9911 }
9912 break;
9913 }
9914 Ok(())
9915 }
9916
9917 async fn repair_merge_review(&mut self) -> Result<bool> {
9920 use crate::run::MergeRepairPhase as Phase;
9921 crate::pr_head::ensure_owner(&self.state)?;
9922 let i = self.state.merge_repairs.len() - 1;
9923 let request = self.state.merge_repairs[i].clone();
9924 if request.phase == Phase::Fixing {
9925 let restored = match self.state.winner().cloned() {
9926 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9927 None => Err(anyhow::anyhow!("repair has no winning branch")),
9928 };
9929 let why = match restored {
9930 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9931 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9932 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9933 };
9934 self.hold_merge_repair(i, why)?;
9935 return Ok(false);
9936 }
9937 if request.phase == Phase::Reviewing && self.state.gate_ran {
9942 let winner = self
9943 .state
9944 .winner()
9945 .context("repair has no winning branch")?;
9946 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9947 if self.state.gate_head.as_deref() != Some(head.as_str())
9948 || !git::is_clean(&winner.worktree).await?
9949 {
9950 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9951 return Ok(false);
9952 }
9953 }
9954 let failure = if i >= self.state.config.graph.land_rounds
9955 || self.state.config.graph.review_rounds == 0
9956 {
9957 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9958 } else {
9959 None
9960 };
9961 if let Some(why) = failure {
9962 self.hold_merge_repair(i, why)?;
9963 return Ok(false);
9964 }
9965 if request.phase == Phase::Requested {
9966 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9967 Ok(pr) => pr,
9968 Err(e) => {
9969 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9970 return Ok(false);
9971 }
9972 };
9973 self.state.merge_repairs[i].phase = Phase::Fixing;
9974 self.state.land_approval = None;
9975 self.state.land_armed_head = None;
9976 self.state.save()?;
9977 let evidence = serde_json::to_string_pretty(&request)?;
9980 let reason = format!(
9981 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9982 );
9983 let budget = self.state.config.graph.land_rounds;
9984 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9985 match result {
9986 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9987 Ok(land::Fixed::Deferred) => {
9988 let worktree = self
9992 .state
9993 .winner()
9994 .context("repair has no winning branch")?
9995 .worktree
9996 .clone();
9997 self.graft_restore(&worktree, &request.head).await?;
9998 self.state.merge_repairs[i].phase = Phase::Requested;
9999 self.stop_for_resource()?;
10000 return Ok(false);
10001 }
10002 Ok(land::Fixed::Failed(why)) => {
10003 self.hold_merge_repair(
10004 i,
10005 format!(
10006 "repair failed: {why}; inspect the branch and run an explicit review"
10007 ),
10008 )?;
10009 return Ok(false);
10010 }
10011 Err(e) => {
10012 self.hold_merge_repair(
10013 i,
10014 format!(
10015 "repair failed: {e:#}; inspect the branch and run an explicit review"
10016 ),
10017 )?;
10018 return Ok(false);
10019 }
10020 }
10021 self.begin_merge_repair_review(i)?;
10025 }
10026 self.review_loop().await?;
10027 self.gate().await?;
10028 if self.resource_halt()? || self.state.parked {
10029 return Ok(false);
10030 }
10031 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10032 != Some(RunStatus::Gating)
10033 || !self.state.gate_allows_progress()
10034 {
10035 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10036 return Ok(false);
10037 }
10038 let winner = self
10041 .state
10042 .winner()
10043 .cloned()
10044 .context("repair has no winning branch")?;
10045 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10046 if self.state.gate_head.as_deref() != Some(after.as_str())
10047 || !git::is_clean(&winner.worktree).await?
10048 {
10049 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10050 return Ok(false);
10051 }
10052 let expected = self
10053 .state
10054 .pr_pushed_head
10055 .as_deref()
10056 .unwrap_or(&request.head)
10057 .to_owned();
10058 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10059 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10062 return Ok(false);
10063 }
10064 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10065 let refspec = format!("{after}:refs/heads/{public}");
10066 let push = git::git_raw(
10067 &winner.worktree,
10068 &["push", &lease, &self.state.config.merge.remote, &refspec],
10069 )
10070 .await;
10071 match push {
10072 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10073 Ok(out) => {
10074 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10075 return Ok(false);
10076 }
10077 Err(e) => {
10078 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10079 return Ok(false);
10080 }
10081 }
10082 self.state.merge_repairs[i].phase = Phase::Complete;
10083 self.state.status = RunStatus::Landing;
10084 self.state.event(
10085 "repair",
10086 "new review and gate completed; fresh owner merge decision required",
10087 );
10088 self.state.save()?;
10089 Ok(true)
10090 }
10091
10092 fn merge_repair_artifacts(&self) -> PathBuf {
10093 let artifacts = agent::artifacts_dir(&self.state.dir());
10094 match self
10095 .state
10096 .merge_repairs
10097 .last()
10098 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10099 {
10100 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10101 None => artifacts,
10102 }
10103 }
10104
10105 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10106 if let Some(e2e) = self.state.head_e2e.take() {
10107 self.state
10108 .verification_history
10109 .push(crate::run::VerificationEvidence {
10110 superseded_by: None,
10111 archived_at: Some(Timestamp::now()),
10112 fixes_spent: Some(self.state.gate_fixes.len()),
10113 gate_head: None,
10114 gate_ran: false,
10115 gate: Vec::new(),
10116 e2e: Some(e2e),
10117 });
10118 }
10119 self.state.reviews.clear();
10120 self.state.gate.clear();
10121 self.state.gate_ran = false;
10122 self.state.gate_head = None;
10123 self.state.gate_fixes.clear();
10124 self.state.gate_repair_e2e = None;
10125 self.state.pre_gate.clear();
10126 self.state.pre_gate_commit = None;
10127 self.state.contested_handoff = None;
10128 self.state.land_reruns = None;
10129 self.state.land_approval = None;
10130 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10131 self.state.status = RunStatus::Reviewing;
10132 self.state.save()
10133 }
10134
10135 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10136 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10137 self.state.merge_repairs[index].failure = Some(why.clone());
10138 self.state.status = RunStatus::Blocked;
10139 self.state.event("repair", why);
10140 self.state.save()
10141 }
10142
10143 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10147 if let Some(existing) = self.state.seats.get(key)
10148 && existing.agent == agent
10149 {
10150 return existing.clone();
10151 }
10152 let fresh = if self.state.seats.contains_key(key) {
10155 handover_seat(key, agent, self.state.next_seat_seed())
10156 } else {
10157 SeatState::new(key, agent, self.state.seed)
10158 };
10159 self.state.seats.insert(key.to_owned(), fresh.clone());
10160 fresh
10161 }
10162
10163 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10169 match self.state.seats.get(key) {
10170 Some(s) if s.agent != spec.id => {
10171 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10172 }
10173 _ => spec,
10174 }
10175 }
10176
10177 fn view(&self, c: &Candidate) -> CandidateView {
10179 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10180 .unwrap_or_default();
10181 let (patch, _) = blind::sanitize_patch(
10182 &format!("candidate {} patch", c.label),
10183 &raw,
10184 &self.state.config.blind,
10185 );
10186 CandidateView {
10187 label: c.label,
10188 branch: c.branch.clone(),
10189 summary: c.summary.clone(),
10190 stat: c.stat.clone(),
10191 patch,
10192 }
10193 }
10194
10195 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10197 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10198 prompt::judge(
10199 "(see above)",
10200 &views,
10201 self.roles.judges.len(),
10202 base_short,
10203 "en",
10204 )
10205 }
10206
10207 fn vote_prompt_full(
10213 &self,
10214 j: usize,
10215 labels: &[char],
10216 language: &str,
10217 candidates: &[Candidate],
10218 base_short: &str,
10219 ) -> String {
10220 let mut text = format!(
10221 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10222 prompt::final_vote(labels, language),
10223 self.state.instruction,
10224 self.candidate_block(candidates, base_short)
10225 );
10226 if let Some(own) = self
10227 .state
10228 .judgements
10229 .get(j)
10230 .filter(|r| !r.ranking.is_empty())
10231 {
10232 let reasons = own
10233 .reasons
10234 .iter()
10235 .map(|(k, v)| format!("- {k}: {v}"))
10236 .collect::<Vec<_>>()
10237 .join("\n");
10238 text.push_str(&format!(
10239 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10240 own.ranking.iter().collect::<String>(),
10241 if reasons.is_empty() {
10242 ""
10243 } else {
10244 ", because:\n"
10245 }
10246 ));
10247 }
10248 if !self.state.deliberation.is_empty() {
10249 text.push_str("\n# What was argued before this vote\n");
10250 for t in self.transcript(&[], j) {
10251 text.push_str(&format!(
10252 "\n## {}{}\n\n{}\n",
10253 t.who,
10254 if t.is_self { " (you)" } else { "" },
10255 t.body.trim()
10256 ));
10257 }
10258 }
10259 text
10260 }
10261
10262 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10269 let mut turns = Vec::new();
10270 for j in &self.state.judgements {
10271 if j.ranking.is_empty() {
10272 continue;
10273 }
10274 let reasons = j
10275 .reasons
10276 .iter()
10277 .map(|(k, v)| format!("- {k}: {v}"))
10278 .collect::<Vec<_>>()
10279 .join("\n");
10280 turns.push(Turn {
10281 who: format!("Judge {} (opening ranking)", j.judge),
10282 is_self: j.judge == self_idx + 1,
10283 body: format!(
10284 "Ranked {}{}{reasons}",
10285 j.ranking.iter().collect::<String>(),
10286 if reasons.is_empty() {
10287 ""
10288 } else {
10289 ", because:\n"
10290 }
10291 ),
10292 });
10293 }
10294 for t in self
10295 .state
10296 .deliberation
10297 .iter()
10298 .flat_map(|r| r.turns.iter())
10299 .chain(current)
10300 {
10301 turns.push(Turn {
10302 who: format!("Judge {}", t.judge),
10303 is_self: t.judge == self_idx + 1,
10304 body: t.body.clone(),
10305 });
10306 }
10307 turns
10308 }
10309}
10310
10311fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10313 agent::has_session(spec.kind, seat, sessions)
10314}
10315
10316fn next_untried_in_roster<'a>(
10337 roster: &'a [AgentSpec],
10338 start: usize,
10339 tried: &BTreeSet<String>,
10340) -> Option<&'a AgentSpec> {
10341 roster
10342 .get(start + 1..)?
10343 .iter()
10344 .find(|s| !tried.contains(&s.id))
10345}
10346
10347fn pick_successor<'a>(
10368 roster: &'a [AgentSpec],
10369 start: usize,
10370 tried: &BTreeSet<String>,
10371 carried: Option<&BTreeSet<String>>,
10372 others: &BTreeSet<String>,
10373) -> Option<&'a AgentSpec> {
10374 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10375 roster
10376 .get(start + 1..)
10377 .and_then(|tail| {
10378 tail.iter()
10379 .filter(free)
10380 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10381 })
10382 .or_else(|| {
10383 carried
10384 .is_some()
10385 .then(|| roster.iter().find(free))
10386 .flatten()
10387 })
10388 .or_else(|| match carried {
10389 Some(c) => next_for_seat(roster, start, tried, c),
10390 None => next_untried_in_roster(roster, start, tried),
10391 })
10392}
10393
10394fn next_for_seat<'a>(
10404 roster: &'a [AgentSpec],
10405 start: usize,
10406 round_tried: &BTreeSet<String>,
10407 carried_failed: &BTreeSet<String>,
10408) -> Option<&'a AgentSpec> {
10409 roster
10410 .get(start + 1..)?
10411 .iter()
10412 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10413 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10414}
10415
10416fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10422 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10423 return spec;
10424 };
10425 let ok = |id: &str| !h.failed.contains(id);
10426 if let Some(last) = h.last_ok.as_deref()
10427 && ok(last)
10428 && let Some(s) = roster.iter().find(|s| s.id == last)
10429 {
10430 return s.clone();
10431 }
10432 if ok(&spec.id) {
10433 return spec;
10434 }
10435 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10436 roster
10437 .get(from + 1..)
10438 .into_iter()
10439 .flatten()
10440 .chain(roster.iter())
10441 .find(|s| ok(&s.id))
10442 .cloned()
10443 .unwrap_or(spec)
10444}
10445
10446pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10450 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10451}
10452
10453const TIMED_OUT: &str = "timed out";
10456
10457impl FailClass {
10458 fn of(out: &AgentOutcome) -> Option<Self> {
10460 match out {
10461 AgentOutcome::Ok(_) => None,
10462 AgentOutcome::Quota(_) => Some(Self::Quota),
10463 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10464 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10465 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10466 AgentOutcome::ResourceBlocked(_) => None,
10469 }
10470 }
10471
10472 fn stem_word(&self) -> &'static str {
10474 match self {
10475 Self::Quota => "quota",
10476 _ => "handover",
10477 }
10478 }
10479}
10480
10481fn failure_signature(msg: &str) -> String {
10485 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10486 if let Some(code) = line
10489 .strip_prefix("process exited with no usable output (exit code ")
10490 .and_then(|s| s.strip_suffix(')'))
10491 .and_then(|s| s.parse::<i32>().ok())
10492 {
10493 line = format!("exited with some({code}) and no usable output");
10494 } else if matches!(
10495 line.as_str(),
10496 "process exited with no usable output (no exit code reported)"
10497 | "process exited with no usable output (exit code unavailable)"
10498 ) {
10499 line = "exited with none and no usable output".to_owned();
10500 }
10501 let mut out = Vec::new();
10502 for word in line.split_whitespace() {
10503 if word.contains('/') || word.contains('\\') {
10504 out.push("<path>".to_owned());
10505 continue;
10506 }
10507 let mut w = String::new();
10508 let mut in_digits = false;
10509 for c in word.chars() {
10510 if c.is_ascii_digit() {
10511 if !in_digits {
10512 w.push('#');
10513 }
10514 in_digits = true;
10515 } else {
10516 in_digits = false;
10517 w.push(c);
10518 }
10519 }
10520 out.push(w);
10521 }
10522 out.join(" ").chars().take(120).collect()
10523}
10524
10525fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10532 match cur {
10533 FailClass::Quota | FailClass::Timeout => true,
10534 FailClass::Other(_) => prev != Some(cur),
10535 }
10536}
10537
10538fn fail_reason(out: &AgentOutcome) -> String {
10540 match out {
10541 AgentOutcome::Ok(_) => String::new(),
10542 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10543 AgentOutcome::Dropped(o) => format!(
10544 "the CLI dropped the stream ({})",
10545 o.dropped
10546 .as_ref()
10547 .map(|d| d.why.as_str())
10548 .unwrap_or("it ended without delivering its answer")
10549 ),
10550 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10551 e.lines().next().unwrap_or("").chars().take(160).collect()
10552 }
10553 }
10554}
10555
10556pub(crate) fn record_handover(
10559 state: &mut RunState,
10560 node: &str,
10561 seat: &str,
10562 from: &str,
10563 to: &str,
10564 class: &FailClass,
10565 reason: &str,
10566) {
10567 let message = if *class == FailClass::Quota {
10568 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10569 } else {
10570 format!("{seat}: handed over {from} -> {to} ({reason})")
10571 };
10572 state.event(node, message);
10573 state.handovers.push(Handover {
10574 at: Timestamp::now(),
10575 node: node.to_owned(),
10576 seat: seat.to_owned(),
10577 from: from.to_owned(),
10578 to: to.to_owned(),
10579 reason: reason.to_owned(),
10580 });
10581}
10582
10583fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10597 commands.iter().any(|c| c.exit_code.is_none())
10598}
10599
10600fn verified_noop_claim(
10613 usable: bool,
10614 commands: &[agent::CommandEvidence],
10615 text: &str,
10616) -> Option<String> {
10617 (usable && !has_unconfirmed_command(commands))
10618 .then(|| verdict::verified_noop(text))
10619 .flatten()
10620}
10621
10622fn short(commit: &str) -> String {
10623 commit.chars().take(7).collect()
10624}
10625
10626fn make_executable(path: &Path) -> Result<()> {
10627 #[cfg(unix)]
10628 {
10629 use std::os::unix::fs::PermissionsExt as _;
10630 let mut perms = std::fs::metadata(path)?.permissions();
10631 perms.set_mode(0o755);
10632 std::fs::set_permissions(path, perms)?;
10633 }
10634 #[cfg(not(unix))]
10635 {
10636 let _ = path;
10637 }
10638 Ok(())
10639}
10640
10641struct WaveCtx<'a> {
10648 run: &'a str,
10651 node: &'a str,
10653 prompts: &'a Prompts,
10654 cache: Option<&'a Path>,
10656 round: Option<usize>,
10659 carry_seats: bool,
10663 pauses: Option<(&'a Pause, &'a Pause)>,
10666}
10667
10668fn repository_wait(
10670 state: &RunState,
10671 pauses: Option<(&Pause, &Pause)>,
10672) -> crate::repo_lock::WaitPolicy {
10673 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10674 let inherited = crate::repo_lock::WaitPolicy::current();
10675 crate::repo_lock::WaitPolicy::new(
10676 Duration::from_secs(state.config.graph.timeout_resource_wait),
10677 move || {
10678 inherited.cancelled()
10679 || pauses
10680 .as_ref()
10681 .is_some_and(|(a, b)| a.parked() || b.parked())
10682 },
10683 )
10684}
10685
10686fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10687 if error.is::<crate::repo_lock::ResourceBlocked>() {
10688 state.resource_block = Some(crate::run::ResourceBlock {
10689 cause: crate::run::ResourceBlockCause::Other,
10690 node: "repository".into(),
10691 kind: if error.to_string().contains("disk pressure") {
10692 "disk pressure"
10693 } else if error.to_string().contains("disk admission") {
10694 "disk admission"
10695 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10696 "incremental cache admission"
10697 } else {
10698 "repository coordination"
10699 }
10700 .into(),
10701 detail: error.to_string(),
10702 at: Timestamp::now(),
10703 });
10704 state.event("resource", error.to_string());
10705 }
10706}
10707
10708async fn coordinate_operation(
10709 state: &mut RunState,
10710 cwd: &Path,
10711 wait: &crate::repo_lock::WaitPolicy,
10712) -> Result<Option<crate::repo_lock::SeatLease>> {
10713 let result =
10714 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10715 .await;
10716 if let Err(e) = &result {
10717 record_repository_block(state, e);
10718 }
10719 result
10720}
10721
10722async fn run_one(
10724 job: SeatJob,
10725 sem: Arc<Semaphore>,
10726 ctx: &WaveCtx<'_>,
10727 state: &mut RunState,
10728 attempt: usize,
10729) -> (SeatState, AgentOutcome) {
10730 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10731 .await
10732 .pop()
10733 .expect("one job in, one result out");
10734 (seat, out)
10735}
10736
10737async fn wave(
10743 jobs: Vec<SeatJob>,
10744 sem: Arc<Semaphore>,
10745 ctx: &WaveCtx<'_>,
10746 state: &mut RunState,
10747 attempt: usize,
10748) -> Vec<(usize, SeatState, AgentOutcome)> {
10749 let WaveCtx {
10750 run,
10751 node,
10752 prompts,
10753 cache,
10754 round,
10755 carry_seats: _,
10756 pauses,
10757 } = *ctx;
10758 for job in &jobs {
10759 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10760 }
10761 if let Err(e) = state.save() {
10762 tracing::warn!("could not persist in-progress seats: {e:#}");
10767 }
10768 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10784 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10788 let cache_guard = if let Some(cache_dir) = cache {
10789 if jobs_had_a_writer {
10790 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10791 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10792 match acquire_cache_lease(
10793 state,
10794 cache_dir,
10795 &owner,
10796 resource_limit,
10797 node,
10798 node,
10799 &cancel,
10800 )
10801 .await
10802 {
10803 Ok(Some(g)) => Some(g),
10804 Ok(None) => {
10807 if let Some(block) = &mut state.resource_block {
10808 block.cause = crate::run::ResourceBlockCause::Cancelled {
10809 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10810 };
10811 }
10812 return block_wave(
10813 jobs,
10814 state,
10815 node,
10816 attempt,
10817 "parked while waiting for the shared build cache",
10818 );
10819 }
10820 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10823 Err(e) => {
10824 let why = format!("{e:#}");
10828 state.event(
10829 "resource",
10830 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10831 );
10832 return block_wave(jobs, state, node, attempt, &why);
10833 }
10834 }
10835 } else {
10836 None
10837 }
10838 } else {
10839 None
10840 };
10841 state.seats_spawning(node, attempt);
10844 if let Err(e) = state.save() {
10845 tracing::warn!("could not persist the start of a wave: {e:#}");
10846 }
10847 let slot_limit = resource_limit
10851 + jobs
10852 .iter()
10853 .map(|j| j.timeout)
10854 .max()
10855 .unwrap_or_default()
10856 .saturating_mul(jobs.len() as u32);
10857 let mut set = tokio::task::JoinSet::new();
10858 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10861 let overlay = prompts.overlay(node);
10862 let repository_wait = repository_wait(state, pauses);
10863 for (i, mut job) in jobs.into_iter().enumerate() {
10864 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10865 if cache.is_some() {
10866 job.prompt.push('\n');
10867 job.prompt
10868 .push_str(&prompt::build_cache_note(node, job.allow_write));
10869 }
10870 let sem = Arc::clone(&sem);
10871 let run = run.to_owned();
10872 let node = node.to_owned();
10873 let attachments = if node == "implement" {
10876 state.attachments.clone()
10877 } else {
10878 Vec::new()
10879 };
10880 let cache = cache
10891 .filter(|_| job.allow_write && cache_guard.is_some())
10892 .map(Path::to_path_buf);
10893 let spawned_tx = spawned_tx.clone();
10894 let repository_wait = repository_wait.clone();
10895 let coordination = state.config.coordination.clone();
10896 let fetched_base = state.fetched_base.clone();
10897 set.spawn(async move {
10898 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10902 Ok(p) => p,
10903 Err(_) => {
10904 return (
10905 i,
10906 job.seat,
10907 AgentOutcome::ResourceBlocked(format!(
10908 "no parallel slot became free within {}s",
10909 slot_limit.as_secs()
10910 )),
10911 false,
10912 );
10913 }
10914 };
10915 spawned_tx.send(job.seat.key.clone()).ok();
10916 let mut seat = job.seat;
10917 let out = repository_wait
10918 .scope(agent::invoke(
10919 &job.spec,
10920 &mut seat,
10921 &Invocation {
10922 coordination: coordination.as_ref(),
10923 fetched_base: fetched_base.as_ref(),
10924 cwd: &job.cwd,
10925 prompt: &job.prompt,
10926 timeout: job.timeout,
10927 allow_write: job.allow_write,
10928 sessions: job.sessions,
10929 artifacts: &job.artifacts,
10930 stem: &job.stem,
10931 run: &run,
10932 node: &node,
10933 cache_dir: cache.as_deref(),
10934 attachments: &attachments,
10935 writable: &[],
10936 },
10937 ))
10938 .await;
10939 let interrupted = out
10940 .as_ref()
10941 .err()
10942 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10943 let out = match out {
10944 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10945 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10946 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10954 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10955 Ok(o) => AgentOutcome::Failed(format!(
10956 "process exited with no usable output ({})",
10957 crate::notices::exit_status(o.exit_code)
10958 )),
10959 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10960 AgentOutcome::ResourceBlocked(e.to_string())
10961 }
10962 Err(e) => AgentOutcome::Failed(e.to_string()),
10963 };
10964 (i, seat, out, interrupted)
10965 });
10966 }
10967 drop(spawned_tx);
10968 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10969 loop {
10970 let joined = tokio::select! {
10971 Some(key) = spawned_rx.recv() => {
10972 state.seat_spawned(&key);
10974 if let Err(e) = state.save() {
10975 tracing::warn!("could not persist a seat's start: {e:#}");
10976 }
10977 continue;
10978 }
10979 joined = set.join_next() => joined,
10980 };
10981 let Some(joined) = joined else {
10982 break;
10983 };
10984 let (i, seat, out, interrupted) = match joined {
10985 Ok(v) => v,
10986 Err(e) => {
10990 tracing::error!("agent task panicked: {e}");
10991 continue;
10992 }
10993 };
10994 state.seat_finished(&seat.key);
10995 if interrupted
10996 && node == "implement"
10997 && let Some(label) = state
10998 .candidates
10999 .iter()
11000 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11001 .map(|candidate| candidate.label)
11002 {
11003 state.candidate_preservation.insert(
11004 label,
11005 "interrupted implementation awaiting persistence".into(),
11006 );
11007 state.interrupted_implementations.insert(label);
11008 }
11009 if let AgentOutcome::ResourceBlocked(why) = &out
11010 && state.resource_block.is_none()
11011 {
11012 state.resource_block = Some(crate::run::ResourceBlock {
11013 cause: crate::run::ResourceBlockCause::Other,
11014 node: node.to_owned(),
11015 kind: if why.contains("disk pressure") {
11016 "disk pressure"
11017 } else if why.contains("disk admission") {
11018 "disk admission"
11019 } else if why.contains(crate::incremental::BLOCKED) {
11020 "incremental cache admission"
11021 } else if why.contains("repository coordination") {
11022 "repository coordination"
11023 } else {
11024 "parallel slot"
11025 }
11026 .to_owned(),
11027 detail: why.clone(),
11028 at: Timestamp::now(),
11029 });
11030 let activity = if interrupted {
11031 "agent execution interrupted"
11032 } else {
11033 "no agent was started"
11034 };
11035 state.event("resource", format!("{node}: {activity} - {why}"));
11036 }
11037 record_jobs(state, node, round, &seat.key, &out);
11038 if let Err(e) = state.save() {
11039 tracing::warn!("could not persist a seat's completion: {e:#}");
11040 }
11041 if collected.len() <= i {
11042 collected.resize_with(i + 1, || None);
11043 }
11044 collected[i] = Some((i, seat, out));
11045 }
11046 if state
11052 .active
11053 .values()
11054 .any(|a| a.node == node && a.attempt == attempt)
11055 {
11056 state
11057 .active
11058 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11059 if let Err(e) = state.save() {
11060 tracing::warn!("could not persist the end of a wave: {e:#}");
11061 }
11062 }
11063 if let Some(cache_dir) = cache
11070 && jobs_had_a_writer
11071 {
11072 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11073 }
11074 if let Some(guard) = cache_guard {
11075 guard.release();
11076 }
11077 collected.into_iter().flatten().collect()
11078}
11079
11080fn record_jobs(
11091 state: &mut RunState,
11092 node: &str,
11093 round: Option<usize>,
11094 seat: &str,
11095 out: &AgentOutcome,
11096) {
11097 let commands: &[agent::CommandEvidence] = match out {
11098 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11099 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11100 };
11101 let checked_at = Timestamp::now();
11102 for c in commands {
11103 state.jobs.push(JobRecord {
11104 node: node.to_owned(),
11105 round,
11106 seat: seat.to_owned(),
11107 id: c.id.clone(),
11108 description: c.description.clone(),
11109 checked_at,
11110 status: match c.exit_code {
11111 Some(0) => JobStatus::Completed,
11112 Some(_) => JobStatus::Failed,
11113 None => JobStatus::Unknown,
11114 },
11115 exit_code: c.exit_code,
11116 result_summary: c.result_summary.clone(),
11117 source: c.source.clone(),
11118 });
11119 }
11120}
11121
11122fn round_is_clean(
11143 blocking: usize,
11144 e2e_ok: bool,
11145 answered: usize,
11146 expected: usize,
11147 quota_missing: usize,
11148 policy: IncompleteReviewPolicy,
11149) -> bool {
11150 if blocking != 0 || !e2e_ok {
11151 return false;
11152 }
11153 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11154 return true;
11155 }
11156 answered > 0 && expected - answered <= quota_missing
11157}
11158
11159pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11183 if max_rounds == 0
11184 || reviews.last().is_some_and(|r| {
11185 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11186 })
11187 {
11188 return Some(RunStatus::Gating);
11189 }
11190 let last = reviews.last()?;
11191 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11192 if reviews.len() < max_rounds && !stagnant {
11193 return None;
11194 }
11195 if last.incomplete() && last.blocking == 0 {
11196 return Some(RunStatus::Blocked);
11197 }
11198 if last.e2e_status() == E2eStatus::ResourceBlocked
11199 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11200 {
11201 return None;
11202 }
11203 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11204 RunStatus::Gating
11205 } else {
11206 RunStatus::Blocked
11207 })
11208}
11209
11210pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11225 if nudged {
11226 (full / 4).max(Duration::from_secs(120)).min(full)
11227 } else {
11228 full
11229 }
11230}
11231
11232#[allow(clippy::too_many_arguments)]
11261async fn ask_json_wave<T>(
11262 jobs: Vec<SeatJob>,
11263 sem: Arc<Semaphore>,
11264 retries: usize,
11265 roster: &[AgentSpec],
11266 ctx: &WaveCtx<'_>,
11267 losses: &mut Vec<QuotaLoss>,
11268 state: &mut RunState,
11269 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11270) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11271where
11272 T: serde::de::DeserializeOwned + Send + 'static,
11273{
11274 ask_wave_with(
11275 jobs,
11276 sem,
11277 retries,
11278 roster,
11279 ctx,
11280 losses,
11281 state,
11282 &|text: &str| {
11283 let v = verdict::extract_json::<T>(text)?;
11284 validate(&v)?;
11285 Ok(v)
11286 },
11287 )
11288 .await
11289}
11290
11291#[allow(clippy::too_many_arguments)]
11300async fn ask_wave_with<T>(
11301 jobs: Vec<SeatJob>,
11302 sem: Arc<Semaphore>,
11303 retries: usize,
11304 roster: &[AgentSpec],
11305 ctx: &WaveCtx<'_>,
11306 losses: &mut Vec<QuotaLoss>,
11307 state: &mut RunState,
11308 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11309) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11310where
11311 T: Send + 'static,
11312{
11313 let n = jobs.len();
11314 let originals: Vec<SeatJob> = jobs;
11315 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11316 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11317 let mut nudges: Vec<usize> = vec![0; n];
11324 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11328 let mut tried: Vec<BTreeSet<String>> = specs
11329 .iter()
11330 .map(|s| BTreeSet::from([s.id.clone()]))
11331 .collect();
11332 let starts: Vec<usize> = specs
11333 .iter()
11334 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11335 .collect();
11336 let carry = ctx.carry_seats && !roster.is_empty();
11337 let carried: Vec<BTreeSet<String>> = originals
11340 .iter()
11341 .map(|j| {
11342 state
11343 .seat_history
11344 .get(&j.seat.key)
11345 .filter(|_| carry)
11346 .map(|h| h.failed.clone())
11347 .unwrap_or_default()
11348 })
11349 .collect();
11350 let mut prev: Vec<Option<FailClass>> = originals
11351 .iter()
11352 .map(|j| {
11353 state
11354 .seat_history
11355 .get(&j.seat.key)
11356 .filter(|_| carry)
11357 .and_then(|h| h.last_fail.clone())
11358 })
11359 .collect();
11360 let next_agent =
11361 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11362 let others: BTreeSet<String> = specs
11363 .iter()
11364 .enumerate()
11365 .filter(|(j, _)| *j != i)
11366 .map(|(_, s)| s.id.clone())
11367 .collect();
11368 pick_successor(
11369 roster,
11370 starts[i],
11371 tried,
11372 carry.then(|| &carried[i]),
11373 &others,
11374 )
11375 .cloned()
11376 };
11377 let mut fresh: Vec<Option<String>> = vec![None; n];
11378 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11379 let mut pending: Vec<usize> = (0..n).collect();
11380
11381 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11386 for round in 0..max_rounds {
11387 if pending.is_empty() {
11388 break;
11389 }
11390 let mut batch = Vec::with_capacity(pending.len());
11391 let mut renudged: Vec<&str> = Vec::new();
11392 for &i in &pending {
11393 let src = &originals[i];
11394 let full: &str = match &src.handover {
11399 Some(h) if specs[i].id != src.spec.id => h,
11400 _ => &src.prompt,
11401 };
11402 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11405 (
11406 full.to_owned(),
11407 src.timeout,
11408 format!("{}-{word}-{}", src.stem, specs[i].id),
11409 )
11410 } else if nudges[i] == 0 {
11411 (full.to_owned(), src.timeout, src.stem.clone())
11412 } else {
11413 renudged.push(src.seat.key.as_str());
11414 let why = done[i]
11415 .as_ref()
11416 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11417 .unwrap_or_else(|| "no parsable answer".to_owned());
11418 let nudge = prompt::nudge(&why);
11419 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11420 let prompt = if nudged {
11421 nudge
11422 } else {
11423 format!("{full}\n\n---\n\n{nudge}")
11424 };
11425 (
11426 prompt,
11427 retry_budget(src.timeout, nudged),
11428 format!("{}-retry{}", src.stem, nudges[i]),
11429 )
11430 };
11431 batch.push(SeatJob {
11432 spec: specs[i].clone(),
11433 seat: seats[i].clone(),
11434 cwd: src.cwd.clone(),
11435 prompt,
11436 timeout,
11437 allow_write: src.allow_write,
11438 sessions: src.sessions,
11439 artifacts: src.artifacts.clone(),
11440 stem,
11441 handover: None,
11442 });
11443 }
11444
11445 if !renudged.is_empty() {
11446 state.event(
11447 ctx.node,
11448 format!("retry {round}: re-asking {}", renudged.join(", ")),
11449 );
11450 }
11451 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11452 let mut still = Vec::new();
11453 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11454 seats[i] = seat;
11455 if let AgentOutcome::ResourceBlocked(why) = &out {
11458 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11459 continue;
11460 }
11461 let class = FailClass::of(&out);
11462 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11465 && nudges[i] < retries
11466 && !roster.is_empty();
11467 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11468 let next = should_hand_over(prev[i].as_ref(), &cur)
11469 .then(|| next_agent(i, &tried[i], &specs))
11470 .flatten();
11471 if carry {
11472 let h = state
11473 .seat_history
11474 .entry(originals[i].seat.key.clone())
11475 .or_default();
11476 h.failed.insert(specs[i].id.clone());
11477 h.last_fail = Some(cur.clone());
11478 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11479 h.last_ok = None;
11480 }
11481 if let Err(e) = state.save() {
11484 tracing::warn!("could not persist a seat's failure history: {e:#}");
11485 }
11486 }
11487 if let Some(next) = next {
11488 record_handover(
11489 state,
11490 ctx.node,
11491 &originals[i].seat.key,
11492 &specs[i].id,
11493 &next.id,
11494 &cur,
11495 &fail_reason(&out),
11496 );
11497 tried[i].insert(next.id.clone());
11498 prev[i] = Some(cur.clone());
11499 seats[i] =
11500 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11501 specs[i] = next;
11502 fresh[i] = Some(cur.stem_word().to_owned());
11503 nudges[i] = 0;
11504 done[i] = Some(Err(anyhow::anyhow!(
11505 "handed over after: {}",
11506 fail_reason(&out)
11507 )));
11508 still.push(i);
11509 continue;
11510 }
11511 }
11512 if carry
11513 && !nudge_first
11514 && let Some(cur) = class.clone()
11515 {
11516 let h = state
11518 .seat_history
11519 .entry(originals[i].seat.key.clone())
11520 .or_default();
11521 h.failed.insert(specs[i].id.clone());
11522 h.last_fail = Some(cur);
11523 }
11524 let parsed = match out {
11525 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11526 AgentOutcome::Quota(o) => {
11527 last_quota[i] = o.quota.clone();
11528 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11529 }
11530 AgentOutcome::Dropped(o) => {
11534 let why = o
11535 .dropped
11536 .as_ref()
11537 .map(|d| d.why.as_str())
11538 .unwrap_or("the CLI ended the stream without delivering its answer");
11539 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11540 }
11541 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11542 Err(anyhow::anyhow!(e))
11543 }
11544 };
11545 let quota = class == Some(FailClass::Quota);
11546 let failed = parsed.is_err();
11547 if !failed && matches!(ctx.node, "judge" | "vote") {
11548 state
11551 .quota
11552 .retain(|loss| loss.seat != originals[i].seat.key);
11553 }
11554 done[i] = Some(parsed);
11555 if carry && !failed {
11556 let h = state
11557 .seat_history
11558 .entry(originals[i].seat.key.clone())
11559 .or_default();
11560 h.failed.remove(&specs[i].id);
11561 h.last_ok = Some(specs[i].id.clone());
11562 h.last_fail = None;
11563 }
11564 let agent_failure = class.is_some()
11572 && !nudge_first
11573 && !roster.is_empty()
11574 && next_agent(i, &tried[i], &specs).is_some();
11575 if failed && !quota && !agent_failure && nudges[i] < retries {
11576 nudges[i] += 1;
11577 still.push(i);
11578 }
11579 }
11580 pending = still;
11581 }
11582
11583 for (i, q) in last_quota.into_iter().enumerate() {
11586 if let Some(quota) = q {
11587 losses.push(QuotaLoss::from_report(
11588 originals[i].seat.key.clone(),
11589 ctx.node,
11590 Some("a),
11591 ));
11592 }
11593 }
11594
11595 if state.status != RunStatus::Stalled
11598 && ctx.node != "deliberate"
11599 && n > 0
11600 && losses.len() == n
11601 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11602 {
11603 block_machine_outage(state, ctx.node, losses);
11604 state.quota.append(losses);
11605 }
11606
11607 seats
11608 .into_iter()
11609 .zip(done)
11610 .zip(nudges)
11611 .map(|((seat, res), attempts)| {
11612 (
11613 seat,
11614 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11615 attempts,
11616 )
11617 })
11618 .collect()
11619}
11620
11621pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11624 let mut auth: Vec<_> = losses
11625 .iter()
11626 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11627 .map(|loss| loss.agent.clone())
11628 .collect();
11629 auth.sort();
11630 auth.dedup();
11631 let (kind, detail) = if auth.is_empty() {
11632 (
11633 "agent usage limit",
11634 "All seats reached their agent usage limits; resume after usage is available"
11635 .to_owned(),
11636 )
11637 } else {
11638 (
11639 "agent authentication",
11640 format!(
11641 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11642 auth.join(", ")
11643 ),
11644 )
11645 };
11646 state.resource_block = Some(crate::run::ResourceBlock {
11647 node: node.to_owned(),
11648 kind: kind.to_owned(),
11649 cause: crate::run::ResourceBlockCause::Other,
11650 detail: detail.clone(),
11651 at: Timestamp::now(),
11652 });
11653 state.event(node, detail);
11654}
11655
11656fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11658 results
11659 .iter()
11660 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11661}
11662
11663fn block_wave(
11667 jobs: Vec<SeatJob>,
11668 state: &mut RunState,
11669 node: &str,
11670 attempt: usize,
11671 why: &str,
11672) -> Vec<(usize, SeatState, AgentOutcome)> {
11673 state.resource_wait = None;
11674 state
11675 .active
11676 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11677 if let Err(e) = state.save() {
11678 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11679 }
11680 jobs.into_iter()
11681 .enumerate()
11682 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11683 .collect()
11684}
11685
11686async fn acquire_cache_lease(
11699 state: &mut RunState,
11700 cache_dir: &Path,
11701 owner: &crate::cache::Owner,
11702 budget: Duration,
11703 node: &str,
11704 context: &str,
11705 cancel: &(dyn Fn() -> bool + Sync),
11706) -> Result<Option<crate::cache::Guard>> {
11707 let home = crate::run::home();
11708 let started = Instant::now();
11709 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11710 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11711 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11712 Err(e) => {
11713 state.event(
11714 "verify",
11715 format!("{context}: could not check the shared build cache: {e:#}"),
11716 );
11717 if let Err(e2) = state.save() {
11718 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11719 }
11720 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11721 }
11722 };
11723 state.event(
11724 "verify",
11725 format!(
11726 "{context}: waiting for the shared build cache at {} ({})",
11727 cache_dir.display(),
11728 busy.describe()
11729 ),
11730 );
11731 state.resource_wait = Some(crate::run::ResourceWait {
11734 node: context.to_owned(),
11735 kind: "shared build cache".to_owned(),
11736 holder: busy.describe(),
11737 since: Timestamp::now(),
11738 limit_secs: budget.as_secs(),
11739 });
11740 if let Err(e) = state.save() {
11741 tracing::warn!("could not persist a cache wait: {e:#}");
11742 }
11743 let remaining = budget.saturating_sub(started.elapsed());
11744 let waited = crate::cache::wait_for_unless(
11745 &home,
11746 cache_dir,
11747 owner,
11748 remaining,
11749 Duration::from_secs(5),
11750 cancel,
11751 )
11752 .await;
11753 state.resource_wait = None;
11754 let cause = match &waited {
11755 Ok(Some(_)) => None,
11756 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11757 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11758 Some(crate::run::ResourceBlockCause::Timeout {
11759 waited_secs: started.elapsed().as_secs(),
11760 limit_secs: budget.as_secs(),
11761 })
11762 }
11763 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11764 };
11765 if let Some(cause) = cause {
11766 let detail = match &waited {
11767 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11768 Err(error) => format!("{error:#}"),
11769 };
11770 state.resource_block = Some(crate::run::ResourceBlock {
11771 cause,
11772 node: node.to_owned(),
11773 kind: "shared build cache".into(),
11774 detail,
11775 at: Timestamp::now(),
11776 });
11777 state.event(
11778 "resource",
11779 state.resource_block.as_ref().unwrap().description(),
11780 );
11781 }
11782 if let Err(error) = state.save() {
11783 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11784 }
11785 waited
11786}
11787
11788#[derive(Debug)]
11793struct CacheCheckError(String);
11794
11795impl std::fmt::Display for CacheCheckError {
11796 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11797 f.write_str(&self.0)
11798 }
11799}
11800
11801impl std::error::Error for CacheCheckError {}
11802
11803fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11805 state
11806 .resource_block
11807 .get_or_insert_with(|| crate::run::ResourceBlock {
11808 cause: crate::run::ResourceBlockCause::Other,
11809 node: node.to_owned(),
11810 kind: "shared build cache".into(),
11811 detail: format!("{error:#}"),
11812 at: Timestamp::now(),
11813 });
11814 if let Err(error) = state.save() {
11815 tracing::warn!("could not persist cache resource block: {error:#}");
11816 }
11817}
11818
11819#[allow(clippy::too_many_arguments)]
11840async fn with_cache_lease<'s, F, Fut>(
11841 state: &'s mut RunState,
11842 cache_dir: Option<&Path>,
11843 node: &str,
11844 seat: &str,
11845 worktree: &Path,
11846 head: &str,
11847 budget: Duration,
11848 context: &str,
11849 wait: &crate::repo_lock::WaitPolicy,
11850 body: F,
11851) -> Result<(Vec<CommandOutcome>, bool)>
11852where
11853 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11854 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11855{
11856 let started = Instant::now();
11859 let home = crate::run::home();
11860 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11861 let guard = if let Some(cache_dir) = cache_dir {
11862 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11863 match acquire_cache_lease(
11864 state,
11865 cache_dir,
11866 &owner,
11867 resource_limit,
11868 node,
11869 context,
11870 &|| wait.cancelled(),
11871 )
11872 .await
11873 .and_then(|g| {
11874 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11875 }) {
11876 Ok(g) => Some(g),
11877 Err(e) => {
11878 record_cache_error(state, node, &e);
11879 return Ok((
11880 vec![CommandOutcome {
11881 command: "(waiting for the shared build cache)".into(),
11882 code: None,
11883 output_tail: e.to_string(),
11884 duration_ms: started.elapsed().as_millis() as u64,
11885 resource_blocked: true,
11886 }],
11887 false,
11888 ));
11889 }
11890 }
11891 } else {
11892 None
11893 };
11894 let _coordination = coordinate_operation(state, worktree, wait).await?;
11895 if let Some(cache_dir) = cache_dir {
11896 let identity = crate::cache::Identity::new(worktree, head);
11897 if let Err(e) = crate::cache::ensure_fresh(
11898 &home,
11899 cache_dir,
11900 &identity,
11901 &state.config.verify.cargo_manifests(),
11902 ) {
11903 record_cache_error(state, node, &e);
11904 state.event(
11905 "verify",
11906 format!(
11907 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11908 worktree.display(),
11909 short(head)
11910 ),
11911 );
11912 if let Some(guard) = guard {
11913 guard.release();
11914 }
11915 return Ok((
11916 vec![CommandOutcome {
11917 command: "(confirming the shared build cache is fresh)".into(),
11918 code: None,
11919 output_tail: e.to_string(),
11920 duration_ms: started.elapsed().as_millis() as u64,
11921 resource_blocked: true,
11922 }],
11923 false,
11924 ));
11925 }
11926 }
11927 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11928 if !timed_out_pids.is_empty() {
11929 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11930 }
11931 drop(_coordination);
11934 if let Some(guard) = guard {
11935 guard.release();
11936 }
11937 Ok((outcomes, retried))
11938}
11939
11940async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11952 wait_for_pids_with(
11953 pids,
11954 crate::proc::pid_alive,
11955 LEASE_RELEASE_POLL,
11956 LEASE_RELEASE_MAX_WAIT,
11957 )
11958 .await;
11959}
11960
11961async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11967 pids: &[u32],
11968 alive: F,
11969 poll: Duration,
11970 max_wait: Duration,
11971) {
11972 let deadline = Instant::now() + max_wait;
11973 loop {
11974 if pids.iter().all(|&pid| !alive(pid)) {
11975 return;
11976 }
11977 if Instant::now() >= deadline {
11978 return;
11979 }
11980 tokio::time::sleep(poll).await;
11981 }
11982}
11983
11984fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11991 outcomes.iter().any(|o| o.resource_blocked)
11992}
11993
11994pub(crate) enum GateFix {
11996 Retry,
11998 Stop,
12001 Defer,
12004}
12005
12006pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12014 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12015 red.peek().is_some()
12016 && red.all(|o| {
12017 !o.resource_blocked
12018 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12019 && !o.output_tail.trim().is_empty()
12020 })
12021}
12022
12023fn e2e_outcome_label(o: &CommandOutcome) -> String {
12027 if o.ok() {
12028 return "pass".to_owned();
12029 }
12030 let reason = if o.build_failed() {
12031 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12032 } else {
12033 format!("FAIL ({:?})", o.code)
12034 };
12035 format!(
12036 "{reason}\n{}",
12037 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12038 )
12039}
12040
12041async fn run_e2e_with_retry(
12049 state: &mut RunState,
12050 shell: &[String],
12051 commands: &[String],
12052 worktree: &Path,
12053 timeout: Duration,
12054 context: &str,
12055) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12056 let (mut e2e, mut timed_out_pids) = run_commands(
12057 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12058 )
12059 .await;
12060 for o in &e2e {
12061 state.event(
12062 "verify",
12063 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12064 );
12065 }
12066 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12070 if verify_retried {
12071 state.event(
12072 "verify",
12073 format!(
12074 "{context}: verify could not build/link, not a test result — retrying once \
12075 before concluding"
12076 ),
12077 );
12078 let retried = run_commands(
12079 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12080 )
12081 .await;
12082 e2e = retried.0;
12083 timed_out_pids.extend(retried.1);
12086 for o in &e2e {
12087 state.event(
12088 "verify",
12089 format!(
12090 "{context}: retry `{}` -> {}",
12091 o.command,
12092 e2e_outcome_label(o)
12093 ),
12094 );
12095 }
12096 }
12097 (e2e, verify_retried, timed_out_pids)
12098}
12099
12100#[allow(clippy::too_many_arguments)]
12116async fn run_commands(
12117 state: &mut RunState,
12118 node: &str,
12119 task: &str,
12120 attempt: usize,
12121 shell: &[String],
12122 commands: &[String],
12123 cwd: &Path,
12124 timeout: Duration,
12125) -> (Vec<CommandOutcome>, Vec<u32>) {
12126 if commands.is_empty() {
12127 return (Vec::new(), Vec::new());
12132 }
12133 let mut out = Vec::new();
12134 let mut timed_out_pids = Vec::new();
12135 let total = commands.len();
12136 for (idx, command) in commands.iter().enumerate() {
12137 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12138 if let Err(e) = state.save() {
12139 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12140 }
12141 let started = Instant::now();
12142 let mut cmd = tokio::process::Command::new(&shell[0]);
12143 cmd.quiet();
12144 for key in crate::worker::INVOCATION_ENV {
12148 cmd.env_remove(key);
12149 }
12150 cmd.args(&shell[1..])
12151 .arg(command)
12152 .current_dir(cwd)
12153 .stdin(std::process::Stdio::null())
12154 .stdout(std::process::Stdio::piped())
12155 .stderr(std::process::Stdio::piped())
12156 .kill_on_drop(true);
12157 let mut outputs = Vec::new();
12158 let protected = (|| -> Result<()> {
12159 crate::incremental::configure(&mut cmd, cwd)?;
12160 let mut paths = vec![cwd.to_owned()];
12161 if let Some(cache) = state.config.cache_dir() {
12162 paths.push(cache);
12163 }
12164 outputs.extend(crate::retention::OutputLease::admit(
12165 &crate::run::home(),
12166 &paths,
12167 state.config.disk.min_free_bytes,
12168 )?);
12169 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12170 outputs.push(guard);
12171 }
12172 if let Some(cache) = state.config.cache_dir() {
12173 outputs.push(crate::retention::OutputLease::cache(
12174 &crate::run::home(),
12175 &cache,
12176 false,
12177 )?);
12178 }
12179 Ok(())
12180 })();
12181 if let Err(e) = protected {
12182 let detail = format!("execution admission unavailable: {e:#}");
12183 state.resource_block = Some(crate::run::ResourceBlock {
12184 cause: crate::run::ResourceBlockCause::Other,
12185 node: node.to_owned(),
12186 kind: if detail.contains("disk admission") {
12187 "disk admission"
12188 } else {
12189 "output ownership"
12190 }
12191 .to_owned(),
12192 detail: detail.clone(),
12193 at: Timestamp::now(),
12194 });
12195 out.push(CommandOutcome {
12196 command: command.clone(),
12197 code: None,
12198 output_tail: detail,
12199 duration_ms: 0,
12200 resource_blocked: true,
12201 });
12202 break;
12203 }
12204 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12205 let (code, body) = match spawned {
12206 Ok(mut running) => {
12207 let pid = running.child().id();
12208 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12209 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12210 let mut paths = vec![cwd.to_owned()];
12211 if let Some(cache) = state.config.cache_dir() {
12212 paths.push(cache);
12213 }
12214 let mut pressure = None;
12215 let status = tokio::select! {
12216 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12217 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12218 pressure = Some(reason); None
12219 }
12220 };
12221 let (code, failure) = match status {
12222 Some(Ok(Ok(status))) => (
12223 status.code(),
12224 status
12225 .code()
12226 .is_none()
12227 .then(|| format!("command terminated: {status}")),
12228 ),
12229 Some(Ok(Err(error))) => {
12230 (None, Some(format!("failed waiting for command: {error}")))
12231 }
12232 Some(Err(_)) => {
12233 if !running.coordinated()
12237 && let Some(pid) = pid
12238 {
12239 timed_out_pids.push(pid);
12240 }
12241 (
12242 None,
12243 Some(format!("timed out after {}s", timeout.as_secs())),
12244 )
12245 }
12246 None => (None, None),
12247 };
12248 running.finish().await;
12249 if let Some(reason) = pressure {
12250 state.resource_block = Some(crate::run::ResourceBlock {
12251 cause: crate::run::ResourceBlockCause::Other,
12252 node: node.to_owned(),
12253 kind: "disk pressure".to_owned(),
12254 detail: reason.clone(),
12255 at: Timestamp::now(),
12256 });
12257 out.push(CommandOutcome {
12258 command: command.clone(),
12259 code: None,
12260 output_tail: reason,
12261 duration_ms: started.elapsed().as_millis() as u64,
12262 resource_blocked: true,
12263 });
12264 break;
12265 }
12266 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12267 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12268 if let Some(failure) = failure {
12269 body.push('\n');
12270 body.push_str(&failure);
12271 }
12272 (code, body)
12273 }
12274 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12275 };
12276 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12277 if !body.is_empty() {
12278 match crate::run::write_verification_artifact(state, task, &body) {
12282 Ok(name) if summary != body => {
12283 summary = format!("[full output: artifacts/{name}]\n{summary}");
12284 }
12285 Ok(_) => {}
12286 Err(e) => {
12287 let warning = format!("could not save the full {task} output: {e:#}");
12288 tracing::warn!("{warning}");
12289 state.event(node, warning);
12290 if summary != body {
12291 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12292 }
12293 }
12294 }
12295 }
12296 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12297 out.push(CommandOutcome {
12298 command: command.clone(),
12299 code,
12300 output_tail: summary,
12301 duration_ms: started.elapsed().as_millis() as u64,
12302 resource_blocked,
12303 });
12304 if resource_blocked {
12305 state.resource_block = Some(crate::run::ResourceBlock {
12306 cause: crate::run::ResourceBlockCause::Other,
12307 node: node.to_owned(),
12308 kind: "incremental cache admission".into(),
12309 detail: body,
12310 at: Timestamp::now(),
12311 });
12312 break;
12313 }
12314 }
12315 state.task_finished(task);
12316 if let Err(e) = state.save() {
12317 tracing::warn!("could not persist the end of {task}: {e:#}");
12318 }
12319 (out, timed_out_pids)
12320}
12321
12322fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12334 let repo = repo.display();
12335 match style {
12336 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12337 MergeStyle::Squash => {
12338 let subject = message
12341 .lines()
12342 .next()
12343 .unwrap_or(branch)
12344 .replace(['\\', '"', '$', '`'], "");
12345 format!(
12346 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12347 )
12348 }
12349 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12350 }
12351}
12352
12353const PR_TITLE_MAX: usize = 240;
12364
12365fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12376 let line = queue::first_line(text)?;
12377 let chars: Vec<char> = line.chars().collect();
12378 let (kept, mark) = if chars.len() <= max {
12379 (line.to_owned(), "")
12380 } else if let Some(end) = sentence_end(&chars, max) {
12381 (chars[..end].iter().collect::<String>(), "")
12382 } else {
12383 let room = max.saturating_sub(3);
12384 let cut = if chars[room].is_whitespace() {
12387 room
12388 } else {
12389 chars[..room]
12390 .iter()
12391 .rposition(|c| c.is_whitespace())
12392 .unwrap_or(room)
12393 };
12394 let head: String = chars[..cut].iter().collect();
12395 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12396 (head.to_owned(), "...")
12397 };
12398 if kept.is_empty() {
12399 return None;
12400 }
12401 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12402 return None;
12403 }
12404 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12405 return None;
12406 }
12407 Some(format!("{kept}{mark}"))
12408}
12409
12410fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12414 const MIN: usize = 20;
12415 for i in MIN..max.min(chars.len()) {
12416 if !matches!(chars[i], '.' | '!' | '?') {
12417 continue;
12418 }
12419 let Some(&next) = chars.get(i + 1) else {
12420 continue;
12421 };
12422 if !next.is_whitespace() {
12423 continue;
12424 }
12425 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12426 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12427 continue;
12428 }
12429 let word: String = chars[..i]
12430 .iter()
12431 .rev()
12432 .take_while(|c| !c.is_whitespace())
12433 .collect::<Vec<_>>()
12434 .into_iter()
12435 .rev()
12436 .collect();
12437 let word = word.to_ascii_lowercase();
12438 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12439 continue;
12440 }
12441 let end = chars[..i]
12442 .iter()
12443 .rposition(|c| !c.is_whitespace())
12444 .map_or(i, |p| p + 1);
12445 return Some(end);
12446 }
12447 None
12448}
12449
12450struct PrMessage {
12455 title: String,
12456 body: String,
12457}
12458
12459impl PrMessage {
12460 fn commit_message(&self) -> String {
12464 format!("{}\n\n{}", self.title, self.body)
12465 }
12466}
12467
12468fn title_marker(line: &str) -> Option<&str> {
12470 let line = line.trim();
12471 let head = line.get(..6)?;
12472 head.eq_ignore_ascii_case("title:")
12473 .then(|| line[6..].trim())
12474}
12475
12476fn summary_title(summary: &str) -> Option<String> {
12481 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12482 let raw = title_marker(first)?;
12483 if raw.is_empty() {
12484 return None;
12485 }
12486 let title = queue::title_from(raw, PR_TITLE_MAX);
12487 let lower = title.to_ascii_lowercase();
12488 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12489 return None;
12490 }
12491 Some(title)
12492}
12493
12494const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12496
12497pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12499
12500pub(crate) fn is_review_run(state: &RunState) -> bool {
12504 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12505}
12506
12507fn review_title(state: &RunState) -> Option<String> {
12512 english_subject(
12513 state.reviewed_commits.as_ref()?.first()?,
12514 state.config.graph.github_english(),
12515 )
12516}
12517
12518fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12521 let raw = raw.trim();
12522 if raw.is_empty() {
12523 return None;
12524 }
12525 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12526 return None;
12527 }
12528 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12529 return None;
12530 }
12531 let title = queue::title_from(raw, PR_TITLE_MAX);
12532 let lower = title.to_ascii_lowercase();
12533 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12534 return None;
12535 }
12536 Some(title)
12537}
12538
12539#[derive(Debug, Clone, PartialEq, Eq)]
12542struct BranchFacts {
12543 commits: Vec<(String, String)>,
12545 stat: String,
12547}
12548
12549const STAT_FILE_LINES: usize = 25;
12551const COMMITS_MAX_CHARS: usize = 20_000;
12553
12554async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12557 let commits = git::commit_log(repo, base, branch).await.ok()?;
12558 if commits.is_empty() {
12559 return None;
12560 }
12561 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12562 let lines: Vec<&str> = stat.lines().collect();
12563 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12564 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12565 let more = format!(" ... {omitted} more file(s)");
12566 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12567 kept.push(&more);
12568 kept.push(lines[lines.len() - 1]);
12569 kept.join("\n")
12570 } else {
12571 lines.join("\n")
12572 };
12573 Some(BranchFacts { commits, stat })
12574}
12575
12576fn markdown_safe(text: &str) -> String {
12579 text.replace("</details>", "</details>")
12580 .replace("\x60\x60\x60", "~~~")
12581}
12582
12583fn neutral_title(state: &RunState, winner: char) -> String {
12584 format!(
12585 "chore: land candidate {} of run {}",
12586 winner.to_ascii_uppercase(),
12587 state.id
12588 )
12589}
12590
12591pub(crate) async fn repair_landing_gate(
12594 state: &mut RunState,
12595 winner: &Candidate,
12596 outcomes: &[CommandOutcome],
12597) -> Result<GateFix> {
12598 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12599 return Ok(GateFix::Stop);
12600 }
12601 let mut runner = Runner {
12602 roles: state.config.resolve_roles()?,
12603 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12604 state: state.clone(),
12605 pause: Pause::new(),
12606 interrupt: Pause::new(),
12607 };
12608 let result = runner.gate_fix_round(winner, outcomes).await;
12609 *state = runner.state;
12610 result
12611}
12612
12613pub(crate) async fn verify_current_head(
12616 state: &mut RunState,
12617 winner: &Candidate,
12618 wait: &crate::repo_lock::WaitPolicy,
12619) -> Result<bool> {
12620 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12621 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12623 state
12624 .verification_history
12625 .push(crate::run::VerificationEvidence {
12626 superseded_by: Some(head.clone()),
12627 archived_at: Some(Timestamp::now()),
12628 fixes_spent: Some(state.gate_fixes.len()),
12629 gate_head: None,
12630 gate_ran: false,
12631 gate: Vec::new(),
12632 e2e: state.head_e2e.take(),
12633 });
12634 }
12635 if let Some(e) = &state.head_e2e
12636 && e.head == head
12637 {
12638 if e.passed {
12639 return Ok(true);
12640 }
12641 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12642 state.status = RunStatus::Blocked;
12643 state.save()?;
12644 return Ok(false);
12645 }
12646 } else if state.reviews.last().is_some_and(|r| {
12647 r.verified_head.as_deref() == Some(head.as_str())
12648 && (r.e2e_status() == E2eStatus::Passed
12649 || (r.e2e_status() == E2eStatus::NotConfigured
12650 && state.config.verify.e2e.is_empty()))
12651 }) || state
12652 .gate_repair_e2e
12653 .as_ref()
12654 .is_some_and(|e| e.head == head && e.passed)
12655 {
12656 return Ok(true);
12657 }
12658 if let Some(e) = &state.gate_repair_e2e
12659 && e.head == head
12660 && !e.passed
12661 && !e.outcomes.is_empty()
12662 && !verify_inconclusive(&e.outcomes)
12663 {
12664 state.status = RunStatus::Blocked;
12665 state.save()?;
12666 return Ok(false);
12667 }
12668 state.head_e2e = Some(crate::run::GateRepairE2e {
12669 head: head.clone(),
12670 passed: false,
12671 outcomes: Vec::new(),
12672 });
12673 state.save()?;
12674 let commands = state.config.verify.e2e.clone();
12675 let shell = state.config.shell();
12676 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12677 let cache_dir = state.config.cache_dir();
12678 let outcomes = if commands.is_empty() {
12679 Vec::new()
12680 } else {
12681 let (outcomes, _) = with_cache_lease(
12682 state,
12683 cache_dir.as_deref(),
12684 "e2e",
12685 "e2e",
12686 &winner.worktree,
12687 &head,
12688 timeout,
12689 "current head e2e",
12690 wait,
12691 |state, budget| async move {
12692 run_e2e_with_retry(
12693 state,
12694 &shell,
12695 &commands,
12696 &winner.worktree,
12697 budget,
12698 "current head e2e",
12699 )
12700 .await
12701 },
12702 )
12703 .await?;
12704 outcomes
12705 };
12706 let unresolved = verify_inconclusive(&outcomes);
12707 let passed = !unresolved
12708 && outcomes.iter().all(CommandOutcome::ok)
12709 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12710 if !outcomes.is_empty() && !unresolved {
12711 state.record_base_progress(&head);
12712 }
12713 state.head_e2e = Some(crate::run::GateRepairE2e {
12714 head: head.clone(),
12715 passed,
12716 outcomes,
12717 });
12718 if !passed && !unresolved {
12719 state.status = RunStatus::Blocked;
12720 state.event(
12721 "e2e",
12722 format!("verify.e2e failed on head {head}; nothing published"),
12723 );
12724 }
12725 state.save()?;
12726 Ok(passed)
12727}
12728
12729pub(crate) async fn check_gate_repair_e2e(
12732 state: &mut RunState,
12733 winner: &Candidate,
12734 wait: &crate::repo_lock::WaitPolicy,
12735) -> Result<GateFix> {
12736 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12737 return Ok(GateFix::Retry);
12738 }
12739 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12740 if state
12741 .gate_repair_e2e
12742 .as_ref()
12743 .is_some_and(|r| r.head == head && r.passed)
12744 {
12745 return Ok(GateFix::Retry);
12746 }
12747 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12748 head: head.clone(),
12749 passed: false,
12750 outcomes: Vec::new(),
12751 });
12752 state.save()?;
12753 let commands = state.config.verify.e2e.clone();
12754 let outcomes = if commands.is_empty() {
12755 Vec::new()
12756 } else {
12757 let shell = state.config.shell();
12758 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12759 let cache_dir = state.config.cache_dir();
12760 let (outcomes, _) = with_cache_lease(
12761 state,
12762 cache_dir.as_deref(),
12763 "e2e",
12764 "e2e",
12765 &winner.worktree,
12766 &head,
12767 timeout,
12768 "gate repair e2e",
12769 wait,
12770 |state, budget| async move {
12771 run_e2e_with_retry(
12772 state,
12773 &shell,
12774 &commands,
12775 &winner.worktree,
12776 budget,
12777 "gate repair e2e",
12778 )
12779 .await
12780 },
12781 )
12782 .await?;
12783 outcomes
12784 };
12785 let deferred = verify_inconclusive(&outcomes);
12786 let passed = outcomes.iter().all(CommandOutcome::ok)
12787 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12788 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12789 head,
12790 passed,
12791 outcomes,
12792 });
12793 state.save()?;
12794 if deferred {
12795 Ok(GateFix::Defer)
12796 } else if !passed {
12797 state.event(
12798 "gate",
12799 "verify.e2e failed after gate repair; nothing published",
12800 );
12801 Ok(GateFix::Stop)
12802 } else {
12803 Ok(GateFix::Retry)
12804 }
12805}
12806
12807pub(crate) async fn run_landing_gate(
12809 state: &mut RunState,
12810 winner: &Candidate,
12811 wait: &crate::repo_lock::WaitPolicy,
12812) -> Result<Vec<CommandOutcome>> {
12813 let shell = state.config.shell();
12814 let gate_commands = state.config.verify.gate.clone();
12815 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12816 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12817 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12818 Vec::new()
12819 } else {
12820 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12821 let cache_dir = state.config.cache_dir();
12822 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12823 let (outcomes, _) = with_cache_lease(
12824 state,
12825 cache_dir.as_deref(),
12826 "gate",
12827 "gate",
12828 &winner.worktree,
12829 &head,
12830 timeout,
12831 "final gate",
12832 wait,
12833 |state, budget| {
12834 let shell = shell.clone();
12835 let gate_commands = gate_commands.clone();
12836 let winner = winner.clone();
12837 async move {
12838 Runner::run_pre_gate(state, &winner).await;
12839 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12841 let (outcomes, timed_out_pids) = run_commands(
12842 state,
12843 "gate",
12844 "gate",
12845 0,
12846 &shell,
12847 &gate_commands,
12848 &winner.worktree,
12849 budget,
12850 )
12851 .await;
12852 (outcomes, false, timed_out_pids)
12853 }
12854 },
12855 )
12856 .await?;
12857 outcomes
12858 };
12859 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12860 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12861 let mut outcomes = outcomes;
12862 outcomes.push(CommandOutcome {
12863 command: "gate head stability".into(), code: None,
12864 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12865 duration_ms: 0, resource_blocked: false,
12866 });
12867 return Ok(outcomes);
12868 }
12869 if outcomes.is_empty() {
12870 state.event("gate", "no gate commands configured; not tested");
12875 }
12876 for o in &outcomes {
12877 state.event(
12878 "gate",
12879 format!(
12880 "`{}` -> {}",
12881 o.command,
12882 if o.ok() {
12883 "pass".to_owned()
12884 } else {
12885 format!(
12886 "FAIL ({:?})\n{}",
12887 o.code,
12888 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12889 )
12890 }
12891 ),
12892 );
12893 }
12894 Ok(outcomes)
12895}
12896
12897pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12903 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12904 ""
12905 } else {
12906 pr_title
12907 }
12908}
12909
12910pub fn landing_subject_source(state: &RunState) -> String {
12914 if is_review_run(state) {
12915 let winner = state.candidates.first().map_or('A', |c| c.label);
12916 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12917 }
12918 state.instruction.clone()
12919}
12920
12921fn summary_without_title(summary: &str) -> String {
12924 let mut lines = summary.trim().lines().peekable();
12925 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12926 lines.next();
12927 }
12928 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12929}
12930
12931#[cfg(test)]
12943fn pr_message(state: &RunState, winner: char) -> PrMessage {
12944 pr_message_with(state, winner, None)
12945}
12946
12947#[cfg(test)]
12950fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12951 let raw = pr_message_raw(
12952 state,
12953 winner,
12954 facts,
12955 crate::pr_outcome::FinalHead::Unreported,
12956 ChangeNote::NotAsked,
12957 );
12958 let id = crate::scrub::Identity::current();
12959 PrMessage {
12960 title: crate::scrub::scrub(&raw.title, &id),
12961 body: crate::scrub::scrub(&raw.body, &id),
12962 }
12963}
12964
12965#[derive(Debug, Clone, Copy)]
12967enum ChangeNote<'a> {
12968 NotAsked,
12970 Summary(&'a crate::run::ChangeSummary),
12972 Unavailable(&'a str),
12974}
12975
12976fn pr_message_raw(
12977 state: &RunState,
12978 winner: char,
12979 facts: Option<&BranchFacts>,
12980 head: crate::pr_outcome::FinalHead<'_>,
12981 change: ChangeNote<'_>,
12982) -> PrMessage {
12983 if !matches!(change, ChangeNote::NotAsked) {
12984 let (title, mut body) = match change {
12985 ChangeNote::Summary(c) => (
12986 c.title.clone(),
12987 format!("## Summary\n\n{}\n\n", c.body.trim()),
12988 ),
12989 ChangeNote::Unavailable(why) => (
12990 neutral_title(state, winner),
12991 format!(
12992 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12993 ),
12994 ),
12995 ChangeNote::NotAsked => unreachable!(),
12996 };
12997 body.push_str(&public_pr_evidence(state, head));
12998 body.push_str(&format!(
12999 "\n---\nmagi:run/{} magi:candidate-{}\n",
13000 state.id,
13001 winner.to_ascii_lowercase()
13002 ));
13003 return PrMessage { title, body };
13004 }
13005 pr_preview(state, winner, facts, head)
13006}
13007
13008fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13011 let mut body = crate::pr_outcome::review_rounds(state);
13012 body.push_str(&crate::pr_outcome::final_verification(state, head));
13013 let open = state.last_round_findings();
13014 if !open.is_empty() {
13015 body.push_str("## Open review findings\n\n");
13016 for f in open {
13017 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13018 }
13019 body.push('\n');
13020 }
13021 if let Some(c) = state.change_summary.as_ref()
13022 && head == crate::pr_outcome::FinalHead::At(&c.head)
13023 {
13024 body.push_str(&format!(
13025 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13026 c.base, c.head
13027 ));
13028 }
13029 body
13030}
13031
13032fn pr_preview(
13034 state: &RunState,
13035 winner: char,
13036 facts: Option<&BranchFacts>,
13037 head: crate::pr_outcome::FinalHead<'_>,
13038) -> PrMessage {
13039 let summary = state
13040 .candidates
13041 .iter()
13042 .find(|c| c.label == winner)
13043 .map(|c| c.summary.as_str())
13044 .unwrap_or_default();
13045 let review = is_review_run(state);
13048 let title = if review {
13049 facts
13050 .and_then(|f| {
13051 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13052 })
13053 .or_else(|| review_title(state))
13054 .or_else(|| {
13055 state
13056 .candidates
13057 .iter()
13058 .find(|c| c.label == winner)
13059 .filter(|c| !c.branch.starts_with("magi/"))
13060 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13061 })
13062 .unwrap_or_else(|| neutral_title(state, winner))
13063 } else {
13064 summary_title(summary).unwrap_or_else(|| {
13065 english_title(
13066 &state.instruction,
13067 PR_TITLE_MAX,
13068 state.config.graph.github_english(),
13069 )
13070 .unwrap_or_else(|| neutral_title(state, winner))
13071 })
13072 };
13073
13074 let mut body = String::new();
13075 let what = summary_without_title(summary);
13076 if !what.is_empty() {
13077 body.push_str("## Summary\n\n");
13078 body.push_str(&what);
13079 body.push_str("\n\n");
13080 }
13081
13082 body.push_str(&crate::pr_outcome::review_rounds(state));
13084 body.push_str(&crate::pr_outcome::final_verification(state, head));
13085 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13086
13087 let open = state.open_findings();
13088 if !open.is_empty() {
13089 body.push_str("## Open review findings\n\n");
13090 for f in &open {
13091 body.push_str(&format!(
13092 "- `{}` [{:?}] {}\n",
13093 f.id, f.severity, "Context retained in local review records."
13094 ));
13095 }
13096 body.push('\n');
13097 }
13098
13099 if let Some(fix) = fix
13100 && !fix.rejected.is_empty()
13101 {
13102 body.push_str("## Declined by the fixer\n\n");
13103 for r in &fix.rejected {
13104 body.push_str(&format!(
13105 "- `{}`: {}\n",
13106 r.id, "Reason retained in local review records."
13107 ));
13108 }
13109 body.push('\n');
13110 }
13111
13112 if review {
13113 body.push_str("## Commits under review\n\n");
13115 if let Some(facts) = facts {
13116 let mut left = COMMITS_MAX_CHARS;
13117 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13118 let mut entry = format!("- {}\n", markdown_safe(subject));
13119 for l in markdown_safe(text).lines() {
13120 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13121 }
13122 if left == 0 {
13123 body.push_str(&format!(
13124 "- ... {} more commit(s)\n",
13125 facts.commits.len() - i
13126 ));
13127 break;
13128 }
13129 if entry.len() > left {
13130 let mut end = left;
13133 while !entry.is_char_boundary(end) {
13134 end -= 1;
13135 }
13136 entry.truncate(end);
13137 entry.push_str("\n ... (truncated)\n");
13138 left = 0;
13139 } else {
13140 left -= entry.len();
13141 }
13142 body.push_str(&entry);
13143 }
13144 if !facts.stat.trim().is_empty() {
13145 body.push_str(&format!(
13146 "\n## Diff stat\n\n```\n{}\n```\n",
13147 markdown_safe(facts.stat.trim())
13148 ));
13149 }
13150 } else {
13151 match &state.reviewed_commits {
13152 Some(subjects) => {
13153 for s in subjects {
13154 body.push_str(&format!("- {}\n", s.trim()));
13155 }
13156 }
13157 None => {
13158 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13161 body.push_str(rest.trim());
13162 body.push('\n');
13163 }
13164 }
13165 }
13166 } else {
13167 let task = state.instruction.trim();
13168 let task = if task.is_empty() {
13169 "(empty task)"
13170 } else {
13171 task
13172 };
13173 body.push_str(&format!(
13174 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13175 task.replace("</details>", "</details>")
13176 ));
13177 }
13178
13179 body.push_str(&format!(
13180 "\n---\nmagi:run/{} magi:candidate-{}\n",
13181 state.id,
13182 winner.to_ascii_lowercase()
13183 ));
13184
13185 PrMessage { title, body }
13186}
13187
13188fn seeded_instruction(state: &RunState) -> String {
13191 let mut inputs = Vec::new();
13192 if let Some(start) = &state.sources.start_from {
13193 inputs.push(format!(
13194 "Start from `{}` at `{}` (including ancestry).",
13195 start.token, start.sha
13196 ));
13197 }
13198 for pick in &state.sources.pick_commits {
13199 inputs.push(format!(
13200 "Pick `{}` at `{}` (single commit).",
13201 pick.token, pick.sha
13202 ));
13203 }
13204 if inputs.is_empty() {
13205 state.instruction.clone()
13206 } else {
13207 format!(
13208 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13209 state.instruction,
13210 inputs.join("\n")
13211 )
13212 }
13213}
13214
13215async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13219 let base = &state.base_branch;
13220 let mut against = base.clone();
13221 if mode == MergeMode::Pr {
13222 let remote = &state.config.merge.remote;
13226 let tracking = format!("{remote}/{base}");
13227 let fetched = git::fetch(repo, remote, base).await;
13228 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13229 against = tracking;
13230 } else {
13231 return false;
13232 }
13233 }
13234 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13235}
13236
13237fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13240 let mut detail = format!(
13241 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13242 nothing to open a pull request for"
13243 );
13244 detail.push_str(
13245 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13246 );
13247 detail
13248}
13249
13250#[derive(Debug, PartialEq, Eq)]
13253enum PrPlan {
13254 Create,
13255 Adopt { url: String, title: String },
13256 Stop(String),
13257}
13258
13259fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13262 match found {
13263 Ok(land::OpenPr::None) => PrPlan::Create,
13264 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13265 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13266 "several open pull requests exist for this branch, not picking one: {}",
13267 urls.join(" ")
13268 )),
13269 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13270 }
13271}
13272
13273pub(crate) async fn gh_pr_create(
13275 cwd: &Path,
13276 base: &str,
13277 head: &str,
13278 title: &str,
13279 body: &str,
13280) -> Result<String> {
13281 let out = tokio::process::Command::new("gh")
13282 .args([
13283 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13284 ])
13285 .current_dir(cwd)
13286 .quiet()
13287 .stdin(std::process::Stdio::null())
13288 .output()
13289 .await
13290 .context("spawn gh")?;
13291 if out.status.success() {
13292 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13293 } else {
13294 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13295 }
13296}
13297
13298async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13300 match std::fs::symlink_metadata(path) {
13301 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13302 git::worktree_remove_clean(repo, path).await?;
13305 return Ok(());
13306 }
13307 Err(e) => return Err(e.into()),
13308 Ok(_) => {}
13309 }
13310 if !git::is_clean(path).await? {
13311 bail!(
13312 "worktree has uncommitted work; refusing removal: {}",
13313 path.display()
13314 );
13315 }
13316 if !git::worktree_remove_clean(repo, path).await? {
13317 bail!("worktree removal refused: {}", path.display());
13318 }
13319 Ok(())
13320}
13321
13322pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13331 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13332 bail!("refusing fold: {reason}");
13333 }
13334 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13335 bail!("candidate preservation incomplete; refusing cleanup");
13336 }
13337 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13341 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13342 let _driver = crate::driver::maintenance(home, &state.id)?;
13343 let mut output_guards = Vec::new();
13344 for entry in std::fs::read_dir(state.worktree_root())
13345 .into_iter()
13346 .flatten()
13347 .flatten()
13348 {
13349 if entry.path().join(".git").is_file() {
13350 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13351 }
13352 }
13353 let repo = state.repo.clone();
13354 let wait = repository_wait(state, None);
13357 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13358 Ok(lease) => lease,
13359 Err(e) => {
13360 state.save_under(home)?;
13361 return Err(e);
13362 }
13363 };
13364 let _source_outputs = if state.status == RunStatus::Merged
13367 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13368 {
13369 verify_merged_fold_sources(state, drop_winner, home).await?
13370 } else {
13371 Vec::new()
13372 };
13373 let root = state.worktree_root();
13374 let winner = state.tally.as_ref().map(|t| t.winner);
13375 let mut removed = Vec::new();
13376
13377 for i in 0..state.candidates.len() {
13378 let c = state.candidates[i].clone();
13379 let is_winner = Some(c.label) == winner;
13380 if is_winner && !drop_winner {
13381 continue;
13382 }
13383 let existed = c.worktree.exists();
13384 remove_preserved_worktree(&repo, &c.worktree).await?;
13387 if existed {
13388 removed.push(c.worktree.to_string_lossy().into_owned());
13389 }
13390 let handed_over = state.released_branches.contains(&c.branch);
13393 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13394 if !git::branch_delete(&repo, &c.branch).await? {
13395 bail!("candidate branch removal refused");
13396 }
13397 removed.push(c.branch.clone());
13398 }
13399 state.candidates[i].folded = true;
13400 }
13401
13402 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13403 let path = name.path();
13404 let keep = !drop_winner
13405 && winner.is_some_and(|w| {
13406 path.file_name()
13407 .is_some_and(|n| n == format!("cand-{w}").as_str())
13408 });
13409 if keep {
13410 continue;
13411 }
13412 remove_preserved_worktree(&repo, &path).await?;
13413 removed.push(path.to_string_lossy().into_owned());
13414 }
13415
13416 remove_if_empty(&root);
13425
13426 if state.enabled_worktree_config && drop_winner {
13427 git::release_worktree_config(&repo).await.ok();
13431 state.enabled_worktree_config = false;
13432 }
13433 state.save_under(home)?;
13434 Ok(removed)
13435}
13436
13437async fn verify_merged_fold_sources(
13438 state: &RunState,
13439 drop_winner: bool,
13440 home: &Path,
13441) -> Result<Vec<std::fs::File>> {
13442 let winner = state.tally.as_ref().map(|t| t.winner);
13443 let mut heads = BTreeSet::new();
13444 for candidate in &state.candidates {
13445 if Some(candidate.label) == winner && !drop_winner {
13446 continue;
13447 }
13448 anyhow::ensure!(
13449 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13450 "candidate recovery source is outside the inventoried run root: {}",
13451 candidate.worktree.display()
13452 );
13453 match std::fs::symlink_metadata(&candidate.worktree) {
13454 Ok(_) => {}
13455 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13456 Err(error) => return Err(error.into()),
13457 }
13458 if !state.released_branches.contains(&candidate.branch)
13459 && git::branch_exists(&state.repo, &candidate.branch).await?
13460 {
13461 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13462 }
13463 }
13464 let mut cargo_guards = Vec::new();
13465 let entries = match std::fs::read_dir(state.worktree_root()) {
13466 Ok(entries) => Some(entries),
13467 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13468 Err(error) => return Err(error.into()),
13469 };
13470 for entry in entries.into_iter().flatten() {
13471 let path = entry?.path();
13472 let metadata = std::fs::symlink_metadata(&path)?;
13473 anyhow::ensure!(
13474 metadata.is_dir() && !metadata.file_type().is_symlink(),
13475 "non-directory or symlink recovery source retained: {}",
13476 path.display()
13477 );
13478 if !drop_winner
13479 && winner.is_some_and(|w| {
13480 path.file_name()
13481 .is_some_and(|name| name == format!("cand-{w}").as_str())
13482 })
13483 {
13484 continue;
13485 }
13486 anyhow::ensure!(
13487 path.join(".git").is_file(),
13488 "unregistered recovery source retained: {}",
13489 path.display()
13490 );
13491 anyhow::ensure!(
13492 git::git(
13493 &path,
13494 &[
13495 "status",
13496 "--porcelain",
13497 "--untracked-files=all",
13498 "--ignored",
13499 "--",
13500 ".",
13501 ":(exclude)target"
13502 ]
13503 )
13504 .await?
13505 .is_empty(),
13506 "source changes or ignored recovery files retained: {}",
13507 path.display()
13508 );
13509 match std::fs::symlink_metadata(path.join("target")) {
13510 Ok(_) => {
13511 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13512 }
13513 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13514 Err(error) => return Err(error.into()),
13515 }
13516 for checkpoint in [
13517 "rebase-merge",
13518 "rebase-apply",
13519 "sequencer",
13520 "CHERRY_PICK_HEAD",
13521 "MERGE_HEAD",
13522 "REBASE_HEAD",
13523 "REVERT_HEAD",
13524 "BISECT_LOG",
13525 "AUTO_MERGE",
13526 "MERGE_MSG",
13527 ] {
13528 let location = git::git(
13529 &path,
13530 &[
13531 "rev-parse",
13532 "--path-format=absolute",
13533 "--git-path",
13534 checkpoint,
13535 ],
13536 )
13537 .await?;
13538 anyhow::ensure!(
13539 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13540 if e.kind() == std::io::ErrorKind::NotFound),
13541 "Git recovery checkpoint retained: {checkpoint}"
13542 );
13543 }
13544 heads.insert(git::rev_parse(&path, "HEAD").await?);
13545 }
13546 crate::fold_evidence::preserve(state, home, &heads).await?;
13547
13548 Ok(cargo_guards)
13549}
13550
13551fn remove_if_empty(dir: &Path) {
13562 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13563 std::fs::remove_dir(dir).ok();
13564 }
13565}
13566
13567pub fn worst_open(state: &RunState) -> Option<Severity> {
13569 state
13570 .reviews
13571 .last()?
13572 .reviews
13573 .iter()
13574 .flat_map(|r| r.findings.iter())
13575 .map(|f| f.severity)
13576 .max()
13577}
13578
13579async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13583 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13584 git::acquire_worktree_config(repo).await?;
13585 state.enabled_worktree_config = true;
13586 }
13587 Ok(())
13588}
13589
13590#[cfg(test)]
13591mod tests {
13592 #[tokio::test]
13593 async fn launcher_base_fetch_requires_remote_success() {
13594 let tmp = tempfile::tempdir().unwrap();
13595 let repo = tmp.path().join("repo");
13596 let remote = tmp.path().join("remote");
13597 std::fs::create_dir(&repo).unwrap();
13598 std::fs::create_dir(&remote).unwrap();
13599 git::git(&remote, &["init", "--bare", "-b", "main"])
13600 .await
13601 .unwrap();
13602 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13603 git::git(
13604 &repo,
13605 &[
13606 "-c",
13607 "user.name=Fixture",
13608 "-c",
13609 "user.email=fixture@example.invalid",
13610 "commit",
13611 "--allow-empty",
13612 "-m",
13613 "fixture",
13614 ],
13615 )
13616 .await
13617 .unwrap();
13618 git::git(
13619 &repo,
13620 &["remote", "add", "origin", remote.to_str().unwrap()],
13621 )
13622 .await
13623 .unwrap();
13624 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13625 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13626 assert_eq!(
13627 resolve_base(&repo, "main", "origin", None)
13628 .await
13629 .unwrap()
13630 .commit,
13631 head
13632 );
13633 #[cfg(unix)]
13634 {
13635 let policy: crate::config::Coordination = toml::from_str(
13636 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13637 ).unwrap();
13638 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13639 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13640 let held = crate::repo_lock::SeatLease::operation_with(
13641 &repo,
13642 Some(&policy),
13643 &crate::repo_lock::WaitPolicy::default(),
13644 )
13645 .await
13646 .unwrap();
13647 assert!(held.is_some());
13648 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13649 tokio::pin!(fetch);
13650 assert!(
13651 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13652 .await
13653 .is_err()
13654 );
13655 assert!(
13656 !repo.join(".git/FETCH_HEAD").exists(),
13657 "no fetch before admission"
13658 );
13659 drop(held);
13660 let fetched = fetch.await.unwrap();
13661 assert_eq!(fetched.commit, head);
13662 assert_eq!(fetched.remote, "origin");
13663 assert_eq!(fetched.branch, "main");
13664 }
13665 git::git(
13667 &repo,
13668 &[
13669 "remote",
13670 "set-url",
13671 "origin",
13672 tmp.path().join("missing").to_str().unwrap(),
13673 ],
13674 )
13675 .await
13676 .unwrap();
13677 let error = resolve_base(&repo, "main", "origin", None)
13678 .await
13679 .unwrap_err()
13680 .to_string();
13681 assert!(error.contains("cannot read origin/main"));
13682 assert!(error.contains("refusing to branch off"));
13683 }
13684
13685 #[test]
13686 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13687 let leaked = vec!["chore(deps): update a crate".to_owned()];
13688 let own = "fix(daemon): apply a chosen action";
13689 assert!(should_retitle("", own, &leaked));
13690 assert!(should_retitle(
13691 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13692 own,
13693 &leaked
13694 ));
13695 assert!(should_retitle(
13696 "chore: land candidate A of run 1",
13697 own,
13698 &leaked
13699 ));
13700 assert!(should_retitle(
13701 "magi: candidate A (uncommitted work)",
13702 own,
13703 &leaked
13704 ));
13705 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13706 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13707 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13708 }
13709
13710 #[test]
13711 fn pr_merge_plan_creates_adopts_or_stops() {
13712 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13713 assert_eq!(
13714 pr_merge_plan(Ok(land::OpenPr::One {
13715 url: "u".into(),
13716 title: "t".into()
13717 })),
13718 PrPlan::Adopt {
13719 url: "u".into(),
13720 title: "t".into()
13721 }
13722 );
13723 let PrPlan::Stop(many) =
13724 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13725 else {
13726 panic!("many must stop");
13727 };
13728 assert!(many.contains('a') && many.contains('b'));
13729 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13730 panic!("a failed lookup must stop");
13731 };
13732 assert!(err.contains("bad token"));
13733 }
13734
13735 use super::*;
13736 use crate::run::GateStatus;
13737 use std::collections::BTreeMap;
13738 use std::time::Duration;
13739
13740 fn conductor() -> AgentSpec {
13741 AgentSpec {
13742 id: "conductor".to_owned(),
13743 kind: crate::config::AgentKind::Command,
13744 model: None,
13745 command: vec!["true".to_owned()],
13746 extra_args: Vec::new(),
13747 env: BTreeMap::new(),
13748 prompt_delivery: None,
13749 max_turn_secs: None,
13750 }
13751 }
13752
13753 fn spec(id: &str) -> AgentSpec {
13754 AgentSpec {
13755 id: id.to_owned(),
13756 kind: crate::config::AgentKind::Command,
13757 model: None,
13758 command: vec!["true".to_owned()],
13759 extra_args: Vec::new(),
13760 env: BTreeMap::new(),
13761 prompt_delivery: None,
13762 max_turn_secs: None,
13763 }
13764 }
13765
13766 fn ids(xs: &[&str]) -> BTreeSet<String> {
13767 xs.iter().map(|s| (*s).to_owned()).collect()
13768 }
13769
13770 #[test]
13771 fn pick_successor_skips_an_agent_another_seat_holds() {
13772 let roster = [spec("a"), spec("b"), spec("c")];
13774 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13775 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13776 }
13777
13778 #[test]
13779 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13780 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13782 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13783 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13784 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13785 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13786 }
13787
13788 #[test]
13789 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13790 let roster = [spec("a"), spec("b")];
13791 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13792 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13793 let carried = ids(&["b"]);
13794 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13795 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13796 }
13797
13798 #[test]
13799 fn pick_successor_returns_none_without_any_untried_successor() {
13800 let roster = [spec("a"), spec("b")];
13801 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13802 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13803 let carried = ids(&["a"]);
13804 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13805 }
13806
13807 #[test]
13808 fn pick_successor_rescue_avoids_another_seats_occupant() {
13809 let roster = [spec("a"), spec("b"), spec("c")];
13811 let carried = ids(&["b", "c"]);
13812 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13813 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13814 }
13815
13816 #[test]
13817 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13818 let roster = [spec("a"), spec("b"), spec("c")];
13819 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13820 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13821 }
13822
13823 #[test]
13824 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13825 let roster = [spec("a"), spec("b"), spec("c")];
13826 let failed = ids(&["a", "b", "c"]);
13827 let mut tried = ids(&["b"]);
13829 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13830 assert_eq!(first.id, "a");
13831 tried.insert(first.id.clone());
13832 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13833 assert_eq!(second.id, "c");
13834 tried.insert(second.id.clone());
13835 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13836 }
13837
13838 #[test]
13839 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13840 let roster = [spec("a"), spec("b")];
13841 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13842 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13843 }
13844
13845 #[test]
13846 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13847 let roster = [spec("a"), spec("b"), spec("c")];
13848 let h = SeatHistory {
13849 failed: ids(&["a"]),
13850 last_ok: Some("b".to_owned()),
13851 last_fail: None,
13852 };
13853 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13854 let gone = SeatHistory {
13856 last_ok: Some("zzz".to_owned()),
13857 ..h.clone()
13858 };
13859 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13860 let failed = SeatHistory {
13861 failed: ids(&["a", "b"]),
13862 last_ok: Some("b".to_owned()),
13863 last_fail: None,
13864 };
13865 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13866 }
13867
13868 #[test]
13869 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13870 let first = SeatState::new("review-1", "alpha", 7);
13871 let next = handover_seat("review-1", "gamma", 7);
13872 assert_ne!(first.claude_session, next.claude_session);
13873 }
13874
13875 #[test]
13876 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13877 let mut state = state_with_summary("x", "y");
13878 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13879 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13880 assert_ne!(a.claude_session, b.claude_session);
13881 }
13882
13883 #[test]
13884 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13885 let roster = [spec("a"), spec("b")];
13886 let h = SeatHistory {
13887 failed: ids(&["a", "b"]),
13888 ..SeatHistory::default()
13889 };
13890 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13891 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13892 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13893 }
13894
13895 #[test]
13902 fn failure_signature_ignores_numbers_and_paths() {
13903 assert_eq!(
13904 failure_signature("exited with Some(2) and no usable output"),
13905 failure_signature("exited with Some(137) and no usable output")
13906 );
13907 assert_eq!(
13908 failure_signature("process exited with no usable output (exit code 2)"),
13909 failure_signature("process exited with no usable output (exit code 137)")
13910 );
13911 for (old, new) in [
13912 (
13913 "exited with Some(2) and no usable output",
13914 "process exited with no usable output (exit code 2)",
13915 ),
13916 (
13917 "exited with None and no usable output",
13918 "process exited with no usable output (exit code unavailable)",
13919 ),
13920 ] {
13921 let previous = FailClass::Other(failure_signature(old));
13922 let current = FailClass::Other(failure_signature(new));
13923 assert_eq!(previous, current, "resumption retains the failure class");
13924 assert!(!should_hand_over(Some(&previous), ¤t));
13925 }
13926 assert_eq!(
13927 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13928 failure_signature("cannot open /var/x.txt: denied")
13929 );
13930 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13931 }
13932
13933 #[test]
13934 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13935 let other = FailClass::Other("x".into());
13936 assert!(should_hand_over(None, &FailClass::Quota));
13937 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13938 assert!(should_hand_over(
13939 Some(&FailClass::Timeout),
13940 &FailClass::Timeout
13941 ));
13942 assert!(should_hand_over(None, &other));
13943 assert!(!should_hand_over(Some(&other), &other));
13944 assert!(should_hand_over(
13945 Some(&other),
13946 &FailClass::Other("y".into())
13947 ));
13948 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13950 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13951 }
13952
13953 #[test]
13954 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13955 let a = SeatState::new("judge-1", "alpha", 7);
13956 let b = handover_seat("judge-1", "beta", 7);
13957 assert_ne!(a.claude_session, b.claude_session);
13958 assert_eq!(b.turns, 0);
13959 }
13960
13961 #[test]
13962 fn a_timeout_is_classified_apart_from_other_failures() {
13963 assert_eq!(
13964 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13965 Some(FailClass::Timeout)
13966 );
13967 assert!(matches!(
13968 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13969 Some(FailClass::Other(_))
13970 ));
13971 }
13972
13973 #[test]
13974 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13975 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13976 let tried = BTreeSet::from(["beta".to_owned()]);
13977 let next = next_untried_in_roster(&roster, 1, &tried);
13980 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13981 }
13982
13983 #[test]
13984 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13985 let roster = vec![spec("alpha"), spec("beta")];
13986 let tried = BTreeSet::from(["beta".to_owned()]);
13987 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13991 }
13992
13993 #[test]
13994 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13995 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13996 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13997 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14001 }
14002
14003 #[test]
14004 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14005 let roster = vec![spec("a"), spec("a"), spec("b")];
14006 let tried = BTreeSet::from(["a".to_owned()]);
14007 let next = next_untried_in_roster(&roster, 0, &tried);
14008 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14009 }
14010
14011 #[test]
14012 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14013 let roster = vec![spec("a"), spec("b")];
14014 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14015 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14016 }
14017
14018 #[test]
14019 fn remove_if_empty_only_ever_takes_a_bare_directory() {
14020 let dir = tempfile::tempdir().unwrap();
14021 let bay = dir.path().join("ffff");
14022
14023 remove_if_empty(&bay);
14025 assert!(!bay.exists());
14026
14027 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14030 remove_if_empty(&bay);
14031 assert!(bay.exists(), "non-empty directory must survive");
14032
14033 std::fs::remove_dir(bay.join("cand-A")).unwrap();
14035 remove_if_empty(&bay);
14036 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14037 }
14038
14039 #[test]
14048 fn a_full_panel_that_found_nothing_is_clean() {
14049 assert!(round_is_clean(
14050 0,
14051 true,
14052 2,
14053 2,
14054 0,
14055 IncompleteReviewPolicy::Block
14056 ));
14057 }
14058
14059 #[test]
14060 fn a_missing_seat_is_never_clean_under_the_default_policy() {
14061 assert!(!round_is_clean(
14062 0,
14063 true,
14064 1,
14065 2,
14066 0,
14067 IncompleteReviewPolicy::Block
14068 ));
14069 }
14070
14071 #[test]
14072 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14073 assert!(!round_is_clean(
14074 1,
14075 true,
14076 1,
14077 2,
14078 0,
14079 IncompleteReviewPolicy::Warn
14080 ));
14081 }
14082
14083 #[test]
14084 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14085 assert!(round_is_clean(
14086 0,
14087 true,
14088 1,
14089 2,
14090 0,
14091 IncompleteReviewPolicy::Warn
14092 ));
14093 }
14094
14095 #[test]
14096 fn a_full_panel_with_an_open_finding_is_not_clean() {
14097 assert!(!round_is_clean(
14098 1,
14099 true,
14100 2,
14101 2,
14102 0,
14103 IncompleteReviewPolicy::Block
14104 ));
14105 }
14106
14107 #[test]
14108 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14109 assert!(!round_is_clean(
14110 0,
14111 false,
14112 2,
14113 2,
14114 0,
14115 IncompleteReviewPolicy::Block
14116 ));
14117 }
14118
14119 #[test]
14126 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14127 assert!(round_is_clean(
14130 0,
14131 true,
14132 1,
14133 2,
14134 1,
14135 IncompleteReviewPolicy::Block
14136 ));
14137 }
14138
14139 #[test]
14140 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14141 assert!(!round_is_clean(
14144 0,
14145 true,
14146 1,
14147 2,
14148 0,
14149 IncompleteReviewPolicy::Block
14150 ));
14151 }
14152
14153 #[test]
14154 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14155 assert!(!round_is_clean(
14156 1,
14157 true,
14158 1,
14159 2,
14160 1,
14161 IncompleteReviewPolicy::Block
14162 ));
14163 assert!(!round_is_clean(
14164 0,
14165 false,
14166 1,
14167 2,
14168 1,
14169 IncompleteReviewPolicy::Block
14170 ));
14171 }
14172
14173 #[test]
14174 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14175 assert!(!round_is_clean(
14179 0,
14180 true,
14181 0,
14182 2,
14183 2,
14184 IncompleteReviewPolicy::Block
14185 ));
14186 }
14187
14188 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14189 CommandOutcome {
14190 command: "test".to_owned(),
14191 code,
14192 output_tail: String::new(),
14193 duration_ms: 0,
14194 resource_blocked,
14195 }
14196 }
14197
14198 #[cfg(unix)]
14199 #[tokio::test]
14200 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14201 ask_test_home();
14202 let temp = tempfile::tempdir().unwrap();
14203 let mut runner = runner_at(RunStatus::Gating);
14204 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14205 let commands = vec![
14206 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14207 "touch must-not-run".to_owned(),
14208 ];
14209 let (outcomes, retried, pids) = run_e2e_with_retry(
14210 &mut runner.state,
14211 &shell,
14212 &commands,
14213 temp.path(),
14214 Duration::from_secs(5),
14215 "fixture",
14216 )
14217 .await;
14218 assert_eq!(outcomes.len(), 1);
14219 assert!(outcomes[0].resource_blocked);
14220 assert!(!outcomes[0].build_failed());
14221 assert!(!retried);
14222 assert!(pids.is_empty());
14223 assert!(!temp.path().join("must-not-run").exists());
14224 assert_eq!(
14225 runner.state.resource_block.as_ref().unwrap().kind,
14226 "incremental cache admission"
14227 );
14228 }
14229
14230 #[test]
14231 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14232 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14233 assert!(
14234 !verify_inconclusive(&[outcome(Some(1), false)]),
14235 "an ordinary failure is still evidence about the patch"
14236 );
14237 assert!(verify_inconclusive(&[outcome(None, true)]));
14238 assert!(
14239 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14240 "one inconclusive outcome taints the whole batch"
14241 );
14242 assert!(!verify_inconclusive(&[]));
14243 }
14244
14245 #[tokio::test]
14246 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14247 let calls = std::sync::atomic::AtomicUsize::new(0);
14251 let started = Instant::now();
14252 wait_for_pids_with(
14253 &[123],
14254 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14255 Duration::from_millis(5),
14256 Duration::from_secs(5),
14257 )
14258 .await;
14259 assert!(
14260 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14261 "must keep checking rather than deciding on the first answer"
14262 );
14263 assert!(
14264 started.elapsed() < Duration::from_secs(1),
14265 "must return the moment it is confirmed dead, not wait out the ceiling"
14266 );
14267 }
14268
14269 #[tokio::test]
14270 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14271 let started = Instant::now();
14272 wait_for_pids_with(
14273 &[123],
14274 |_| true, Duration::from_millis(5),
14276 Duration::from_millis(30),
14277 )
14278 .await;
14279 let elapsed = started.elapsed();
14280 assert!(
14281 elapsed >= Duration::from_millis(30),
14282 "must not give up before its own ceiling: {elapsed:?}"
14283 );
14284 assert!(
14285 elapsed < Duration::from_secs(1),
14286 "must not wait past its own ceiling either: {elapsed:?}"
14287 );
14288 }
14289
14290 #[tokio::test]
14291 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14292 let started = Instant::now();
14293 wait_for_pids_with(
14294 &[],
14295 |_| true,
14296 Duration::from_secs(5),
14297 Duration::from_secs(5),
14298 )
14299 .await;
14300 assert!(
14301 started.elapsed() < Duration::from_millis(200),
14302 "an empty pid list has nothing to confirm"
14303 );
14304 }
14305
14306 fn review_round(
14312 clean: bool,
14313 blocking: usize,
14314 answered: usize,
14315 expected: usize,
14316 progressed: bool,
14317 e2e_ok: bool,
14318 ) -> ReviewRound {
14319 ReviewRound {
14320 round: 1,
14321 head: "h".to_owned(),
14322 verified_head: None,
14323 verified_at: None,
14324 reviews: Vec::new(),
14325 e2e: vec![CommandOutcome {
14326 command: "test".to_owned(),
14327 code: Some(if e2e_ok { 0 } else { 1 }),
14328 output_tail: String::new(),
14329 duration_ms: 0,
14330 resource_blocked: false,
14331 }],
14332 verify_retried: false,
14333 e2e_deferred: false,
14334 e2e_defer_reason: None,
14335 fix: None,
14336 blocking,
14337 answered,
14338 expected,
14339 clean,
14340 progressed,
14341 vote_split: false,
14342 reconsideration: Vec::new(),
14343 verdict: None,
14344 }
14345 }
14346
14347 #[test]
14348 fn review_conclusion_is_none_when_nothing_has_run() {
14349 assert_eq!(review_conclusion(&[], 3), None);
14350 }
14351
14352 #[test]
14353 fn review_conclusion_is_none_while_rounds_remain() {
14354 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14355 assert_eq!(review_conclusion(&rounds, 3), None);
14356 }
14357
14358 #[test]
14359 fn review_conclusion_is_gating_once_a_round_is_clean() {
14360 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14361 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14362 }
14363
14364 #[test]
14365 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14366 let rounds = vec![
14367 review_round(false, 1, 2, 2, true, true),
14368 review_round(false, 1, 2, 2, true, true),
14369 ];
14370 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14371 }
14372
14373 #[test]
14374 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14375 let rounds = vec![
14376 review_round(false, 1, 2, 2, true, true),
14377 review_round(false, 1, 2, 2, true, false),
14378 ];
14379 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14380 }
14381
14382 #[test]
14383 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14384 let mut blocked = review_round(false, 1, 2, 2, true, false);
14391 blocked.e2e[0].resource_blocked = true;
14392 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14393 assert_eq!(review_conclusion(&rounds, 2), None);
14394 }
14395
14396 #[test]
14397 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14398 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14400 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14401 }
14402
14403 #[test]
14404 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14405 let rounds = vec![
14406 review_round(false, 1, 2, 2, false, true),
14407 review_round(false, 1, 2, 2, false, true),
14408 ];
14409 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14410 }
14411
14412 fn secs(n: u64) -> Duration {
14413 Duration::from_secs(n)
14414 }
14415
14416 #[tokio::test]
14419 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14420 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14421 if std::env::var_os(CHILD).is_none() {
14422 let output = std::process::Command::new(std::env::current_exe().unwrap())
14426 .args([
14427 "--exact",
14428 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14429 "--nocapture",
14430 ])
14431 .env(CHILD, "1")
14432 .env_remove("MAGI_WORKER_HOME")
14433 .env_remove("MAGI_WORKER_ID")
14434 .output()
14435 .unwrap();
14436 assert!(
14437 output.status.success()
14438 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14439 "{}\n{}",
14440 String::from_utf8_lossy(&output.stdout),
14441 String::from_utf8_lossy(&output.stderr)
14442 );
14443 return;
14444 }
14445 ask_test_home();
14446 let tmp = tempfile::tempdir().unwrap();
14447 let repo = tmp.path().join("repo");
14448 std::fs::create_dir_all(&repo).unwrap();
14449 init_repo(&repo);
14450 let g = |args: &[&str]| {
14451 let out = std::process::Command::new("git")
14452 .args(args)
14453 .current_dir(&repo)
14454 .quiet()
14455 .output()
14456 .unwrap();
14457 assert!(
14458 out.status.success(),
14459 "{}",
14460 String::from_utf8_lossy(&out.stderr)
14461 );
14462 };
14463 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14464 g(&["checkout", "-b", "historical"]);
14465 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14466 g(&["add", "."]);
14467 g(&["commit", "-m", "old alternative"]);
14468 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14469 g(&["checkout", "-b", "magi/task-selected", &base]);
14470 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14471 g(&["add", "."]);
14472 g(&["commit", "-m", "measurement infrastructure"]);
14473 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14474 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14475 g(&["add", "."]);
14476 g(&["commit", "-m", "dependent implementation"]);
14477 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14478 g(&["checkout", "main"]);
14479 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14480 g(&["add", "."]);
14481 g(&["commit", "-m", "unrelated base change"]);
14482 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14483 let origin = tmp.path().join("origin.git");
14484 g(&[
14485 "clone",
14486 "--bare",
14487 repo.to_str().unwrap(),
14488 origin.to_str().unwrap(),
14489 ]);
14490 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14491 g(&["fetch", "origin"]);
14492 let selection = crate::pr_recovery::Selection {
14493 url: "https://example.test/project/pull/1".into(),
14494 branch: "magi/task-selected".into(),
14495 head: head.clone(),
14496 source_head: None,
14497 source_run: None,
14498 source_proof: None,
14499 disposition: Some(crate::pr_recovery::Disposition::Review),
14500 instruction: "preserve measured behavior".into(),
14501 approved_at: jiff::Timestamp::now(),
14502 };
14503 let mut template = runner_at(RunStatus::Reviewing);
14504 template.state.repo = repo.clone();
14505 template.state.base_commit = current.clone();
14506 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14507 template.state.origin = Some(crate::run::Origin::queue("selected"));
14508 template.state.pr_head = Some(selection.branch.clone());
14509 template.state.pr_recovery = Some(selection.clone());
14510 let mut runner =
14511 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14512 .await
14513 .unwrap();
14514 let wt = runner.state.winner().unwrap().worktree.clone();
14515 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14516 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14517 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14518 assert!(runner.state.seeds.is_empty());
14519 assert_eq!(runner.state.instruction, "preserve measured behavior");
14520 runner.sync_to_base().await.unwrap();
14521 assert_eq!(
14522 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14523 "prerequisite"
14524 );
14525 assert_eq!(
14526 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14527 "measured implementation"
14528 );
14529 assert_eq!(
14530 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14531 "current main change"
14532 );
14533 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14534 assert_eq!(
14535 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14536 head
14537 );
14538
14539 let mut partial = runner_at(RunStatus::Reviewing);
14541 partial.state.repo = repo.clone();
14542 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14543 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14544 run: Some("exit 1".into()),
14545 ..Default::default()
14546 }];
14547 let selection = runner.state.pr_recovery.clone().unwrap();
14548 partial.state.pr_recovery = Some(selection.clone());
14549 partial.state.pr_head = Some(selection.branch.clone());
14550 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14551 let id = partial.state.id.clone();
14552 let error = match Runner::open_selected(
14553 &repo,
14554 partial.state,
14555 partial.roles,
14556 1,
14557 runner.state.base_commit.clone(),
14558 selection,
14559 )
14560 .await
14561 {
14562 Ok(_) => panic!("setup must fail"),
14563 Err(e) => e,
14564 };
14565 assert_eq!(
14566 error
14567 .downcast_ref::<crate::run::StartupFailure>()
14568 .unwrap()
14569 .run,
14570 id
14571 );
14572 let failed = RunState::load(&id).unwrap();
14573 assert_eq!(failed.status, RunStatus::Failed);
14574 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14575 assert!(Runner::resume(&id).is_err());
14576 runner.state.status = RunStatus::Ready;
14577 runner.state.driver_exited = true;
14578 runner.state.save().unwrap();
14579 let mut retry = runner_at(RunStatus::Reviewing);
14580 retry.state.repo = repo.clone();
14581 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14582 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14583 reselected.approved_at = jiff::Timestamp::now();
14584 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14585 retry.state.pr_head = Some(reselected.branch.clone());
14586 retry.state.pr_recovery = Some(reselected.clone());
14587 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14588 .await
14589 .unwrap();
14590 let restarted = Runner::open_selected(
14591 &repo,
14592 retry.state,
14593 retry.roles,
14594 1,
14595 runner.state.base_commit.clone(),
14596 reselected,
14597 )
14598 .await
14599 .unwrap();
14600 assert_eq!(
14601 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14602 .await
14603 .unwrap(),
14604 head
14605 );
14606 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14607 assert_eq!(
14608 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14609 head
14610 );
14611 }
14612
14613 #[tokio::test]
14614 #[cfg(unix)]
14615 async fn fallback_and_interrupted_restoration_require_coordination() {
14616 use std::os::fd::AsRawFd as _;
14617 ask_test_home();
14618 let tmp = tempfile::tempdir().unwrap();
14619 let mut runner = registered_runner(tmp.path()).await;
14620 runner.state.config.graph.timeout_resource_wait = 1;
14621 let winner = runner.state.candidates[0].clone();
14622 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14623 let leftover = winner.worktree.join("leftover");
14624 std::fs::write(&leftover, "untrusted edits").unwrap();
14625 let next = tmp.path().join("next-fixer");
14626 let mut failed = spec("first");
14627 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14628 let mut successor = spec("second");
14629 successor.command = vec![
14630 "sh".into(),
14631 "-c".into(),
14632 format!("touch '{}'; echo done", next.display()),
14633 ];
14634 let attempts = vec![
14635 (failed.clone(), "first".into()),
14636 (successor.clone(), "second".into()),
14637 ];
14638 let lock_path = runner.state.repo.join(".git/updater.lock");
14639 let external = std::fs::File::create(&lock_path).unwrap();
14640 assert_eq!(
14641 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14642 0
14643 );
14644 let artifacts = tmp.path().join("artifacts");
14645 let build = |spec, seat| SeatJob {
14646 spec,
14647 seat,
14648 cwd: winner.worktree.clone(),
14649 prompt: "fixture".into(),
14650 timeout: Duration::from_secs(10),
14651 allow_write: true,
14652 sessions: false,
14653 artifacts: artifacts.clone(),
14654 stem: "fix".into(),
14655 handover: None,
14656 };
14657 let (_, _, out) = runner
14658 .ask_fixer_among(
14659 attempts.clone(),
14660 "graft",
14661 None,
14662 Some((&winner.worktree, &head)),
14663 build,
14664 )
14665 .await;
14666 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14667 assert!(leftover.exists(), "contention must prevent clean");
14668 assert!(!next.exists());
14669 assert!(runner.state.handovers.is_empty());
14670 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14672 from_run: "earlier".into(),
14673 generation: 1,
14674 mode: crate::recovery::Mode::VerifyRepair,
14675 reasons: Vec::new(),
14676 reviewed_head: None,
14677 rounds: Vec::new(),
14678 open_findings: Vec::new(),
14679 evidence: None,
14680 criteria_inherited: true,
14681 repair: crate::recovery::Repair::Started {
14682 before: head.clone(),
14683 },
14684 });
14685 assert!(
14686 runner
14687 .recovery_repair(&winner)
14688 .await
14689 .unwrap_err()
14690 .is::<crate::repo_lock::ResourceBlocked>()
14691 );
14692 assert!(matches!(
14693 runner.state.recovery.as_ref().unwrap().repair,
14694 crate::recovery::Repair::Started { .. }
14695 ));
14696 assert!(leftover.exists());
14697 drop(external);
14698 runner.recovery_repair(&winner).await.unwrap();
14699 assert!(!leftover.exists());
14700 assert!(matches!(
14701 runner.state.recovery.as_ref().unwrap().repair,
14702 crate::recovery::Repair::Done { .. }
14703 ));
14704 std::fs::remove_file(&lock_path).unwrap();
14706 std::fs::create_dir(&lock_path).unwrap();
14707 std::fs::write(&leftover, "untrusted edits").unwrap();
14708 let (_, _, out) = runner
14709 .ask_fixer_among(
14710 attempts,
14711 "graft",
14712 None,
14713 Some((&winner.worktree, &head)),
14714 build,
14715 )
14716 .await;
14717 assert!(matches!(out, AgentOutcome::Failed(_)));
14718 assert!(leftover.exists());
14719 assert!(!next.exists());
14720 assert_eq!(
14721 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14722 head
14723 );
14724 }
14725
14726 #[cfg(unix)]
14727 async fn registered_runner(root: &Path) -> Runner {
14728 repository_runner(root, crate::config::test_coordination()).await
14729 }
14730
14731 async fn repository_runner(
14732 root: &Path,
14733 coordination: Option<crate::config::Coordination>,
14734 ) -> Runner {
14735 ask_test_home();
14736 let repo = root.join("repo");
14737 std::fs::create_dir_all(&repo).unwrap();
14738 init_repo(&repo);
14739 if coordination.is_some() {
14740 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14741 }
14742 let mut runner = runner_at(RunStatus::Prep);
14743 runner.state.repo = repo.clone();
14744 runner.state.config.coordination = coordination;
14745 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14746 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14747 runner.state.config.graph.advise = false;
14748 runner.roles.implementers = vec![spec("implement")];
14749 runner.roles.judges = vec![spec("judge")];
14750 runner.prep().await.unwrap();
14751 runner
14752 }
14753
14754 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14755
14756 #[tokio::test]
14757 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14758 let tmp = tempfile::tempdir().unwrap();
14759 let mut runner = repository_runner(tmp.path(), None).await;
14760 let prompts = runner.state.config.prompts.clone();
14761 let run = runner.state.id.clone();
14762 let ctx = WaveCtx {
14763 carry_seats: false,
14764 pauses: None,
14765 run: &run,
14766 node: "advise",
14767 prompts: &prompts,
14768 cache: None,
14769 round: None,
14770 };
14771 let jobs: Vec<_> = ["alpha", "beta"]
14772 .into_iter()
14773 .map(|id| SeatJob {
14774 spec: scripted(id, AUTH_OUTAGE),
14775 seat: SeatState::new(id, id, 7),
14776 prompt: "propose".into(),
14777 cwd: runner.state.candidates[0].worktree.clone(),
14778 timeout: Duration::from_secs(10),
14779 allow_write: false,
14780 sessions: false,
14781 artifacts: agent::artifacts_dir(&runner.state.dir()),
14782 stem: id.into(),
14783 handover: None,
14784 })
14785 .collect();
14786 let mut losses = Vec::new();
14787 let results = ask_wave_with(
14788 jobs.clone(),
14789 runner.sem.clone(),
14790 2,
14791 &[],
14792 &ctx,
14793 &mut losses,
14794 &mut runner.state,
14795 &|_: &str| Ok(()),
14796 )
14797 .await;
14798 assert!(
14799 results
14800 .iter()
14801 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14802 );
14803 assert_eq!(runner.state.quota.len(), 2);
14804 assert!(
14805 runner
14806 .state
14807 .quota
14808 .iter()
14809 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14810 );
14811 assert!(!runner.state.advise_attempted);
14812 runner.stop_for_resource().unwrap();
14813 assert_eq!(runner.state.status, RunStatus::Blocked);
14814 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14815 assert_eq!(
14816 runner.state.resource_block.as_ref().unwrap().kind,
14817 "agent authentication"
14818 );
14819 runner.state.resource_block = None;
14822 runner.state.quota.clear();
14823 let mut mixed = jobs;
14824 mixed[1].spec.command = vec!["false".into()];
14825 let results = ask_wave_with(
14826 mixed,
14827 runner.sem.clone(),
14828 0,
14829 &[],
14830 &ctx,
14831 &mut losses,
14832 &mut runner.state,
14833 &|_: &str| Ok(()),
14834 )
14835 .await;
14836 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14837 assert!(runner.state.resource_block.is_none());
14838 assert_eq!(losses.len(), 1);
14839 }
14840
14841 #[tokio::test]
14842 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14843 let tmp = tempfile::tempdir().unwrap();
14844 let mut runner = repository_runner(tmp.path(), None).await;
14845 let implementer = scripted("implement", AUTH_OUTAGE);
14846 runner.roles.implementers = vec![implementer.clone()];
14847 runner.state.config.agents = vec![implementer];
14848 runner.implement().await.unwrap();
14849 assert_eq!(runner.state.status, RunStatus::Blocked);
14850 assert_eq!(runner.state.quota.len(), 1);
14851 assert_eq!(
14852 runner.state.quota[0].cause,
14853 agent::QuotaCause::Authentication
14854 );
14855 let candidate = &runner.state.candidates[0];
14856 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14857 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14858 }
14859
14860 #[tokio::test]
14861 #[cfg(unix)]
14862 async fn parked_implementer_persists_completed_work() {
14863 for registered in [false, true] {
14864 let tmp = tempfile::tempdir().unwrap();
14865 let mut runner = registered_runner(tmp.path()).await;
14866 if !registered {
14867 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14868 }
14869 let marker = tmp.path().join("started");
14870 let release = tmp.path().join("release");
14871 let mut implementer = spec("implement");
14872 implementer.command = vec![
14873 "sh".into(),
14874 "-c".into(),
14875 format!(
14876 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14877 marker.display(),
14878 release.display()
14879 ),
14880 ];
14881 runner.roles.implementers = vec![implementer.clone()];
14882 runner.state.config.agents = vec![implementer];
14883 let pause = runner.pause.clone();
14884 let (result, ()) = tokio::join!(runner.implement(), async {
14885 tokio::time::timeout(Duration::from_secs(10), async {
14886 while !marker.exists() {
14887 tokio::time::sleep(Duration::from_millis(10)).await;
14888 }
14889 })
14890 .await
14891 .unwrap();
14892 pause.park();
14893 std::fs::write(&release, "go").unwrap();
14894 });
14895 result.unwrap();
14896 let candidate = &runner.state.candidates[0];
14897 assert!(candidate.commits > 0, "{candidate:?}");
14898 assert!(candidate.summary.contains("saved-result"));
14899 assert_eq!(candidate.files, 1);
14900 let before = candidate.commits;
14901 runner.implement().await.unwrap();
14902 assert_eq!(runner.state.candidates[0].commits, before);
14903 }
14904 }
14905
14906 #[tokio::test]
14907 #[cfg(unix)]
14908 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14909 let tmp = tempfile::tempdir().unwrap();
14910 let mut runner = registered_runner(tmp.path()).await;
14911 let winner = runner.state.candidates[0].clone();
14912 let cache = tmp.path().join("cache");
14913 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14914 runner.state.config.graph.timeout_resource_wait = 0;
14915 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14916 let crate::cache::AcquireOutcome::Acquired(held) =
14917 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14918 else {
14919 panic!("cache must be free")
14920 };
14921 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14922 from_run: "earlier".into(),
14923 generation: 1,
14924 mode: crate::recovery::Mode::VerifyRepair,
14925 reasons: Vec::new(),
14926 reviewed_head: None,
14927 rounds: Vec::new(),
14928 open_findings: Vec::new(),
14929 evidence: None,
14930 criteria_inherited: true,
14931 repair: crate::recovery::Repair::NotStarted,
14932 });
14933 runner.recovery_repair(&winner).await.unwrap();
14934 assert!(matches!(
14935 runner.state.recovery.as_ref().unwrap().repair,
14936 crate::recovery::Repair::WaitingForRestore { .. }
14937 ));
14938 drop(held);
14939 runner.state.resource_block = None;
14940 runner.state.config.graph.timeout_resource_wait = 1;
14941 runner.recovery_repair(&winner).await.unwrap();
14942 assert!(matches!(
14943 runner.state.recovery.as_ref().unwrap().repair,
14944 crate::recovery::Repair::Done { .. }
14945 ));
14946 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14947 }
14948
14949 #[tokio::test]
14950 #[cfg(unix)]
14951 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14952 use std::os::fd::AsRawFd as _;
14953 for graft in [false, true] {
14954 let tmp = tempfile::tempdir().unwrap();
14955 let mut runner = registered_runner(tmp.path()).await;
14956 runner.state.config.graph.timeout_resource_wait = 1;
14957 let winner = runner.state.candidates[0].clone();
14958 let marker = tmp.path().join("fixer-started");
14959 let mut fixer = spec("fixer");
14960 fixer.command = vec![
14961 "sh".into(),
14962 "-c".into(),
14963 format!("echo started >> '{}'; echo done", marker.display()),
14964 ];
14965 runner.roles.fixer = Some(vec![fixer]);
14966 if graft {
14967 let mut loser = winner.clone();
14968 loser.label = 'B';
14969 loser.index = 1;
14970 runner.state.candidates.push(loser);
14971 runner.state.config.graph.graft_runner_up = true;
14972 runner.state.tally = Some(Tally {
14973 first_choice: BTreeMap::new(),
14974 borda: BTreeMap::new(),
14975 winner: winner.label,
14976 rankings: 1,
14977 unanimous_initial: true,
14978 deliberated: false,
14979 changed_votes: 0,
14980 unanimous_final: true,
14981 tie_break: None,
14982 judges: 1,
14983 present: 1,
14984 quorum: 1,
14985 met_quorum: true,
14986 uncontested: None,
14987 });
14988 runner.state.judgements.push(crate::run::Judgement {
14989 judge: 1,
14990 seat: "judge-1".into(),
14991 agent: "judge".into(),
14992 ranking: vec![winner.label, 'B'],
14993 reasons: BTreeMap::new(),
14994 confidence: None,
14995 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14996 order: vec![0, 1],
14997 failed: None,
14998 duration_ms: 0,
14999 });
15000 } else {
15001 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15002 from_run: "earlier".into(),
15003 generation: 1,
15004 mode: crate::recovery::Mode::VerifyRepair,
15005 reasons: Vec::new(),
15006 reviewed_head: None,
15007 rounds: Vec::new(),
15008 open_findings: Vec::new(),
15009 evidence: None,
15010 criteria_inherited: true,
15011 repair: crate::recovery::Repair::NotStarted,
15012 });
15013 }
15014 let reader = crate::repo_lock::SeatLease::for_checkout(
15015 &winner.worktree,
15016 false,
15017 crate::config::test_coordination().as_ref(),
15018 )
15019 .await
15020 .unwrap();
15021 let pause = runner.pause.clone();
15022 let (result, ()) = tokio::join!(
15023 async {
15024 if graft {
15025 runner.graft().await
15026 } else {
15027 runner.recovery_repair(&winner).await
15028 }
15029 },
15030 async {
15031 tokio::time::sleep(Duration::from_millis(100)).await;
15032 pause.park();
15033 }
15034 );
15035 result.unwrap();
15036 assert!(!marker.exists(), "a blocked seat must not spawn");
15037 runner.state = RunState::load(&runner.state.id).unwrap();
15039 if graft {
15040 assert_eq!(
15041 runner.state.graft.as_ref().unwrap().outcome,
15042 GraftOutcome::WaitingForRestore
15043 );
15044 } else {
15045 assert!(matches!(
15046 runner.state.recovery.as_ref().unwrap().repair,
15047 crate::recovery::Repair::WaitingForRestore { .. }
15048 ));
15049 }
15050 drop(reader);
15051 runner.pause = Pause::new();
15052 runner.state.resource_block = None;
15053 let external =
15055 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15056 assert_eq!(
15057 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15058 0
15059 );
15060 let leftover = winner.worktree.join("leftover");
15061 std::fs::write(&leftover, "earlier fallback edits").unwrap();
15062 let result = if graft {
15063 runner.graft().await
15064 } else {
15065 runner.recovery_repair(&winner).await
15066 };
15067 assert!(
15068 result
15069 .unwrap_err()
15070 .is::<crate::repo_lock::ResourceBlocked>()
15071 );
15072 assert!(leftover.exists());
15073 assert!(!marker.exists());
15074 drop(external);
15075 runner.state.resource_block = None;
15076 let result = if graft {
15077 runner.graft().await
15078 } else {
15079 runner.recovery_repair(&winner).await
15080 };
15081 result.unwrap();
15082 assert!(!leftover.exists(), "restore must precede retry");
15083 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15084 if graft {
15085 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15086 } else {
15087 assert!(matches!(
15088 runner.state.recovery.as_ref().unwrap().repair,
15089 crate::recovery::Repair::Done { .. }
15090 ));
15091 }
15092 }
15093 }
15094
15095 #[tokio::test]
15096 #[cfg(unix)]
15097 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15098 use std::os::fd::AsRawFd as _;
15099 let tmp = tempfile::tempdir().unwrap();
15100 let mut runner = registered_runner(tmp.path()).await;
15101 runner.state.config.graph.timeout_resource_wait = 1;
15102 let winner = runner.state.candidates[0].clone();
15103 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15104 runner.state.tally = Some(Tally {
15105 first_choice: BTreeMap::new(),
15106 borda: BTreeMap::new(),
15107 winner: winner.label,
15108 rankings: 0,
15109 unanimous_initial: false,
15110 deliberated: false,
15111 changed_votes: 0,
15112 unanimous_final: false,
15113 tie_break: None,
15114 judges: 0,
15115 present: 0,
15116 quorum: 0,
15117 met_quorum: true,
15118 uncontested: Some("fixture".into()),
15119 });
15120 runner.state.graft = Some(GraftRecord {
15121 pre: pre.clone(),
15122 post: None,
15123 lines: None,
15124 limit: None,
15125 agent: None,
15126 outcome: GraftOutcome::Running,
15127 requested: Vec::new(),
15128 adoption: None,
15129 adoption_error: None,
15130 });
15131 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15132 git::commit_all(&winner.worktree, "unfinished graft")
15133 .await
15134 .unwrap();
15135 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15136 let leftover = winner.worktree.join("leftover");
15137 std::fs::write(&leftover, "untrusted edits").unwrap();
15138 let path = runner.state.repo.join(".git/updater.lock");
15139 let external = std::fs::File::create(&path).unwrap();
15140 assert_eq!(
15141 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15142 0
15143 );
15144 assert!(
15145 runner
15146 .graft()
15147 .await
15148 .unwrap_err()
15149 .is::<crate::repo_lock::ResourceBlocked>()
15150 );
15151 assert!(matches!(
15152 runner.state.graft.as_ref().unwrap().outcome,
15153 GraftOutcome::Running
15154 ));
15155 assert_eq!(
15156 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15157 unfinished
15158 );
15159 assert!(leftover.exists());
15160 drop(external);
15161 std::fs::remove_file(&path).unwrap();
15162 std::fs::create_dir(&path).unwrap();
15163 assert!(runner.graft().await.is_err());
15164 assert!(matches!(
15165 runner.state.graft.as_ref().unwrap().outcome,
15166 GraftOutcome::Running
15167 ));
15168 assert_eq!(
15169 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15170 unfinished
15171 );
15172 assert!(leftover.exists());
15173 std::fs::remove_dir(&path).unwrap();
15174 runner.graft().await.unwrap();
15175 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15176 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15177 assert!(!leftover.exists());
15178 }
15179
15180 #[tokio::test]
15181 #[cfg(unix)]
15182 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15183 use std::os::fd::AsRawFd as _;
15184 let tmp = tempfile::tempdir().unwrap();
15185 let mut runner = registered_runner(tmp.path()).await;
15186 runner.state.prep_complete = Some(false);
15187 runner.state.config.graph.timeout_resource_wait = 1;
15188 runner.state.save().unwrap();
15189 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15190 assert_eq!(
15191 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15192 0
15193 );
15194 runner.execute().await.unwrap();
15195 let disk = RunState::load(&runner.state.id).unwrap();
15196 assert!(disk.driver_exited);
15197 assert_eq!(disk.driver_pid, None);
15198 assert_eq!(runner.state.status, RunStatus::Blocked);
15199 assert_eq!(
15200 runner.state.resource_block.as_ref().unwrap().kind,
15201 "repository coordination"
15202 );
15203 assert!(runner.state.prep_failure.is_none());
15204 assert!(runner.state.seats.is_empty());
15205 runner.pause.park();
15206 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15209 .await
15210 .unwrap()
15211 .unwrap();
15212 assert!(runner.state.parked);
15213 assert!(runner.state.prep_failure.is_none());
15214 let disk = RunState::load(&runner.state.id).unwrap();
15215 assert!(disk.driver_exited);
15216 assert_eq!(disk.driver_pid, None);
15217 }
15218
15219 #[tokio::test]
15220 #[cfg(unix)]
15221 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15222 use std::os::fd::AsRawFd as _;
15223 let tmp = tempfile::tempdir().unwrap();
15224 let mut runner = registered_runner(tmp.path()).await;
15225 runner.state.prep_complete = Some(false);
15226 runner.state.config.graph.timeout_resource_wait = 30;
15227 runner.state.save().unwrap();
15228 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15229 assert_eq!(
15230 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15231 0
15232 );
15233 assert!(
15234 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15235 .await
15236 .is_err()
15237 );
15238 let disk = RunState::load(&runner.state.id).unwrap();
15239 assert!(disk.driver_exited);
15240 assert_eq!(disk.driver_pid, None);
15241 assert!(disk.status.resumable());
15242 assert!(runner.state.driver_exited);
15243 assert_eq!(runner.state.driver_pid, None);
15244 runner.state.save().unwrap();
15245 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15246 }
15247
15248 #[tokio::test]
15249 #[cfg(unix)]
15250 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15251 let tmp = tempfile::tempdir().unwrap();
15252 let mut runner = registered_runner(tmp.path()).await;
15253 let winner = runner.state.candidates[0].clone();
15254 let cache = tmp.path().join("cache");
15255 let started = tmp.path().join("pre-gate-started");
15256 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15257 runner.state.config.verify.gate =
15258 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15259 let home = crate::run::home();
15260 let owner =
15261 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15262 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15263 crate::cache::AcquireOutcome::Acquired(g) => g,
15264 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15265 };
15266 let mut gate = Box::pin(runner.run_gate(&winner));
15267 assert!(
15268 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15269 .await
15270 .is_err()
15271 );
15272 assert!(!started.exists(), "pre-gate must wait for the cache too");
15273 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15277 let editor = crate::repo_lock::SeatLease::checkout_with(
15278 &winner.worktree,
15279 true,
15280 crate::config::test_coordination().as_ref(),
15281 &wait,
15282 )
15283 .await
15284 .unwrap();
15285 drop(editor);
15286 guard.release();
15287 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15288 .await
15289 .unwrap()
15290 .unwrap();
15291 drop(gate);
15292 assert!(outcomes.iter().all(CommandOutcome::ok));
15293 assert!(started.exists());
15294 }
15295
15296 #[tokio::test]
15297 #[cfg(unix)]
15298 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15299 use std::os::fd::AsRawFd as _;
15300
15301 ask_test_home();
15302 let tmp = tempfile::tempdir().unwrap();
15303 let repo = tmp.path().join("repo");
15304 std::fs::create_dir_all(&repo).unwrap();
15305 init_repo(&repo);
15306 let common = repo.join(".git");
15307 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15308 let mut runner = runner_at(RunStatus::Prep);
15309 runner.state.repo = repo.clone();
15310 runner.state.config.coordination = crate::config::test_coordination();
15311 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15312 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15313 runner.state.config.graph.advise = false;
15314 runner.roles.implementers = vec![spec("implement")];
15315 runner.roles.judges = vec![spec("judge")];
15316 runner.prep().await.unwrap();
15317 let winner = runner.state.candidates[0].clone();
15318 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15321 let completed = tmp.path().join("seat-completed");
15322 runner.roles.implementers[0].command = vec![
15323 "sh".into(),
15324 "-c".into(),
15325 format!("touch '{}'; echo completed", completed.display()),
15326 ];
15327 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15328 assert_eq!(
15329 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15330 0
15331 );
15332 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15333 let mut implement = Box::pin(runner.implement());
15334 tokio::select! {
15335 result = &mut implement => panic!("rescue must wait: {result:?}"),
15336 result = tokio::time::timeout(Duration::from_secs(10), async {
15337 while !completed.exists() {
15338 tokio::time::sleep(Duration::from_millis(20)).await;
15339 }
15340 }) => result.unwrap(),
15341 }
15342 assert!(
15343 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15344 .await
15345 .is_err()
15346 );
15347 assert_eq!(
15348 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15349 before
15350 );
15351 assert_eq!(
15352 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15353 0
15354 );
15355 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15356 .await
15357 .unwrap()
15358 .unwrap();
15359 drop(implement);
15360 assert_ne!(
15361 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15362 before
15363 );
15364 assert!(git::is_clean(&winner.worktree).await.unwrap());
15365
15366 let started = tmp.path().join("pre-gate-started");
15367 let proceed = tmp.path().join("pre-gate-proceed");
15368 let checked = tmp.path().join("gate-ran");
15369 runner.state.config.verify.pre_gate = vec![format!(
15370 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15371 started.display(),
15372 proceed.display()
15373 )];
15374 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15375 assert_eq!(
15376 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15377 0
15378 );
15379 let mut gate = Box::pin(runner.run_gate(&winner));
15380 assert!(
15381 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15382 .await
15383 .is_err()
15384 );
15385 assert!(!started.exists());
15386 assert!(!checked.exists());
15387 assert!(git::is_clean(&winner.worktree).await.unwrap());
15388 assert_eq!(
15389 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15390 0
15391 );
15392 tokio::select! {
15393 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15394 result = tokio::time::timeout(Duration::from_secs(10), async {
15395 while !started.exists() {
15396 tokio::time::sleep(Duration::from_millis(20)).await;
15397 }
15398 }) => result.unwrap(),
15399 }
15400 assert_eq!(
15401 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15402 -1,
15403 "mechanical edits must hold the real lock"
15404 );
15405 std::fs::write(proceed, "ready").unwrap();
15406 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15407 .await
15408 .unwrap()
15409 .unwrap();
15410 drop(gate);
15411 assert!(outcomes.iter().all(CommandOutcome::ok));
15412 assert!(checked.exists());
15413 assert!(runner.state.pre_gate_commit.is_some());
15414 assert!(git::is_clean(&winner.worktree).await.unwrap());
15415 assert_eq!(
15416 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15417 0,
15418 "gate completion must release coordination"
15419 );
15420 drop(outsider);
15421 std::fs::remove_file(common.join("updater.lock")).unwrap();
15422 std::fs::create_dir(common.join("updater.lock")).unwrap();
15423 std::fs::remove_file(&started).unwrap();
15424 std::fs::remove_file(&checked).unwrap();
15425 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15426 let error = runner.run_gate(&winner).await.unwrap_err();
15427 assert!(
15428 error
15429 .to_string()
15430 .contains("acquire mandatory repository lock")
15431 );
15432 assert!(
15433 !started.exists(),
15434 "failed acquisition must prevent pre-gate"
15435 );
15436 assert!(!checked.exists(), "failed acquisition must prevent gate");
15437 assert!(git::is_clean(&winner.worktree).await.unwrap());
15438 assert_eq!(
15439 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15440 head
15441 );
15442 }
15443
15444 #[tokio::test]
15445 #[cfg(unix)]
15446 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15447 use std::os::fd::AsRawFd as _;
15448 use std::time::Duration;
15449
15450 ask_test_home();
15451 let tmp = tempfile::tempdir().unwrap();
15452 let repo = tmp.path().join("repo");
15453 std::fs::create_dir_all(&repo).unwrap();
15454 init_repo(&repo);
15455 let common = repo.join(".git");
15456 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15457 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15458 assert_eq!(
15459 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15460 0
15461 );
15462 let mut runner = runner_at(RunStatus::Prep);
15463 runner.state.repo = repo.clone();
15464 runner.state.config.coordination = crate::config::test_coordination();
15465 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15466 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15467 runner.state.config.graph.advise = true;
15468 runner.state.config.graph.advisors = 1;
15469 runner.roles.implementers = vec![spec("implement")];
15470 runner.roles.judges = vec![spec("judge")];
15471 let root = runner.state.worktree_root();
15472 let started = tmp.path().join("setup-started");
15473 let proceed = tmp.path().join("setup-proceed");
15474 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15475 run: Some(format!(
15476 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15477 started.display(),
15478 proceed.display()
15479 )),
15480 ..Default::default()
15481 }];
15482 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15483 let config = git::git(&repo, &["config", "--local", "--list"])
15484 .await
15485 .unwrap();
15486 let mut prep = Box::pin(runner.prep());
15487 assert!(
15488 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15489 .await
15490 .is_err()
15491 );
15492 assert!(
15493 !root.exists(),
15494 "contention must prevent even checkout creation"
15495 );
15496 assert!(!started.exists(), "contention must prevent setup");
15497 assert_eq!(
15498 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15499 branches
15500 );
15501 assert_eq!(
15502 git::git(&repo, &["config", "--local", "--list"])
15503 .await
15504 .unwrap(),
15505 config
15506 );
15507 drop(lock);
15508 tokio::select! {
15509 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15510 result = tokio::time::timeout(Duration::from_secs(10), async {
15511 while !started.exists() {
15512 tokio::time::sleep(Duration::from_millis(20)).await;
15513 }
15514 }) => result.unwrap(),
15515 }
15516 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15517 assert_eq!(
15518 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15519 -1
15520 );
15521 assert!(
15522 tokio::time::timeout(
15523 Duration::from_millis(100),
15524 crate::repo_lock::SeatLease::for_checkout(
15525 &repo,
15526 true,
15527 crate::config::test_coordination().as_ref()
15528 )
15529 )
15530 .await
15531 .is_err(),
15532 "preparation must also exclude local editors"
15533 );
15534 std::fs::write(proceed, "ready").unwrap();
15535 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15536 .await
15537 .unwrap()
15538 .unwrap();
15539 drop(prep);
15540 assert_eq!(runner.state.prep_complete, Some(true));
15541 assert!(root.join("judge-1").exists());
15542 assert!(root.join("advisor-1").exists());
15543 assert_eq!(
15544 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15545 0
15546 );
15547 }
15548
15549 #[tokio::test]
15550 #[cfg(unix)]
15551 async fn preparation_lock_failure_prevents_repository_changes() {
15552 ask_test_home();
15553 let tmp = tempfile::tempdir().unwrap();
15554 let repo = tmp.path().join("repo");
15555 std::fs::create_dir_all(&repo).unwrap();
15556 init_repo(&repo);
15557 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15558 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15561 let mut runner = runner_at(RunStatus::Prep);
15562 runner.state.repo = repo.clone();
15563 runner.state.config.coordination = crate::config::test_coordination();
15564 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15565 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15566 runner.roles.implementers = vec![spec("implement")];
15567 let error = runner.prep().await.unwrap_err();
15568 assert!(
15569 error
15570 .to_string()
15571 .contains("acquire mandatory repository lock")
15572 );
15573 assert!(!runner.state.worktree_root().exists());
15574 assert!(runner.state.candidates.is_empty());
15575 assert!(!repo.join(".git/config.worktree").exists());
15576 assert_eq!(
15577 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15578 "* main"
15579 );
15580 }
15581
15582 #[tokio::test]
15583 async fn preparation_records_creation_intent_before_git_can_fail() {
15584 ask_test_home();
15585 let tmp = tempfile::tempdir().unwrap();
15586 let repo = tmp.path().join("repo");
15587 std::fs::create_dir_all(&repo).unwrap();
15588 init_repo(&repo);
15589 let mut runner = runner_at(RunStatus::Prep);
15590 runner.state.repo = repo.clone();
15591 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15592 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15593 runner.state.config.blind.commit_msg_hook = false;
15594 runner.roles.implementers = vec![spec("one")];
15595 let label = blind::assign_labels(1, runner.state.seed)[0];
15596 let branch = runner.state.branch_for(label);
15597 git::git(&repo, &["branch", &branch]).await.unwrap();
15600 assert!(runner.prep().await.is_err());
15601 let saved = RunState::load(&runner.state.id).unwrap();
15602 assert!(saved.candidates.is_empty());
15603 let event = saved
15604 .events
15605 .iter()
15606 .find(|e| e.node == "prep-intent")
15607 .unwrap();
15608 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15609 assert_eq!(intent["branch"], branch);
15610 assert_eq!(intent["start"], runner.state.base_commit);
15611 assert_eq!(
15612 intent["worktree"],
15613 runner
15614 .state
15615 .worktree_root()
15616 .join(format!("cand-{label}"))
15617 .to_string_lossy()
15618 .as_ref()
15619 );
15620 assert_eq!(intent["picks"], serde_json::json!([]));
15621 }
15622
15623 #[tokio::test]
15624 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15625 ask_test_home();
15626 let tmp = tempfile::tempdir().unwrap();
15627 let repo = tmp.path().join("repo");
15628 std::fs::create_dir_all(&repo).unwrap();
15629 init_repo(&repo);
15630 let mut runner = runner_at(RunStatus::Prep);
15631 runner.state.repo = repo.clone();
15632 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15633 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15634 runner.state.sources.pick_commits.push(refs::Input {
15635 token: "missing".into(),
15636 sha: "0".repeat(40),
15637 provenance: refs::Provenance::ExplicitCli,
15638 });
15639 assert!(runner.prep().await.is_err());
15640 assert!(runner.state.candidates.is_empty());
15641 assert!(!runner.state.worktree_root().exists());
15642 }
15643
15644 #[tokio::test]
15645 async fn prose_and_legacy_inference_never_import_into_candidates() {
15646 ask_test_home();
15647 let tmp = tempfile::tempdir().unwrap();
15648 let repo = tmp.path().join("repo");
15649 std::fs::create_dir_all(&repo).unwrap();
15650 init_repo(&repo);
15651 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15652 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15653 .await
15654 .unwrap();
15655 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15656 git::git(&repo, &["add", "."]).await.unwrap();
15657 git::git(&repo, &["commit", "-m", "historical implementation"])
15658 .await
15659 .unwrap();
15660 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15661 git::git(&repo, &["checkout", "main"]).await.unwrap();
15662 for instruction in [
15663 format!("Example: magi/history/A at {source}"),
15664 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15665 format!("Never import magi/history/A or {source}"),
15666 format!("Conversation answer: I mentioned {source} only as background"),
15667 ] {
15668 let mut runner = runner_at(RunStatus::Prep);
15669 runner.state.repo = repo.clone();
15670 runner.state.base_commit = base.clone();
15671 runner.state.instruction = instruction.clone();
15672 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15673 runner.state.config.blind.commit_msg_hook = false;
15674 runner.state.seeds = vec![refs::Seed {
15675 token: "magi/history/A".into(),
15676 sha: source.clone(),
15677 kind: refs::SeedKind::Unmerged,
15678 branch: true,
15679 detail: String::new(),
15680 }];
15681 runner.roles.implementers = vec![spec("one")];
15682 assert_eq!(seeded_instruction(&runner.state), instruction);
15683 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15685 legacy.as_object_mut().unwrap().remove("sources");
15686 runner.state = serde_json::from_value(legacy).unwrap();
15687 assert_eq!(runner.state.sources, refs::Sources::default());
15688 runner.state.save().unwrap();
15689 runner.state = RunState::load(&runner.state.id).unwrap();
15690 runner.prep().await.unwrap();
15691 for candidate in &runner.state.candidates {
15692 assert_eq!(
15693 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15694 base
15695 );
15696 assert!(!candidate.worktree.join("unwanted").exists());
15697 }
15698 }
15699 }
15700
15701 #[tokio::test]
15702 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15703 ask_test_home();
15704 let tmp = tempfile::tempdir().unwrap();
15705 let repo = tmp.path().join("repo");
15706 std::fs::create_dir_all(&repo).unwrap();
15707 init_repo(&repo);
15708 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15709 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15710 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15711 git::git(&repo, &["add", "."]).await.unwrap();
15712 git::git(&repo, &["commit", "-m", "required seed"])
15713 .await
15714 .unwrap();
15715 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15716 git::git(&repo, &["checkout", "main"]).await.unwrap();
15717 let mut runner = runner_at(RunStatus::Prep);
15718 runner.state.repo = repo.clone();
15719 runner.state.base_commit = base.clone();
15720 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15721 runner.state.config.graph.advise = false;
15722 runner.state.config.blind.commit_msg_hook = false;
15723 runner.state.sources =
15724 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15725 .await
15726 .unwrap();
15727 runner.roles.implementers = vec![spec("one"), spec("two")];
15728 runner.roles.judges = vec![spec("judge")];
15729 let label = blind::assign_labels(2, runner.state.seed)[0];
15730 let branch = runner.state.branch_for(label);
15731 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15732 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15733 .await
15734 .unwrap();
15735 runner.state.candidates.push(Candidate {
15736 index: 0,
15737 label,
15738 agent: "one".into(),
15739 branch,
15740 worktree,
15741 summary: String::new(),
15742 stat: String::new(),
15743 files: 0,
15744 commits: 0,
15745 empty: false,
15746 failed: None,
15747 verified_noop: None,
15748 duration_ms: 0,
15749 folded: false,
15750 });
15751 runner.state.save().unwrap();
15752 runner.state = RunState::load(&runner.state.id).unwrap();
15754 assert_eq!(runner.state.prep_complete, Some(false));
15755 runner.prep().await.unwrap();
15756 assert_eq!(runner.state.prep_complete, Some(true));
15757 assert_eq!(runner.state.candidates.len(), 2);
15758 for candidate in &runner.state.candidates {
15759 assert_eq!(
15760 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15761 git::tree_of(&repo, &seed).await.unwrap()
15762 );
15763 assert_eq!(
15764 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15765 "required commit"
15766 );
15767 }
15768 assert!(runner.state.worktree_root().join("judge-1").exists());
15769 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15770 runner.prep().await.unwrap();
15771 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15772
15773 runner.state.prep_complete = Some(false);
15774 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15775 run: Some(format!(
15776 "test -f '{}'",
15777 tmp.path().join("dependency-ready").display()
15778 )),
15779 ..Default::default()
15780 }];
15781 assert!(runner.prep().await.is_err());
15782 assert!(runner.state.prep_failure.is_some());
15783 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15784 runner.state = RunState::load(&runner.state.id).unwrap();
15785 assert!(
15786 runner.prep().await.is_err(),
15787 "correcting the cause alone must not auto-retry"
15788 );
15789 let failed_at = runner.state.events.last().unwrap().at;
15790 assert!(
15791 runner.retry_preparation(failed_at).is_err(),
15792 "old approvals cannot unlock the hold"
15793 );
15794 let mut task = crate::queue::Task::new(
15795 "retry".into(),
15796 "retry setup".into(),
15797 repo.clone(),
15798 crate::queue::Source::Human,
15799 );
15800 task.start(runner.state.id.clone());
15801 task.hold_machine(Some("setup failed".into()));
15802 task.release_operator("cli", None);
15803 runner
15804 .retry_preparation(task.resumption.as_ref().unwrap().at)
15805 .unwrap();
15806 runner.prep().await.unwrap();
15807 assert!(runner.state.prep_failure.is_none());
15808 assert_eq!(runner.state.prep_complete, Some(true));
15809 assert_eq!(runner.state.candidates.len(), 2);
15810 assert!(runner.state.worktree_root().join("judge-1").exists());
15811 for candidate in &runner.state.candidates {
15812 assert_eq!(
15813 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15814 git::tree_of(&repo, &seed).await.unwrap()
15815 );
15816 }
15817 }
15818
15819 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15820 let mut heads = Vec::new();
15821 for candidate in candidates {
15822 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15823 }
15824 heads
15825 }
15826
15827 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15828 ask_test_home();
15829 let tmp = tempfile::tempdir().unwrap();
15830 let repo = tmp.path().join("repo");
15831 std::fs::create_dir(&repo).unwrap();
15832 init_repo(&repo);
15833 let mut runner = runner_at(RunStatus::Reviewing);
15834 runner.state.repo = repo.clone();
15835 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15836 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15837 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15838 let branch = format!("candidate-{label}");
15839 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15840 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15841 .await
15842 .unwrap();
15843 if label == 'B' {
15844 std::fs::write(worktree.join("change"), "winner").unwrap();
15845 git::rescue_commit(&worktree, "candidate change")
15846 .await
15847 .unwrap();
15848 }
15849 runner.state.candidates.push(Candidate {
15850 index,
15851 label,
15852 agent: "alpha".into(),
15853 branch,
15854 worktree,
15855 summary: "completed reply".into(),
15856 stat: String::new(),
15857 files: usize::from(label == 'B'),
15858 commits: usize::from(label == 'B'),
15859 empty: label == 'A',
15860 failed: None,
15861 verified_noop: None,
15862 duration_ms: 1,
15863 folded: false,
15864 });
15865 }
15866 runner.state.judge_skipped = true;
15867 runner.state.tally = Some(Tally {
15868 first_choice: BTreeMap::new(),
15869 borda: BTreeMap::new(),
15870 winner: 'B',
15871 rankings: 0,
15872 unanimous_initial: false,
15873 deliberated: false,
15874 changed_votes: 0,
15875 unanimous_final: false,
15876 tie_break: None,
15877 judges: 0,
15878 present: 0,
15879 quorum: 0,
15880 met_quorum: true,
15881 uncontested: Some("only B produced a change".into()),
15882 });
15883 runner.state.save().unwrap();
15884 (tmp, runner)
15885 }
15886
15887 #[tokio::test]
15888 #[cfg(unix)]
15889 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15890 for (solo, failed_checkpoint) in
15891 [(true, false), (false, false), (true, true), (false, true)]
15892 {
15893 let (_tmp, mut runner) = preservation_resume_fixture().await;
15894 if solo {
15895 runner.state.candidates.retain(|c| c.label == 'A');
15896 }
15897 runner.state.candidates[0].empty = false;
15898 runner.state.candidates[0].summary.clear();
15899 runner.state.tally = None;
15900 runner.state.judge_skipped = false;
15901 runner.state.status = RunStatus::Implementing;
15902 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15903 let mut implementer = spec("alpha");
15904 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15905 runner.roles.implementers = vec![implementer];
15906 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15907 if failed_checkpoint {
15908 std::fs::create_dir_all(&patch).unwrap();
15909 }
15910 runner.state.save().unwrap();
15911 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15912 drop(pressure);
15913 runner.state = RunState::load(&runner.state.id).unwrap();
15914 assert_eq!(runner.state.status, RunStatus::Blocked);
15915 assert!(runner.state.interrupted_implementations.contains(&'A'));
15916 assert_eq!(runner.state.candidates[0].commits, 0);
15917 assert!(!runner.state.candidates[0].empty);
15918 assert!(runner.state.candidates[0].failed.is_none());
15919 if failed_checkpoint {
15920 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15921 std::fs::remove_dir(patch).unwrap();
15922 }
15923 runner.state.resource_block = None;
15924 runner.state.config.disk.min_free_bytes = 0;
15925 runner.roles.implementers[0].command = vec![
15926 "sh".into(),
15927 "-c".into(),
15928 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15929 ];
15930 runner.implement().await.unwrap();
15931 assert!(runner.state.interrupted_implementations.is_empty());
15932 assert!(runner.state.candidate_preservation.is_empty());
15933 assert_eq!(runner.state.candidates[0].commits, 1);
15934 assert!(!runner.state.candidates[0].empty);
15935 assert!(
15936 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15937 .unwrap()
15938 .contains("retried empty interrupted seat")
15939 );
15940 if !solo {
15941 assert_eq!(runner.state.candidates[1].commits, 1);
15942 }
15943 }
15944 }
15945
15946 #[tokio::test]
15947 #[cfg(unix)]
15948 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15949 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15950 let (_tmp, mut runner) = preservation_resume_fixture().await;
15951 runner.state.candidates.retain(|c| c.label == 'A');
15952 runner.state.candidates[0].empty = false;
15953 runner.state.candidates[0].summary.clear();
15954 runner.state.tally = None;
15955 runner.state.judge_skipped = false;
15956 runner.state.status = RunStatus::Implementing;
15957 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15960 let wt = runner.state.candidates[0].worktree.clone();
15961 std::fs::write(
15962 wt.join("interrupted-source"),
15963 "retain implementation work\n",
15964 )
15965 .unwrap();
15966 if committed {
15967 git::rescue_commit(&wt, "implementation checkpoint")
15968 .await
15969 .unwrap();
15970 }
15971 let lock = git::git(
15972 &wt,
15973 &[
15974 "rev-parse",
15975 "--path-format=absolute",
15976 "--git-path",
15977 "index.lock",
15978 ],
15979 )
15980 .await
15981 .unwrap();
15982 if block_rescue {
15983 std::fs::write(&lock, "fixture-owned lock").unwrap();
15984 }
15985 let mut implementer = spec("alpha");
15986 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15987 runner.roles.implementers = vec![implementer];
15988 runner.state.save().unwrap();
15989 let result = runner.implement().await;
15990 drop(pressure);
15991 assert_eq!(result.is_err(), block_rescue);
15992 assert_eq!(
15993 runner.state.resource_block.as_ref().unwrap().kind,
15994 "disk pressure"
15995 );
15996 assert!(
15997 runner
15998 .state
15999 .events
16000 .iter()
16001 .any(|e| e.message.contains("agent execution interrupted"))
16002 );
16003 assert!(
16004 runner
16005 .state
16006 .dir()
16007 .join("artifacts/candidate-A-recovery/staged.patch")
16008 .exists()
16009 );
16010 if block_rescue {
16011 let saved = RunState::load(&runner.state.id).unwrap();
16012 assert!(saved.candidate_preservation.contains_key(&'A'));
16013 assert!(!saved.candidates[0].empty);
16014 assert_eq!(
16015 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16016 "retain implementation work\n"
16017 );
16018 std::fs::remove_file(lock).unwrap();
16019 runner.state = saved;
16020 } else {
16021 assert_eq!(runner.state.status, RunStatus::Blocked);
16022 assert_eq!(runner.state.candidates[0].commits, 1);
16023 }
16024 runner.roles.implementers.clear();
16027 runner.state.config.disk.min_free_bytes = 0;
16028 runner.state.resource_block = None;
16029 runner.implement().await.unwrap();
16030 assert!(runner.state.candidate_preservation.is_empty());
16031 assert_eq!(runner.state.candidates[0].commits, 1);
16032 assert!(!runner.state.candidates[0].empty);
16033 assert!(git::is_clean(&wt).await.unwrap());
16034 assert!(
16035 crate::run::read_artifact(&runner.state, "cand-A.patch")
16036 .unwrap()
16037 .contains("retain implementation work")
16038 );
16039 }
16040 }
16041
16042 #[tokio::test]
16043 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16044 let (_tmp, mut runner) = preservation_resume_fixture().await;
16045 runner.implement().await.unwrap();
16047 assert!(runner.state.candidate_preservation.is_empty());
16048 assert!(
16049 !runner
16050 .state
16051 .dir()
16052 .join("artifacts/candidate-A-recovery")
16053 .exists()
16054 );
16055 runner.fold_losers().await.unwrap();
16056 assert!(!runner.state.candidates[0].worktree.exists());
16057 for schema in [14, crate::run::SCHEMA] {
16058 runner.state.schema = schema;
16059 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16060 runner.state.save().unwrap();
16061 runner.state = RunState::load(&runner.state.id).unwrap();
16062 runner.implement().await.unwrap();
16063 assert!(runner.state.candidate_preservation.is_empty());
16064 assert!(runner.state.candidates[0].folded);
16065 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16066 }
16067 }
16068
16069 #[tokio::test]
16070 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16071 let (_tmp, mut runner) = preservation_resume_fixture().await;
16072 runner.state.candidates.retain(|c| c.label == 'A');
16073 runner.state.tally = None;
16074 runner.state.judge_skipped = false;
16075 let candidate = &mut runner.state.candidates[0];
16076 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16077 candidate.failed = Some("deployment validation unavailable".into());
16078 std::fs::write(
16079 candidate.worktree.join("repair"),
16080 "future launcher policy\n",
16081 )
16082 .unwrap();
16083 runner
16084 .state
16085 .candidate_preservation
16086 .insert('A', "awaiting persistence".into());
16087 runner.persist_candidates().await.unwrap();
16088 let candidate = &runner.state.candidates[0];
16089 assert!(!candidate.empty);
16090 assert_eq!(candidate.commits, 1);
16091 assert_eq!(candidate.files, 1);
16092 assert!(candidate.failed.is_none());
16093 assert!(candidate.summary.contains("EPERM"));
16094 assert!(
16095 crate::run::read_artifact(&runner.state, "cand-A.patch")
16096 .unwrap()
16097 .contains("future launcher policy")
16098 );
16099 assert!(runner.state.candidate_preservation.is_empty());
16100 }
16101
16102 #[tokio::test]
16103 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16104 let (_tmp, mut runner) = preservation_resume_fixture().await;
16105 let wt = runner.state.candidates[0].worktree.clone();
16106 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16107 runner.state.schema = 14;
16108 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16109 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16110 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16111 for _ in 0..2 {
16112 assert!(runner.implement().await.is_err());
16113 assert_eq!(runner.state.status, RunStatus::Blocked);
16114 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16115 assert!(
16116 runner.state.candidates[0].empty,
16117 "old verdict must remain unusable"
16118 );
16119 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16120 assert!(runner.state.judge_skipped);
16121 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16122 assert!(
16123 crate::run::read_artifact(&runner.state, "cand-A.patch")
16124 .unwrap()
16125 .contains("owned edits")
16126 );
16127 assert!(wt.exists());
16128 assert_eq!(
16129 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16130 .await
16131 .unwrap(),
16132 1
16133 );
16134 runner.state = RunState::load(&runner.state.id).unwrap();
16135 }
16136 }
16137
16138 #[tokio::test]
16139 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16140 let (_tmp, mut runner) = preservation_resume_fixture().await;
16141 let wt = runner.state.candidates[0].worktree.clone();
16142 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16143 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16144 let index_lock = PathBuf::from(
16145 git::git(
16146 &wt,
16147 &[
16148 "rev-parse",
16149 "--path-format=absolute",
16150 "--git-path",
16151 "index.lock",
16152 ],
16153 )
16154 .await
16155 .unwrap(),
16156 );
16157 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16158 runner.state.schema = 14;
16159 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16160 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16161 assert!(runner.implement().await.is_err());
16162 assert!(runner.after_implement().is_err());
16163 assert!(runner.fold_losers().await.is_err());
16164 assert_eq!(
16165 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16166 "failed seat edits"
16167 );
16168 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16169 runner.state = RunState::load(&runner.state.id).unwrap();
16170 std::fs::remove_file(index_lock).unwrap();
16171 assert!(runner.implement().await.is_err());
16173 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16174 assert!(runner.state.candidates[0].empty);
16175 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16176 assert!(
16177 crate::run::read_artifact(&runner.state, "cand-A.patch")
16178 .unwrap()
16179 .contains("failed seat edits")
16180 );
16181 assert!(wt.exists());
16182 }
16183
16184 #[tokio::test]
16185 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16186 let (_tmp, mut runner) = preservation_resume_fixture().await;
16187 let repo = runner.state.repo.clone();
16188 let common = git::git(
16189 &repo,
16190 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16191 )
16192 .await
16193 .unwrap();
16194 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16195 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16196 assert!(runner.fold_losers().await.is_err());
16197 assert!(!runner.state.candidates[0].worktree.exists());
16198 assert!(!runner.state.candidates[0].folded);
16199 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16200 runner.state = RunState::load(&runner.state.id).unwrap();
16201 std::fs::remove_file(branch_lock).unwrap();
16202 let judge = runner.state.worktree_root().join("judge-1");
16203 git::worktree_add_detached(&repo, &judge, "main")
16204 .await
16205 .unwrap();
16206 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16207 runner.roles.judges = vec![spec("judge")];
16208 assert!(runner.fold_losers().await.is_err());
16209 runner.state = RunState::load(&runner.state.id).unwrap();
16210 assert!(
16211 runner.state.candidates[0].folded,
16212 "candidate progress saved before judge failure"
16213 );
16214 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16215 assert!(judge.join("uncommitted").exists());
16216 std::fs::remove_file(judge.join("uncommitted")).unwrap();
16217 runner.fold_losers().await.unwrap();
16218 assert!(!judge.exists());
16219 runner.fold_losers().await.unwrap();
16220 }
16221
16222 #[tokio::test]
16223 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16224 let (_tmp, mut runner) = preservation_resume_fixture().await;
16225 runner.state.candidates.retain(|c| c.label == 'B');
16226 runner.state.tally = None;
16227 runner.state.judge_skipped = false;
16228 runner.state.prep_complete = Some(true);
16229 runner.state.config.graph.advise = false;
16230 runner.state.config.graph.selected_implementer = Some("alpha".into());
16231 let wt = runner.state.candidates[0].worktree.clone();
16232 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16233 runner
16234 .state
16235 .candidate_preservation
16236 .insert('B', "awaiting persistence".into());
16237 let index_lock = git::git(
16238 &wt,
16239 &[
16240 "rev-parse",
16241 "--path-format=absolute",
16242 "--git-path",
16243 "index.lock",
16244 ],
16245 )
16246 .await
16247 .unwrap();
16248 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16249 runner.state.save().unwrap();
16250 assert!(runner.execute_graph().await.is_err());
16251 assert_eq!(runner.state.status, RunStatus::Blocked);
16252 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16253 std::fs::remove_file(index_lock).unwrap();
16254 for _ in 0..2 {
16257 runner.state = RunState::load(&runner.state.id).unwrap();
16258 runner.pause.park();
16259 runner.execute_graph().await.unwrap();
16260 assert_eq!(runner.state.status, RunStatus::Implementing);
16261 assert!(runner.state.candidate_preservation.is_empty());
16262 assert_eq!(runner.state.candidates[0].commits, 2);
16263 assert!(runner.state.tally.is_none());
16264 }
16265 runner.implement().await.unwrap();
16266 assert_eq!(runner.state.status, RunStatus::Judging);
16267 runner.state.status = RunStatus::Blocked;
16269 runner.state.candidates[0].failed = Some("implementation failed".into());
16270 runner.state.save().unwrap();
16271 let error = runner.execute_graph().await.unwrap_err();
16272 assert!(
16273 error
16274 .to_string()
16275 .contains("selected implementation is held")
16276 );
16277 }
16278
16279 #[tokio::test]
16280 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16281 let (_tmp, mut runner) = preservation_resume_fixture().await;
16282 let repo = runner.state.repo.clone();
16283 let candidate = runner.state.candidates[0].clone();
16284 let registered = git::worktree_holding(&repo, &candidate.branch)
16287 .await
16288 .unwrap()
16289 .unwrap();
16290 assert_eq!(
16291 registered.canonicalize().unwrap(),
16292 candidate.worktree.canonicalize().unwrap()
16293 );
16294 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16295 assert_eq!(
16296 git::worktree_holding(&repo, &candidate.branch)
16297 .await
16298 .unwrap(),
16299 Some(registered)
16300 );
16301 assert!(
16302 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16303 "the stale registration initially keeps the branch checked out"
16304 );
16305 runner.fold_losers().await.unwrap();
16306 assert!(runner.state.candidates[0].folded);
16307 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16308 assert_eq!(
16309 git::worktree_holding(&repo, &candidate.branch)
16310 .await
16311 .unwrap(),
16312 None
16313 );
16314 runner.state = RunState::load(&runner.state.id).unwrap();
16315 runner.fold_losers().await.unwrap();
16316 }
16317
16318 #[tokio::test]
16319 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16320 let (_tmp, mut runner) = preservation_resume_fixture().await;
16321 let repo = runner.state.repo.clone();
16322 let candidate = runner.state.candidates[0].clone();
16323 let winner = runner.state.candidates[1].clone();
16324 let registered = git::worktree_holding(&repo, &candidate.branch)
16325 .await
16326 .unwrap()
16327 .unwrap();
16328 assert_eq!(
16329 registered.canonicalize().unwrap(),
16330 candidate.worktree.canonicalize().unwrap()
16331 );
16332 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16333 assert_eq!(
16334 git::worktree_holding(&repo, &candidate.branch)
16335 .await
16336 .unwrap(),
16337 Some(registered)
16338 );
16339 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16340
16341 for _ in 0..2 {
16342 fold_run(&mut runner.state, false, &crate::run::home())
16343 .await
16344 .unwrap();
16345 runner.state = RunState::load(&runner.state.id).unwrap();
16346 assert!(runner.state.candidates[0].folded);
16347 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16348 assert_eq!(
16349 git::worktree_holding(&repo, &candidate.branch)
16350 .await
16351 .unwrap(),
16352 None
16353 );
16354 assert!(!runner.state.candidates[1].folded);
16355 assert!(winner.worktree.exists());
16356 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16357 }
16358 }
16359
16360 #[tokio::test]
16361 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16362 ask_test_home();
16363 let tmp = tempfile::tempdir().unwrap();
16364 let repo = tmp.path().join("repo");
16365 std::fs::create_dir(&repo).unwrap();
16366 init_repo(&repo);
16367 let mut runner = runner_at(RunStatus::Implementing);
16368 runner.state.repo = repo.clone();
16369 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16370 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16371 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16372 let branch = format!("candidate-{label}");
16373 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16374 git::worktree_add_branch(&repo, &wt, &branch, "main")
16375 .await
16376 .unwrap();
16377 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16378 git::git(&wt, &["add", "README.md"]).await.unwrap();
16379 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16380 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16381 runner.state.candidates.push(Candidate {
16382 index,
16383 label,
16384 agent: "alpha".to_owned(),
16385 branch,
16386 worktree: wt,
16387 summary: "completed implementation".to_owned(),
16388 stat: String::new(),
16389 files: 0,
16390 commits: 0,
16391 empty: false,
16392 failed: None,
16393 verified_noop: None,
16394 duration_ms: 42,
16395 folded: false,
16396 });
16397 runner
16398 .state
16399 .candidate_preservation
16400 .insert(label, "awaiting persistence".to_owned());
16401 }
16402 let wt = runner.state.candidates[0].worktree.clone();
16403 let index_lock = git::git(
16404 &wt,
16405 &[
16406 "rev-parse",
16407 "--path-format=absolute",
16408 "--git-path",
16409 "index.lock",
16410 ],
16411 )
16412 .await
16413 .unwrap();
16414 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16415 runner.state.save().unwrap();
16416 assert!(runner.persist_candidates().await.is_err());
16417 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16418 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16419 assert_eq!(
16420 runner.state.candidates[1].commits, 1,
16421 "other results are still collected"
16422 );
16423 assert!(
16424 !runner.state.candidates[0].empty,
16425 "failed rescue is not no change"
16426 );
16427 assert_eq!(
16428 runner.state.candidates[0].summary,
16429 "completed implementation"
16430 );
16431 assert!(runner.after_implement().is_err());
16432 let home = tmp.path().join("fold-home");
16433 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16434 assert!(wt.join("new-file").exists());
16435 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16436 assert!(recovery.join("staged.patch").exists());
16437 assert!(recovery.join("unstaged.patch").exists());
16438 assert_eq!(
16439 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16440 "untracked edits\n"
16441 );
16442 runner.state.begin_pass();
16443 runner.state.end_pass_parked(ParkCause::Upgrade {
16444 from: "1.0".into(),
16445 to: Some("1.1".into()),
16446 });
16447 runner.state.save().unwrap();
16448 runner.state = RunState::load(&runner.state.id).unwrap();
16449 assert!(matches!(
16450 runner.state.passes.last().unwrap().end,
16451 Some(crate::run::PassEnd::Parked {
16452 cause: ParkCause::Upgrade { .. }
16453 })
16454 ));
16455 std::fs::remove_file(index_lock).unwrap();
16456 runner.implement().await.unwrap();
16458 assert!(runner.state.candidate_preservation.is_empty());
16459 assert!(
16460 runner
16461 .state
16462 .candidates
16463 .iter()
16464 .all(|c| c.commits == 1 && !c.empty)
16465 );
16466 assert!(git::is_clean(&wt).await.unwrap());
16467 assert!(
16468 crate::run::read_artifact(&runner.state, "cand-A.patch")
16469 .unwrap()
16470 .contains("later tracked edits")
16471 );
16472 }
16473
16474 #[tokio::test]
16475 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16476 ask_test_home();
16477 let tmp = tempfile::tempdir().unwrap();
16478 let repo = tmp.path().join("repo");
16479 std::fs::create_dir(&repo).unwrap();
16480 init_repo(&repo);
16481 let wt = tmp.path().join("candidate");
16482 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16483 .await
16484 .unwrap();
16485 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16486 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16487 assert!(wt.join("uncommitted").exists());
16488 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16489 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16490 let mut runner = runner_at(RunStatus::Implementing);
16491 runner.state.repo = repo.clone();
16492 runner.state.base_commit = "missing-base".to_owned();
16493 runner.state.candidates.push(Candidate {
16494 index: 0,
16495 label: 'A',
16496 agent: "alpha".into(),
16497 branch: "candidate".into(),
16498 worktree: wt.clone(),
16499 summary: "completed".into(),
16500 stat: String::new(),
16501 files: 0,
16502 commits: 0,
16503 empty: false,
16504 failed: None,
16505 verified_noop: None,
16506 duration_ms: 1,
16507 folded: false,
16508 });
16509 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16510 assert!(
16511 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16512 .await
16513 .is_err()
16514 );
16515 assert!(!runner.state.candidates[0].folded);
16516 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16517 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16518 runner
16519 .state
16520 .candidate_preservation
16521 .insert('A', "pending".into());
16522 assert!(runner.persist_candidates().await.is_err());
16523 assert!(!runner.state.candidates[0].empty);
16524 assert!(!runner.state.candidates[0].folded);
16525 assert!(wt.exists());
16526 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16527 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16528 std::fs::create_dir_all(&patch_path).unwrap();
16529 assert!(
16530 runner.persist_candidates().await.is_err(),
16531 "artifact failure stays pending"
16532 );
16533 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16534 assert!(!runner.state.candidates[0].empty);
16535 std::fs::remove_dir(patch_path).unwrap();
16536 runner.persist_candidates().await.unwrap();
16537 assert!(
16538 runner.state.candidates[0].empty,
16539 "only successful inspection confirms no change"
16540 );
16541 }
16542
16543 fn init_repo(dir: &Path) {
16544 let run = |args: &[&str]| {
16545 let out = std::process::Command::new("git")
16546 .args(args)
16547 .current_dir(dir)
16548 .quiet()
16549 .output()
16550 .expect("spawn git");
16551 assert!(
16552 out.status.success(),
16553 "git {args:?} failed: {}",
16554 String::from_utf8_lossy(&out.stderr)
16555 );
16556 };
16557 run(&["init", "-b", "main"]);
16558 run(&["config", "user.name", "magi test"]);
16559 run(&["config", "user.email", "magi@example.com"]);
16560 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16561 run(&["add", "-A"]);
16562 run(&["commit", "-m", "init"]);
16563 }
16564
16565 fn ask_test_home() {
16573 crate::run::pin_test_home();
16574 }
16575
16576 #[test]
16577 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16578 ask_test_home();
16579 let mut runner = runner_at(RunStatus::Prep);
16580 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16581 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16584 let error = guard
16585 .finish(
16586 &mut runner.state,
16587 Err(
16588 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16589 .into(),
16590 ),
16591 )
16592 .unwrap_err();
16593 assert_eq!(
16594 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16595 std::io::ErrorKind::PermissionDenied
16596 );
16597 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16598 let message = format!("{error:#}");
16599 assert!(message.contains("original refusal"));
16600 assert!(!message.contains("could not save terminal"));
16601 assert_eq!(
16602 error.chain().count(),
16603 2,
16604 "only startup attribution wraps the original error"
16605 );
16606 let saved = RunState::load(&runner.state.id).unwrap();
16607 assert_eq!(saved.status, RunStatus::Blocked);
16608 assert!(!saved.startup.unwrap().complete);
16609 }
16610
16611 #[test]
16612 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16613 ask_test_home();
16614 let mut runner = runner_at(RunStatus::Prep);
16615 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16616 let id = runner.state.id.clone();
16617 drop(guard);
16618 let saved = RunState::load(&id).unwrap();
16619 assert_eq!(saved.status, RunStatus::Blocked);
16620 assert!(!saved.startup.as_ref().unwrap().complete);
16621 assert!(
16622 saved
16623 .startup
16624 .unwrap()
16625 .failure
16626 .unwrap()
16627 .contains("interrupted")
16628 );
16629 assert!(Runner::resume(&id).is_err());
16630 }
16631
16632 #[test]
16633 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16634 ask_test_home();
16635 let mut runner = runner_at(RunStatus::Prep);
16636 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16637 let error = guard
16638 .finish(
16639 &mut runner.state,
16640 Err(
16641 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16642 .into(),
16643 ),
16644 )
16645 .unwrap_err();
16646 assert_eq!(
16647 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16648 std::io::ErrorKind::PermissionDenied
16649 );
16650 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16651 let mut task = crate::queue::Task::new(
16652 "review".into(),
16653 "criteria".into(),
16654 runner.state.repo.clone(),
16655 crate::queue::Source::Human,
16656 );
16657 task.record_startup_failure(failure.clone());
16658 task.record_startup_failure(failure.clone());
16659 assert_eq!(task.startup_failures.len(), 1);
16660 assert!(task.runs.is_empty());
16661 assert_eq!(task.attempts, 0);
16662 }
16663
16664 #[cfg(unix)]
16665 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16666 ask_test_home();
16667 let tmp = tempfile::tempdir().unwrap();
16668 let repo = tmp.path().join("repo");
16669 std::fs::create_dir(&repo).unwrap();
16670 init_repo(&repo);
16671 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16672 let origin = tmp.path().join("origin.git");
16673 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16674 .await
16675 .unwrap();
16676 git::git(
16677 &repo,
16678 &["remote", "add", "origin", origin.to_str().unwrap()],
16679 )
16680 .await
16681 .unwrap();
16682 git::git(&repo, &["checkout", "-b", "fix/retained"])
16683 .await
16684 .unwrap();
16685 std::fs::write(repo.join("retained"), "retained change").unwrap();
16686 git::git(&repo, &["add", "."]).await.unwrap();
16687 git::git(&repo, &["commit", "-m", "retained implementation"])
16688 .await
16689 .unwrap();
16690 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16691 git::git(&repo, &["checkout", "main"]).await.unwrap();
16692 std::fs::write(repo.join("base-change"), "new base").unwrap();
16693 git::git(&repo, &["add", "."]).await.unwrap();
16694 git::git(&repo, &["commit", "-m", "base moved"])
16695 .await
16696 .unwrap();
16697 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16698 git::git(&repo, &["checkout", "fix/retained"])
16699 .await
16700 .unwrap();
16701 let mut runner = runner_at(RunStatus::Voting);
16702 runner.state.repo = repo.clone();
16703 runner.state.base_commit = base;
16704 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16705 runner.state.config.disk.min_free_bytes = 0;
16706 runner.state.config.graph.review_rounds = 1;
16707 runner.state.config.verify.e2e = vec!["true".into()];
16708 runner.state.config.verify.gate = vec!["true".into()];
16709 runner.state.candidates = vec![Candidate {
16710 index: 0,
16711 label: 'A',
16712 agent: "fixture".into(),
16713 branch: "fix/retained".into(),
16714 worktree: repo.clone(),
16715 summary: String::new(),
16716 stat: String::new(),
16717 files: 1,
16718 commits: 1,
16719 empty: false,
16720 failed: None,
16721 verified_noop: None,
16722 duration_ms: 0,
16723 folded: false,
16724 }];
16725 runner.state.tally = Some(Tally {
16726 first_choice: BTreeMap::new(),
16727 borda: BTreeMap::new(),
16728 winner: 'A',
16729 rankings: 0,
16730 unanimous_initial: true,
16731 deliberated: false,
16732 changed_votes: 0,
16733 unanimous_final: true,
16734 tie_break: None,
16735 judges: 0,
16736 present: 0,
16737 quorum: 0,
16738 met_quorum: true,
16739 uncontested: Some("review-only".into()),
16740 });
16741 let mut task = crate::queue::Task::new(
16742 "fixture".into(),
16743 "fixture".into(),
16744 repo,
16745 crate::queue::Source::Human,
16746 );
16747 task.link_run(&runner.state.id);
16748 crate::queue::Queue::open().put(&mut task).unwrap();
16749 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16750 runner.state.base_sync = Some(BaseSync {
16751 tip: String::new(),
16752 behind: 1,
16753 attempts: BASE_SYNC_ROUNDS,
16754 conflict: None,
16755 already_in: None,
16756 });
16757 runner.sync_to_base().await.unwrap();
16758 assert_eq!(runner.state.status, RunStatus::Blocked);
16759 (tmp, runner, head, task.id)
16760 }
16761
16762 #[cfg(unix)]
16763 #[tokio::test]
16764 async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16765 for routed in [false, true] {
16766 let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16767 let home = crate::run::home();
16768 let queue = crate::queue::Queue::open();
16769 let task = queue.get(&owner).unwrap();
16770 runner.state.task_instruction = Some(task.instruction.clone());
16771 runner.state.driver_exited = true;
16772 runner.state.save().unwrap();
16773 let questions = crate::ask::Questions::at(home.join("questions"));
16774 let mut q = crate::ask::Question::new(
16775 runner.state.id.clone(),
16776 "base-recovery".into(),
16777 "owner".into(),
16778 "Recover bounded base movement?".into(),
16779 String::new(),
16780 vec!["recover".into()],
16781 );
16782 q.actions.insert(
16783 "recover".into(),
16784 crate::ask::ChoiceAction::RecoverBase {
16785 run: runner.state.id.clone(),
16786 stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16787 episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16788 },
16789 );
16790 q.routed_task = Some(owner.clone());
16791 q.answer(crate::ask::Answer::Choice("recover".into()))
16792 .unwrap();
16793 questions.put(&mut q).unwrap();
16794 crate::base_recovery_control::recover(
16795 &queue,
16796 &home,
16797 crate::base_recovery_control::Request {
16798 run: runner.state.id.clone(),
16799 head: head.clone(),
16800 actor: "decisions".into(),
16801 request: q.id.clone(),
16802 decision: Some((owner.clone(), q.answered_at.unwrap())),
16803 expected_stop: Some((
16804 runner.state.base_sync_recovery.stop.clone().unwrap(),
16805 runner.state.base_sync_recovery.episodes.last().unwrap().id,
16806 )),
16807 },
16808 )
16809 .await
16810 .unwrap();
16811 if routed {
16812 questions
16813 .update(&q.id, |q| {
16814 q.routed_task = Some("another-owner".into());
16815 Ok(())
16816 })
16817 .unwrap();
16818 } else {
16819 let mut task = queue.get(&owner).unwrap();
16820 task.hold_manual(Some("new owner decision".into()));
16821 task.status = crate::queue::TaskStatus::Queued;
16823 queue.put(&mut task).unwrap();
16824 }
16825 runner.state = RunState::load(&runner.state.id).unwrap();
16826 runner.sync_to_base().await.unwrap();
16827 let approval = runner
16828 .state
16829 .base_sync_recovery
16830 .authorizations
16831 .last()
16832 .unwrap();
16833 assert!(approval.consumed);
16834 assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
16835 "routing"
16836 } else {
16837 "owner decision"
16838 }));
16839 assert_eq!(
16840 runner.state.base_sync.as_ref().unwrap().attempts,
16841 BASE_SYNC_ROUNDS
16842 );
16843 assert_eq!(
16844 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16845 head
16846 );
16847 }
16848 }
16849
16850 #[cfg(unix)]
16851 #[tokio::test]
16852 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16853 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16854 if std::env::var_os(CHILD).is_none() {
16855 let home = tempfile::tempdir().unwrap();
16856 let output = std::process::Command::new(std::env::current_exe().unwrap())
16857 .args([
16858 "--exact",
16859 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16860 "--nocapture",
16861 ])
16862 .env(CHILD, "1")
16863 .env("MAGI_HOME", home.path())
16864 .env("MAGI_WORKER_HOME", home.path())
16865 .env("MAGI_WORKER_ID", "abcdef")
16866 .output()
16867 .unwrap();
16868 assert!(
16869 output.status.success()
16870 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16871 "{}\n{}",
16872 String::from_utf8_lossy(&output.stdout),
16873 String::from_utf8_lossy(&output.stderr)
16874 );
16875 return;
16876 }
16877 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16878 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16879 let home = crate::run::home();
16880 let queue = crate::queue::Queue::open();
16881 let mut owner = queue.get(&task).unwrap();
16882 owner.release_operator("test-release", None);
16883 owner.start(runner.state.id.clone());
16884 queue.put(&mut owner).unwrap();
16885 runner.state.driver_exited = true;
16886 runner.state.task_instruction = Some(owner.instruction.clone());
16887 runner.state.save().unwrap();
16888 let _claim = queue.claim(&task).unwrap();
16889 let reserve = |id: &str| {
16890 let directory = home.join("workers/jobs").join(id);
16891 std::fs::create_dir_all(&directory).unwrap();
16892 let job = crate::worker::Job {
16893 id: id.into(),
16894 sequence: 1,
16895 kind: "task".into(),
16896 key: format!("task:{task}"),
16897 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16898 generation: "fixture-generation".into(),
16899 version: None,
16900 status: crate::worker::Status::Running,
16901 result: None,
16902 pid: Some(std::process::id()),
16903 created_at: 0,
16904 finished_at: None,
16905 };
16906 std::fs::write(
16907 directory.join("job.json"),
16908 serde_json::to_vec(&job).unwrap(),
16909 )
16910 .unwrap();
16911 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16912 };
16913 let _own = reserve("abcdef");
16914 let other = reserve("fedcba");
16915 assert!(
16916 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16917 .await
16918 .is_err()
16919 );
16920 assert!(
16921 RunState::load(&runner.state.id)
16922 .unwrap()
16923 .base_sync_recovery
16924 .authorizations
16925 .is_empty()
16926 );
16927 drop(other);
16928 assert!(crate::daemon::is_working_on(
16929 &home,
16930 &runner.state.id,
16931 Timestamp::now()
16932 ));
16933 assert!(
16934 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16935 );
16936 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16937 .await
16938 .unwrap();
16939 assert!(
16940 RunState::load(&runner.state.id)
16941 .unwrap()
16942 .base_recovery_pending()
16943 );
16944 }
16945
16946 #[cfg(unix)]
16947 #[tokio::test]
16948 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16949 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16950 let home = crate::run::home();
16951 runner.state.driver_pid = Some(std::process::id());
16952 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16953 runner.state.driver_exited = false;
16954 runner.state.begin_pass();
16955 runner.state.save().unwrap();
16956 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16957 driver.record(&runner.state).unwrap();
16958 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16959 drop(driver);
16961 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16962 assert_eq!(
16963 runner.state.liveness_under(&home, false),
16964 crate::run::Liveness::Dead
16965 );
16966 let queue = crate::queue::Queue::open();
16967 let _claim = queue.claim(&task).unwrap();
16968 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16969 .await
16970 .unwrap();
16971 let disk = RunState::load(&runner.state.id).unwrap();
16972 assert!(!disk.driver_exited);
16973 assert!(disk.base_recovery_pending());
16974 }
16975
16976 #[cfg(unix)]
16977 #[tokio::test]
16978 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16979 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16980 if std::env::var_os(CHILD).is_none() {
16981 use std::os::unix::fs::PermissionsExt;
16982 let tmp = tempfile::tempdir().unwrap();
16983 let gh = tmp.path().join("gh");
16984 std::fs::write(&gh, r#"#!/bin/sh
16985case "$1 $2" in
16986 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16987 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16988 *) exit 2 ;;
16989esac
16990"#).unwrap();
16991 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16992 let mut paths = vec![tmp.path().to_path_buf()];
16993 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16994 let output = std::process::Command::new(std::env::current_exe().unwrap())
16995 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16996 .env(CHILD, "1")
16997 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16998 .env("PATH", std::env::join_paths(paths).unwrap())
16999 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17000 .output().unwrap();
17001 assert!(
17002 output.status.success()
17003 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17004 "{}\n{}",
17005 String::from_utf8_lossy(&output.stdout),
17006 String::from_utf8_lossy(&output.stderr)
17007 );
17008 return;
17009 }
17010 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17011 let repo = runner.state.repo.clone();
17012 git::git(&repo, &["checkout", "main"]).await.unwrap();
17013 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17014 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17015 git::git(&repo, &["checkout", "fix/retained"])
17016 .await
17017 .unwrap();
17018 runner.state.pr = Some(crate::run::PrRecord {
17019 url: "https://example.test/project/pull/1".into(),
17020 number: 1,
17021 state: "open".into(),
17022 checks: "unknown".into(),
17023 round: 0,
17024 rounds: 1,
17025 red_at_merge: Vec::new(),
17026 });
17027 runner.sync_to_base().await.unwrap();
17028 assert_eq!(runner.state.status, RunStatus::Blocked);
17029 assert_eq!(
17030 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17031 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17032 );
17033 assert!(runner.state.base_sync_closure_pending());
17034 assert!(crate::handover::can_resume(&runner.state));
17035 let queue = crate::queue::Queue::open();
17036 assert!(!crate::base_sync_recovery::held_task(
17037 &queue,
17038 &queue.get(&task).unwrap()
17039 ));
17040 assert!(runner.state.authorize_base_sync(&task).is_err());
17041 runner.state.base_sync_recovery.stop = None;
17043 runner.state.status = RunStatus::Voting;
17044 runner.state.gate_ran = true;
17045 runner.state.config.graph.review_rounds = 0;
17046 runner.state.save().unwrap();
17047 assert!(crate::handover::can_resume(&runner.state));
17048 runner.state = RunState::load(&runner.state.id).unwrap();
17049 runner.execute_graph().await.unwrap();
17050 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17051 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17052 assert_eq!(
17053 runner.state.base_sync.as_ref().unwrap().attempts,
17054 BASE_SYNC_ROUNDS
17055 );
17056 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17057 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17058 assert!(runner.state.reviews.is_empty());
17059 assert!(runner.state.gate.is_empty());
17060 }
17061
17062 #[cfg(unix)]
17063 #[tokio::test]
17064 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17065 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17066 runner.state.driver_exited = true;
17067 runner.state.save().unwrap();
17068 let home = crate::run::home();
17069 let queue = crate::queue::Queue::open();
17070 let _claim = queue.claim(&task).unwrap();
17071 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17072 assert!(
17073 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17074 .await
17075 .is_err()
17076 );
17077 drop(lease);
17078 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17079 assert!(
17080 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17081 .await
17082 .is_err()
17083 );
17084 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17085 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17086 .await
17087 .unwrap();
17088 let disk = RunState::load(&runner.state.id).unwrap();
17089 assert!(disk.base_recovery_pending());
17090 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17091 assert!(
17092 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17093 .await
17094 .is_err()
17095 );
17096 assert_eq!(
17097 RunState::load(&runner.state.id)
17098 .unwrap()
17099 .base_sync_recovery
17100 .authorizations
17101 .len(),
17102 1
17103 );
17104 let mut owner = queue.get(&task).unwrap();
17105 owner.runs.push("20260101-000000-ffff".into());
17106 queue.put(&mut owner).unwrap();
17107 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17108 .await
17109 .unwrap_err();
17110 assert!(error.to_string().contains("run task ownership changed"));
17111 }
17112
17113 #[cfg(unix)]
17114 #[tokio::test]
17115 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17116 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17117 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17118 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17119 legacy["status"] = serde_json::json!("voting");
17120 runner.state = serde_json::from_value(legacy).unwrap();
17121 runner.state.save().unwrap();
17122 for _ in 0..3 {
17123 runner.execute_graph().await.unwrap();
17124 assert_eq!(runner.state.status, RunStatus::Blocked);
17125 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17126 assert!(!crate::handover::can_resume(&runner.state));
17127 assert_eq!(
17128 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17129 head
17130 );
17131 assert!(runner.state.reviews.is_empty());
17132 assert!(!runner.state.gate_ran);
17133 }
17134 let notice = crate::notices::run_ended(&runner.state).unwrap();
17135 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17136 runner.state.begin_pass();
17137 runner.state.status = RunStatus::Voting;
17138 runner.deliberate().await.unwrap();
17139 runner.vote().await.unwrap();
17140 runner.finish_after_tally().await.unwrap();
17141 assert_eq!(runner.state.status, RunStatus::Blocked);
17142 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17143 assert_eq!(
17144 crate::notices::run_ended(&runner.state).unwrap().since,
17145 notice.since
17146 );
17147 }
17148
17149 #[cfg(unix)]
17150 #[tokio::test]
17151 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17152 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17153 for detail in [
17154 "fix/retained conflicts with origin/main and did not rebase: conflict",
17155 "rebased fix/retained locally but origin refused the push",
17156 "unknown failure",
17157 ] {
17158 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17159 runner.state.record_base_sync_stop(&head);
17160 assert!(runner.state.authorize_base_sync(&task).is_err());
17161 runner.state.status = RunStatus::Voting;
17162 runner.execute_graph().await.unwrap();
17163 assert_eq!(runner.state.status, RunStatus::Blocked);
17164 assert!(!crate::handover::can_resume(&runner.state));
17165 assert_eq!(
17166 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17167 head
17168 );
17169 }
17170 }
17171
17172 #[cfg(unix)]
17173 #[tokio::test]
17174 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17175 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17176 runner.state.authorize_base_sync(&task).unwrap();
17177 assert!(runner.state.consume_base_recovery(&head));
17178 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17179 runner.state.reserve_base_operation(&head, &tip);
17180 runner.state.save().unwrap();
17181 runner.state = RunState::load(&runner.state.id).unwrap();
17182 assert!(!runner.state.consume_base_recovery(&head));
17183 runner.sync_to_base().await.unwrap();
17184 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17185 assert_eq!(runner.state.base_episode_used(), 1);
17186 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17187 assert!(
17188 runner
17189 .state
17190 .base_sync_recovery
17191 .operations
17192 .last()
17193 .unwrap()
17194 .finished
17195 );
17196 assert!(!runner.state.base_sync_blocked());
17197 runner.sync_to_base().await.unwrap();
17198 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17199 }
17200
17201 #[cfg(unix)]
17202 #[tokio::test]
17203 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17204 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17205 let original = runner
17207 .state
17208 .base_sync
17209 .as_ref()
17210 .unwrap()
17211 .conflict
17212 .clone()
17213 .unwrap();
17214 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17215 original
17216 .replace("synchronization(s)", "rebase(s)")
17217 .replace("synchronizing", "rebasing"),
17218 );
17219 runner.state.base_sync_recovery = Default::default();
17220 runner.state.record_base_sync_stop(&head);
17221 assert_eq!(
17222 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17223 crate::base_sync_recovery::StopKind::Exhausted
17224 );
17225 runner.state.authorize_base_sync(&task).unwrap();
17226 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17227 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17228 .as_object_mut()
17229 .unwrap();
17230 auth.remove("interval");
17231 auth.remove("scope");
17232 runner.state = serde_json::from_value(legacy).unwrap();
17233 assert!(runner.state.consume_base_recovery(&head));
17234 assert_eq!(runner.state.base_episode_used(), 3);
17235 runner.sync_to_base().await.unwrap();
17236 assert_eq!(runner.state.base_episode_used(), 4);
17237 runner.state.resource_block = Some(crate::run::ResourceBlock {
17238 node: "gate".into(),
17239 kind: "cache".into(),
17240 detail: "legacy wait".into(),
17241 at: Timestamp::now(),
17242 cause: crate::run::ResourceBlockCause::Unrecorded,
17243 });
17244 runner.state.save().unwrap();
17245 assert!(
17246 runner
17247 .state
17248 .base_sync_recovery
17249 .episodes
17250 .last()
17251 .unwrap()
17252 .boundary
17253 .is_none()
17254 );
17255 runner.state.resource_block = None;
17256 advance_base_fixture(&runner).await;
17257 runner.sync_to_base().await.unwrap();
17258 assert!(runner.state.base_sync_blocked());
17259 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17260 }
17261
17262 #[cfg(unix)]
17263 async fn advance_base_fixture(runner: &Runner) {
17264 let repo = &runner.state.repo;
17265 for args in [
17266 vec!["checkout", "main"],
17267 vec!["commit", "--allow-empty", "-m", "base advanced"],
17268 vec!["push", "origin", "main"],
17269 vec!["checkout", "fix/retained"],
17270 ] {
17271 git::git(repo, &args).await.unwrap();
17272 }
17273 }
17274
17275 #[cfg(unix)]
17276 fn fresh_base_interval_fixture(runner: &mut Runner) {
17277 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17278 runner.state.base_sync.as_mut().unwrap().conflict = None;
17279 runner.state.base_sync_recovery = Default::default();
17280 runner.state.status = RunStatus::Voting;
17281 }
17282
17283 #[cfg(unix)]
17284 #[tokio::test]
17285 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17286 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17287 fresh_base_interval_fixture(&mut runner);
17288 runner.state.begin_pass();
17289 runner.sync_to_base().await.unwrap();
17290 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17291 runner.state.resource_block = Some(crate::run::ResourceBlock {
17292 node: "gate".into(),
17293 kind: "build cache".into(),
17294 detail: "busy".into(),
17295 at: Timestamp::now(),
17296 cause: crate::run::ResourceBlockCause::Timeout {
17297 waited_secs: 1,
17298 limit_secs: 1,
17299 },
17300 });
17301 runner.state.save().unwrap();
17302 let queue = crate::queue::Queue::open();
17303 let mut owner = queue.get(&task).unwrap();
17304 runner.state.task_instruction = Some(owner.instruction.clone());
17305 owner.release_operator("fixture-owner", None);
17306 queue.put(&mut owner).unwrap();
17307 let approval = owner.resumption.clone().unwrap();
17308 runner.state.resource_block = None;
17309 runner.state.save().unwrap();
17310 runner.state = RunState::load(&runner.state.id).unwrap();
17311 advance_base_fixture(&runner).await;
17312 runner.sync_to_base().await.unwrap();
17313 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17314 assert_eq!(runner.state.base_episode_used(), 1);
17315 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17316 assert!(
17317 runner.state.base_sync_recovery.episodes[0]
17318 .boundary
17319 .as_ref()
17320 .unwrap()
17321 .consumed
17322 );
17323 assert!(
17324 !runner
17325 .state
17326 .renew_base_episode(&task, &approval, &head, "base")
17327 );
17328 for _ in 0..3 {
17329 owner.release_operator("fixture-owner", None);
17330 queue.put(&mut owner).unwrap();
17331 runner.state = RunState::load(&runner.state.id).unwrap();
17332 runner.state.begin_pass();
17333 runner.sync_to_base().await.unwrap();
17334 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17335 assert_eq!(runner.state.base_episode_used(), 1);
17336 }
17337 runner.state.resource_block = Some(crate::run::ResourceBlock {
17338 node: "gate".into(),
17339 kind: "disk pressure".into(),
17340 detail: "free space required".into(),
17341 at: Timestamp::now(),
17342 cause: crate::run::ResourceBlockCause::Other,
17343 });
17344 runner.state.end_pass_parked(ParkCause::Interrupt {
17345 reason: "disk pressure".into(),
17346 });
17347 runner.state.save().unwrap();
17348 owner.release_operator("fixture-owner-after-disk", None);
17349 queue.put(&mut owner).unwrap();
17350 runner.state.resource_block = None;
17351 advance_base_fixture(&runner).await;
17352 runner.sync_to_base().await.unwrap();
17353 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17354 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17355 assert_eq!(runner.state.base_episode_used(), 1);
17356 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17357 }
17358
17359 #[cfg(unix)]
17360 #[tokio::test]
17361 async fn base_recovery_continuous_advancement_stops_at_four() {
17362 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17363 fresh_base_interval_fixture(&mut runner);
17364 for spent in 1..=BASE_SYNC_ROUNDS {
17365 runner.sync_to_base().await.unwrap();
17366 assert!(!runner.state.base_sync_blocked());
17367 assert_eq!(runner.state.base_episode_used(), spent);
17368 advance_base_fixture(&runner).await;
17369 }
17370 runner.sync_to_base().await.unwrap();
17371 assert_eq!(
17372 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17373 crate::base_sync_recovery::StopKind::Exhausted
17374 );
17375 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17376 assert!(stop.unverified_sync);
17377 let action = runner.state.base_sync_next_action();
17378 assert!(action.contains("never reached its gate"));
17379 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17380 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17381 runner.state = RunState::load(&runner.state.id).unwrap();
17382 runner.sync_to_base().await.unwrap();
17383 assert_eq!(runner.state.base_episode_used(), 4);
17384 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17385 }
17386
17387 #[cfg(unix)]
17388 #[tokio::test]
17389 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17390 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17391 runner.state.ensure_base_episode();
17392 runner.state.record_base_progress(&head);
17394 runner.state.base_sync_recovery.stop = None;
17395 runner.state.record_base_sync_stop(&head);
17396 assert!(
17397 !runner
17398 .state
17399 .base_sync_recovery
17400 .stop
17401 .as_ref()
17402 .unwrap()
17403 .unverified_sync
17404 );
17405
17406 runner.state.finish_base_operation(Some(&head));
17407 runner.state.gate_ran = true;
17409 runner.state.gate_head = Some("older-head".into());
17410 runner.state.base_sync_recovery.stop = None;
17411 runner.state.record_base_sync_stop(&head);
17412 runner.state.save().unwrap();
17413 runner.state = RunState::load(&runner.state.id).unwrap();
17414 assert!(
17415 runner
17416 .state
17417 .base_sync_recovery
17418 .stop
17419 .as_ref()
17420 .unwrap()
17421 .unverified_sync
17422 );
17423
17424 runner.state.gate_head = Some(head.clone());
17425 runner.state.base_sync_recovery.stop = None;
17426 runner.state.record_base_sync_stop(&head);
17427 assert!(
17428 !runner
17429 .state
17430 .base_sync_recovery
17431 .stop
17432 .as_ref()
17433 .unwrap()
17434 .unverified_sync
17435 );
17436 assert!(
17437 !runner
17438 .state
17439 .base_sync_next_action()
17440 .contains("never reached its gate")
17441 );
17442
17443 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17445 legacy["base_sync_recovery"]["stop"]
17446 .as_object_mut()
17447 .unwrap()
17448 .remove("unverified_sync");
17449 legacy["base_sync_recovery"]["episodes"][0]
17450 .as_object_mut()
17451 .unwrap()
17452 .remove("last_sync_head");
17453 runner.state = serde_json::from_value(legacy).unwrap();
17454 assert!(
17455 !runner
17456 .state
17457 .base_sync_recovery
17458 .stop
17459 .as_ref()
17460 .unwrap()
17461 .unverified_sync
17462 );
17463 assert!(
17464 runner.state.base_sync_recovery.episodes[0]
17465 .last_sync_head
17466 .is_none()
17467 );
17468 }
17469
17470 #[cfg(unix)]
17471 #[tokio::test]
17472 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17473 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17474 runner.state.authorize_base_sync(&task).unwrap();
17475 assert!(
17476 runner
17477 .state
17478 .base_sync_next_action()
17479 .contains("Approval recorded")
17480 );
17481 assert!(
17482 runner
17483 .state
17484 .base_sync_next_action()
17485 .contains("4/4 in episode 0, 4 lifetime")
17486 );
17487 assert!(runner.state.consume_base_recovery(&head));
17488 assert!(
17489 runner
17490 .state
17491 .base_sync_next_action()
17492 .contains("0/4 in episode 1, 4 lifetime")
17493 );
17494 }
17495
17496 #[cfg(unix)]
17497 #[tokio::test]
17498 async fn base_recovery_changed_head_invalidates_without_spending() {
17499 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17500 runner.state.authorize_base_sync(&task).unwrap();
17501 git::git(
17502 &runner.state.repo,
17503 &["commit", "--allow-empty", "-m", "external head change"],
17504 )
17505 .await
17506 .unwrap();
17507 runner.sync_to_base().await.unwrap();
17508 let a = runner
17509 .state
17510 .base_sync_recovery
17511 .authorizations
17512 .last()
17513 .unwrap();
17514 assert!(a.consumed);
17515 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17516 runner.state = RunState::load(&runner.state.id).unwrap();
17517 assert!(
17518 runner
17519 .state
17520 .base_sync_next_action()
17521 .contains("Approval invalidated")
17522 );
17523 assert!(!runner.state.base_recovery_pending());
17524 assert!(runner.state.base_sync_blocked());
17525 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17526 }
17527
17528 #[cfg(unix)]
17529 #[tokio::test]
17530 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17531 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17532 runner.state.driver_exited = true;
17533 runner.state.save().unwrap();
17534 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17535 .await
17536 .unwrap();
17537 runner.state = RunState::load(&runner.state.id).unwrap();
17538 advance_base_fixture(&runner).await;
17539 runner.sync_to_base().await.unwrap();
17540 let a = runner
17541 .state
17542 .base_sync_recovery
17543 .authorizations
17544 .last()
17545 .unwrap();
17546 assert!(
17547 a.consumed
17548 && a.invalidated
17549 .as_deref()
17550 .unwrap()
17551 .contains("fetched base changed")
17552 );
17553 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17554 assert_eq!(
17555 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17556 head
17557 );
17558 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17559 .await
17560 .unwrap();
17561 runner.state = RunState::load(&runner.state.id).unwrap();
17562 runner.sync_to_base().await.unwrap();
17563 assert!(!runner.state.base_sync_blocked());
17564 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17565 }
17566
17567 #[cfg(unix)]
17568 #[tokio::test]
17569 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17570 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17571 runner.state.authorize_base_sync(&task).unwrap();
17572 assert!(runner.state.consume_base_recovery(&head));
17573 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17574 runner.state.reserve_base_operation(&head, &tip);
17575 runner.state.save().unwrap();
17576 let scratch = runner.state.dir().join("base-sync");
17577 assert_eq!(
17578 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17579 .await
17580 .unwrap(),
17581 crate::rebase::Rebased::Applied
17582 );
17583 runner.state = RunState::load(&runner.state.id).unwrap();
17584 runner.sync_to_base().await.unwrap();
17585 assert!(
17586 runner
17587 .state
17588 .base_sync_recovery
17589 .operations
17590 .last()
17591 .unwrap()
17592 .finished
17593 );
17594 assert_eq!(runner.state.base_episode_used(), 1);
17595 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17596 assert!(!runner.state.base_sync_blocked());
17597 }
17598
17599 #[cfg(unix)]
17600 #[tokio::test]
17601 async fn base_recovery_crash_preserves_untracked_files() {
17602 for ignored in [false, true] {
17603 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17604 runner.state.authorize_base_sync(&task).unwrap();
17605 assert!(runner.state.consume_base_recovery(&head));
17606 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17607 runner.state.reserve_base_operation(&head, &tip);
17608 runner.state.save().unwrap();
17609 let scratch = runner.state.dir().join("base-sync");
17610 assert_eq!(
17611 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17612 .await
17613 .unwrap(),
17614 crate::rebase::Rebased::Applied
17615 );
17616 let repo = &runner.state.repo;
17617 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17618 assert_ne!(result, head);
17619 if ignored {
17620 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17621 .await
17622 .unwrap();
17623 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17624 }
17625 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17626 assert!(
17627 git::git_raw(repo, &["diff", "--quiet", &head])
17628 .await
17629 .unwrap()
17630 .ok()
17631 );
17632 runner.state = RunState::load(&runner.state.id).unwrap();
17634 runner.sync_to_base().await.unwrap();
17635 assert!(runner.state.base_sync_blocked());
17636 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17637 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17638 assert_eq!(
17639 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17640 result
17641 );
17642 assert_eq!(
17643 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17644 "retained work\n"
17645 );
17646 assert!(
17647 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17648 .await
17649 .unwrap()
17650 .ok()
17651 );
17652 }
17653 }
17654
17655 #[cfg(unix)]
17656 #[tokio::test]
17657 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17658 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17659 runner.state.authorize_base_sync(&task).unwrap();
17660 assert!(runner.state.consume_base_recovery(&head));
17661 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17662 runner.state.reserve_base_operation(&head, &tip);
17663 runner.state.save().unwrap();
17664 let scratch = runner.state.dir().join("base-sync");
17665 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17666 .await
17667 .unwrap();
17668 git::sync_to_head(&runner.state.repo).await.unwrap();
17669 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17670 git::git(&runner.state.repo, &["add", "foreign"])
17671 .await
17672 .unwrap();
17673 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17674 .await
17675 .unwrap();
17676 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17677 runner.state = RunState::load(&runner.state.id).unwrap();
17678 runner.sync_to_base().await.unwrap();
17679 assert!(runner.state.base_sync_blocked());
17680 assert!(
17681 !runner
17682 .state
17683 .base_sync_recovery
17684 .operations
17685 .last()
17686 .unwrap()
17687 .finished
17688 );
17689 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17690 assert_eq!(
17691 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17692 foreign_head
17693 );
17694 }
17695
17696 #[cfg(unix)]
17697 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17698 {
17699 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17700 fresh_base_interval_fixture(&mut runner);
17701 let repo = &runner.state.repo;
17702 git::git(repo, &["checkout", "main"]).await.unwrap();
17703 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17704 git::git(repo, &["add", "retained"]).await.unwrap();
17705 git::git(repo, &["commit", "-m", "upstream retained change"])
17706 .await
17707 .unwrap();
17708 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17709 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17710 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17711 let scratch = runner.state.dir().join("base-sync");
17712 assert!(matches!(
17713 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17714 .await
17715 .unwrap(),
17716 git::RebaseStart::Conflicted(_)
17717 ));
17718 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17719 agent: "fixture".into(),
17720 paths: vec!["retained".into()],
17721 from: Some(head.clone()),
17722 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17723 index_outside: None,
17724 finished: false,
17725 error: None,
17726 launcher: None,
17727 infra: None,
17728 outage: None,
17729 });
17730 (tmp, runner, head, tip, scratch)
17731 }
17732
17733 #[cfg(unix)]
17734 #[tokio::test]
17735 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17736 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17737 std::fs::write(
17740 scratch.join("retained"),
17741 "upstream and retained resolution\n",
17742 )
17743 .unwrap();
17744 runner.state.resource_block = Some(crate::run::ResourceBlock {
17745 node: "rebase".into(),
17746 kind: "repository coordination".into(),
17747 detail: "busy".into(),
17748 at: Timestamp::now(),
17749 cause: crate::run::ResourceBlockCause::Other,
17750 });
17751 runner.state.save().unwrap();
17752 let newer = git::git(
17755 &runner.state.repo,
17756 &[
17757 "commit-tree",
17758 "main^{tree}",
17759 "-p",
17760 "main",
17761 "-m",
17762 "base advanced during wait",
17763 ],
17764 )
17765 .await
17766 .unwrap();
17767 git::git(
17768 &runner.state.repo,
17769 &[
17770 "push",
17771 "origin",
17772 &format!("{}:refs/heads/main", newer.trim()),
17773 ],
17774 )
17775 .await
17776 .unwrap();
17777 runner.state = RunState::load(&runner.state.id).unwrap();
17778 runner.state.resource_block = None;
17779 runner.sync_to_base().await.unwrap();
17780 assert!(!runner.state.base_sync_blocked());
17781 let op = &runner.state.base_sync_recovery.operations[0];
17782 assert_eq!(op.head, head);
17783 assert_eq!(op.tip, tip);
17784 assert!(op.finished);
17785 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17786 assert_eq!(runner.state.rebase_fixes.len(), 1);
17787 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17788 .await
17789 .unwrap();
17790 assert_ne!(latest, tip);
17791 runner.state = RunState::load(&runner.state.id).unwrap();
17792 runner.sync_to_base().await.unwrap();
17793 assert!(!runner.state.base_sync_blocked());
17794 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17795 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17796 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17797 }
17798
17799 #[cfg(unix)]
17800 #[tokio::test]
17801 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17802 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17803 runner.state.reserve_base_operation(&head, &tip);
17804 runner.state.save().unwrap();
17805 std::fs::write(
17806 scratch.join("retained"),
17807 "upstream and retained resolution\n",
17808 )
17809 .unwrap();
17810 git::git(&scratch, &["add", "retained"]).await.unwrap();
17811 git::git(
17812 &scratch,
17813 &["-c", "core.editor=true", "rebase", "--continue"],
17814 )
17815 .await
17816 .unwrap();
17817 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17818 assert!(
17819 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17820 .await
17821 .unwrap()
17822 .is_empty()
17823 );
17824 assert_eq!(
17825 crate::rebase::finish(
17826 &mut runner.state,
17827 &scratch,
17828 "fix/retained",
17829 &head,
17830 &tip,
17831 &["retained".into()],
17832 ""
17833 )
17834 .await
17835 .unwrap(),
17836 crate::rebase::Rebased::Applied
17837 );
17838 assert!(!scratch.exists());
17839 runner.state = RunState::load(&runner.state.id).unwrap();
17841 let op = &runner.state.base_sync_recovery.operations[0];
17842 assert!(!op.finished);
17843 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17844 runner.sync_to_base().await.unwrap();
17845 assert!(!runner.state.base_sync_blocked());
17846 assert!(runner.state.base_sync_recovery.operations[0].finished);
17847 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17848 assert_eq!(runner.state.rebase_fixes.len(), 1);
17849 let winner = runner.state.winner().unwrap().clone();
17850 let wait = runner.repository_wait();
17851 assert!(
17852 verify_current_head(&mut runner.state, &winner, &wait)
17853 .await
17854 .unwrap()
17855 );
17856 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17857 }
17858
17859 #[cfg(unix)]
17860 #[tokio::test]
17861 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17862 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17863 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17864 runner.state.reserve_base_operation(&head, &tip);
17865 runner.state.save().unwrap();
17866 std::fs::write(
17867 scratch.join("retained"),
17868 "upstream and retained resolution\n",
17869 )
17870 .unwrap();
17871 git::git(&scratch, &["add", "retained"]).await.unwrap();
17872 git::git(
17873 &scratch,
17874 &["-c", "core.editor=true", "rebase", "--continue"],
17875 )
17876 .await
17877 .unwrap();
17878 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17879 assert!(!git::rebase_in_progress(&scratch).await);
17880 assert!(
17881 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17882 .await
17883 .unwrap()
17884 .is_empty()
17885 );
17886 match damage {
17887 Some("record") => {
17888 let path =
17889 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17890 .await
17891 .unwrap();
17892 std::fs::write(scratch.join(path), "").unwrap();
17893 }
17894 Some("foreign") => {
17895 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17896 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17897 git::git(&scratch, &["commit", "-m", "foreign work"])
17898 .await
17899 .unwrap();
17900 }
17901 Some("dirty") => {
17902 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17903 }
17904 _ => {}
17905 }
17906 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17907 runner.state = RunState::load(&runner.state.id).unwrap();
17909 assert!(
17910 runner.state.base_sync_recovery.operations[0]
17911 .validated_head
17912 .is_none()
17913 );
17914 runner.sync_to_base().await.unwrap();
17915 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17916 assert_eq!(runner.state.base_episode_used(), 1);
17917 assert_eq!(runner.state.rebase_fixes.len(), 1);
17918 assert_eq!(
17919 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17920 retained_head
17921 );
17922 if damage.is_some() {
17923 assert!(runner.state.base_sync_blocked());
17924 assert!(scratch.exists());
17925 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17926 assert!(
17927 runner.state.base_sync_recovery.operations[0]
17928 .validated_head
17929 .is_none()
17930 );
17931 } else {
17932 assert!(!runner.state.base_sync_blocked());
17933 assert!(!scratch.exists());
17934 assert!(runner.state.base_sync_recovery.operations[0].finished);
17935 assert_eq!(
17936 runner.state.base_sync_recovery.operations[0]
17937 .validated_head
17938 .as_deref(),
17939 Some(result.as_str())
17940 );
17941 let winner = runner.state.winner().unwrap().clone();
17942 let wait = runner.repository_wait();
17943 assert!(
17944 verify_current_head(&mut runner.state, &winner, &wait)
17945 .await
17946 .unwrap()
17947 );
17948 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17949 runner.state = RunState::load(&runner.state.id).unwrap();
17950 runner.sync_to_base().await.unwrap();
17951 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17952 }
17953 }
17954 }
17955
17956 #[cfg(unix)]
17957 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17958 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17959 runner.state.prep_complete = Some(true);
17961 runner.state.base_sync = None;
17962 runner.state.base_sync_recovery = Default::default();
17963 runner.state.config.graph.gate_fix_rounds = 1;
17964 let mut review = review_round(true, 0, 1, 1, false, true);
17965 review.head = head.clone();
17966 review.verified_head = Some(head.clone());
17967 runner.state.reviews = vec![review];
17968 runner.state.gate = vec![CommandOutcome {
17969 command: "old gate".into(),
17970 code: Some(1),
17971 output_tail: "old failure".into(),
17972 duration_ms: 0,
17973 resource_blocked: false,
17974 }];
17975 runner.state.gate_head = Some(head.clone());
17976 runner.state.gate_ran = true;
17977 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17978 agent: "fixture".into(),
17979 failed: runner.state.gate.clone(),
17980 notes: "spent".into(),
17981 committed: false,
17982 error: None,
17983 }];
17984 runner.state.land_approval = Some(crate::run::LandApproval {
17985 question: "old approval".into(),
17986 head: head.clone(),
17987 });
17988 runner.state.status = RunStatus::Blocked;
17989 (tmp, runner, head)
17990 }
17991
17992 #[cfg(unix)]
17993 #[tokio::test]
17994 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17995 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17996 let (tmp, mut runner, old) = resumed_gate_fixture().await;
17997 let phases = tmp.path().join("phases");
17998 runner.state.config.verify.e2e = vec![format!(
17999 "echo e2e >> '{}'; {}",
18000 phases.display(),
18001 if e2e_ok {
18002 "true"
18003 } else {
18004 "echo new-e2e-failure; false"
18005 }
18006 )];
18007 runner.state.config.verify.gate = vec![format!(
18008 "echo gate >> '{}'; {}",
18009 phases.display(),
18010 if gate_ok {
18011 "true"
18012 } else {
18013 "echo new-gate-failure; false"
18014 }
18015 )];
18016 runner.state.save().unwrap();
18017 runner.state = RunState::load(&runner.state.id).unwrap();
18018 runner.sync_to_base().await.unwrap();
18019 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18020 assert_ne!(head, old);
18021 runner.state = RunState::load(&runner.state.id).unwrap();
18023 runner.execute_graph().await.unwrap();
18024 assert_eq!(runner.state.gate_fixes.len(), 1);
18025 assert_eq!(runner.state.reviews.len(), 1);
18026 assert_eq!(runner.state.reviews[0].head, old);
18027 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18028 assert_eq!(runner.state.verification_history.len(), 1);
18029 let history = &runner.state.verification_history[0];
18030 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18031 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18032 assert!(history.archived_at.is_some());
18033 assert_eq!(history.fixes_spent, Some(1));
18034 assert_eq!(history.gate[0].output_tail, "old failure");
18035 let archive_events: Vec<_> = runner
18036 .state
18037 .events
18038 .iter()
18039 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18040 .collect();
18041 assert_eq!(archive_events.len(), 1);
18042 assert!(archive_events[0].message.contains(&short(&old)));
18043 assert!(archive_events[0].message.contains(&short(&head)));
18044 assert!(
18045 archive_events[0]
18046 .message
18047 .contains("1 gate-fix round(s) already spent stay spent")
18048 );
18049 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18050 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18051 if e2e_ok {
18052 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18053 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18054 } else {
18055 assert!(!runner.state.gate_ran);
18056 }
18057 assert_eq!(
18058 runner.state.status,
18059 if e2e_ok && gate_ok {
18060 RunStatus::Ready
18061 } else {
18062 RunStatus::Blocked
18063 }
18064 );
18065 if !e2e_ok || !gate_ok {
18066 let cause = runner.state.verification_blocking_cause().unwrap();
18067 assert!(cause.contains(&head));
18068 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18069 runner.state.event("land", "rebased successfully");
18071 let notice = crate::notices::run_ended(&runner.state).unwrap();
18072 assert!(!format!("{notice:?}").contains("rebased successfully"));
18073 assert!(runner.state.merge.is_none());
18074 }
18075 let before = std::fs::read_to_string(&phases).unwrap();
18076 runner.state.save().unwrap();
18077 runner.state = RunState::load(&runner.state.id).unwrap();
18078 runner.review_loop().await.unwrap();
18079 runner.gate().await.unwrap();
18080 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18081 assert_eq!(runner.state.verification_history.len(), 1);
18082 let history = &runner.state.verification_history[0];
18083 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18084 assert!(history.archived_at.is_some());
18085 assert_eq!(history.fixes_spent, Some(1));
18086 }
18087 }
18088
18089 #[cfg(unix)]
18090 #[tokio::test]
18091 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18092 let (tmp, mut runner, head) = resumed_gate_fixture().await;
18093 let marker = tmp.path().join("gate-ran");
18094 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18095 runner.review_loop().await.unwrap();
18096 runner.gate().await.unwrap();
18097 assert_eq!(runner.state.status, RunStatus::Blocked);
18098 assert!(!marker.exists());
18099 assert!(runner.state.verification_history.is_empty());
18100 runner.state.gate_head = None;
18101 runner.gate().await.unwrap();
18102 assert!(marker.exists());
18103 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18104 assert_eq!(runner.state.verification_history.len(), 1);
18105 assert_eq!(runner.state.gate_fixes.len(), 1);
18106 }
18107
18108 #[cfg(unix)]
18109 #[tokio::test]
18110 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18111 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18112 runner.sync_to_base().await.unwrap();
18113 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18114 runner.state.reconcile_verification_head(&head);
18115 for outcomes in [
18117 Vec::new(),
18118 vec![CommandOutcome {
18119 command: "true".into(),
18120 code: None,
18121 output_tail: "cache unavailable".into(),
18122 duration_ms: 0,
18123 resource_blocked: true,
18124 }],
18125 ] {
18126 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18127 head: head.clone(),
18128 passed: false,
18129 outcomes,
18130 });
18131 runner.state.gate_ran = false;
18132 runner.state.gate.clear();
18133 runner.state.save().unwrap();
18134 runner.state = RunState::load(&runner.state.id).unwrap();
18135 runner.merge().await.unwrap();
18136 assert!(runner.state.merge.is_none());
18137 runner.review_loop().await.unwrap();
18138 runner.gate().await.unwrap();
18139 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18140 assert!(runner.state.gate_allows_progress());
18141 assert_eq!(runner.state.gate_fixes.len(), 1);
18142 assert!(runner.state.merge.is_none());
18143 }
18144 git::git(
18146 &runner.state.repo,
18147 &["commit", "--allow-empty", "-m", "external update"],
18148 )
18149 .await
18150 .unwrap();
18151 runner.merge().await.unwrap();
18152 assert!(runner.state.merge.is_none());
18153 assert_eq!(runner.state.status, RunStatus::Blocked);
18154 }
18155
18156 #[cfg(unix)]
18157 #[tokio::test]
18158 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18159 for waiting_on_gate in [false, true] {
18160 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18161 runner.sync_to_base().await.unwrap();
18162 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18163 runner.state.reconcile_verification_head(&head);
18164 if waiting_on_gate {
18165 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18166 head: head.clone(),
18167 passed: true,
18168 outcomes: Vec::new(),
18169 });
18170 }
18171 let cache = tmp.path().join("cache");
18172 runner.state.config.verify.e2e =
18173 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18174 runner.state.config.verify.gate = vec!["true".into()];
18175 runner.state.config.graph.timeout_resource_wait = 1;
18176 let owner =
18177 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18178 let guard =
18179 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18180 crate::cache::AcquireOutcome::Acquired(g) => g,
18181 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18182 };
18183 runner.review_loop().await.unwrap();
18184 runner.gate().await.unwrap();
18185 assert!(!runner.state.gate_ran);
18186 assert_eq!(runner.state.gate_fixes.len(), 1);
18187 assert!(runner.state.resource_block.is_some());
18188 runner.merge().await.unwrap();
18189 assert!(runner.state.merge.is_none());
18190 drop(guard);
18191 runner.state = RunState::load(&runner.state.id).unwrap();
18192 runner.state.resource_block = None;
18194 runner.review_loop().await.unwrap();
18195 runner.gate().await.unwrap();
18196 assert!(runner.state.gate_allows_progress());
18197 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18198 assert_eq!(runner.state.gate_fixes.len(), 1);
18199 }
18200 }
18201
18202 #[cfg(unix)]
18203 #[tokio::test]
18204 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18205 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18206 runner.sync_to_base().await.unwrap();
18207 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18208 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18209 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18211 runner.review_loop().await.unwrap();
18212 runner.gate().await.unwrap();
18213 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18214 assert_ne!(before, after);
18215 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18216 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18217 assert!(!runner.state.gate_allows_progress());
18218 runner.merge().await.unwrap();
18219 assert!(runner.state.merge.is_none());
18220 assert!(
18221 runner
18222 .state
18223 .verification_history
18224 .iter()
18225 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18226 );
18227
18228 runner.state.reviews[0].clean = false;
18230 runner.state.reviews[0].answered = 0;
18231 runner.state.config.verify.e2e = vec!["true".into()];
18232 runner.review_loop().await.unwrap();
18233 runner.gate().await.unwrap();
18234 runner.merge().await.unwrap();
18235 assert_eq!(runner.state.status, RunStatus::Blocked);
18236 assert!(runner.state.merge.is_none());
18237 assert!(
18238 runner
18239 .state
18240 .verification_blocking_cause()
18241 .unwrap()
18242 .contains("review blocked")
18243 );
18244 }
18245
18246 #[cfg(unix)]
18247 #[tokio::test]
18248 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18249 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18250 runner.state.gate_head = None;
18251 runner.state.config.verify.gate =
18252 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18253 runner.review_loop().await.unwrap();
18254 runner.gate().await.unwrap();
18255 assert_ne!(
18256 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18257 old
18258 );
18259 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18260 assert!(!runner.state.gate_allows_progress());
18261 assert_eq!(runner.state.status, RunStatus::Blocked);
18262 runner.merge().await.unwrap();
18263 assert!(runner.state.merge.is_none());
18264 }
18265
18266 #[cfg(unix)]
18267 #[tokio::test]
18268 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18269 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18270 let mut old_review = review_round(true, 0, 1, 1, false, true);
18271 old_review.head = head.clone();
18272 old_review.verified_head = Some(head.clone());
18273 runner.state.reviews = vec![old_review];
18274 runner.state.gate = vec![CommandOutcome {
18275 command: "old gate".into(),
18276 code: None,
18277 output_tail: "timed out after 2400s".into(),
18278 duration_ms: 2_400_000,
18279 resource_blocked: false,
18280 }];
18281 runner.state.gate_ran = true;
18282 runner.state.gate_head = Some(head.clone());
18283 runner.state.land_approval = Some(crate::run::LandApproval {
18284 question: "old-head-approval".into(),
18285 head: head.clone(),
18286 });
18287 runner.state.authorize_base_sync(&task).unwrap();
18288 runner.sync_to_base().await.unwrap();
18289 assert!(!runner.state.base_sync_blocked());
18290 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18291 assert_ne!(head, fresh);
18292 assert_eq!(runner.state.reviews.len(), 1);
18293 assert!(!runner.state.gate_ran);
18294 let saved = &runner.state.base_sync_recovery.authorizations[0];
18295 assert_eq!(saved.reviews[0].head, head);
18296 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18297 assert!(saved.gate_ran);
18298 assert!(!saved.gate[0].ok());
18299 runner.review_loop().await.unwrap();
18300 runner.gate().await.unwrap();
18301 assert_eq!(runner.state.reviews[0].head, head);
18302 assert_eq!(
18303 runner.state.reviews[0].verified_head.as_deref(),
18304 Some(head.as_str())
18305 );
18306 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18307 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18308 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18309 assert!(runner.state.gate_ran);
18310 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18311 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18312 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18313 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18314 assert_eq!(
18315 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18316 "retained change"
18317 );
18318 }
18319
18320 #[cfg(unix)]
18321 #[tokio::test]
18322 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18323 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18324 let queue = crate::queue::Queue::open();
18325 let mut owner = queue.get(&task).unwrap();
18326 owner.release_operator("test-resume", None);
18327 queue.put(&mut owner).unwrap();
18328 let generation = owner.resumption.as_ref().unwrap().generation;
18329 runner.state.task_instruction = Some(owner.instruction.clone());
18330 runner.state.driver_exited = true;
18331 runner.state.save().unwrap();
18332 let home = crate::run::home();
18333 let _claim = queue.claim(&task).unwrap();
18334 owner.resumption.as_mut().unwrap().at =
18335 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18336 queue.put(&mut owner).unwrap();
18337 assert!(
18338 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18339 .await
18340 .is_err(),
18341 "an approval from before the stop grants nothing"
18342 );
18343 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18344 queue.put(&mut owner).unwrap();
18345 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18346 .await
18347 .unwrap();
18348 assert!(
18349 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18350 .await
18351 .is_err()
18352 );
18353 runner.state = RunState::load(&runner.state.id).unwrap();
18354 assert_eq!(
18355 runner.state.base_sync_recovery.authorizations[0].generation,
18356 Some(generation)
18357 );
18358 runner.sync_to_base().await.unwrap();
18359 let repo = runner.state.repo.clone();
18360 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18361 git::git(&repo, &["checkout", "main"]).await.unwrap();
18362 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18363 git::git(&repo, &["add", "."]).await.unwrap();
18364 git::git(&repo, &["commit", "-m", "move base again"])
18365 .await
18366 .unwrap();
18367 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18368 git::git(&repo, &["checkout", "fix/retained"])
18369 .await
18370 .unwrap();
18371 runner.sync_to_base().await.unwrap();
18372 assert!(!runner.state.base_sync_blocked());
18373 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18374 assert_eq!(runner.state.base_episode_used(), 2);
18375 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18376 assert_ne!(fresh, head);
18377 assert!(!runner.state.base_recovery_pending());
18378 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18380 queue.put(&mut owner).unwrap();
18381 assert!(
18382 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18383 .await
18384 .is_err()
18385 );
18386 runner.state = RunState::load(&runner.state.id).unwrap();
18387 runner.sync_to_base().await.unwrap();
18388 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18389 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18390 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18391 }
18392
18393 #[cfg(unix)]
18394 #[tokio::test]
18395 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18396 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18397 runner.state.base_sync = None;
18398 runner.state.status = RunStatus::Blocked;
18399 let mut second = runner.state.candidates[0].clone();
18400 second.label = 'B';
18401 runner.state.candidates.push(second);
18402 assert!(runner.state.votes.is_empty());
18403 assert!(runner.state.deliberation.is_empty());
18404 runner.deliberate().await.unwrap();
18405 runner.vote().await.unwrap();
18406 assert_eq!(runner.state.status, RunStatus::Blocked);
18407 assert!(runner.state.votes.is_empty());
18408 assert!(runner.state.deliberation.is_empty());
18409 }
18410
18411 fn runner_at(status: RunStatus) -> Runner {
18414 let mut state = RunState::new(
18415 PathBuf::from("/nonexistent/repo"),
18416 "main".to_owned(),
18417 "deadbeef".to_owned(),
18418 "task".to_owned(),
18419 Config::default(),
18420 );
18421 state.config.disk.min_free_bytes = 0;
18423 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18426 state.id = format!(
18427 "20000101-000000-{:04x}",
18428 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18429 );
18430 state.status = status;
18431 Runner {
18432 state,
18433 roles: ResolvedRoles {
18434 implementers: Vec::new(),
18435 judges: Vec::new(),
18436 reviewers: Vec::new(),
18437 fixer: None,
18438 conductor: conductor(),
18439 implementer_roster: Vec::new(),
18440 judge_roster: Vec::new(),
18441 reviewer_roster: Vec::new(),
18442 },
18443 sem: Arc::new(Semaphore::new(1)),
18444 pause: Pause::new(),
18445 interrupt: Pause::new(),
18446 }
18447 }
18448
18449 #[tokio::test]
18450 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18451 crate::run::pin_test_home();
18452 let temp = tempfile::tempdir().unwrap();
18453 let root = temp.path().canonicalize().unwrap();
18454 let repo = root.join("repo");
18455 std::fs::create_dir(&repo).unwrap();
18456 init_repo(&repo);
18457 git::git(&repo, &["config", "core.autocrlf", "false"])
18458 .await
18459 .unwrap();
18460 let mut runner = runner_at(RunStatus::Blocked);
18461 runner.state.repo = repo.clone();
18462 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18463 runner.state.config.agents = vec![conductor()];
18464 runner.state.config.graph.selected_implementer = Some("conductor".into());
18465 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18466 let (shell, command, setup_output) = if cfg!(windows) {
18467 (
18468 vec!["cmd".into(), "/C".into()],
18469 "echo ready>seat-ready",
18470 "ready\r\n",
18471 )
18472 } else {
18473 (
18474 vec!["sh".into(), "-c".into()],
18475 "printf ready > seat-ready",
18476 "ready",
18477 )
18478 };
18479 runner.state.config.verify.shell = Some(shell);
18480 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18481 run: Some(command.into()),
18482 ..Default::default()
18483 }];
18484 runner.state.save().unwrap();
18485 let path = runner.state.worktree_root().join("judge-1");
18486 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18487 .await
18488 .unwrap();
18489 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18490 git::git(&path, &["add", "."]).await.unwrap();
18491 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18492 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18493 .await
18494 .unwrap();
18495 git::git(&path, &["checkout", "--detach", &head])
18496 .await
18497 .unwrap();
18498 let evidence = crate::fold_evidence::snapshot(
18499 &runner.state,
18500 &crate::run::home().canonicalize().unwrap(),
18501 &head,
18502 64 * 1024 * 1024,
18503 )
18504 .await
18505 .unwrap();
18506 let mut contents = BTreeMap::new();
18507 contents.insert(
18508 repo.join("logs/HEAD"),
18509 format!("{head} {head} fixture\n").into_bytes(),
18510 );
18511 let record = serde_json::json!({
18512 "version":1, "run":runner.state.id, "source":path, "head":head,
18513 "evidence":evidence, "admin":{}, "admin_contents":contents,
18514 "history":{head.clone():evidence}, "phase":"evicted"
18515 });
18516 std::fs::write(
18517 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18518 serde_json::to_vec(&record).unwrap(),
18519 )
18520 .unwrap();
18521 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18522 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18523 .await
18524 .unwrap();
18525 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18526 assert!(
18527 !git::git_raw(&repo, &["cat-file", "-e", &head])
18528 .await
18529 .unwrap()
18530 .ok()
18531 );
18532 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18533 let error = resumed.execute().await.unwrap_err();
18534 assert!(
18535 error
18536 .to_string()
18537 .contains("selected implementation is held"),
18538 "{error:#}"
18539 );
18540 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18541 assert_eq!(
18542 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18543 setup_output
18544 );
18545 assert_eq!(
18546 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18547 "unrelated seat"
18548 );
18549 let error = resumed.execute().await.unwrap_err();
18550 assert!(
18551 error
18552 .to_string()
18553 .contains("selected implementation is held"),
18554 "{error:#}"
18555 );
18556 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18557 git::git(&path, &["add", "."]).await.unwrap();
18558 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18559 let next = git::git(
18560 &path,
18561 &[
18562 "commit-tree",
18563 &next_tree,
18564 "-p",
18565 &head,
18566 "-m",
18567 "next review checkpoint",
18568 ],
18569 )
18570 .await
18571 .unwrap();
18572 git::git(&path, &["checkout", "--detach", &next])
18573 .await
18574 .unwrap();
18575 resumed.state.reviews.push(
18576 serde_json::from_value(serde_json::json!({
18577 "round":1, "head":next, "reviews":[]
18578 }))
18579 .unwrap(),
18580 );
18581 resumed.state.save().unwrap();
18582 let error = resumed.execute().await.unwrap_err();
18583 assert!(
18584 error
18585 .to_string()
18586 .contains("selected implementation is held"),
18587 "{error:#}"
18588 );
18589 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18590 }
18591
18592 #[tokio::test]
18593 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18594 crate::run::pin_test_home();
18595 let mut runner = runner_at(RunStatus::Gating);
18596 runner.state.save().unwrap();
18597 let id = runner.state.id.clone();
18598 let passes = runner.state.passes.len();
18599 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18600 let mut claiming = Box::pin(runner.claim_driver());
18601 std::future::poll_fn(|cx| {
18602 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18603 std::task::Poll::Ready(())
18604 })
18605 .await;
18606 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18607 drop(owner);
18608 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18609 .await
18610 .unwrap()
18611 .unwrap();
18612 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18613 assert!(execution.runner.state.resource_block.is_none());
18614 assert_eq!(
18615 execution.runner.state.liveness(false),
18616 crate::run::Liveness::Live
18617 );
18618 drop(execution);
18619 assert!(RunState::load(&id).unwrap().driver_exited);
18620 }
18621
18622 #[tokio::test]
18623 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18624 crate::run::pin_test_home();
18625 let mut runner = runner_at(RunStatus::Gating);
18626 runner.state.save().unwrap();
18627 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18630 assert!(runner.claim_driver().await.is_err());
18631 assert!(runner.state.driver_exited);
18632 assert_eq!(runner.state.driver_pid, None);
18633 let disk = RunState::load(&runner.state.id).unwrap();
18634 assert!(disk.driver_exited);
18635 assert_eq!(disk.driver_pid, None);
18636 assert_eq!(disk.status, RunStatus::Gating);
18637 }
18638
18639 #[tokio::test]
18640 async fn an_early_graph_error_releases_its_driver() {
18641 crate::run::pin_test_home();
18642 let mut runner = runner_at(RunStatus::Blocked);
18643 runner.state.config.graph.selected_implementer = Some("selected".into());
18644 runner.state.save().unwrap();
18645 assert!(runner.execute().await.is_err());
18646 let disk = RunState::load(&runner.state.id).unwrap();
18647 assert!(disk.driver_exited);
18648 assert_eq!(disk.driver_pid, None);
18649 assert_eq!(disk.status, RunStatus::Blocked);
18650 }
18651
18652 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18653 crate::run::MergeRepair {
18654 question: "repair-question".into(),
18655 head: "old-head".into(),
18656 pr: "https://github.com/o/r/pull/42".into(),
18657 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18658 gate: vec![outcome(Some(0), false)],
18659 gate_head: Some("old-head".into()),
18660 gate_fixes: Vec::new(),
18661 pre_gate: Vec::new(),
18662 phase,
18663 failure: None,
18664 }
18665 }
18666
18667 #[tokio::test]
18668 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18669 crate::run::pin_test_home();
18670 let mut runner = runner_at(RunStatus::Landing);
18671 runner.state.merge_repairs.push(merge_repair_request(
18672 crate::run::MergeRepairPhase::Requested,
18673 ));
18674 runner.state.save().unwrap();
18675 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18676 assert!(
18677 runner
18678 .execute()
18679 .await
18680 .unwrap_err()
18681 .to_string()
18682 .contains("already running")
18683 );
18684 assert_eq!(
18685 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18686 crate::run::MergeRepairPhase::Requested
18687 );
18688 drop(claim);
18689 let mut disk = runner.state.clone();
18690 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18691 disk.save().unwrap();
18692 assert!(
18693 runner
18694 .execute()
18695 .await
18696 .unwrap_err()
18697 .to_string()
18698 .contains("decision changed")
18699 );
18700 assert_eq!(
18701 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18702 crate::run::MergeRepairPhase::Complete
18703 );
18704 }
18705
18706 #[test]
18707 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18708 crate::run::pin_test_home();
18709 let mut runner = runner_at(RunStatus::Landing);
18710 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18711 runner.state.reviews = request.reviews.clone();
18712 runner.state.gate = request.gate.clone();
18713 runner.state.gate_ran = true;
18714 runner.state.land_approval = Some(crate::run::LandApproval {
18715 question: "old".into(),
18716 head: "old-head".into(),
18717 });
18718 runner.state.merge_repairs.push(request);
18719 runner.begin_merge_repair_review(0).unwrap();
18720 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18721 assert!(!runner.state.gate_status().ok());
18722 assert!(runner.state.land_approval.is_none());
18723 let disk = RunState::load(&runner.state.id).unwrap();
18724 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18725 assert!(disk.merge_repairs[0].gate[0].ok());
18726 assert_eq!(
18727 disk.merge_repairs[0].phase,
18728 crate::run::MergeRepairPhase::Reviewing
18729 );
18730 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18731 assert!(disk.reviews.is_empty());
18732 }
18733
18734 #[tokio::test]
18735 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18736 crate::run::pin_test_home();
18737 for phase in [
18738 crate::run::MergeRepairPhase::Fixing,
18739 crate::run::MergeRepairPhase::Requested,
18740 ] {
18741 let mut runner = runner_at(RunStatus::Landing);
18742 runner.state.config.graph.land_rounds = 0;
18743 runner.state.merge_repairs.push(merge_repair_request(phase));
18744 runner.state.save().unwrap();
18745 assert!(!runner.repair_merge_review().await.unwrap());
18746 assert_eq!(runner.state.status, RunStatus::Blocked);
18747 assert_eq!(
18748 runner.state.merge_repairs[0].phase,
18749 crate::run::MergeRepairPhase::Failed
18750 );
18751 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18752 assert!(
18753 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18754 "interrupted"
18755 } else {
18756 "budget exhausted"
18757 })
18758 );
18759 assert!(runner.state.merge.is_none());
18760 }
18761 }
18762
18763 #[test]
18767 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18768 crate::run::pin_test_home();
18769 let mut runner = runner_at(RunStatus::Implementing);
18770 let interrupt = Pause::new();
18771 runner.watch_interrupt(interrupt.clone());
18772
18773 interrupt.park_because("task a1b2 asked to run first");
18774
18775 assert!(runner.park_here().expect("park_here"));
18776 assert!(runner.state.parked);
18777 let last = runner.state.events.last().expect("a park event");
18778 assert_eq!(last.node, "park");
18779 assert!(
18780 last.message.contains("task a1b2 asked to run first"),
18781 "expected the interrupt reason in {:?}",
18782 last.message
18783 );
18784 }
18785
18786 #[test]
18789 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18790 crate::run::pin_test_home();
18791 let mut runner = runner_at(RunStatus::Implementing);
18792 runner.state.begin_pass();
18793 let shutdown = Pause::new();
18794 runner.on_pause(shutdown.clone());
18795 let upgrade = ParkCause::Upgrade {
18796 from: "0.121.0".to_owned(),
18797 to: Some("0.121.1".to_owned()),
18798 };
18799 shutdown.park_for(upgrade.clone());
18800 shutdown.park_for(ParkCause::Operator);
18801
18802 assert!(runner.park_here().expect("park_here"));
18803 assert_eq!(
18804 runner.state.passes[0].end,
18805 Some(crate::run::PassEnd::Parked { cause: upgrade })
18806 );
18807 let last = runner.state.events.last().expect("a park event");
18808 assert!(last.message.contains("0.121.0"), "{}", last.message);
18809 }
18810
18811 #[tokio::test]
18812 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18813 crate::run::pin_test_home();
18814 let mut runner = runner_at(RunStatus::Reviewing);
18815 runner.state.save().unwrap();
18816 let mut execution = runner.claim_driver().await.expect("first execution");
18817 assert_eq!(execution.runner.state.passes.len(), 1);
18818 execution
18819 .driver
18820 .finish(&mut execution.runner.state)
18821 .await
18822 .unwrap();
18823 }
18824
18825 #[tokio::test]
18826 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18827 crate::run::pin_test_home();
18828 let mut runner = runner_at(RunStatus::Reviewing);
18829 runner.state.config.agents = vec![conductor()];
18830 runner.state.config.graph.timeout_resource_wait = 0;
18831 runner.state.save().unwrap();
18832 let id = runner.state.id.clone();
18833 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18834 assert!(runner.claim_driver().await.is_err());
18835 drop(owner);
18836 drop(runner);
18837 let mut resumed = Runner::resume(&id).unwrap();
18838 assert_eq!(resumed.state.driver_claimed, Some(false));
18839 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18840 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18841 let mut execution = resumed
18842 .claim_driver()
18843 .await
18844 .expect("retry initial claim after reload");
18845 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18846 assert_eq!(execution.runner.state.passes.len(), 1);
18847 execution
18848 .driver
18849 .finish(&mut execution.runner.state)
18850 .await
18851 .unwrap();
18852 }
18853
18854 #[tokio::test]
18855 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18856 crate::run::pin_test_home();
18857 let mut runner = runner_at(RunStatus::Reviewing);
18858 runner.state.config.agents = vec![conductor()];
18859 runner.state.save().unwrap();
18860 let id = runner.state.id.clone();
18861 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18862 let mut resumed = Runner::resume(&id).unwrap();
18863 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18864 assert!(resumed.claim_driver().await.is_err());
18865 assert!(RunState::load(&id).unwrap().passes.is_empty());
18866 drop(lease);
18867 let mut execution = resumed.claim_driver().await.unwrap();
18868 execution
18869 .driver
18870 .finish(&mut execution.runner.state)
18871 .await
18872 .unwrap();
18873 }
18874
18875 #[tokio::test]
18876 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18877 crate::run::pin_test_home();
18878 let mut runner = runner_at(RunStatus::Reviewing);
18879 let state = &mut runner.state;
18880 state.config.agents = vec![conductor()];
18881 state.driver_claimed = None;
18882 state.driver_pid = Some(std::process::id());
18883 state.driver_started_at = None;
18884 state.begin_pass();
18885 state.save().unwrap();
18886 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18887 std::fs::write(
18888 state.dir().join("driver.json"),
18889 serde_json::to_vec(&serde_json::json!({
18890 "pid": state.driver_pid, "started_at": null,
18891 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18892 }))
18893 .unwrap(),
18894 )
18895 .unwrap();
18896 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18897 drop(lease);
18898 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18899 let mut resumed = Runner::resume(&state.id).unwrap();
18900 assert!(!resumed.state.driver_exited);
18901 let mut execution = resumed
18902 .claim_driver()
18903 .await
18904 .expect("lease proves previous execution ended");
18905 assert_eq!(execution.runner.state.passes.len(), 2);
18906 execution
18907 .driver
18908 .finish(&mut execution.runner.state)
18909 .await
18910 .unwrap();
18911 }
18912
18913 #[tokio::test]
18914 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18915 crate::run::pin_test_home();
18916 let mut runner = runner_at(RunStatus::Reviewing);
18917 let mut json = serde_json::to_value(&runner.state).unwrap();
18918 json.as_object_mut().unwrap().remove("driver_claimed");
18919 runner.state = serde_json::from_value(json).unwrap();
18920 assert_eq!(runner.state.driver_claimed, None);
18921 runner.state.save().unwrap();
18922 let mut execution = runner
18923 .claim_driver()
18924 .await
18925 .expect("the execution lease, not a missing marker, excludes a second driver");
18926 assert_eq!(execution.runner.state.passes.len(), 1);
18927 execution
18928 .driver
18929 .finish(&mut execution.runner.state)
18930 .await
18931 .unwrap();
18932 }
18933
18934 #[tokio::test]
18935 async fn driver_admission_refuses_a_proven_live_driver() {
18936 crate::run::pin_test_home();
18937 let mut runner = runner_at(RunStatus::Reviewing);
18938 let pid = std::process::id();
18939 runner.state.driver_pid = Some(pid);
18940 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18941 assert!(runner.state.driver_started_at.is_some());
18942 runner.state.save().unwrap();
18943 let error = runner
18944 .claim_driver()
18945 .await
18946 .err()
18947 .expect("a live driver must refuse");
18948 assert!(error.to_string().contains("live driver"), "{error:#}");
18949 let saved = RunState::load(&runner.state.id).unwrap();
18950 assert_eq!(saved.driver_pid, Some(pid));
18951 assert_eq!(saved.status, RunStatus::Reviewing);
18952 assert!(saved.passes.is_empty());
18953 }
18954
18955 #[test]
18956 fn parking_resource_halt_persists_the_pass_once() {
18957 crate::run::pin_test_home();
18958 let mut runner = runner_at(RunStatus::Reviewing);
18959 runner.state.begin_pass();
18960 runner.state.resource_block = Some(crate::run::ResourceBlock {
18961 node: "fix".into(),
18962 kind: "shared build cache".into(),
18963 detail: "cancelled".into(),
18964 at: Timestamp::now(),
18965 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18966 });
18967 let cause = ParkCause::Upgrade {
18968 from: "old".into(),
18969 to: Some("new".into()),
18970 };
18971 runner.pause.park_for(cause.clone());
18972 runner.stop_for_resource().unwrap();
18975 assert!(runner.resource_halt().unwrap());
18976 assert!(runner.park_here().unwrap());
18977 let saved = RunState::load(&runner.state.id).unwrap();
18978 assert!(saved.parked);
18979 assert_eq!(saved.status, RunStatus::Reviewing);
18980 assert!(saved.passes[0].ended_at.is_some());
18981 assert_eq!(
18982 saved.passes[0].end,
18983 Some(crate::run::PassEnd::Parked {
18984 cause: cause.clone()
18985 })
18986 );
18987 assert_eq!(
18988 saved.resource_block.unwrap().cause,
18989 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18990 );
18991 assert_eq!(
18992 saved
18993 .events
18994 .iter()
18995 .filter(|event| event.node == "park")
18996 .count(),
18997 1
18998 );
18999 }
19000
19001 #[test]
19009 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19010 crate::run::pin_test_home();
19011 let mut runner = runner_at(RunStatus::Implementing);
19012 let shutdown = Pause::new();
19013 runner.on_pause(shutdown.clone());
19014 let interrupt = Pause::new();
19015 runner.watch_interrupt(interrupt.clone());
19016
19017 assert!(!runner.park_here().expect("park_here"));
19019 assert!(!runner.state.parked);
19020
19021 interrupt.park_because("test");
19023 assert!(!shutdown.parked());
19024 assert!(runner.park_here().expect("park_here"));
19025 }
19026
19027 #[tokio::test]
19041 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19042 crate::run::pin_test_home();
19043 let mut runner = runner_at(RunStatus::Implementing);
19044 let interrupt = Pause::new();
19045 runner.watch_interrupt(interrupt.clone());
19046
19047 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19048 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19049
19050 let node = async move {
19054 started_tx.send(()).expect("send started");
19055 finish_rx.await.expect("recv finish");
19056 "node finished"
19057 };
19058
19059 let interrupter = async move {
19060 started_rx.await.expect("recv started");
19061 interrupt.park_because("higher-priority task waiting");
19063 tokio::task::yield_now().await;
19067 finish_tx.send(()).expect("send finish");
19068 };
19069
19070 let (node_result, ()) = tokio::join!(node, interrupter);
19071 assert_eq!(
19072 node_result, "node finished",
19073 "the in-flight call ran to completion"
19074 );
19075
19076 assert!(runner.park_here().expect("park_here"));
19079 assert!(runner.state.parked);
19080 }
19081
19082 #[test]
19088 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19089 crate::run::pin_test_home();
19090 let mut runner = runner_at(RunStatus::Judging);
19091 runner.state.config.agents = vec![conductor()];
19095 runner.state.candidates = vec![Candidate {
19096 index: 0,
19097 label: 'A',
19098 agent: "alpha".to_owned(),
19099 branch: "magi/x/A".to_owned(),
19100 worktree: PathBuf::from("/nonexistent/worktree"),
19101 summary: "did the thing".to_owned(),
19102 stat: "1 file changed".to_owned(),
19103 files: 1,
19104 commits: 1,
19105 empty: false,
19106 failed: None,
19107 verified_noop: None,
19108 duration_ms: 1234,
19109 folded: false,
19110 }];
19111 let run_id = runner.state.id.clone();
19112
19113 let interrupt = Pause::new();
19114 runner.watch_interrupt(interrupt.clone());
19115 interrupt.park_because("task c3d4 asked to run first");
19116 assert!(runner.park_here().expect("park_here"));
19117
19118 let resumed = Runner::resume(&run_id).expect("resume");
19119 assert_eq!(resumed.state.candidates.len(), 1);
19120 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19121 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19122 assert_eq!(resumed.state.status, runner.state.status);
19123 assert!(
19124 resumed.state.parked,
19125 "still parked until `execute` actually walks the graph again"
19126 );
19127 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19128 }
19129
19130 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19132 let mut q = ask::Question::new(
19133 run.to_owned(),
19134 "implement".to_owned(),
19135 "impl-A".to_owned(),
19136 "Which storage backend should the cache use?".to_owned(),
19137 String::new(),
19138 vec!["SQLite".to_owned(), "Redis".to_owned()],
19139 );
19140 store.put(&mut q).unwrap();
19141 q
19142 }
19143
19144 #[test]
19145 fn a_failed_runs_open_question_is_abandoned() {
19146 ask_test_home();
19147 let store = ask::Questions::open();
19148 let mut runner = runner_at(RunStatus::Failed);
19149 let run = runner.state.id.clone();
19150 let q = ask_open_question(&store, &run);
19151
19152 runner.settle_questions();
19153
19154 let back = store.get(&q.id).unwrap();
19155 assert!(
19156 !back.status.open(),
19157 "the seat that asked died with the run; nobody is left to read an answer"
19158 );
19159 assert!(
19160 back.detail.contains(&run) && back.detail.contains("failed"),
19161 "the reason names what the run became, not just that it is gone: {}",
19162 back.detail
19163 );
19164 }
19165
19166 #[test]
19167 fn a_merged_runs_open_question_is_abandoned_too() {
19168 ask_test_home();
19169 let store = ask::Questions::open();
19170 for status in [RunStatus::Merged, RunStatus::Ready] {
19173 let mut runner = runner_at(status);
19174 let run = runner.state.id.clone();
19175 let q = ask_open_question(&store, &run);
19176
19177 runner.settle_questions();
19178
19179 let back = store.get(&q.id).unwrap();
19180 assert!(
19181 !back.status.open(),
19182 "{status:?} run's question must not outlive the run"
19183 );
19184 }
19185 }
19186
19187 #[test]
19188 fn a_still_resumable_runs_open_question_is_left_alone() {
19189 ask_test_home();
19190 let store = ask::Questions::open();
19191 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19197 let mut runner = runner_at(status);
19198 let run = runner.state.id.clone();
19199 let q = ask_open_question(&store, &run);
19200
19201 runner.settle_questions();
19202
19203 let back = store.get(&q.id).unwrap();
19204 assert!(
19205 back.status.open(),
19206 "{status:?} is still alive; the question must still be waiting"
19207 );
19208 }
19209 }
19210
19211 #[test]
19212 fn settle_questions_never_touches_an_already_answered_question() {
19213 ask_test_home();
19214 let store = ask::Questions::open();
19215 let mut runner = runner_at(RunStatus::Failed);
19216 let run = runner.state.id.clone();
19217 let mut q = ask_open_question(&store, &run);
19218 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19219 .unwrap();
19220 store.put(&mut q).unwrap();
19221
19222 runner.settle_questions();
19227 runner.settle_questions();
19228
19229 let back = store.get(&q.id).unwrap();
19230 assert_eq!(
19231 back.status,
19232 ask::QuestionStatus::Answered,
19233 "a real answer is a decision on record, never overwritten by a sweep"
19234 );
19235 }
19236
19237 #[tokio::test]
19248 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19249 crate::run::pin_test_home();
19250 let tmp = tempfile::tempdir().expect("tempdir");
19251 let repo = tmp.path().join("repo");
19252 std::fs::create_dir_all(&repo).unwrap();
19253 init_repo(&repo);
19254
19255 let mut config = Config::default();
19256 config.graph.worktree_root = Some(tmp.path().join("wt"));
19257
19258 let mut state = RunState::new(
19259 repo.clone(),
19260 "main".to_owned(),
19261 "deadbeef".to_owned(),
19262 "task".to_owned(),
19263 config,
19264 );
19265 let root = state.worktree_root();
19266 let wt_a = root.join("cand-A");
19267 let wt_b = root.join("cand-B");
19268 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19269 .await
19270 .expect("worktree A");
19271 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19272 .await
19273 .expect("worktree B");
19274
19275 state.candidates = vec![
19276 Candidate {
19277 index: 0,
19278 label: 'A',
19279 agent: "alpha".to_owned(),
19280 branch: "magi/x/A".to_owned(),
19281 worktree: wt_a.clone(),
19282 summary: String::new(),
19283 stat: String::new(),
19284 files: 0,
19285 commits: 0,
19286 empty: false,
19287 failed: None,
19288 verified_noop: None,
19289 duration_ms: 0,
19290 folded: false,
19291 },
19292 Candidate {
19293 index: 1,
19294 label: 'B',
19295 agent: "beta".to_owned(),
19296 branch: "magi/x/B".to_owned(),
19297 worktree: wt_b.clone(),
19298 summary: String::new(),
19299 stat: String::new(),
19300 files: 0,
19301 commits: 0,
19302 empty: false,
19303 failed: None,
19304 verified_noop: None,
19305 duration_ms: 0,
19306 folded: false,
19307 },
19308 ];
19309 state.tally = Some(Tally {
19310 first_choice: BTreeMap::from([('A', 1)]),
19311 borda: BTreeMap::new(),
19312 winner: 'A',
19313 rankings: 1,
19314 unanimous_initial: true,
19315 deliberated: false,
19316 changed_votes: 0,
19317 unanimous_final: true,
19318 tie_break: None,
19319 judges: 1,
19320 present: 1,
19321 quorum: 1,
19322 met_quorum: true,
19323 uncontested: None,
19324 });
19325 state.status = RunStatus::Ready;
19326
19327 fold_run(&mut state, false, &crate::run::home())
19328 .await
19329 .expect("fold_run");
19330
19331 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19332 assert!(
19333 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19334 "the unmerged winner's branch survives"
19335 );
19336 assert!(
19337 !state.candidates[0].folded,
19338 "the winner is not marked folded"
19339 );
19340
19341 assert!(!wt_b.exists(), "the loser's worktree is removed");
19342 assert!(
19343 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19344 "the loser's branch is removed"
19345 );
19346 assert!(state.candidates[1].folded, "the loser is marked folded");
19347 }
19348
19349 #[tokio::test]
19350 #[cfg(unix)]
19351 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19352 use std::os::fd::AsRawFd as _;
19353 use std::time::Duration;
19354
19355 let tmp = tempfile::tempdir().unwrap();
19356 let repo = tmp.path().join("repo");
19357 std::fs::create_dir_all(&repo).unwrap();
19358 init_repo(&repo);
19359 let common = repo.join(".git");
19360 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19361 let home = tmp.path().join("home");
19362 let mut config = Config::default();
19363 config.coordination = crate::config::test_coordination();
19364 config.graph.worktree_root = Some(tmp.path().join("wt"));
19365 config.graph.timeout_resource_wait = 1;
19366 let mut state = RunState::new(
19367 repo.clone(),
19368 "main".into(),
19369 "deadbeef".into(),
19370 "task".into(),
19371 config,
19372 );
19373 let root = state.worktree_root();
19374 let candidate = root.join("cand-A");
19375 let reviewer = root.join("review-1");
19376 let branch = state.branch_for('A');
19377 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19378 .await
19379 .unwrap();
19380 git::worktree_add_detached(&repo, &reviewer, "main")
19381 .await
19382 .unwrap();
19383 git::acquire_worktree_config(&repo).await.unwrap();
19384 state.enabled_worktree_config = true;
19385 state.status = RunStatus::Ready;
19386 state.candidates.push(Candidate {
19387 index: 0,
19388 label: 'A',
19389 agent: "implement".into(),
19390 branch: branch.clone(),
19391 worktree: candidate.clone(),
19392 summary: String::new(),
19393 stat: String::new(),
19394 files: 0,
19395 commits: 0,
19396 empty: false,
19397 failed: None,
19398 verified_noop: None,
19399 duration_ms: 0,
19400 folded: false,
19401 });
19402 state.save_under(&home).unwrap();
19403 let config_before = std::fs::read(common.join("config")).unwrap();
19404 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19405 .await
19406 .unwrap();
19407 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19410 assert_eq!(
19411 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19412 0
19413 );
19414 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19415 assert!(
19416 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19417 .await
19418 .is_err()
19419 );
19420 assert!(candidate.exists());
19421 assert!(reviewer.exists());
19422 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19423 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19424 assert_eq!(
19425 git::git(&repo, &["worktree", "list", "--porcelain"])
19426 .await
19427 .unwrap(),
19428 worktrees_before
19429 );
19430 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19431 .await
19432 .unwrap()
19433 .unwrap_err();
19434 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19435 drop(fold);
19436 assert!(candidate.exists() && reviewer.exists());
19437 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19438 assert_eq!(
19439 state.resource_block.as_ref().unwrap().kind,
19440 "repository coordination"
19441 );
19442 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19443 drop(external);
19444 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19445 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19446 .await
19447 .unwrap()
19448 .unwrap();
19449 drop(fold);
19450 assert!(removed.contains(&branch));
19451 assert!(!root.exists());
19452 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19453 assert!(state.candidates[0].folded);
19454 assert!(!state.enabled_worktree_config);
19455 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19456 let enabled = git::git_raw(
19457 &repo,
19458 &["config", "--local", "--get", "extensions.worktreeConfig"],
19459 )
19460 .await
19461 .unwrap();
19462 assert!(!enabled.ok(), "the last config reference was released");
19463 }
19464
19465 #[tokio::test]
19468 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19469 let tmp = tempfile::tempdir().expect("tempdir");
19470 let repo = tmp.path().join("repo");
19471 std::fs::create_dir_all(&repo).unwrap();
19472 init_repo(&repo);
19473 let home = tmp.path().join("home");
19474
19475 let mut config = Config::default();
19476 config.graph.worktree_root = Some(tmp.path().join("wt"));
19477 let mut state = RunState::new(
19478 repo.clone(),
19479 "main".to_owned(),
19480 "deadbeef".to_owned(),
19481 "task".to_owned(),
19482 config,
19483 );
19484 git::git(&repo, &["branch", "magi/x/A", "main"])
19486 .await
19487 .expect("branch");
19488 state.candidates = vec![Candidate {
19489 index: 0,
19490 label: 'A',
19491 agent: "alpha".to_owned(),
19492 branch: "magi/x/A".to_owned(),
19493 worktree: state.worktree_root().join("cand-A"),
19494 summary: String::new(),
19495 stat: String::new(),
19496 files: 0,
19497 commits: 0,
19498 empty: false,
19499 failed: None,
19500 verified_noop: None,
19501 duration_ms: 0,
19502 folded: true,
19503 }];
19504 state.released_to = Some("20260901-000000-new1".to_owned());
19505 state.released_branches = vec!["magi/x/A".to_owned()];
19506
19507 fold_run(&mut state, true, &home).await.expect("fold_run");
19508 state.status = RunStatus::Merged;
19509 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19510 state.merge = Some(crate::run::MergeOutcome {
19511 mode: MergeMode::Pr,
19512 ok: true,
19513 detail: "confirmed merged".into(),
19514 empty: false,
19515 });
19516 fold_run(&mut state, true, &home)
19517 .await
19518 .expect("merged fold");
19519
19520 assert!(
19521 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19522 "the handed-over branch survives a fold"
19523 );
19524 }
19525
19526 #[tokio::test]
19530 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19531 let tmp = tempfile::tempdir().expect("tempdir");
19532 let repo = tmp.path().join("repo");
19533 std::fs::create_dir_all(&repo).unwrap();
19534 init_repo(&repo);
19535 let run = |args: &[&str]| {
19536 let out = std::process::Command::new("git")
19537 .quiet()
19538 .args(args)
19539 .current_dir(&repo)
19540 .output()
19541 .expect("spawn git");
19542 assert!(out.status.success(), "git {args:?}");
19543 };
19544 run(&["branch", "magi/x/A"]);
19545 run(&["checkout", "-q", "-b", "magi/x/B"]);
19546 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19547 run(&["add", "-A"]);
19548 run(&["commit", "-q", "-m", "work"]);
19549 run(&["checkout", "-q", "main"]);
19550
19551 let probe = RunState::new(
19555 repo.clone(),
19556 "main".to_owned(),
19557 "deadbeef".to_owned(),
19558 "task".to_owned(),
19559 Config::default(),
19560 );
19561 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19562 let bare = tmp.path().join("origin.git");
19563 let out = std::process::Command::new("git")
19564 .quiet()
19565 .args(["init", "-q", "--bare"])
19566 .arg(&bare)
19567 .output()
19568 .expect("spawn git");
19569 assert!(out.status.success(), "git init --bare");
19570 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19571 run(&["push", "-q", "origin", "main"]);
19572
19573 let mut state = RunState::new(
19574 repo.clone(),
19575 "main".to_owned(),
19576 "deadbeef".to_owned(),
19577 "task".to_owned(),
19578 Config::default(),
19579 );
19580 state.seeds = vec![refs::Seed {
19581 token: "magi/27b2/A".to_owned(),
19582 kind: refs::SeedKind::Unresolved,
19583 sha: String::new(),
19584 branch: true,
19585 detail: "no branch or commit named magi/27b2/A".to_owned(),
19586 }];
19587
19588 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19589 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19590 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19591 let detail = empty_candidate_detail(&state, "main");
19592 assert!(detail.starts_with("empty candidate"), "{detail}");
19593 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19594 assert!(detail.contains("--start-from"), "{detail}");
19595 }
19596
19597 #[tokio::test]
19606 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19607 let tmp = tempfile::tempdir().expect("tempdir");
19608 let repo = tmp.path().join("repo");
19609 std::fs::create_dir_all(&repo).unwrap();
19610 init_repo(&repo);
19611
19612 let mut config = Config::default();
19613 config.merge.mode = MergeMode::Local;
19614
19615 let mut state = RunState::new(
19616 repo.clone(),
19617 "main".to_owned(),
19618 "deadbeef".to_owned(),
19619 "task".to_owned(),
19620 config,
19621 );
19622 state.candidates = vec![Candidate {
19623 index: 0,
19624 label: 'A',
19625 agent: "alpha".to_owned(),
19626 branch: "does-not-exist".to_owned(),
19627 worktree: repo.clone(),
19628 summary: String::new(),
19629 stat: String::new(),
19630 files: 0,
19631 commits: 0,
19632 empty: false,
19633 failed: None,
19634 verified_noop: None,
19635 duration_ms: 0,
19636 folded: false,
19637 }];
19638 state.tally = Some(Tally {
19639 first_choice: BTreeMap::from([('A', 1)]),
19640 borda: BTreeMap::new(),
19641 winner: 'A',
19642 rankings: 1,
19643 unanimous_initial: true,
19644 deliberated: false,
19645 changed_votes: 0,
19646 unanimous_final: true,
19647 tie_break: None,
19648 judges: 0,
19649 present: 0,
19650 quorum: 0,
19651 met_quorum: true,
19652 uncontested: Some("only candidate A produced a change".to_owned()),
19653 });
19654 state.reviews = vec![ReviewRound {
19655 round: 1,
19656 head: "deadbeef".to_owned(),
19657 verified_head: None,
19658 verified_at: None,
19659 reviews: Vec::new(),
19660 e2e: Vec::new(),
19661 fix: None,
19662 blocking: 0,
19663 answered: 0,
19664 expected: 0,
19665 clean: true,
19666 verify_retried: false,
19667 e2e_deferred: false,
19668 e2e_defer_reason: None,
19669 progressed: false,
19670 vote_split: false,
19671 reconsideration: Vec::new(),
19672 verdict: None,
19673 }];
19674 state.gate = vec![CommandOutcome {
19675 command: "test".to_owned(),
19676 code: Some(0),
19677 output_tail: String::new(),
19678 duration_ms: 0,
19679 resource_blocked: false,
19680 }];
19681 state.gate_ran = true;
19682 state.status = RunStatus::Ready;
19687 state.merge = Some(MergeOutcome {
19688 mode: MergeMode::Local,
19689 ok: false,
19690 detail: "already concluded".to_owned(),
19691 empty: false,
19692 });
19693
19694 let mut runner = Runner {
19695 state,
19696 roles: ResolvedRoles {
19697 implementers: Vec::new(),
19698 judges: Vec::new(),
19699 reviewers: Vec::new(),
19700 fixer: None,
19701 conductor: conductor(),
19702 implementer_roster: Vec::new(),
19703 judge_roster: Vec::new(),
19704 reviewer_roster: Vec::new(),
19705 },
19706 sem: Arc::new(Semaphore::new(1)),
19707 pause: Pause::new(),
19708 interrupt: Pause::new(),
19709 };
19710
19711 runner.merge().await.expect("merge");
19712
19713 assert_eq!(
19714 runner.state.status,
19715 RunStatus::Ready,
19716 "a concluded run's status must not change on reentry"
19717 );
19718 assert_eq!(
19719 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19720 Some("already concluded"),
19721 "merge must not run again once the node already recorded an outcome"
19722 );
19723 }
19724
19725 #[tokio::test]
19734 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19735 let tmp = tempfile::tempdir().expect("tempdir");
19736 let repo = tmp.path().join("repo");
19737 std::fs::create_dir_all(&repo).unwrap();
19738 init_repo(&repo);
19739
19740 let mut config = Config::default();
19741 config.merge.mode = MergeMode::Local;
19742
19743 let mut state = RunState::new(
19744 repo.clone(),
19745 "main".to_owned(),
19746 "deadbeef".to_owned(),
19747 "task".to_owned(),
19748 config,
19749 );
19750 state.candidates = vec![Candidate {
19751 index: 0,
19752 label: 'A',
19753 agent: "alpha".to_owned(),
19754 branch: "does-not-exist".to_owned(),
19755 worktree: repo.clone(),
19756 summary: String::new(),
19757 stat: String::new(),
19758 files: 0,
19759 commits: 0,
19760 empty: false,
19761 failed: None,
19762 verified_noop: None,
19763 duration_ms: 0,
19764 folded: false,
19765 }];
19766 state.tally = Some(Tally {
19767 first_choice: BTreeMap::from([('A', 1)]),
19768 borda: BTreeMap::new(),
19769 winner: 'A',
19770 rankings: 1,
19771 unanimous_initial: true,
19772 deliberated: false,
19773 changed_votes: 0,
19774 unanimous_final: true,
19775 tie_break: None,
19776 judges: 0,
19777 present: 0,
19778 quorum: 0,
19779 met_quorum: true,
19780 uncontested: Some("only candidate A produced a change".to_owned()),
19781 });
19782 state.reviews = vec![ReviewRound {
19783 round: 1,
19784 head: "deadbeef".to_owned(),
19785 verified_head: None,
19786 verified_at: None,
19787 reviews: Vec::new(),
19788 e2e: Vec::new(),
19789 fix: None,
19790 blocking: 0,
19791 answered: 0,
19792 expected: 0,
19793 clean: true,
19794 verify_retried: false,
19795 e2e_deferred: false,
19796 e2e_defer_reason: None,
19797 progressed: false,
19798 vote_split: false,
19799 reconsideration: Vec::new(),
19800 verdict: None,
19801 }];
19802 state.gate = Vec::new();
19804 state.gate_ran = false;
19805 state.status = RunStatus::Gating;
19806
19807 let mut runner = Runner {
19808 state,
19809 roles: ResolvedRoles {
19810 implementers: Vec::new(),
19811 judges: Vec::new(),
19812 reviewers: Vec::new(),
19813 fixer: None,
19814 conductor: conductor(),
19815 implementer_roster: Vec::new(),
19816 judge_roster: Vec::new(),
19817 reviewer_roster: Vec::new(),
19818 },
19819 sem: Arc::new(Semaphore::new(1)),
19820 pause: Pause::new(),
19821 interrupt: Pause::new(),
19822 };
19823
19824 runner.merge().await.expect("merge");
19825
19826 assert!(
19827 runner.state.merge.is_none(),
19828 "an empty gate must never be read as a passing one: {:?}",
19829 runner.state.merge
19830 );
19831 }
19832
19833 #[tokio::test]
19840 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19841 let tmp = tempfile::tempdir().expect("tempdir");
19842 let repo = tmp.path().join("repo");
19843 std::fs::create_dir_all(&repo).unwrap();
19844 init_repo(&repo);
19845
19846 let config = Config::default();
19848
19849 let mut state = RunState::new(
19850 repo.clone(),
19851 "main".to_owned(),
19852 "deadbeef".to_owned(),
19853 "task".to_owned(),
19854 config,
19855 );
19856 state.candidates = vec![Candidate {
19857 index: 0,
19858 label: 'A',
19859 agent: "alpha".to_owned(),
19860 branch: "does-not-exist".to_owned(),
19861 worktree: repo.clone(),
19862 summary: String::new(),
19863 stat: String::new(),
19864 files: 0,
19865 commits: 0,
19866 empty: false,
19867 failed: None,
19868 verified_noop: None,
19869 duration_ms: 0,
19870 folded: false,
19871 }];
19872 state.tally = Some(Tally {
19873 first_choice: BTreeMap::from([('A', 1)]),
19874 borda: BTreeMap::new(),
19875 winner: 'A',
19876 rankings: 1,
19877 unanimous_initial: true,
19878 deliberated: false,
19879 changed_votes: 0,
19880 unanimous_final: true,
19881 tie_break: None,
19882 judges: 0,
19883 present: 0,
19884 quorum: 0,
19885 met_quorum: true,
19886 uncontested: Some("only candidate A produced a change".to_owned()),
19887 });
19888 state.reviews = vec![ReviewRound {
19889 round: 1,
19890 head: "deadbeef".to_owned(),
19891 verified_head: None,
19892 verified_at: None,
19893 reviews: Vec::new(),
19894 e2e: Vec::new(),
19895 fix: None,
19896 blocking: 0,
19897 answered: 0,
19898 expected: 0,
19899 clean: true,
19900 verify_retried: false,
19901 e2e_deferred: false,
19902 e2e_defer_reason: None,
19903 progressed: false,
19904 vote_split: false,
19905 reconsideration: Vec::new(),
19906 verdict: None,
19907 }];
19908
19909 let mut runner = Runner {
19910 state,
19911 roles: ResolvedRoles {
19912 implementers: Vec::new(),
19913 judges: Vec::new(),
19914 reviewers: Vec::new(),
19915 fixer: None,
19916 conductor: conductor(),
19917 implementer_roster: Vec::new(),
19918 judge_roster: Vec::new(),
19919 reviewer_roster: Vec::new(),
19920 },
19921 sem: Arc::new(Semaphore::new(1)),
19922 pause: Pause::new(),
19923 interrupt: Pause::new(),
19924 };
19925
19926 runner.gate().await.expect("gate");
19927 assert!(
19928 runner.state.gate_ran,
19929 "zero configured commands is still a real attempt, not an unrun gate"
19930 );
19931 assert!(runner.state.gate.is_empty());
19932 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19933 assert_ne!(
19934 runner.state.status,
19935 RunStatus::Blocked,
19936 "a gate with nothing to check must not read as failed"
19937 );
19938
19939 runner.merge().await.expect("merge");
19940 assert_eq!(
19941 runner.state.status,
19942 RunStatus::Ready,
19943 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19944 );
19945 }
19946
19947 #[tokio::test]
19958 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19959 crate::run::pin_test_home();
19960 let home = crate::run::home();
19961
19962 let tmp = tempfile::tempdir().expect("tempdir");
19963 let repo = tmp.path().join("repo");
19964 std::fs::create_dir_all(&repo).unwrap();
19965 init_repo(&repo);
19966 let cache_dir = tmp.path().join("target");
19969
19970 let mut config = Config::default();
19971 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19972 config.graph.timeout_verify = Some(2);
19976 config.graph.timeout_resource_wait = 0;
19977
19978 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19979 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19980 .expect("no io error acquiring directly")
19981 {
19982 crate::cache::AcquireOutcome::Acquired(g) => g,
19983 crate::cache::AcquireOutcome::Busy(b) => {
19984 panic!("expected the direct acquire to win the lease first: {b:?}")
19985 }
19986 };
19987
19988 let mut state = RunState::new(
19989 repo.clone(),
19990 "main".to_owned(),
19991 "deadbeef".to_owned(),
19992 "task".to_owned(),
19993 config,
19994 );
19995 state.candidates = vec![Candidate {
19996 index: 0,
19997 label: 'A',
19998 agent: "alpha".to_owned(),
19999 branch: "does-not-exist".to_owned(),
20000 worktree: repo.clone(),
20001 summary: String::new(),
20002 stat: String::new(),
20003 files: 0,
20004 commits: 0,
20005 empty: false,
20006 failed: None,
20007 verified_noop: None,
20008 duration_ms: 0,
20009 folded: false,
20010 }];
20011 state.tally = Some(Tally {
20012 first_choice: BTreeMap::from([('A', 1)]),
20013 borda: BTreeMap::new(),
20014 winner: 'A',
20015 rankings: 1,
20016 unanimous_initial: true,
20017 deliberated: false,
20018 changed_votes: 0,
20019 unanimous_final: true,
20020 tie_break: None,
20021 judges: 0,
20022 present: 0,
20023 quorum: 0,
20024 met_quorum: true,
20025 uncontested: Some("only candidate A produced a change".to_owned()),
20026 });
20027 state.reviews = vec![ReviewRound {
20028 round: 1,
20029 head: "deadbeef".to_owned(),
20030 verified_head: None,
20031 verified_at: None,
20032 reviews: Vec::new(),
20033 e2e: Vec::new(),
20034 fix: None,
20035 blocking: 0,
20036 answered: 0,
20037 expected: 0,
20038 clean: true,
20039 verify_retried: false,
20040 e2e_deferred: false,
20041 e2e_defer_reason: None,
20042 progressed: false,
20043 vote_split: false,
20044 reconsideration: Vec::new(),
20045 verdict: None,
20046 }];
20047
20048 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20049 state.reviews[0].e2e = vec![CommandOutcome {
20050 command: state.config.verify.e2e[0].clone(),
20051 code: Some(0),
20052 output_tail: String::new(),
20053 duration_ms: 0,
20054 resource_blocked: false,
20055 }];
20056 let mut runner = Runner {
20057 state,
20058 roles: ResolvedRoles {
20059 implementers: Vec::new(),
20060 judges: Vec::new(),
20061 reviewers: Vec::new(),
20062 fixer: None,
20063 conductor: conductor(),
20064 implementer_roster: Vec::new(),
20065 judge_roster: Vec::new(),
20066 reviewer_roster: Vec::new(),
20067 },
20068 sem: Arc::new(Semaphore::new(1)),
20069 pause: Pause::new(),
20070 interrupt: Pause::new(),
20071 };
20072
20073 runner.gate().await.expect("gate");
20074 assert!(runner.state.resource_block.is_none());
20075 assert!(
20076 !runner
20077 .state
20078 .events
20079 .iter()
20080 .any(|event| { event.message.contains("waiting for the shared build cache") }),
20081 "a gate with nothing to run must never request the held cache lease"
20082 );
20083 assert!(
20084 runner.state.gate_ran,
20085 "zero commands is still a real, immediate attempt"
20086 );
20087 assert!(runner.state.gate.is_empty());
20088 assert_ne!(
20089 runner.state.status,
20090 RunStatus::Blocked,
20091 "must not read as resource-blocked on a lease it never asked for"
20092 );
20093 }
20094
20095 #[cfg(unix)]
20098 #[tokio::test]
20099 async fn verification_is_standalone_inside_a_live_worker() {
20100 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20101 if std::env::var_os(CHILD).is_none() {
20102 let home = tempfile::tempdir().unwrap();
20103 let generation = crate::worker::pin_current(home.path()).unwrap();
20104 let output = std::process::Command::new(std::env::current_exe().unwrap())
20105 .args([
20106 "--exact",
20107 "graph::tests::verification_is_standalone_inside_a_live_worker",
20108 "--nocapture",
20109 ])
20110 .env(CHILD, "1")
20111 .env("MAGI_HOME", home.path())
20112 .env("MAGI_WORKER_HOME", home.path())
20113 .env("MAGI_WORKER_ID", "abcdef")
20114 .env("MAGI_WORKER_GENERATION", &generation.id)
20115 .env("MAGI_WORKER_EXE", &generation.executable)
20116 .env("MAGI_RUN", "verification-parent")
20117 .env("MAGI_NODE", "gate")
20118 .env("MAGI_SEAT", "worker-fixture")
20119 .env("MAGI_PROCESS_OWNER", "fixture-owner")
20120 .env("MAGI_NO_AUTOUPDATE", "0")
20121 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20122 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20123 .env(
20124 "VERIFY_EXPECT_TOOLCHAIN",
20125 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20126 )
20127 .env(
20128 "VERIFY_EXPECT_CACHE",
20129 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20130 )
20131 .output()
20132 .unwrap();
20133 assert!(
20134 output.status.success()
20135 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20136 "{}\n{}",
20137 String::from_utf8_lossy(&output.stdout),
20138 String::from_utf8_lossy(&output.stderr)
20139 );
20140 return;
20141 }
20142 let home = crate::run::home();
20143 let directory = home.join("workers/jobs/abcdef");
20144 std::fs::create_dir_all(&directory).unwrap();
20145 let job = crate::worker::Job {
20146 id: "abcdef".into(),
20147 sequence: 1,
20148 kind: "task".into(),
20149 key: "task:verification".into(),
20150 payload: serde_json::json!({}),
20151 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20152 version: Some(env!("CARGO_PKG_VERSION").into()),
20153 status: crate::worker::Status::Running,
20154 result: None,
20155 pid: Some(std::process::id()),
20156 created_at: 0,
20157 finished_at: None,
20158 };
20159 std::fs::write(
20160 directory.join("job.json"),
20161 serde_json::to_vec(&job).unwrap(),
20162 )
20163 .unwrap();
20164 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20165 let assert_worker = || {
20166 assert_eq!(
20167 crate::worker::current_job(&home).unwrap().unwrap().id,
20168 job.id
20169 );
20170 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20171 assert_eq!(
20172 crate::worker::helper_executable(&home).unwrap(),
20173 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20174 );
20175 assert!(
20176 crate::updater::Checker::new(&crate::config::Update {
20177 mode: crate::config::UpdateMode::Notify,
20178 interval: None,
20179 auto_restart: false,
20180 })
20181 .is_none(),
20182 "the driver remains pinned, even while tests are standalone"
20183 );
20184 };
20185 assert_worker();
20186 let repo = home.join("repo");
20187 std::fs::create_dir_all(&repo).unwrap();
20188 init_repo(&repo);
20189 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20192 test -z "${MAGI_WORKER_GENERATION+x}" &&
20193 test -z "${MAGI_WORKER_HOME+x}" &&
20194 test -z "${MAGI_WORKER_EXE+x}" &&
20195 test "$MAGI_HOME/repo" -ef . &&
20196 test "$MAGI_RUN" = verification-parent &&
20197 test "$MAGI_NODE" = gate &&
20198 test "$MAGI_SEAT" = worker-fixture &&
20199 test -n "$MAGI_PROCESS_OWNER" &&
20200 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20201 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20202 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20203 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20204 assert!(
20206 !std::process::Command::new("sh")
20207 .args(["-c", probe])
20208 .current_dir(&repo)
20209 .status()
20210 .unwrap()
20211 .success()
20212 );
20213 let unrelated_home = tempfile::tempdir().unwrap();
20214 assert!(
20215 crate::worker::current_job(unrelated_home.path()).is_err(),
20216 "an inherited worker id cannot be resolved in a unit fixture's home"
20217 );
20218 let mut runner = runner_at(RunStatus::Gating);
20219 runner.state.repo = repo.clone();
20220 runner.state.config.disk.min_free_bytes = 0;
20221 runner.state.config.coordination = crate::config::test_coordination();
20222 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20223 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20224 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20225 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20226 let winner = Candidate {
20227 index: 0,
20228 label: 'A',
20229 agent: "fixture".into(),
20230 branch: "main".into(),
20231 worktree: repo.clone(),
20232 summary: String::new(),
20233 stat: String::new(),
20234 files: 0,
20235 commits: 0,
20236 empty: false,
20237 failed: None,
20238 verified_noop: None,
20239 duration_ms: 0,
20240 folded: false,
20241 };
20242 let (e2e, retried, pids) = run_e2e_with_retry(
20243 &mut runner.state,
20244 &["sh".into(), "-c".into()],
20245 &[format!("{probe} && printf initial")],
20246 &repo,
20247 Duration::from_secs(10),
20248 "initial",
20249 )
20250 .await;
20251 assert!(!retried && pids.is_empty());
20252 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20253 assert_eq!(e2e[0].output_tail, "initial");
20254 let gate = runner.run_gate(&winner).await.unwrap();
20255 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20256 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20259 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20260 passed: false,
20261 outcomes: Vec::new(),
20262 });
20263 runner.state.save().unwrap();
20264 let mut resumed = RunState::load(&runner.state.id).unwrap();
20265 let wait = crate::repo_lock::WaitPolicy::default();
20266 assert!(matches!(
20267 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20268 .await
20269 .unwrap(),
20270 GateFix::Retry
20271 ));
20272 let saved = RunState::load(&resumed.id).unwrap();
20273 let repair = saved.gate_repair_e2e.unwrap();
20274 assert!(repair.passed);
20275 assert_eq!(repair.outcomes.len(), 1);
20276 assert!(repair.outcomes[0].ok());
20277 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20278 .await
20279 .unwrap();
20280 assert_eq!(landing.len(), 1);
20281 assert!(landing[0].ok(), "{landing:?}");
20282 assert_eq!(
20283 std::fs::read_to_string(repo.join("phases")).unwrap(),
20284 "pre\ngate\ne2e\npre\ngate\n"
20285 );
20286 assert_worker();
20287 }
20288
20289 #[cfg(unix)]
20290 #[tokio::test]
20291 async fn verification_keeps_the_panic_and_the_full_output() {
20292 crate::run::pin_test_home();
20293 let tmp = tempfile::tempdir().unwrap();
20294 let mut config = Config::default();
20295 config.disk.min_free_bytes = 0;
20296 let mut state = RunState::new(
20297 tmp.path().to_owned(),
20298 "main".into(),
20299 "base".into(),
20300 "verification evidence".into(),
20301 config,
20302 );
20303 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20304 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20305 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20306 let mut names = Vec::new();
20307 for _ in 0..2 {
20308 let (outcomes, _) = run_commands(
20309 &mut state,
20310 "gate",
20311 "gate",
20312 0,
20313 &["sh".into(), "-c".into()],
20314 &[script.into()],
20315 tmp.path(),
20316 Duration::from_secs(60),
20317 )
20318 .await;
20319 let tail = &outcomes[0].output_tail;
20320 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20321 assert!(tail.contains("boom assertion"));
20322 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20323 let line = tail.lines().next().unwrap();
20324 names.push(line.to_owned());
20325 }
20326 assert_ne!(names[0], names[1], "the second run must not overwrite");
20327 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20328 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20329 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20330
20331 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20333 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20334 let (outcomes, _) = run_commands(
20335 &mut state,
20336 "gate",
20337 "gate",
20338 0,
20339 &["sh".into(), "-c".into()],
20340 &["echo failure-output; exit 7".into()],
20341 tmp.path(),
20342 Duration::from_secs(10),
20343 )
20344 .await;
20345 assert_eq!(outcomes[0].code, Some(7));
20346 assert!(state.events.iter().any(|e| {
20347 e.node == "gate" && e.message.contains("could not save the full gate output:")
20348 }));
20349 }
20350
20351 #[cfg(unix)]
20352 #[tokio::test]
20353 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20354 crate::run::pin_test_home();
20355 let tmp = tempfile::tempdir().unwrap();
20356 let mut config = Config::default();
20357 config.disk.min_free_bytes = 0;
20358 let mut state = RunState::new(
20359 tmp.path().to_owned(),
20360 "main".into(),
20361 "base".into(),
20362 "verification evidence".into(),
20363 config,
20364 );
20365 let (outcomes, pids) = run_commands(
20366 &mut state,
20367 "gate",
20368 "gate",
20369 1,
20370 &["sh".into(), "-c".into()],
20371 &[
20372 "printf before-timeout; exec sleep 30".into(),
20373 "kill -TERM $$".into(),
20374 "exit 7".into(),
20375 "printf success".into(),
20376 ],
20377 tmp.path(),
20378 Duration::from_secs(1),
20379 )
20380 .await;
20381 wait_for_timed_out_children_to_die(&pids).await;
20382 assert_eq!(
20383 pids.len(),
20384 1,
20385 "unleased timeout retains its death-check PID"
20386 );
20387 assert_eq!(outcomes.len(), 4);
20388 assert_eq!(outcomes[0].code, None);
20389 assert!(outcomes[0].output_tail.contains("before-timeout"));
20390 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20391 assert_eq!(outcomes[1].code, None);
20392 assert!(outcomes[1].output_tail.contains("command terminated:"));
20393 assert!(!outcomes[1].output_tail.contains("timed out"));
20394 assert_eq!(outcomes[2].code, Some(7));
20395 assert!(!outcomes[2].output_tail.contains("timed out"));
20396 assert_eq!(outcomes[3].code, Some(0));
20397 assert_eq!(outcomes[3].output_tail, "success");
20398 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20399 }
20400
20401 #[tokio::test]
20411 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20412 crate::run::pin_test_home();
20413
20414 let tmp = tempfile::tempdir().expect("tempdir");
20415 let repo = tmp.path().join("repo");
20416 std::fs::create_dir_all(&repo).unwrap();
20417 init_repo(&repo);
20418
20419 let mut config = Config::default();
20420 config.disk.min_free_bytes = 0;
20421 config.verify.gate = vec![
20422 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20423 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20424 .to_owned(),
20425 ];
20426
20427 let mut state = RunState::new(
20428 repo.clone(),
20429 "main".to_owned(),
20430 "deadbeef".to_owned(),
20431 "task".to_owned(),
20432 config,
20433 );
20434 let run_id = state.id.clone();
20435 state.candidates = vec![Candidate {
20436 index: 0,
20437 label: 'A',
20438 agent: "alpha".to_owned(),
20439 branch: "does-not-exist".to_owned(),
20440 worktree: repo.clone(),
20441 summary: String::new(),
20442 stat: String::new(),
20443 files: 0,
20444 commits: 0,
20445 empty: false,
20446 failed: None,
20447 verified_noop: None,
20448 duration_ms: 0,
20449 folded: false,
20450 }];
20451 state.tally = Some(Tally {
20452 first_choice: BTreeMap::from([('A', 1)]),
20453 borda: BTreeMap::new(),
20454 winner: 'A',
20455 rankings: 1,
20456 unanimous_initial: true,
20457 deliberated: false,
20458 changed_votes: 0,
20459 unanimous_final: true,
20460 tie_break: None,
20461 judges: 0,
20462 present: 0,
20463 quorum: 0,
20464 met_quorum: true,
20465 uncontested: Some("only candidate A produced a change".to_owned()),
20466 });
20467 state.reviews = vec![ReviewRound {
20468 round: 1,
20469 head: "deadbeef".to_owned(),
20470 verified_head: None,
20471 verified_at: None,
20472 reviews: Vec::new(),
20473 e2e: Vec::new(),
20474 fix: None,
20475 blocking: 0,
20476 answered: 0,
20477 expected: 0,
20478 clean: true,
20479 verify_retried: false,
20480 e2e_deferred: false,
20481 e2e_defer_reason: None,
20482 progressed: false,
20483 vote_split: false,
20484 reconsideration: Vec::new(),
20485 verdict: None,
20486 }];
20487
20488 let mut runner = Runner {
20489 state,
20490 roles: ResolvedRoles {
20491 implementers: Vec::new(),
20492 judges: Vec::new(),
20493 reviewers: Vec::new(),
20494 fixer: None,
20495 conductor: conductor(),
20496 implementer_roster: Vec::new(),
20497 judge_roster: Vec::new(),
20498 reviewer_roster: Vec::new(),
20499 },
20500 sem: Arc::new(Semaphore::new(1)),
20501 pause: Pause::new(),
20502 interrupt: Pause::new(),
20503 };
20504
20505 let started_marker = repo.join("started.marker");
20506 let release_marker = repo.join("release.marker");
20507 let poller = tokio::spawn(async move {
20508 for _ in 0..100 {
20513 if started_marker.exists()
20514 && let Ok(s) = crate::run::RunState::load(&run_id)
20515 && let Some(a) = s.active.get("gate")
20516 {
20517 std::fs::write(&release_marker, b"go").expect("release marker");
20518 return Some(a.clone());
20519 }
20520 tokio::time::sleep(Duration::from_millis(50)).await;
20521 }
20522 None
20523 });
20524
20525 runner.gate().await.expect("gate");
20526 let captured = poller.await.expect("poller task");
20527 let captured = captured.expect(
20528 "the poller never saw a `gate` task entry in run.json while the command was \
20529 still blocked on its own release marker",
20530 );
20531
20532 assert_eq!(captured.task.as_deref(), Some("gate"));
20533 assert_eq!(captured.node, "gate");
20534 assert_eq!(captured.index, Some(1));
20535 assert_eq!(captured.total, Some(1));
20536 assert!(
20537 captured
20538 .command
20539 .as_deref()
20540 .is_some_and(|c| c.contains("started.marker")),
20541 "{captured:?}"
20542 );
20543
20544 assert!(
20545 runner.state.active.is_empty(),
20546 "the entry must be cleared once the command actually finished: {:?}",
20547 runner.state.active
20548 );
20549 assert!(runner.state.gate_ran);
20550 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20551 }
20552
20553 #[tokio::test]
20556 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20557 use crate::verdict::{Finding, ReviewVote, Severity};
20558 crate::run::pin_test_home();
20559 let tmp = tempfile::tempdir().expect("tempdir");
20560 let repo = tmp.path().join("repo");
20561 std::fs::create_dir_all(&repo).unwrap();
20562 init_repo(&repo);
20563
20564 for (severity, vote, expect) in [
20565 (Severity::Major, ReviewVote::Reject, true),
20566 (Severity::Minor, ReviewVote::Reject, false),
20567 (Severity::Major, ReviewVote::Approve, false),
20568 ] {
20569 let mut round = review_round(false, 1, 1, 1, false, true);
20570 round.reviews = vec![ReviewRecord {
20571 reviewer: 1,
20572 agent: "alpha".to_owned(),
20573 summary: String::new(),
20574 findings: vec![Finding {
20575 id: "R1-1-1".to_owned(),
20576 severity,
20577 file: None,
20578 line: None,
20579 title: "t".to_owned(),
20580 detail: String::new(),
20581 }],
20582 vote: Some(vote),
20583 failed: None,
20584 duration_ms: 0,
20585 attempts: 0,
20586 }];
20587 let mut state = RunState::new(
20588 repo.clone(),
20589 "main".to_owned(),
20590 "deadbeef".to_owned(),
20591 "task".to_owned(),
20592 Config::default(),
20593 );
20594 state.reviews = vec![round];
20595 let mut runner = Runner {
20596 state,
20597 roles: ResolvedRoles {
20598 implementers: Vec::new(),
20599 judges: Vec::new(),
20600 reviewers: Vec::new(),
20601 fixer: None,
20602 conductor: conductor(),
20603 implementer_roster: Vec::new(),
20604 judge_roster: Vec::new(),
20605 reviewer_roster: Vec::new(),
20606 },
20607 sem: Arc::new(Semaphore::new(1)),
20608 pause: Pause::new(),
20609 interrupt: Pause::new(),
20610 };
20611 let shell = runner.state.config.shell();
20612 runner
20613 .stop_reviewing("round budget spent", &shell, &repo)
20614 .await
20615 .expect("stop_reviewing");
20616 assert_eq!(runner.state.status, RunStatus::Gating);
20617 assert_eq!(
20618 runner.state.contested_handoff.is_some(),
20619 expect,
20620 "{severity:?} + {vote:?}"
20621 );
20622 }
20623 }
20624
20625 #[tokio::test]
20638 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20639 crate::run::pin_test_home();
20640 let home = crate::run::home();
20641
20642 let tmp = tempfile::tempdir().expect("tempdir");
20643 let repo = tmp.path().join("repo");
20644 std::fs::create_dir_all(&repo).unwrap();
20645 init_repo(&repo);
20646 let head = crate::git::rev_parse(&repo, "HEAD")
20647 .await
20648 .expect("rev-parse");
20649 let cache_dir = tmp.path().join("target");
20652
20653 let mut config = Config::default();
20654 config.verify.e2e = vec![format!(
20655 "CARGO_TARGET_DIR='{}' test -f README.md",
20656 cache_dir.display()
20657 )];
20658 config.graph.review_rounds = 1;
20659 config.graph.timeout_verify = Some(2);
20662 config.graph.timeout_resource_wait = 2;
20663
20664 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20665 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20666 .expect("no io error acquiring directly")
20667 {
20668 crate::cache::AcquireOutcome::Acquired(g) => g,
20669 crate::cache::AcquireOutcome::Busy(b) => {
20670 panic!("expected the direct acquire to win the lease first: {b:?}")
20671 }
20672 };
20673
20674 let mut state = RunState::new(
20675 repo.clone(),
20676 "main".to_owned(),
20677 head.clone(),
20678 "task".to_owned(),
20679 config,
20680 );
20681 state.candidates = vec![Candidate {
20682 index: 0,
20683 label: 'A',
20684 agent: "alpha".to_owned(),
20685 branch: "does-not-exist".to_owned(),
20686 worktree: repo.clone(),
20687 summary: String::new(),
20688 stat: String::new(),
20689 files: 0,
20690 commits: 0,
20691 empty: false,
20692 failed: None,
20693 verified_noop: None,
20694 duration_ms: 0,
20695 folded: false,
20696 }];
20697 state.tally = Some(Tally {
20698 first_choice: BTreeMap::from([('A', 1)]),
20699 borda: BTreeMap::new(),
20700 winner: 'A',
20701 rankings: 1,
20702 unanimous_initial: true,
20703 deliberated: false,
20704 changed_votes: 0,
20705 unanimous_final: true,
20706 tie_break: None,
20707 judges: 0,
20708 present: 0,
20709 quorum: 0,
20710 met_quorum: true,
20711 uncontested: Some("only candidate A produced a change".to_owned()),
20712 });
20713 state.reviews = vec![ReviewRound {
20717 round: 1,
20718 head: head.clone(),
20719 verified_head: None,
20720 verified_at: None,
20721 reviews: Vec::new(),
20722 e2e: Vec::new(),
20723 fix: None,
20724 blocking: 1,
20725 answered: 1,
20726 expected: 1,
20727 clean: false,
20728 verify_retried: false,
20729 e2e_deferred: true,
20730 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20731 progressed: false,
20732 vote_split: false,
20733 reconsideration: Vec::new(),
20734 verdict: None,
20735 }];
20736
20737 let mut runner = Runner {
20738 state,
20739 roles: ResolvedRoles {
20740 implementers: Vec::new(),
20741 judges: Vec::new(),
20742 reviewers: Vec::new(),
20743 fixer: None,
20744 conductor: conductor(),
20745 implementer_roster: Vec::new(),
20746 judge_roster: Vec::new(),
20747 reviewer_roster: Vec::new(),
20748 },
20749 sem: Arc::new(Semaphore::new(1)),
20750 pause: Pause::new(),
20751 interrupt: Pause::new(),
20752 };
20753
20754 let shell = runner.state.config.shell();
20755 runner
20756 .stop_reviewing("round budget spent", &shell, &repo)
20757 .await
20758 .expect("stop_reviewing");
20759
20760 let last = runner.state.reviews.last().expect("round record");
20761 assert_eq!(
20762 last.e2e_status(),
20763 E2eStatus::ResourceBlocked,
20764 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20765 failed: {last:?}"
20766 );
20767 assert_eq!(
20768 last.verified_head.as_deref(),
20769 Some(head.as_str()),
20770 "which commit this attempt targeted is known even though nothing finished checking \
20771 it"
20772 );
20773 let first_attempt_at = last
20774 .verified_at
20775 .expect("when this attempt ran is known too");
20776 assert_ne!(
20777 runner.state.status,
20778 RunStatus::Blocked,
20779 "contention is evidence about the machine, not the patch — it must not settle the \
20780 run as blocked: {:?}",
20781 runner.state.status
20782 );
20783 assert!(
20784 !runner
20785 .state
20786 .events
20787 .iter()
20788 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20789 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20790 runner.state.events
20791 );
20792
20793 runner
20798 .stop_reviewing("round budget spent", &shell, &repo)
20799 .await
20800 .expect("stop_reviewing retry");
20801 assert_eq!(
20802 runner.state.reviews.len(),
20803 1,
20804 "no new round was started: {:?}",
20805 runner.state.reviews
20806 );
20807 let last = runner.state.reviews.last().expect("round record");
20808 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20809 assert!(
20810 last.verified_at.expect("still known") > first_attempt_at,
20811 "a second reentry must be a fresh attempt, not a stale copy of the first"
20812 );
20813 assert_ne!(runner.state.status, RunStatus::Blocked);
20814
20815 held.release();
20816 }
20817
20818 #[cfg(unix)]
20819 #[tokio::test]
20820 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20821 ask_test_home();
20822 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20823 let tmp = tempfile::tempdir().unwrap();
20824 let repo = tmp.path().join("repo");
20825 std::fs::create_dir(&repo).unwrap();
20826 init_repo(&repo);
20827 let run_git = |args: &[&str]| {
20828 let out = std::process::Command::new("git")
20829 .args(args)
20830 .current_dir(&repo)
20831 .output()
20832 .unwrap();
20833 assert!(
20834 out.status.success(),
20835 "{args:?}: {}",
20836 String::from_utf8_lossy(&out.stderr)
20837 );
20838 };
20839 let origin = tmp.path().join("origin.git");
20840 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20841 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20842 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20843 run_git(&["add", "spec.rs"]);
20844 run_git(&["commit", "-m", "initial spec"]);
20845 run_git(&["push", "origin", "main"]);
20846 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20847 run_git(&["checkout", "-b", "fix/review"]);
20848 std::fs::write(
20849 repo.join("check.rs"),
20850 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20851 )
20852 .unwrap();
20853 run_git(&["add", "check.rs"]);
20854 run_git(&["commit", "-m", "use spec"]);
20855 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20856 run_git(&["checkout", "main"]);
20857 std::fs::write(
20858 repo.join("spec.rs"),
20859 "struct AgentSpec { max_turn_secs: u64 }\n",
20860 )
20861 .unwrap();
20862 run_git(&["add", "spec.rs"]);
20863 run_git(&["commit", "-m", "extend spec"]);
20864 run_git(&["push", "origin", "main"]);
20865 run_git(&["checkout", "fix/review"]);
20866
20867 let mut runner = runner_at(RunStatus::Reviewing);
20868 runner.state.repo = repo.clone();
20869 runner.state.base_commit = base;
20870 runner.state.pr_head = Some("fix/review".into());
20871 runner.state.config.graph.review_rounds = max_rounds;
20872 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20873 runner.state.config.graph.sessions = false;
20874 runner.state.config.disk.min_free_bytes = 0;
20875 runner.state.config.verify.e2e =
20876 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20877 runner.state.candidates = vec![Candidate {
20878 index: 0,
20879 label: 'A',
20880 agent: "fixture".into(),
20881 branch: "fix/review".into(),
20882 worktree: repo.clone(),
20883 summary: String::new(),
20884 stat: String::new(),
20885 files: 1,
20886 commits: 1,
20887 empty: false,
20888 failed: None,
20889 verified_noop: None,
20890 duration_ms: 0,
20891 folded: false,
20892 }];
20893 runner.state.tally = Some(Tally {
20894 first_choice: BTreeMap::new(),
20895 borda: BTreeMap::new(),
20896 winner: 'A',
20897 rankings: 0,
20898 unanimous_initial: true,
20899 deliberated: false,
20900 changed_votes: 0,
20901 unanimous_final: true,
20902 tie_break: None,
20903 judges: 0,
20904 present: 0,
20905 quorum: 0,
20906 met_quorum: true,
20907 uncontested: Some("review-only".into()),
20908 });
20909 let mut last = review_round(false, 0, 1, 1, false, false);
20910 last.head = reviewed.clone();
20911 last.e2e_deferred = true;
20912 last.e2e[0].resource_blocked = true;
20913 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20914 if max_rounds == 1 {
20915 last.e2e.clear();
20916 }
20917 runner.state.reviews = vec![last];
20918 let calls = tmp.path().join("fix-calls");
20919 let mut fixer = spec("repair");
20920 fixer
20921 .env
20922 .insert("FIX_CALLS".into(), calls.display().to_string());
20923 fixer.command = vec!["sh".into(), "-c".into(), r#"
20924 echo fix >> "$FIX_CALLS"
20925 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20926 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20927 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20928 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20929 "#.into()];
20930 let mut reviewer = spec("reviewer");
20931 reviewer.command = vec![
20932 "sh".into(),
20933 "-c".into(),
20934 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20935 .into(),
20936 ];
20937 runner.roles.fixer = Some(vec![fixer]);
20938 runner.roles.reviewers = vec![reviewer];
20939
20940 runner.sync_to_base().await.unwrap();
20941 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20942 assert_ne!(rebased, reviewed);
20943 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20944 if reentry {
20945 let shell = runner.state.config.shell();
20948 runner
20949 .stop_unresolved_review("deferred verification", &shell, &repo)
20950 .await
20951 .unwrap();
20952 assert!(runner.catchup_repairable());
20953 assert!(!calls.exists());
20954 runner.state = RunState::load(&runner.state.id).unwrap();
20955 assert!(runner.catchup_repairable());
20956 }
20957 runner.review_loop().await.unwrap();
20958 let first = &runner.state.reviews[0];
20959 assert_eq!(first.head, reviewed);
20960 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20961 assert!(
20962 first.verify_retried,
20963 "a reproduced build failure gets exactly one retry: {:?}",
20964 first.e2e
20965 );
20966 assert!(first.e2e[0].output_tail.contains("E0063"));
20967 assert!(!runner.catchup_repairable());
20968 if max_rounds == 2 {
20969 assert_eq!(
20970 runner.state.status,
20971 RunStatus::Gating,
20972 "{}",
20973 crate::report::run(&runner.state)
20974 );
20975 assert_eq!(runner.state.reviews.len(), 2);
20976 assert!(first.fix.as_ref().unwrap().committed);
20977 assert!(runner.state.reviews[1].clean);
20978 assert_eq!(
20979 runner
20980 .state
20981 .reviews
20982 .iter()
20983 .filter(|r| r.fix.is_some())
20984 .count(),
20985 1
20986 );
20987 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20988 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20989 } else {
20990 assert_eq!(runner.state.status, RunStatus::Blocked);
20991 assert!(first.fix.is_none());
20992 assert!(!calls.exists());
20993 let recovery = crate::recovery::assess(
20994 "prior",
20995 Some(&runner.state),
20996 &runner.landing_base(),
20997 crate::recovery::Relation::Diverged,
20998 );
20999 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21000 let reasons = recovery.reasons.join("\n");
21001 assert!(
21002 reasons.contains(&reviewed) && reasons.contains(&rebased),
21003 "{reasons}"
21004 );
21005 assert!(
21006 reasons.contains("base sync to")
21007 && reasons.contains("additional review is required"),
21008 "{reasons}"
21009 );
21010 }
21011 }
21012 }
21013
21014 #[tokio::test]
21026 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21027 crate::run::pin_test_home();
21028 let home = crate::run::home();
21029
21030 let tmp = tempfile::tempdir().expect("tempdir");
21031 let repo = tmp.path().join("repo");
21032 std::fs::create_dir_all(&repo).unwrap();
21033 init_repo(&repo);
21034 let head = crate::git::rev_parse(&repo, "HEAD")
21035 .await
21036 .expect("rev-parse");
21037 let cache_dir = tmp.path().join("target");
21038
21039 let mut config = Config::default();
21040 config.verify.e2e = vec![format!(
21041 "CARGO_TARGET_DIR='{}' test -f README.md",
21042 cache_dir.display()
21043 )];
21044 config.graph.review_rounds = 1;
21045 config.graph.timeout_verify = Some(2);
21046 config.graph.timeout_resource_wait = 2;
21047
21048 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21049 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21050 .expect("no io error acquiring directly")
21051 {
21052 crate::cache::AcquireOutcome::Acquired(g) => g,
21053 crate::cache::AcquireOutcome::Busy(b) => {
21054 panic!("expected the direct acquire to win the lease first: {b:?}")
21055 }
21056 };
21057
21058 let mut state = RunState::new(
21059 repo.clone(),
21060 "main".to_owned(),
21061 head.clone(),
21062 "task".to_owned(),
21063 config,
21064 );
21065 state.candidates = vec![Candidate {
21066 index: 0,
21067 label: 'A',
21068 agent: "alpha".to_owned(),
21069 branch: "does-not-exist".to_owned(),
21070 worktree: repo.clone(),
21071 summary: String::new(),
21072 stat: String::new(),
21073 files: 0,
21074 commits: 0,
21075 empty: false,
21076 failed: None,
21077 verified_noop: None,
21078 duration_ms: 0,
21079 folded: false,
21080 }];
21081 state.tally = Some(Tally {
21082 first_choice: BTreeMap::from([('A', 1)]),
21083 borda: BTreeMap::new(),
21084 winner: 'A',
21085 rankings: 1,
21086 unanimous_initial: true,
21087 deliberated: false,
21088 changed_votes: 0,
21089 unanimous_final: true,
21090 tie_break: None,
21091 judges: 0,
21092 present: 0,
21093 quorum: 0,
21094 met_quorum: true,
21095 uncontested: Some("only candidate A produced a change".to_owned()),
21096 });
21097 state.reviews = vec![ReviewRound {
21101 round: 1,
21102 head: head.clone(),
21103 verified_head: Some(head.clone()),
21104 verified_at: Some(jiff::Timestamp::now()),
21105 reviews: Vec::new(),
21106 e2e: vec![CommandOutcome {
21107 command: format!(
21108 "CARGO_TARGET_DIR='{}' test -f README.md",
21109 cache_dir.display()
21110 ),
21111 code: None,
21112 output_tail: "waiting for the shared build cache".to_owned(),
21113 duration_ms: 0,
21114 resource_blocked: true,
21115 }],
21116 fix: None,
21117 blocking: 1,
21118 answered: 1,
21119 expected: 1,
21120 clean: false,
21121 verify_retried: false,
21122 e2e_deferred: false,
21123 e2e_defer_reason: None,
21124 progressed: false,
21125 vote_split: false,
21126 reconsideration: Vec::new(),
21127 verdict: None,
21128 }];
21129
21130 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21131 let mut runner = Runner {
21132 state,
21133 roles: ResolvedRoles {
21134 implementers: Vec::new(),
21135 judges: Vec::new(),
21136 reviewers: Vec::new(),
21137 fixer: None,
21138 conductor: conductor(),
21139 implementer_roster: Vec::new(),
21140 judge_roster: Vec::new(),
21141 reviewer_roster: Vec::new(),
21142 },
21143 sem: Arc::new(Semaphore::new(1)),
21144 pause: Pause::new(),
21145 interrupt: Pause::new(),
21146 };
21147
21148 runner.review_loop().await.expect("review_loop");
21153
21154 assert_eq!(
21155 runner.state.reviews.len(),
21156 1,
21157 "no new round was started on top of the unresolved one: {:?}",
21158 runner.state.reviews
21159 );
21160 let last = &runner.state.reviews[0];
21161 assert_eq!(
21162 last.e2e_status(),
21163 E2eStatus::ResourceBlocked,
21164 "still contended: {last:?}"
21165 );
21166 assert!(
21167 last.verified_at.expect("still known") > first_attempt_at,
21168 "review_loop must have actually retried the check, not left it exactly as found"
21169 );
21170 assert_ne!(
21171 runner.state.status,
21172 RunStatus::Blocked,
21173 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21174 runner.state.status
21175 );
21176
21177 held.release();
21178 }
21179
21180 #[tokio::test]
21181 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21182 crate::run::pin_test_home();
21183 for status in [RunStatus::Blocked, RunStatus::Gating] {
21184 let mut runner = runner_at(status);
21185 runner.state.config.graph.land = true;
21186 runner.state.gate_ran = true;
21187 runner.state.merge = Some(MergeOutcome {
21188 mode: MergeMode::Pr,
21189 ok: false,
21190 detail: "the fixer produced no commit".into(),
21191 empty: false,
21192 });
21193 let url = "https://example.invalid/o/r/pull/1";
21194 runner.state.pr = Some(crate::run::PrRecord {
21195 url: url.into(),
21196 number: 1,
21197 state: "open".into(),
21198 checks: "red".into(),
21199 round: 1,
21200 rounds: 1,
21201 red_at_merge: Vec::new(),
21202 });
21203 runner.state.save().unwrap();
21204 runner.execute_graph().await.unwrap();
21208 assert_eq!(runner.state.status, RunStatus::Blocked);
21209 assert!(runner.state.land_sync_pending);
21210 assert!(runner.state.events.iter().any(|event|
21211 event.node == "land" && event.message == format!("watching {url}")));
21212 assert_eq!(
21213 runner.state.merge.as_ref().unwrap().detail,
21214 "the fixer produced no commit"
21215 );
21216 }
21217 }
21218
21219 #[tokio::test]
21220 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21221 crate::run::pin_test_home();
21222 let tmp = tempfile::tempdir().expect("tempdir");
21223 let repo = tmp.path().join("repo");
21224 std::fs::create_dir_all(&repo).unwrap();
21225 init_repo(&repo);
21226
21227 let mut config = Config::default();
21228 config.merge.mode = MergeMode::Pr;
21229 config.graph.land = true;
21230 config.graph.land_approval = false;
21231
21232 let mut state = RunState::new(
21233 repo.clone(),
21234 "main".to_owned(),
21235 "deadbeef".to_owned(),
21236 "task".to_owned(),
21237 config,
21238 );
21239 state.candidates = vec![Candidate {
21240 index: 0,
21241 label: 'A',
21242 agent: "alpha".to_owned(),
21243 branch: "does-not-exist".to_owned(),
21244 worktree: repo.clone(),
21245 summary: String::new(),
21246 stat: String::new(),
21247 files: 0,
21248 commits: 0,
21249 empty: false,
21250 failed: None,
21251 verified_noop: None,
21252 duration_ms: 0,
21253 folded: false,
21254 }];
21255 state.tally = Some(Tally {
21256 first_choice: BTreeMap::from([('A', 1)]),
21257 borda: BTreeMap::new(),
21258 winner: 'A',
21259 rankings: 1,
21260 unanimous_initial: true,
21261 deliberated: false,
21262 changed_votes: 0,
21263 unanimous_final: true,
21264 tie_break: None,
21265 judges: 0,
21266 present: 0,
21267 quorum: 0,
21268 met_quorum: true,
21269 uncontested: Some("only candidate A produced a change".to_owned()),
21270 });
21271 state.reviews = vec![ReviewRound {
21272 round: 1,
21273 head: "deadbeef".to_owned(),
21274 verified_head: None,
21275 verified_at: None,
21276 reviews: Vec::new(),
21277 e2e: Vec::new(),
21278 fix: None,
21279 blocking: 0,
21280 answered: 0,
21281 expected: 0,
21282 clean: true,
21283 verify_retried: false,
21284 e2e_deferred: false,
21285 e2e_defer_reason: None,
21286 progressed: false,
21287 vote_split: false,
21288 reconsideration: Vec::new(),
21289 verdict: None,
21290 }];
21291 state.gate = vec![CommandOutcome {
21292 command: "test".to_owned(),
21293 code: Some(0),
21294 output_tail: String::new(),
21295 duration_ms: 0,
21296 resource_blocked: false,
21297 }];
21298 state.gate_ran = true;
21299 state.status = RunStatus::Landing;
21303 state.merge = Some(MergeOutcome {
21304 mode: MergeMode::Pr,
21305 ok: true,
21306 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21307 empty: false,
21308 });
21309
21310 ask_test_home();
21314 state.save().expect("save landing run");
21317 let store = ask::Questions::open();
21318 let q = ask_open_question(&store, &state.id);
21319
21320 let mut runner = Runner {
21321 state,
21322 roles: ResolvedRoles {
21323 implementers: Vec::new(),
21324 judges: Vec::new(),
21325 reviewers: Vec::new(),
21326 fixer: None,
21327 conductor: conductor(),
21328 implementer_roster: Vec::new(),
21329 judge_roster: Vec::new(),
21330 reviewer_roster: Vec::new(),
21331 },
21332 sem: Arc::new(Semaphore::new(1)),
21333 pause: Pause::new(),
21334 interrupt: Pause::new(),
21335 };
21336
21337 runner.execute().await.expect("execute");
21342
21343 assert_eq!(
21344 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21345 Some("https://example.invalid/x/y/pull/1"),
21346 "reentry must not push again or open a second pull request over the \
21347 one `land` is already watching"
21348 );
21349 assert_ne!(
21350 runner.state.status,
21351 RunStatus::Landing,
21352 "land could not actually reach the fake pull request, so it must \
21353 have given up rather than left the run silently parked forever"
21354 );
21355 assert_eq!(runner.state.status, RunStatus::Blocked);
21359 assert!(
21360 store.get(&q.id).unwrap().status.open(),
21361 "Blocked is still alive; settle_questions must have been a no-op here"
21362 );
21363 }
21364
21365 fn state_with_round(round: ReviewRound) -> RunState {
21366 let mut s = RunState::new(
21367 PathBuf::from("/repo"),
21368 "main".to_owned(),
21369 "abc1234".to_owned(),
21370 "add retries".to_owned(),
21371 Config::default(),
21372 );
21373 s.reviews = vec![round];
21374 s
21375 }
21376
21377 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21378 crate::verdict::Finding {
21379 id: id.to_owned(),
21380 severity,
21381 file: None,
21382 line: None,
21383 title: title.to_owned(),
21384 detail: String::new(),
21385 }
21386 }
21387
21388 #[test]
21389 fn pr_body_names_open_findings_and_declined_ones() {
21390 let round = ReviewRound {
21391 round: 2,
21392 head: "deadbee".to_owned(),
21393 verified_head: None,
21394 verified_at: None,
21395 reviews: vec![ReviewRecord {
21396 attempts: 0,
21397 reviewer: 1,
21398 agent: "alpha".to_owned(),
21399 summary: String::new(),
21400 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21401 vote: None,
21402 failed: None,
21403 duration_ms: 0,
21404 }],
21405 e2e: vec![CommandOutcome {
21406 command: "cargo test".to_owned(),
21407 code: Some(0),
21408 output_tail: String::new(),
21409 duration_ms: 0,
21410 resource_blocked: false,
21411 }],
21412 verify_retried: false,
21413 e2e_deferred: false,
21414 e2e_defer_reason: None,
21415 fix: Some(FixRecord {
21416 agent: "alpha".to_owned(),
21417 addressed: Vec::new(),
21418 rejected: vec![crate::verdict::Rejection {
21419 id: "R1-1-1".to_owned(),
21420 why: "not reachable from any caller".to_owned(),
21421 }],
21422 notes: String::new(),
21423 committed: true,
21424 failed: None,
21425 duration_ms: 0,
21426 continuation: None,
21427 }),
21428 blocking: 0,
21429 answered: 1,
21430 expected: 1,
21431 clean: false,
21432 progressed: true,
21433 vote_split: false,
21434 reconsideration: Vec::new(),
21435 verdict: None,
21436 };
21437 let state = state_with_round(round);
21438 let body = pr_message(&state, 'A').body;
21439
21440 assert!(body.contains("add retries"), "the task must still be there");
21441 assert!(body.contains("R2-1-1"), "{body}");
21442 assert!(!body.contains("unused import"), "{body}");
21443 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21444 assert!(
21445 !body.contains("not reachable from any caller"),
21446 "the reason it was declined: {body}"
21447 );
21448 }
21449
21450 #[test]
21451 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21452 let round = ReviewRound {
21453 round: 1,
21454 head: "deadbee".to_owned(),
21455 verified_head: None,
21456 verified_at: None,
21457 reviews: vec![ReviewRecord {
21458 attempts: 0,
21459 reviewer: 1,
21460 agent: "alpha".to_owned(),
21461 summary: String::new(),
21462 findings: Vec::new(),
21463 vote: None,
21464 failed: None,
21465 duration_ms: 0,
21466 }],
21467 e2e: Vec::new(),
21468 verify_retried: false,
21469 e2e_deferred: false,
21470 e2e_defer_reason: None,
21471 fix: None,
21472 blocking: 0,
21473 answered: 1,
21474 expected: 1,
21475 clean: true,
21476 progressed: false,
21477 vote_split: false,
21478 reconsideration: Vec::new(),
21479 verdict: None,
21480 };
21481 let state = state_with_round(round);
21482 let body = pr_message(&state, 'A').body;
21483 assert!(!body.contains("Open review findings"), "{body}");
21484 assert!(!body.contains("Declined"), "{body}");
21485 }
21486
21487 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21488 let mut state = RunState::new(
21489 PathBuf::from("/repo"),
21490 "main".to_owned(),
21491 "abc1234".to_owned(),
21492 instruction.to_owned(),
21493 Config::default(),
21494 );
21495 state.config.disk.min_free_bytes = 0;
21497 state.candidates.push(Candidate {
21498 index: 0,
21499 label: 'A',
21500 agent: "alpha".to_owned(),
21501 branch: "magi/x/A".to_owned(),
21502 worktree: PathBuf::from("/wt"),
21503 summary: summary.to_owned(),
21504 stat: String::new(),
21505 files: 1,
21506 commits: 1,
21507 empty: false,
21508 failed: None,
21509 verified_noop: None,
21510 folded: false,
21511 duration_ms: 0,
21512 });
21513 state
21514 }
21515
21516 fn review_state(subjects: &[&str]) -> RunState {
21517 let mut state = state_with_summary(
21518 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21519 "",
21520 );
21521 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21522 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21523 state
21524 }
21525
21526 #[tokio::test]
21529 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21530 ask_test_home();
21531 let tmp = tempfile::tempdir().unwrap();
21532 let repo = tmp.path().join("repo");
21533 std::fs::create_dir_all(&repo).unwrap();
21534 init_repo(&repo);
21535 let origin = tmp.path().join("origin.git");
21536 let g = |dir: &Path, args: &[&str]| {
21537 let out = std::process::Command::new("git")
21538 .args(args)
21539 .current_dir(dir)
21540 .quiet()
21541 .output()
21542 .expect("spawn git");
21543 assert!(
21544 out.status.success(),
21545 "git {args:?}: {}",
21546 String::from_utf8_lossy(&out.stderr)
21547 );
21548 };
21549 g(
21550 tmp.path(),
21551 &[
21552 "clone",
21553 "--bare",
21554 "-q",
21555 repo.to_str().unwrap(),
21556 origin.to_str().unwrap(),
21557 ],
21558 );
21559 g(
21560 &repo,
21561 &["remote", "add", "origin", origin.to_str().unwrap()],
21562 );
21563 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21564
21565 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21567 g(&repo, &["add", "-A"]);
21568 g(
21569 &repo,
21570 &["commit", "-q", "-m", "chore(deps): update a crate"],
21571 );
21572 g(&repo, &["push", "-q", "origin", "main"]);
21573 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21574 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21575 g(&repo, &["add", "-A"]);
21576 g(
21577 &repo,
21578 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21579 );
21580 g(&repo, &["checkout", "-q", "main"]);
21581
21582 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21583 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21584 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21586 assert_eq!(fallback, c1);
21587
21588 let mut state = review_state(&[
21589 "chore(deps): update a crate",
21590 "fix(daemon): apply a chosen action",
21591 ]);
21592 state.repo = repo.clone();
21593 state.base_branch = "main".to_owned();
21594 state.base_commit = c1;
21595 refresh_reviewed_commits(&mut state, "feat/own").await;
21596 assert_eq!(
21597 state.reviewed_commits,
21598 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21599 );
21600 assert_eq!(
21601 review_title(&state).as_deref(),
21602 Some("fix(daemon): apply a chosen action")
21603 );
21604 assert_eq!(
21605 leaked_subjects(&state, "feat/own").await,
21606 Some(vec!["chore(deps): update a crate".to_owned()])
21607 );
21608 state.config.merge.remote = "nowhere".to_owned();
21611 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21612 }
21613
21614 struct SummaryFixture {
21618 _tmp: tempfile::TempDir,
21619 repo: PathBuf,
21620 reply: PathBuf,
21621 runner: Runner,
21622 }
21623
21624 fn summary_fixture() -> SummaryFixture {
21625 ask_test_home();
21626 let tmp = tempfile::tempdir().unwrap();
21627 let repo = tmp.path().join("repo");
21628 std::fs::create_dir_all(&repo).unwrap();
21629 init_repo(&repo);
21630 let origin = tmp.path().join("origin.git");
21631 let g = |args: &[&str]| {
21632 let out = std::process::Command::new("git")
21633 .args(args)
21634 .current_dir(&repo)
21635 .quiet()
21636 .output()
21637 .expect("spawn git");
21638 assert!(
21639 out.status.success(),
21640 "git {args:?}: {}",
21641 String::from_utf8_lossy(&out.stderr)
21642 );
21643 };
21644 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21645 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21646 g(&["push", "-q", "origin", "main"]);
21647 g(&["checkout", "-q", "-b", "feat/own"]);
21648 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21649 g(&["add", "-A"]);
21650 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21651 let reply = tmp.path().join("reply.txt");
21652 std::fs::write(
21653 &reply,
21654 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21655 )
21656 .unwrap();
21657 let mut runner = runner_at(RunStatus::Gating);
21658 let mut state = review_state(&["feat(own): add the own file"]);
21659 state.repo = repo.clone();
21660 state.base_branch = "main".to_owned();
21661 state.candidates[0].branch = "feat/own".to_owned();
21662 state.candidates[0].worktree = repo.clone();
21663 let mut author = spec("alpha");
21664 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21665 author
21666 .env
21667 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21668 state.config.agents = vec![author];
21669 runner.state = state;
21670 SummaryFixture {
21671 _tmp: tmp,
21672 repo,
21673 reply,
21674 runner,
21675 }
21676 }
21677
21678 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21679 let winner = f.runner.state.candidates[0].clone();
21680 f.runner
21681 .guarded_pr_message(&winner, None, true)
21682 .await
21683 .unwrap()
21684 .expect("an English description never parks")
21685 }
21686
21687 fn clean_round_with_minor_findings() -> ReviewRound {
21688 let mut round = passing_round(2, None);
21689 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21690 round.reviews = vec![ReviewRecord {
21691 attempts: 0,
21692 reviewer: 1,
21693 agent: "alpha".into(),
21694 summary: "内部レビュー".into(),
21695 findings: vec![
21696 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21697 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21698 ],
21699 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21700 failed: None,
21701 duration_ms: 0,
21702 }];
21703 round
21704 }
21705
21706 #[tokio::test]
21707 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21708 for review_only in [false, true] {
21709 let mut f = summary_fixture();
21710 if !review_only {
21711 f.runner.state.candidates[0].agent = "alpha".into();
21712 }
21713 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21714 assert!(f.runner.state.reviews[0].clean);
21715 assert!(f.runner.state.reviews[0].fix.is_none());
21716 assert!(f.runner.state.open_findings().is_empty());
21717 assert_eq!(
21718 crate::pr_outcome::required_finding_ids(&f.runner.state),
21719 ["R2-1-1", "R2-1-2"]
21720 );
21721 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21722 let evidence = crate::pr_outcome::summary_evidence(
21723 &f.runner.state,
21724 crate::pr_outcome::FinalHead::At(&head),
21725 );
21726 assert!(
21727 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21728 );
21729 let omitted = posted(&mut f).await;
21731 assert!(omitted.body.contains("No description"));
21732 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21733 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21734 assert!(f.runner.state.change_summary.is_none());
21735 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21736 let described = posted(&mut f).await;
21737 assert!(described.body.contains("improve documentation for callers"));
21738 assert!(described.body.contains("remaining edge case"));
21739 assert!(described.body.contains("## Open review findings"));
21740 assert!(!described.body.contains("未修正"));
21741 }
21742 }
21743
21744 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21745 let mut f = summary_fixture();
21746 if !review_only {
21747 f.runner.state.candidates[0].agent = "alpha".into();
21748 }
21749 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21750 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21751 fixed
21752 .fix
21753 .as_mut()
21754 .unwrap()
21755 .rejected
21756 .push(crate::verdict::Rejection {
21757 id: "R1-1-1".into(),
21758 why: "呼び出されないコードのため辞退".into(),
21759 });
21760 let mut clean = clean_round_with_minor_findings();
21761 clean.verified_head = Some(head.clone());
21762 f.runner.state.reviews = vec![fixed, clean];
21763 ok_gate(&mut f.runner.state, Some(&head));
21764 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21765 f.runner.state.config.agents.push(scripted("beta", reply));
21766 f.runner.state.config.roles.language_judge =
21767 Some(crate::config::AgentChoice::One("beta".into()));
21768 f
21769 }
21770
21771 #[tokio::test]
21772 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21773 for review_only in [false, true] {
21774 let mut f = rewrite_fixture_with_findings(review_only,
21775 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21776 let m = posted(&mut f).await;
21777 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21778 let expected =
21779 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21780 assert!(m.body.contains(&expected));
21781 assert!(m.body.contains("## Review rounds"));
21782 assert!(
21783 m.body
21784 .contains("Gate (`verify.gate`): passed on the final commit")
21785 );
21786 assert!(
21787 m.body
21788 .contains("Tests (`verify.e2e`): passed on the final commit")
21789 );
21790 let c = f.runner.state.change_summary.as_ref().unwrap();
21791 assert!(
21792 m.body
21793 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21794 );
21795 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21796 assert!(crate::pr_outcome::covers_required_findings(
21797 &f.runner.state,
21798 &m.body
21799 ));
21800 assert!(m.body.contains("documentation remains incomplete"));
21801 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21802 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21803 assert!(f.runner.state.github_text.is_none());
21804 }
21805 }
21806
21807 #[tokio::test]
21808 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21809 for review_only in [false, true] {
21810 for reply in [
21811 "TITLE: fix: public description\nA short English description.",
21812 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21813 ] {
21814 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21815 let winner = f.runner.state.candidates[0].clone();
21816 let posted = f
21817 .runner
21818 .guarded_pr_message(&winner, None, true)
21819 .await
21820 .unwrap();
21821 assert!(posted.is_none());
21822 assert!(f.runner.state.parked);
21823 let withheld = f.runner.state.github_text.as_ref().unwrap();
21824 assert!(
21825 withheld.rewrite.is_none(),
21826 "an incomplete rewrite must not be selectable"
21827 );
21828 let q = ask::Questions::open()
21829 .get(withheld.question.as_ref().unwrap())
21830 .unwrap();
21831 assert!(
21832 !q.choices
21833 .contains(&crate::github_text::USE_REWRITE.to_owned())
21834 );
21835 assert!(
21836 q.detail.contains("R1-1-1")
21837 && q.detail.contains("R2-1-1")
21838 && q.detail.contains("R2-1-2")
21839 );
21840 }
21841 }
21842 }
21843
21844 #[tokio::test]
21845 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21846 for review_only in [false, true] {
21847 let mut f = summary_fixture();
21848 if !review_only {
21849 f.runner.state.candidates[0].agent = "alpha".into();
21850 }
21851 f.runner.state.instruction = "内部タスクの説明".into();
21852 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21853 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21854 f.runner.state.reviews = vec![
21855 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21856 round_with_notes(2, Some("二回目の内部修正報告です。")),
21857 passing_round(3, Some(&head)),
21858 ];
21859 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21860 ok_gate(&mut f.runner.state, Some(&head));
21861 let captured = f.repo.join("public-prompt.txt");
21862 let author = &mut f.runner.state.config.agents[0];
21863 author.command = vec![
21864 "sh".into(),
21865 "-c".into(),
21866 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21867 ];
21868 author.env.insert(
21869 "PROMPT_FILE".into(),
21870 captured.to_string_lossy().into_owned(),
21871 );
21872 let m = posted(&mut f).await;
21873 assert!(m.body.contains("Adds `own.txt`"));
21874 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21875 assert!(m.body.contains("R1-1-1"));
21876 assert!(
21877 m.body
21878 .contains("Gate (`verify.gate`): passed on the final commit")
21879 );
21880 assert!(
21881 m.body
21882 .contains("Tests (`verify.e2e`): passed on the final commit")
21883 );
21884 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21885 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21886 let prompt = std::fs::read_to_string(captured).unwrap();
21887 assert!(prompt.contains(&format!(
21888 "git diff {} {head}",
21889 f.runner.state.change_summary.as_ref().unwrap().base
21890 )));
21891 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21892 assert_eq!(
21893 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21894 "二回目の内部修正報告です。"
21895 );
21896 }
21897 }
21898
21899 #[tokio::test]
21900 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21901 for review_only in [false, true] {
21902 let mut f = summary_fixture();
21903 if !review_only {
21904 f.runner.state.candidates[0].agent = "alpha".into();
21905 }
21906 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21907 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21908 std::fs::write(&f.reply, "no usable description").unwrap();
21909 let failed = posted(&mut f).await;
21910 assert!(
21911 failed
21912 .body
21913 .contains("No description of what this change does could be generated")
21914 );
21915 assert!(failed.body.is_ascii());
21916 assert!(f.runner.state.github_text.is_none());
21917 assert!(f.runner.state.change_summary.is_none());
21918 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21919 let evidence = crate::pr_outcome::summary_evidence(
21920 &f.runner.state,
21921 crate::pr_outcome::FinalHead::At(&head),
21922 );
21923 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21924 assert!(
21925 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21926 .exists()
21927 );
21928 f.runner.state =
21929 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21930 std::fs::write(
21931 &f.reply,
21932 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21933 )
21934 .unwrap();
21935 let retried = posted(&mut f).await;
21936 assert!(retried.body.contains("Adds the committed own file."));
21937 assert_eq!(
21938 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21939 "日本語の修正報告"
21940 );
21941 }
21942 }
21943
21944 #[tokio::test]
21945 async fn same_head_verification_and_findings_invalidate_public_prose() {
21946 let mut f = summary_fixture();
21947 posted(&mut f).await;
21948 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21949 let original = f
21950 .runner
21951 .state
21952 .change_summary
21953 .as_ref()
21954 .unwrap()
21955 .evidence
21956 .clone();
21957 ok_gate(&mut f.runner.state, Some(&head));
21958 std::fs::write(
21959 &f.reply,
21960 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21961 )
21962 .unwrap();
21963 let verified = posted(&mut f).await;
21964 assert!(verified.body.contains("now verified"));
21965 assert_ne!(
21966 f.runner.state.change_summary.as_ref().unwrap().evidence,
21967 original
21968 );
21969 let mut round = round_with_notes(1, Some("内部報告"));
21970 round.reviews.push(ReviewRecord {
21971 attempts: 0,
21972 reviewer: 1,
21973 agent: "alpha".into(),
21974 summary: "内部レビュー".into(),
21975 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21976 vote: None,
21977 failed: None,
21978 duration_ms: 0,
21979 });
21980 round
21981 .fix
21982 .as_mut()
21983 .unwrap()
21984 .rejected
21985 .push(crate::verdict::Rejection {
21986 id: "R1-1-1".into(),
21987 why: "呼び出し元から到達できない".into(),
21988 });
21989 f.runner.state.reviews = vec![round];
21990 let omitted = posted(&mut f).await;
21992 assert!(omitted.body.contains("No description"));
21993 assert!(omitted.body.contains("R1-1-1"));
21994 assert!(!omitted.body.contains("到達"));
21995 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21996 let described = posted(&mut f).await;
21997 assert!(described.body.contains("unreachable from callers"));
21998 assert!(described.body.contains("R1-1-1"));
21999 assert!(!described.body.contains("到達"));
22000 let evidence = f
22001 .runner
22002 .state
22003 .change_summary
22004 .as_ref()
22005 .unwrap()
22006 .evidence
22007 .clone();
22008 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22009 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22010 assert!(posted(&mut f).await.body.contains("rationale updated"));
22011 assert_ne!(
22012 f.runner.state.change_summary.as_ref().unwrap().evidence,
22013 evidence
22014 );
22015 f.runner
22017 .state
22018 .change_summary
22019 .as_mut()
22020 .unwrap()
22021 .evidence
22022 .clear();
22023 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22024 assert!(posted(&mut f).await.body.contains("current specification"));
22025 }
22026
22027 #[tokio::test]
22028 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22029 let mut f = summary_fixture();
22030 let m = posted(&mut f).await;
22031 assert_eq!(m.title, "feat(own): add the own file");
22032 assert!(
22033 m.body
22034 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22035 "{}",
22036 m.body
22037 );
22038 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22039 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22040 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22041 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22042 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22043 let kept = f.runner.state.change_summary.as_ref().unwrap();
22044 assert_eq!(
22045 (kept.head.as_str(), kept.base.as_str()),
22046 (head.as_str(), base.as_str())
22047 );
22048 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22049 }
22050
22051 #[tokio::test]
22052 async fn review_only_summary_receives_the_configured_github_language() {
22053 let mut f = summary_fixture();
22054 f.runner.state.config.graph.github_language = "ja".into();
22055 let captured = f.repo.join("summary-prompt.txt");
22056 let author = &mut f.runner.state.config.agents[0];
22057 author.command = vec![
22058 "sh".into(),
22059 "-c".into(),
22060 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22061 ];
22062 author.env.insert(
22063 "PROMPT_FILE".into(),
22064 captured.to_string_lossy().into_owned(),
22065 );
22066 std::fs::write(
22067 &f.reply,
22068 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22069 )
22070 .unwrap();
22071 let m = posted(&mut f).await;
22072 let prompt = std::fs::read_to_string(captured).unwrap();
22073 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22074 assert!(!prompt.contains("in English"));
22075 assert_eq!(m.title, "feat(own): ファイルを追加");
22076 assert!(m.body.contains("own.txt を追加します。"));
22077 }
22078
22079 #[tokio::test]
22080 async fn review_fixes_come_after_the_whole_change_summary() {
22081 let mut f = summary_fixture();
22082 f.runner.state.reviews = vec![
22083 round_with_notes(1, Some("handled the PATH case")),
22084 round_with_notes(2, None),
22085 ];
22086 let m = posted(&mut f).await;
22087 let summary = m.body.find("## Summary").expect("summary");
22088 let fixes = m.body.find("## Review rounds").expect("review rounds");
22089 assert!(summary < fixes, "{}", m.body);
22090 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22091 assert!(m.body.contains("## Final verification"), "{}", m.body);
22092 }
22093
22094 #[tokio::test]
22095 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22096 let mut f = summary_fixture();
22097 let first = posted(&mut f).await;
22098 let first_head = f.runner.state.change_summary.clone().unwrap().head;
22099 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22101 let again = posted(&mut f).await;
22102 assert_eq!((again.title, again.body), (first.title, first.body));
22103
22104 let g = |args: &[&str]| {
22105 let out = std::process::Command::new("git")
22106 .args(args)
22107 .current_dir(&f.repo)
22108 .quiet()
22109 .output()
22110 .expect("spawn git");
22111 assert!(out.status.success(), "git {args:?}");
22112 };
22113 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22114 g(&["add", "-A"]);
22115 g(&["commit", "-q", "-m", "fix(own): handle more"]);
22116 let second = posted(&mut f).await;
22117 let kept = f.runner.state.change_summary.clone().unwrap();
22118 assert_ne!(kept.head, first_head);
22119 assert!(second.body.contains("Not read yet."), "{}", second.body);
22120 assert!(!second.body.contains("so the first summary is read"));
22121 assert_eq!(second.title, "feat(own): other");
22122 }
22123
22124 #[tokio::test]
22125 async fn an_unavailable_summary_is_stated_and_never_invented() {
22126 let mut f = summary_fixture();
22127 std::fs::write(&f.reply, "no title line here").unwrap();
22128 let m = posted(&mut f).await;
22129 assert!(
22130 m.body
22131 .contains("No description of what this change does could be generated"),
22132 "{}",
22133 m.body
22134 );
22135 assert!(m.body.contains("internal reports remain in local records"));
22136 assert!(f.runner.state.change_summary.is_none());
22137 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22138 let winner = f.runner.state.candidates[0].clone();
22140 let preview = f
22141 .runner
22142 .guarded_pr_message(&winner, None, false)
22143 .await
22144 .unwrap()
22145 .unwrap();
22146 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22147 }
22148
22149 #[test]
22150 fn pr_message_review_single_commit_uses_its_subject() {
22151 let state = review_state(&["feat(nats): per-role user"]);
22152 let m = pr_message(&state, 'A');
22153 assert_eq!(m.title, "feat(nats): per-role user");
22154 assert!(!m.body.contains("Review the work already"), "{}", m.body);
22155 assert!(m.body.contains("## Commits under review"), "{}", m.body);
22156 }
22157
22158 #[test]
22159 fn pr_message_review_multi_commit_takes_the_oldest() {
22160 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22161 let m = pr_message(&state, 'A');
22162 assert_eq!(m.title, "feat: the change");
22163 for s in ["feat: the change", "fix: typo", "fix: again"] {
22164 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22165 }
22166 }
22167
22168 #[test]
22169 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22170 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22171 let mut state = review_state(&[first, "fix: later fixup"]);
22172 state.candidates[0].branch = "機能/ログイン".to_owned();
22173 let m = pr_message(&state, 'A');
22174 assert!(
22175 m.title.starts_with("chore: land candidate A of run"),
22176 "{}",
22177 m.title
22178 );
22179 }
22180 }
22181
22182 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22183 BranchFacts {
22184 commits: commits
22185 .iter()
22186 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22187 .collect(),
22188 stat: stat.to_owned(),
22189 }
22190 }
22191
22192 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22193 let mut r = review_round(true, 0, 1, 1, true, true);
22194 r.round = round;
22195 r.fix = notes.map(|n| FixRecord {
22196 agent: "fixer".to_owned(),
22197 addressed: Vec::new(),
22198 rejected: Vec::new(),
22199 notes: n.to_owned(),
22200 committed: true,
22201 failed: None,
22202 duration_ms: 0,
22203 continuation: None,
22204 });
22205 r
22206 }
22207
22208 #[test]
22209 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22210 let state = review_state(&["ignored"]);
22211 let f = facts(
22212 &[
22213 (
22214 "fix(login): resolve PATH on macOS",
22215 "Login shells skip rc files.",
22216 ),
22217 ("fix: address review", ""),
22218 ],
22219 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22220 );
22221 let m = pr_message_with(&state, 'A', Some(&f));
22222 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22223 assert!(
22224 m.body
22225 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22226 "{}",
22227 m.body
22228 );
22229 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22230 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22231 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22232 for banned in [
22233 "Review the work already",
22234 "no task statement",
22235 "Original task",
22236 ] {
22237 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22238 }
22239 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22240 }
22241
22242 #[test]
22243 fn pr_message_review_truncates_a_huge_first_commit_body() {
22244 let state = review_state(&["ignored"]);
22245 let f = facts(
22246 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22247 "s",
22248 );
22249 let m = pr_message_with(&state, 'A', Some(&f));
22250 assert!(m.body.len() < 30_000, "{}", m.body.len());
22251 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22252 assert!(
22253 m.body.contains("- ... 1 more commit(s)"),
22254 "{}",
22255 m.body.len()
22256 );
22257 assert!(m.body.ends_with("magi:candidate-a\n"));
22258 }
22259
22260 #[test]
22261 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22262 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22263 assert_eq!(m.title, "feat: x");
22264 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22265 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22266 assert!(!m.body.contains("no task statement"), "{}", m.body);
22267 }
22268
22269 #[test]
22270 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22271 let mut state = review_state(&["日本語の件名"]);
22272 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22273 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22274 state.candidates[0].branch = "機能/ログイン".to_owned();
22275 assert!(
22276 pr_message(&state, 'A')
22277 .title
22278 .starts_with("chore: land candidate A")
22279 );
22280 }
22281
22282 #[test]
22283 fn pr_message_review_fixes_survive_a_clean_final_round() {
22284 let mut state = review_state(&["feat: x"]);
22285 state.reviews = vec![
22286 round_with_notes(1, Some("handled the PATH case")),
22287 round_with_notes(2, None),
22288 ];
22289 let body = pr_message(&state, 'A').body;
22290 assert!(
22291 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22292 "{body}"
22293 );
22294 assert!(body.contains("### Round 1"), "{body}");
22295
22296 state.reviews = vec![
22297 round_with_notes(1, Some("first fix")),
22298 round_with_notes(2, Some("")),
22299 round_with_notes(3, Some("second fix")),
22300 round_with_notes(4, None),
22301 ];
22302 let body = pr_message(&state, 'A').body;
22303 assert!(
22304 body.contains("### Round 1") && !body.contains("first fix"),
22305 "{body}"
22306 );
22307 assert!(
22308 body.contains("### Round 3") && !body.contains("second fix"),
22309 "{body}"
22310 );
22311 assert!(body.contains("### Round 2"), "{body}");
22312 }
22313
22314 const FINAL: &str = "abcdef1234567890";
22315
22316 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22317 let mut r = round_with_notes(round, None);
22318 r.verified_head = verified.map(str::to_owned);
22319 r
22320 }
22321
22322 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22323 state.gate = vec![CommandOutcome {
22324 command: "gate".to_owned(),
22325 code: Some(0),
22326 output_tail: String::new(),
22327 duration_ms: 0,
22328 resource_blocked: false,
22329 }];
22330 state.gate_ran = true;
22331 state.gate_head = head.map(str::to_owned);
22332 }
22333
22334 fn body_at(state: &RunState) -> String {
22335 pr_message_raw(
22336 state,
22337 'A',
22338 None,
22339 crate::pr_outcome::FinalHead::At(FINAL),
22340 ChangeNote::NotAsked,
22341 )
22342 .body
22343 }
22344
22345 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22346
22347 #[test]
22348 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22349 let mut state = review_state(&["feat: x"]);
22350 state.reviews = vec![
22351 round_with_notes(1, Some(SANDBOXED)),
22352 passing_round(2, Some(FINAL)),
22353 ];
22354 ok_gate(&mut state, Some(FINAL));
22355 let body = body_at(&state);
22356 assert!(!body.contains("Handled the PATH case."), "{body}");
22357 assert!(!body.contains("could not commit"), "{body}");
22358 assert!(!body.contains("could not run"), "{body}");
22359 assert!(
22360 body.contains("The fix is committed on the branch"),
22361 "{body}"
22362 );
22363 assert!(!body.contains("sandbox"), "{body}");
22364 assert!(
22365 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22366 "{body}"
22367 );
22368 assert!(
22369 body.contains("Gate (`verify.gate`): passed on the final commit"),
22370 "{body}"
22371 );
22372 assert!(
22374 state.reviews[0]
22375 .fix
22376 .as_ref()
22377 .unwrap()
22378 .notes
22379 .contains("could not commit")
22380 );
22381 }
22382
22383 #[test]
22384 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22385 let mut state = review_state(&["feat: x"]);
22386 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22387 r1.fix.as_mut().unwrap().committed = false;
22388 let mut r2 = passing_round(2, Some(FINAL));
22389 r2.e2e[0].code = Some(1);
22390 state.reviews = vec![r1, r2];
22391 let body = body_at(&state);
22392 assert!(!body.contains("could not commit"), "{body}");
22393 assert!(!body.contains("could not run"), "{body}");
22394 assert!(body.contains("This round produced no commit"), "{body}");
22395 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22396 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22397 assert!(!body.contains("passed"), "{body}");
22398 }
22399
22400 #[test]
22401 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22402 let mut state = review_state(&["feat: x"]);
22403 state.reviews = vec![
22404 round_with_notes(1, Some(SANDBOXED)),
22405 passing_round(2, Some("0123456789abcdef")),
22406 ];
22407 ok_gate(&mut state, Some("0123456789abcdef"));
22408 let body = body_at(&state);
22409 assert!(!body.contains("HTTP tests could not run"), "{body}");
22410 assert!(!body.contains("could not commit"), "{body}");
22412 assert!(
22413 body.contains("passed on an earlier commit (`0123456`)"),
22414 "{body}"
22415 );
22416 assert!(!body.contains("passed on the final commit"), "{body}");
22417
22418 ok_gate(&mut state, None);
22420 let body = body_at(&state);
22421 assert!(
22422 body.contains("passed on a commit that was not recorded"),
22423 "{body}"
22424 );
22425 }
22426
22427 #[test]
22428 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22429 let state = state_with_summary(
22430 "add retries to the client",
22431 "TITLE: feat: retries\n\nDid it.",
22432 );
22433 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22434 assert_eq!(m.title, "feat: retries");
22435 assert!(
22436 m.body.contains("<summary>Original task</summary>"),
22437 "{}",
22438 m.body
22439 );
22440 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22441 assert!(
22442 m.body
22443 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22444 "{}",
22445 m.body
22446 );
22447 }
22448
22449 #[test]
22450 fn pr_message_review_bounds_a_long_english_subject() {
22451 let long = format!("feat: {}", "word ".repeat(100));
22452 let m = pr_message(&review_state(&[&long]), 'A');
22453 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22454 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22455 }
22456
22457 #[test]
22458 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22459 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22460 let m = pr_message(&state, 'A');
22461 assert_eq!(m.title, "add retries");
22462 assert!(m.body.contains("<summary>Original task</summary>"));
22463 assert!(!m.body.contains("Commits under review"));
22464 assert_eq!(landing_subject_source(&state), state.instruction);
22465 }
22466
22467 #[test]
22468 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22469 let state = review_state(&["feat: the change", "fix: typo"]);
22470 let source = landing_subject_source(&state);
22471 assert_eq!(land::merge_subject("", &source), "feat: the change");
22472 assert_eq!(
22473 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22474 "feat: the change"
22475 );
22476 assert_eq!(
22478 land::merge_subject("feat: renamed by hand", &source),
22479 "feat: renamed by hand"
22480 );
22481 let blank = review_state(&["日本語"]);
22482 assert!(
22483 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22484 );
22485 }
22486
22487 #[test]
22488 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22489 let state = review_state(&["feat: the change"]);
22490 let source = landing_subject_source(&state);
22491 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22492 assert_eq!(
22493 land::merge_subject(landing_title(&state, old), &source),
22494 "feat: the change"
22495 );
22496 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22497 let task = state_with_summary("add retries", "");
22498 assert_eq!(landing_title(&task, old), old);
22499 }
22500
22501 #[test]
22502 fn pr_message_describes_the_change_not_the_task() {
22503 let state = state_with_summary(
22504 "今回やってほしいこと: results projector を直す",
22505 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22506 );
22507 let m = pr_message(&state, 'A');
22508 assert_eq!(m.title, "fix(web): batch the runs list reads");
22509 assert!(
22510 m.body.starts_with("## Summary\n\n- reads run.json once"),
22511 "{}",
22512 m.body
22513 );
22514 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22515 let task_at = m.body.find("今回やってほしいこと").unwrap();
22516 let details_at = m.body.find("<details>").unwrap();
22517 assert!(
22518 details_at < task_at,
22519 "the task lives inside <details>: {}",
22520 m.body
22521 );
22522 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22523 assert!(m.body.contains("magi:candidate-a"));
22524 }
22525
22526 #[test]
22527 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22528 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22529 let m = pr_message(&state, 'A');
22530 assert_eq!(m.title, "add retries");
22531 assert!(
22532 m.body.contains("## Summary\n\n- did some things"),
22533 "{}",
22534 m.body
22535 );
22536
22537 let none = RunState::new(
22538 PathBuf::from("/repo"),
22539 "main".to_owned(),
22540 "abc1234".to_owned(),
22541 "add retries".to_owned(),
22542 Config::default(),
22543 );
22544 let m = pr_message(&none, 'A');
22545 assert_eq!(m.title, "add retries");
22546 assert!(!m.body.contains("## Summary"), "{}", m.body);
22547 }
22548
22549 #[test]
22550 fn pr_message_refuses_the_candidate_commit_subject() {
22551 for bad in [
22552 "TITLE: magi: candidate A (uncommitted work)",
22553 "TITLE: chore: stuff (uncommitted work)",
22554 "TITLE: ",
22555 ] {
22556 let state = state_with_summary("add retries", bad);
22557 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22558 }
22559 }
22560
22561 #[test]
22562 fn pr_message_bounds_a_very_long_task_and_title() {
22563 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22564 let state = state_with_summary(&long, "- nothing");
22565 let m = pr_message(&state, 'A');
22566 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22567 assert!(!m.title.contains('\n'));
22568
22569 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22570 let m = pr_message(&state, 'A');
22571 assert!(m.title.starts_with("feat: "));
22572 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22573 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22574 }
22575
22576 fn long_title_of(instruction: &str) -> String {
22577 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22578 }
22579
22580 #[test]
22581 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22582 let first = "Make the landing path keep a readable title for long tasks";
22583 let line = format!(
22584 "{first}. {}",
22585 "And then keep going with more words ".repeat(20)
22586 );
22587 let t = long_title_of(&line);
22588 assert_eq!(t, first);
22589 assert!(!t.starts_with("chore: land"));
22590 }
22591
22592 #[test]
22593 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22594 let line = "word ".repeat(200);
22595 let t = long_title_of(&line);
22596 assert!(t.ends_with("word..."), "{t}");
22597 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22598 }
22599
22600 #[test]
22601 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22602 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22603 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22604 }
22605 }
22606
22607 #[test]
22608 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22609 let mut state = state_with_summary("再試行を追加する", "- nothing");
22610 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22611 state.config.graph.github_language = "ja".into();
22612 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22613 }
22614
22615 #[test]
22616 fn pr_message_title_limit_is_exact() {
22617 let at = "a".repeat(PR_TITLE_MAX);
22618 assert_eq!(long_title_of(&at), at);
22619 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22620 assert!(over.ends_with("..."), "{over}");
22621 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22622 }
22623
22624 #[test]
22625 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22626 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22627 let t = long_title_of(&line);
22628 assert!(t.ends_with("..."), "{t}");
22629 assert!(t.is_ascii(), "{t}");
22630 }
22631
22632 #[test]
22633 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22634 let line = format!(
22635 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22636 "plus more filler words ".repeat(20)
22637 );
22638 let t = long_title_of(&line);
22639 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22640 }
22641
22642 #[test]
22643 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22644 let line = format!("\n\n# {}", "title words ".repeat(40));
22645 let state = state_with_summary(&line, "- nothing");
22646 let m = pr_message(&state, 'A');
22647 assert!(m.title.starts_with("title words"), "{}", m.title);
22648 assert_eq!(
22649 land::merge_subject(&m.title, &landing_subject_source(&state)),
22650 m.title
22651 );
22652 assert_eq!(
22654 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22655 "feat: renamed by hand"
22656 );
22657 }
22658
22659 #[tokio::test]
22660 async fn github_text_rewrite_is_bounded_and_falls_back() {
22661 crate::run::pin_test_home();
22662 for (reply, accepted) in [
22663 (
22664 "TITLE: fix: retries\nAdd retries for failed requests.",
22665 true,
22666 ),
22667 (
22668 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22669 false,
22670 ),
22671 (
22672 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22673 true,
22674 ),
22675 (
22676 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22677 true,
22678 ),
22679 (
22680 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22681 true,
22682 ),
22683 (
22684 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22685 false,
22686 ),
22687 ("TITLE: fix: retries\n", false),
22688 ] {
22689 let dir = tempfile::tempdir().unwrap();
22690 let mut runner = runner_at(RunStatus::Gating);
22691 runner.state = state_with_summary(
22692 "add retries",
22693 "TITLE: fix: retries\n日本語の説明を書きます。",
22694 );
22695 runner.state.repo = dir.path().to_owned();
22696 runner.state.candidates[0].worktree = dir.path().to_owned();
22697 let mut author = spec("alpha");
22698 author.command = vec![
22699 "sh".into(),
22700 "-c".into(),
22701 "printf '%s' \"$REWRITE_REPLY\"".into(),
22702 ];
22703 author.env.insert("REWRITE_REPLY".into(), reply.into());
22704 runner.state.config.agents = vec![author];
22705 seed_public_summary(&mut runner);
22706 let winner = runner.state.candidates[0].clone();
22707 let first = runner
22708 .guarded_pr_message(&winner, None, true)
22709 .await
22710 .unwrap();
22711 let message = if accepted {
22712 first.expect("a passing rewrite never asks")
22713 } else {
22714 assert!(first.is_none());
22716 assert!(runner.state.parked);
22717 let store = ask::Questions::open();
22718 let asked: Vec<_> = store
22719 .list()
22720 .into_iter()
22721 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22722 .collect();
22723 assert_eq!(asked.len(), 1);
22724 assert!(!asked[0].detail.contains("secret"));
22725 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22726 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22727 assert_eq!(
22728 asked[0]
22729 .choices
22730 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22731 rewrite.is_some()
22732 );
22733 store
22736 .update(&asked[0].id, |q| {
22737 q.abandon("test");
22738 Ok(())
22739 })
22740 .unwrap();
22741 let again = runner
22742 .guarded_pr_message(&winner, None, true)
22743 .await
22744 .unwrap()
22745 .expect("an abandoned question falls back");
22746 let n = store
22747 .list()
22748 .into_iter()
22749 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22750 .count();
22751 assert_eq!(n, 1, "asked once per rejected text");
22752 again
22753 };
22754 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22755 assert!(!message.body.contains("secret"));
22756 assert!(!message.body.contains("prefix"));
22757 assert!(!message.title.contains("secret"));
22758 assert_eq!(
22759 message.body.contains("Add retries for failed requests."),
22760 accepted
22761 );
22762 assert_eq!(
22763 runner.state.seats["github-text-rewrite"].turns, 1,
22764 "only one rewrite invocation"
22765 );
22766 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22767 if accepted {
22768 assert!(runner.state.github_text.is_none());
22769 assert!(!runner.state.parked);
22770 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22771 assert!(message.body.contains("## Final verification"));
22772 assert!(message.body.contains("Evidence: final diff"));
22773 assert!(
22774 message
22775 .body
22776 .contains(&format!("magi:run/{}", runner.state.id))
22777 );
22778 }
22779 if !accepted {
22780 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22781 assert!(
22782 message
22783 .body
22784 .contains(&format!("magi:run/{}", runner.state.id))
22785 );
22786 }
22787 }
22788 }
22789
22790 fn scripted(id: &str, reply: &str) -> AgentSpec {
22791 let mut a = spec(id);
22792 a.command = vec![
22793 "sh".into(),
22794 "-c".into(),
22795 "printf '%s' \"$REWRITE_REPLY\"".into(),
22796 ];
22797 a.env.insert("REWRITE_REPLY".into(), reply.into());
22798 a
22799 }
22800
22801 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22802 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22803
22804 fn seed_public_summary(runner: &mut Runner) {
22806 let repo = &runner.state.repo;
22807 let g = |args: &[&str]| {
22808 let out = std::process::Command::new("git")
22809 .args(args)
22810 .current_dir(repo)
22811 .quiet()
22812 .output()
22813 .unwrap();
22814 assert!(out.status.success(), "git {args:?}");
22815 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22816 };
22817 g(&["init", "-q", "-b", "main"]);
22818 g(&[
22819 "-c",
22820 "user.name=Test",
22821 "-c",
22822 "user.email=test@example.invalid",
22823 "commit",
22824 "-q",
22825 "--allow-empty",
22826 "-m",
22827 "feat: fixture",
22828 ]);
22829 let head = g(&["rev-parse", "HEAD"]);
22830 g(&["update-ref", "refs/remotes/origin/main", &head]);
22831 let evidence = crate::pr_outcome::summary_evidence(
22832 &runner.state,
22833 crate::pr_outcome::FinalHead::At(&head),
22834 );
22835 let generated = &runner.state.candidates[0].summary;
22836 runner.state.change_summary = Some(crate::run::ChangeSummary {
22837 base: head.clone(),
22838 head,
22839 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22840 title: summary_title(generated).unwrap(),
22841 body: summary_without_title(generated),
22842 });
22843 }
22844
22845 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22846 let mut runner = runner_at(RunStatus::Gating);
22847 runner.state = state_with_summary("add retries", JA_SUMMARY);
22848 if review_only {
22849 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22850 }
22851 runner.state.repo = dir.to_owned();
22852 runner.state.candidates[0].worktree = dir.to_owned();
22853 runner.state.config.agents = agents;
22854 seed_public_summary(&mut runner);
22855 runner
22856 }
22857
22858 fn gate_questions(runner: &Runner) -> usize {
22859 ask::Questions::open()
22860 .list()
22861 .into_iter()
22862 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22863 .count()
22864 }
22865
22866 #[tokio::test]
22867 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22868 crate::run::pin_test_home();
22869 let dir = tempfile::tempdir().unwrap();
22870 let mut runner = rewrite_runner(
22871 dir.path(),
22872 true,
22873 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22874 );
22875 runner.state.config.roles.language_judge =
22876 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22877 let winner = runner.state.candidates[0].clone();
22878 let msg = runner
22879 .guarded_pr_message(&winner, None, true)
22880 .await
22881 .unwrap()
22882 .expect("an English rewrite posts without asking");
22883 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22884 assert!(msg.body.contains("<details>kept</details>"));
22885 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22886 assert_eq!(gate_questions(&runner), 0);
22887 assert!(runner.state.github_text.is_none());
22888 assert!(!runner.state.seats.contains_key("impl-A"));
22889 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22890 }
22891
22892 #[tokio::test]
22893 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22894 crate::run::pin_test_home();
22895 let dir = tempfile::tempdir().unwrap();
22896 let mut runner = rewrite_runner(
22897 dir.path(),
22898 false,
22899 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22900 );
22901 runner.state.config.roles.language_judge =
22902 Some(crate::config::AgentChoice::One("beta".into()));
22903 runner.state.seats.insert(
22904 "impl-A".into(),
22905 SeatState::new("impl-A", "alpha", runner.state.seed),
22906 );
22907 let winner = runner.state.candidates[0].clone();
22908 let msg = runner
22909 .guarded_pr_message(&winner, None, true)
22910 .await
22911 .unwrap()
22912 .expect("rewritten");
22913 assert!(msg.body.contains("<details>kept</details>"));
22914 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22915 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22916 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22917 }
22918
22919 #[tokio::test]
22920 async fn rewrite_reuses_a_resumable_implementer_session() {
22921 crate::run::pin_test_home();
22922 let dir = tempfile::tempdir().unwrap();
22923 let mut runner = rewrite_runner(
22924 dir.path(),
22925 false,
22926 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22927 );
22928 runner.state.config.roles.language_judge =
22929 Some(crate::config::AgentChoice::One("beta".into()));
22930 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22931 seat.turns = 1;
22932 runner.state.seats.insert("impl-A".into(), seat);
22933 let winner = runner.state.candidates[0].clone();
22934 runner
22935 .guarded_pr_message(&winner, None, true)
22936 .await
22937 .unwrap()
22938 .expect("rewritten");
22939 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22940 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22941 }
22942
22943 #[tokio::test]
22944 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22945 crate::run::pin_test_home();
22948 let dir = tempfile::tempdir().unwrap();
22949 let mut exhausted = spec("alpha");
22950 exhausted.command = vec!["false".into()];
22951 let mut runner = rewrite_runner(
22952 dir.path(),
22953 false,
22954 vec![exhausted, scripted("beta", EN_REWRITE)],
22955 );
22956 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22958 "alpha".into(),
22959 "beta".into(),
22960 ]));
22961 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22962 seat.turns = 1;
22963 runner.state.seats.insert("impl-A".into(), seat);
22964 let winner = runner.state.candidates[0].clone();
22965 let msg = runner
22966 .guarded_pr_message(&winner, None, true)
22967 .await
22968 .unwrap()
22969 .expect("the role rewrites after the implementer fails");
22970 assert!(msg.body.contains("`src/net.rs`"));
22971 assert_eq!(
22972 runner.state.seats["impl-A"].turns, 2,
22973 "the implementer was tried once"
22974 );
22975 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22976 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22977 }
22978
22979 #[tokio::test]
22980 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22981 crate::run::pin_test_home();
22982 let dir = tempfile::tempdir().unwrap();
22983 let mut broken = spec("bad");
22984 broken.command = vec!["false".into()];
22985 let mut runner = rewrite_runner(
22986 dir.path(),
22987 true,
22988 vec![
22989 broken,
22990 scripted("noformat", "Add retries."),
22991 scripted("good", EN_REWRITE),
22992 ],
22993 );
22994 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22995 "bad".into(),
22996 "noformat".into(),
22997 "good".into(),
22998 ]));
22999 let winner = runner.state.candidates[0].clone();
23000 let msg = runner
23001 .guarded_pr_message(&winner, None, true)
23002 .await
23003 .unwrap()
23004 .expect("the third agent rewrites");
23005 assert!(msg.body.contains("`src/net.rs`"));
23006 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23007 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23008 }
23009
23010 #[tokio::test]
23011 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23012 crate::run::pin_test_home();
23013 for (agents, role, saved) in [
23015 (
23016 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23017 Some("beta"),
23018 true,
23019 ),
23020 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23021 (Vec::new(), None, false),
23022 ] {
23023 let dir = tempfile::tempdir().unwrap();
23024 let mut runner = rewrite_runner(dir.path(), true, agents);
23025 runner.state.config.roles.language_judge =
23026 role.map(|r| crate::config::AgentChoice::One(r.into()));
23027 let winner = runner.state.candidates[0].clone();
23028 let first = runner
23029 .guarded_pr_message(&winner, None, true)
23030 .await
23031 .unwrap();
23032 assert!(first.is_none(), "withheld text parks the run");
23033 assert!(runner.state.parked);
23034 assert_eq!(gate_questions(&runner), 1);
23035 let g = runner.state.github_text.as_ref().unwrap();
23036 assert_eq!(g.rewrite.is_some(), saved);
23037 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23038 }
23039 }
23040
23041 #[tokio::test]
23042 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23043 crate::run::pin_test_home();
23044 let dir = tempfile::tempdir().unwrap();
23045 let mut runner = runner_at(RunStatus::Gating);
23046 runner.state =
23047 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23048 runner.state.repo = dir.path().to_owned();
23049 runner.state.candidates[0].worktree = dir.path().to_owned();
23050 let mut author = spec("alpha");
23051 author.command = vec![
23052 "sh".into(),
23053 "-c".into(),
23054 "printf '%s' \"$REWRITE_REPLY\"".into(),
23055 ];
23056 author.env.insert(
23057 "REWRITE_REPLY".into(),
23058 format!(
23059 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23060 "English sentence. ".repeat(600)
23061 ),
23062 );
23063 runner.state.config.agents = vec![author];
23064 seed_public_summary(&mut runner);
23065 let winner = runner.state.candidates[0].clone();
23066 assert!(
23067 runner
23068 .guarded_pr_message(&winner, None, true)
23069 .await
23070 .unwrap()
23071 .is_none()
23072 );
23073 let record = runner.state.github_text.clone().unwrap();
23074 let rewrite = record.rewrite.clone().unwrap();
23075 assert_eq!(rewrite.failed, ["body-language"]);
23076 assert!(
23077 rewrite
23078 .body
23079 .ends_with(&format!("magi:run/{}", runner.state.id))
23080 );
23081 let store = ask::Questions::open();
23082 let q = store.get(record.question.as_ref().unwrap()).unwrap();
23083 assert!(
23084 q.choices
23085 .contains(&crate::github_text::USE_REWRITE.to_owned())
23086 );
23087 for label in [
23088 "ORIGINAL",
23089 "FALLBACK",
23090 "REWRITE",
23091 "Rules still failed: body-language",
23092 "truncated",
23093 ] {
23094 assert!(q.detail.contains(label), "{label}");
23095 }
23096 let artifact = std::fs::read_to_string(crate::run::artifact_path(
23097 &runner.state,
23098 &crate::github_text::artifact_name(&record.fingerprint),
23099 ))
23100 .unwrap();
23101 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23102 runner.state =
23104 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23105 store
23106 .update(&q.id, |q| {
23107 q.say("まだ日本語")?;
23108 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23109 })
23110 .unwrap();
23111 assert!(
23112 runner
23113 .guarded_pr_message(&winner, None, true)
23114 .await
23115 .unwrap()
23116 .is_none()
23117 );
23118 let retry = runner.state.github_text.as_ref().unwrap();
23119 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23120 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23121 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23122 assert!(
23123 q.posting_comparison
23124 .as_ref()
23125 .unwrap()
23126 .matches_detail(&q.detail)
23127 );
23128 store
23129 .update(&q.id, |q| {
23130 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23131 })
23132 .unwrap();
23133 let message = runner
23134 .guarded_pr_message(&winner, None, true)
23135 .await
23136 .unwrap()
23137 .unwrap();
23138 assert_eq!(message.title, rewrite.title);
23139 assert_eq!(message.body, rewrite.body);
23140 assert_eq!(
23141 runner.state.github_text.as_ref().unwrap().choice,
23142 Some(GithubTextChoice::Rewrite)
23143 );
23144 let again = runner
23145 .guarded_pr_message(&winner, None, true)
23146 .await
23147 .unwrap()
23148 .unwrap();
23149 assert_eq!(again.title, message.title);
23150 assert_eq!(again.body, message.body);
23151 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23152 assert!(
23153 !runner
23154 .state
23155 .events
23156 .iter()
23157 .any(|e| e.message.contains(&rewrite.title)
23158 || e.message.contains("English sentence."))
23159 );
23160 }
23161
23162 #[tokio::test]
23163 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23164 crate::run::pin_test_home();
23165 let dir = tempfile::tempdir().unwrap();
23166 let mut runner = runner_at(RunStatus::Gating);
23167 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23168 runner.state.repo = dir.path().to_owned();
23169 runner.state.candidates[0].worktree = dir.path().to_owned();
23170 seed_public_summary(&mut runner);
23171 let winner = runner.state.candidates[0].clone();
23172 assert!(
23173 runner
23174 .guarded_pr_message(&winner, None, true)
23175 .await
23176 .unwrap()
23177 .is_none()
23178 );
23179 let record = runner.state.github_text.as_ref().unwrap();
23180 assert!(record.rewrite.is_none());
23181 let mut old = serde_json::to_value(record).unwrap();
23183 old.as_object_mut().unwrap().remove("rewrite");
23184 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23185 assert!(old.rewrite.is_none());
23186 let id = record.question.clone().unwrap();
23187 ask::Questions::open()
23188 .update(&id, |q| {
23189 q.choices.push(crate::github_text::USE_REWRITE.into());
23190 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23191 })
23192 .unwrap();
23193 let message = runner
23194 .guarded_pr_message(&winner, None, true)
23195 .await
23196 .unwrap()
23197 .unwrap();
23198 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23199 assert_eq!(
23200 runner.state.github_text.as_ref().unwrap().choice,
23201 Some(GithubTextChoice::Fallback)
23202 );
23203 }
23204
23205 #[tokio::test]
23206 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23207 crate::run::pin_test_home();
23208 let dir = tempfile::tempdir().unwrap();
23209 let mut runner = runner_at(RunStatus::Gating);
23210 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23211 runner.state.repo = dir.path().to_owned();
23212 runner.state.candidates[0].worktree = dir.path().to_owned();
23213 seed_public_summary(&mut runner);
23214 let winner = runner.state.candidates[0].clone();
23215 assert!(
23216 runner
23217 .guarded_pr_message(&winner, None, true)
23218 .await
23219 .unwrap()
23220 .is_none()
23221 );
23222 let store = ask::Questions::open();
23223 let find = |run: &str| {
23224 store
23225 .list()
23226 .into_iter()
23227 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23228 .unwrap()
23229 };
23230 let run = runner.state.id.clone();
23231 let q = find(&run);
23232 let fp = runner
23233 .state
23234 .github_text
23235 .as_ref()
23236 .unwrap()
23237 .fingerprint
23238 .clone();
23239 let name = crate::github_text::artifact_name(&fp);
23240 let path = crate::run::artifact_path(&runner.state, &name);
23241 assert!(q.detail.contains(&path.display().to_string()));
23242 assert!(q.detail.contains("title-language: title"));
23243 let saved = std::fs::read_to_string(&path).expect("artifact written");
23244 assert!(saved.contains("日本語のタイトル"));
23245 assert!(q.cwd.is_none());
23246 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23247 let say = |id: &str, text: &str, choice: &str| {
23249 store
23250 .update(id, |q| {
23251 q.thread.push(ask::Turn {
23252 who: ask::Who::Operator,
23253 body: text.to_owned(),
23254 at: jiff::Timestamp::now(),
23255 note: None,
23256 });
23257 q.answer(ask::Answer::Choice(choice.to_owned()))
23258 })
23259 .unwrap();
23260 };
23261 say(&q.id, "まだ日本語", "use my text");
23262 assert!(
23263 runner
23264 .guarded_pr_message(&winner, None, true)
23265 .await
23266 .unwrap()
23267 .is_none()
23268 );
23269 let second = find(&run);
23270 assert_ne!(second.id, q.id);
23271 assert!(second.detail.contains("did not pass"));
23272 assert!(second.detail.contains("日本語のタイトル"));
23275 assert!(!second.detail.contains("まだ日本語"));
23276 assert!(second.detail.contains(&path.display().to_string()));
23277 say(&second.id, "fix: retry failed requests", "use my text");
23278 let message = runner
23279 .guarded_pr_message(&winner, None, true)
23280 .await
23281 .unwrap()
23282 .expect("a vetted replacement is posted");
23283 assert_eq!(message.title, "fix: retry failed requests");
23284 assert_eq!(
23285 runner
23286 .state
23287 .github_text
23288 .as_ref()
23289 .unwrap()
23290 .chosen_title
23291 .as_deref(),
23292 Some("fix: retry failed requests")
23293 );
23294 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23295 let again = runner
23297 .guarded_pr_message(&winner, None, true)
23298 .await
23299 .unwrap()
23300 .unwrap();
23301 assert_eq!(again.title, message.title);
23302 assert_eq!(
23303 store
23304 .list()
23305 .into_iter()
23306 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23307 .count(),
23308 2
23309 );
23310 }
23311
23312 #[test]
23313 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23314 let mut state = state_with_summary(
23318 "add retries",
23319 "TITLE: fix(web): batch reads\n- reads run.json once",
23320 );
23321 state.config.graph.language = "ja".to_owned();
23322 let m = pr_message(&state, 'A');
23323 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23324 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23325
23326 let task = "今回やってほしいこと: results projector を直す";
23329 let mut state = state_with_summary(task, "- no title line");
23330 state.config.graph.language = "ja".to_owned();
23331 let m = pr_message(&state, 'A');
23332 assert_eq!(
23333 m.title,
23334 format!("chore: land candidate A of run {}", state.id)
23335 );
23336 assert!(
23337 m.body.contains(&format!(
23338 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23339 )),
23340 "{}",
23341 m.body
23342 );
23343 }
23344
23345 #[test]
23346 fn pr_message_scrubs_home_paths_and_addresses() {
23347 let state = state_with_summary(
23348 "fix it in /Users/someone/src/x",
23349 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23350 );
23351 let m = pr_message(&state, 'A');
23352 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23353 assert!(!m.body.contains(leak), "{}", m.body);
23354 }
23355 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23356 }
23357
23358 #[test]
23359 fn pr_message_survives_a_task_that_closes_details() {
23360 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23361 let m = pr_message(&state, 'A');
23362 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23363 }
23364
23365 #[test]
23366 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23367 let cmd = manual_merge_command(
23368 MergeStyle::Squash,
23369 Path::new("/repo"),
23370 "b",
23371 "fix: \"quoted\" $(x) `y`\n\nbody",
23372 );
23373 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23374 }
23375
23376 #[test]
23377 fn manual_merge_command_matches_the_configured_style() {
23378 let repo = Path::new("/repo");
23379 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23380
23381 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23382 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23383
23384 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23385 assert_eq!(
23386 squash,
23387 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23388 \"Merge magi run 0832 (candidate A)\""
23389 );
23390
23391 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23392 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23393 }
23394
23395 #[test]
23396 fn a_nudge_gets_a_quarter_of_the_budget() {
23397 assert_eq!(retry_budget(secs(1200), true), secs(300));
23399 assert_eq!(retry_budget(secs(3600), true), secs(900));
23400 }
23401
23402 #[test]
23403 fn a_resent_prompt_keeps_the_whole_budget() {
23404 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23407 assert_eq!(retry_budget(secs(60), false), secs(60));
23408 }
23409
23410 #[test]
23411 fn the_floor_never_exceeds_the_original_budget() {
23412 assert_eq!(retry_budget(secs(60), true), secs(60));
23416 assert_eq!(retry_budget(secs(480), true), secs(120));
23417 assert_eq!(retry_budget(secs(0), true), secs(0));
23418 }
23419
23420 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23421 agent::CommandEvidence {
23422 id: "item1".to_owned(),
23423 description: "cargo test".to_owned(),
23424 exit_code,
23425 result_summary: String::new(),
23426 source: "codex".to_owned(),
23427 }
23428 }
23429
23430 #[test]
23431 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23432 assert!(!has_unconfirmed_command(&[]));
23436 }
23437
23438 #[test]
23439 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23440 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23444 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23445 assert!(!has_unconfirmed_command(&[
23446 evidence(Some(0)),
23447 evidence(Some(101))
23448 ]));
23449 }
23450
23451 #[test]
23452 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23453 assert!(has_unconfirmed_command(&[
23454 evidence(Some(0)),
23455 evidence(None)
23456 ]));
23457 }
23458
23459 #[test]
23460 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23461 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23462 assert_eq!(
23463 verified_noop_claim(true, &[], text).as_deref(),
23464 Some("already fixed by b32cfc4, on main.")
23465 );
23466 }
23467
23468 #[test]
23469 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23470 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23473 assert!(verified_noop_claim(false, &[], text).is_none());
23474 }
23475
23476 #[test]
23477 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23478 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23479 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23480 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23482 }
23483
23484 #[test]
23485 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23486 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23487 }
23488
23489 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23492 runner.state.candidates = shape
23493 .iter()
23494 .enumerate()
23495 .map(|(i, &(empty, verified))| Candidate {
23496 index: i,
23497 label: (b'A' + i as u8) as char,
23498 agent: "sonnet".to_owned(),
23499 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23500 worktree: PathBuf::from(format!("/wt/{i}")),
23501 summary: String::new(),
23502 stat: String::new(),
23503 files: 0,
23504 commits: 0,
23505 empty,
23506 failed: None,
23507 verified_noop: verified.map(str::to_owned),
23508 duration_ms: 0,
23509 folded: false,
23510 })
23511 .collect();
23512 }
23513
23514 #[test]
23515 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23516 ask_test_home();
23517 let mut runner = runner_at(RunStatus::Implementing);
23518 set_candidates(
23519 &mut runner,
23520 &[
23521 (true, Some("already on main at b32cfc4")),
23522 (true, Some("same fix, see the existing test")),
23523 ],
23524 );
23525
23526 runner
23527 .after_implement()
23528 .expect("a verified no-op is not an error");
23529
23530 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23531 }
23532
23533 #[test]
23534 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23535 ask_test_home();
23536 let mut runner = runner_at(RunStatus::Implementing);
23537 set_candidates(
23541 &mut runner,
23542 &[(true, Some("already on main at b32cfc4")), (true, None)],
23543 );
23544
23545 let err = runner
23546 .after_implement()
23547 .expect_err("an unverified empty candidate must still fail the run");
23548
23549 assert!(
23550 err.to_string().contains("no candidate produced a change"),
23551 "{err}"
23552 );
23553 assert_eq!(runner.state.status, RunStatus::Failed);
23554 }
23555
23556 #[test]
23557 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23558 ask_test_home();
23559 let mut runner = runner_at(RunStatus::Implementing);
23560 set_candidates(&mut runner, &[(true, None), (true, None)]);
23561
23562 let err = runner
23563 .after_implement()
23564 .expect_err("no candidate declared anything; this is an ordinary failure");
23565
23566 assert!(
23567 err.to_string().contains("no candidate produced a change"),
23568 "{err}"
23569 );
23570 assert_eq!(runner.state.status, RunStatus::Failed);
23571 }
23572}
23573
23574#[derive(Debug)]
23577struct GraftRestoreFailed(String);
23578
23579impl std::fmt::Display for GraftRestoreFailed {
23580 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23581 write!(
23582 f,
23583 "could not restore the winner to its pre-graft commit: {}",
23584 self.0
23585 )
23586 }
23587}
23588
23589impl std::error::Error for GraftRestoreFailed {}
23590
23591#[cfg(test)]
23595mod resource_wait_tests {
23596 use super::*;
23597 use std::collections::BTreeMap;
23598
23599 fn sh(id: &str, script: &str) -> AgentSpec {
23600 AgentSpec {
23601 id: id.to_owned(),
23602 kind: crate::config::AgentKind::Command,
23603 model: None,
23604 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23605 extra_args: Vec::new(),
23606 env: BTreeMap::new(),
23607 prompt_delivery: None,
23608 max_turn_secs: None,
23609 }
23610 }
23611
23612 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23613 SeatJob {
23614 seat: SeatState::new("impl-A", &spec.id, 1),
23615 spec,
23616 cwd: dir.to_path_buf(),
23617 prompt: "do it".to_owned(),
23618 timeout,
23619 allow_write: true,
23620 sessions: false,
23621 artifacts: dir.join("art"),
23622 stem: "impl-A".to_owned(),
23623 handover: None,
23624 }
23625 }
23626
23627 fn state(resource_wait_secs: u64) -> RunState {
23628 let mut config = Config::default();
23629 config.graph.timeout_resource_wait = resource_wait_secs;
23630 RunState::new(
23631 PathBuf::from("/repo"),
23632 "main".to_owned(),
23633 "abc1234".to_owned(),
23634 "task".to_owned(),
23635 config,
23636 )
23637 }
23638
23639 fn hold(cache: &Path) -> crate::cache::Guard {
23640 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23641 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23642 crate::cache::AcquireOutcome::Acquired(g) => g,
23643 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23644 }
23645 }
23646
23647 async fn run(
23648 jobs: Vec<SeatJob>,
23649 sem: Arc<Semaphore>,
23650 cache: Option<&Path>,
23651 st: &mut RunState,
23652 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23653 let prompts = Prompts::default();
23654 let ctx = WaveCtx {
23655 run: "r",
23656 node: "implement",
23657 prompts: &prompts,
23658 cache,
23659 round: None,
23660 carry_seats: false,
23661 pauses: None,
23662 };
23663 wave(jobs, sem, &ctx, st, 0).await
23664 }
23665
23666 #[tokio::test]
23667 async fn verification_cache_wait_preserves_the_command_budget() {
23668 crate::run::pin_test_home();
23669 let tmp = tempfile::tempdir().unwrap();
23670 let cache = tmp.path().join("target");
23671 let guard = hold(&cache);
23672 let release = tokio::spawn(async move {
23673 tokio::time::sleep(Duration::from_millis(100)).await;
23674 guard.release();
23675 });
23676 let mut st = state(10);
23677 let budget = Duration::from_millis(50);
23678 let wait = repository_wait(&st, None);
23679 let (outcomes, _) = with_cache_lease(
23680 &mut st,
23681 Some(&cache),
23682 "gate",
23683 "gate",
23684 tmp.path(),
23685 "abc1234",
23686 budget,
23687 "test gate",
23688 &wait,
23689 |_, available| async move {
23690 assert_eq!(
23691 available, budget,
23692 "contention must not shorten verification"
23693 );
23694 (Vec::new(), false, Vec::new())
23695 },
23696 )
23697 .await
23698 .unwrap();
23699 release.await.unwrap();
23700 assert!(outcomes.is_empty(), "{outcomes:?}");
23701 assert!(
23702 st.events
23703 .iter()
23704 .any(|event| event.message.contains("waiting for"))
23705 );
23706 assert!(st.resource_wait.is_none());
23707 hold(&cache).release();
23709 }
23710
23711 #[tokio::test]
23712 async fn verification_cache_wait_expires_without_starting_commands() {
23713 crate::run::pin_test_home();
23714 let tmp = tempfile::tempdir().unwrap();
23715 let cache = tmp.path().join("target");
23716 let _held = hold(&cache);
23717 let mut st = state(0);
23718 let wait = repository_wait(&st, None);
23719 let (outcomes, _) = with_cache_lease(
23720 &mut st,
23721 Some(&cache),
23722 "gate",
23723 "gate",
23724 tmp.path(),
23725 "abc1234",
23726 Duration::from_secs(60),
23727 "test gate",
23728 &wait,
23729 |_, _| async { panic!("commands must not start without the cache lease") },
23730 )
23731 .await
23732 .unwrap();
23733 assert_eq!(outcomes.len(), 1);
23734 assert!(outcomes[0].resource_blocked);
23735 assert!(st.resource_wait.is_none());
23736 assert!(matches!(
23737 st.resource_block.unwrap().cause,
23738 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23739 ));
23740 }
23741
23742 #[tokio::test]
23743 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23744 crate::run::pin_test_home();
23745 let tmp = tempfile::tempdir().unwrap();
23746 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23747 let cache = tmp.path().join("cache");
23748 let mut state = state(21600);
23749 let wait = repository_wait(&state, None);
23750 let (outcomes, _) = with_cache_lease(
23751 &mut state,
23752 Some(&cache),
23753 "gate",
23754 "gate",
23755 tmp.path(),
23756 "head",
23757 Duration::from_secs(60),
23758 "gate",
23759 &wait,
23760 |_, _| async { panic!("freshness failure must not start verification") },
23761 )
23762 .await
23763 .unwrap();
23764 assert_eq!(outcomes.len(), 1);
23765 assert!(outcomes[0].resource_blocked);
23766 let block = state.resource_block.unwrap();
23767 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23768 assert!(!block.description().contains("timed out"));
23769 }
23770
23771 #[tokio::test]
23772 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23773 crate::run::pin_test_home();
23774 let tmp = tempfile::tempdir().unwrap();
23775 let cache = tmp.path().join("cache");
23776 let _held = hold(&cache);
23777 let mut state = state(21600);
23778 let pause = Pause::new();
23779 let interrupt = Pause::new();
23780 let cause = ParkCause::Operator;
23781 pause.park_for(cause.clone());
23782 let prompts = Prompts::default();
23783 let ctx = WaveCtx {
23784 run: "test",
23785 node: "fix",
23786 prompts: &prompts,
23787 cache: Some(&cache),
23788 round: None,
23789 carry_seats: false,
23790 pauses: Some((&pause, &interrupt)),
23791 };
23792 let result = wave(
23793 vec![job(
23794 sh("writer", "exit 99"),
23795 tmp.path(),
23796 Duration::from_secs(1),
23797 )],
23798 Arc::new(Semaphore::new(1)),
23799 &ctx,
23800 &mut state,
23801 0,
23802 )
23803 .await;
23804 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23805 assert_eq!(result[0].1.turns, 0);
23806 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23807 assert_eq!(
23808 state.resource_block.unwrap().cause,
23809 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23810 );
23811 assert!(state.active.is_empty());
23812 }
23813
23814 #[tokio::test]
23815 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23816 crate::run::pin_test_home();
23817 let tmp = tempfile::tempdir().unwrap();
23818 let cache = tmp.path().join("target-a");
23819 let guard = hold(&cache);
23820 tokio::spawn(async move {
23822 tokio::time::sleep(Duration::from_millis(1500)).await;
23823 guard.release();
23824 });
23825 let mut st = state(60);
23826 let results = run(
23827 vec![job(
23828 sh("alpha", "echo ok"),
23829 tmp.path(),
23830 Duration::from_secs(1),
23831 )],
23832 Arc::new(Semaphore::new(1)),
23833 Some(&cache),
23834 &mut st,
23835 )
23836 .await;
23837 assert!(
23838 matches!(results[0].2, AgentOutcome::Ok(_)),
23839 "the wait must not be charged to the agent"
23840 );
23841 assert!(st.resource_block.is_none());
23842 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23843 assert!(st.resource_wait.is_none() && st.active.is_empty());
23844 }
23845
23846 #[tokio::test]
23847 #[cfg(unix)]
23848 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23849 use std::os::fd::AsRawFd as _;
23850 crate::run::pin_test_home();
23851 let tmp = tempfile::tempdir().unwrap();
23852 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23853 let common = tmp.path().join(".git");
23854 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23855 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23856 assert_eq!(
23857 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23858 0
23859 );
23860 let mut st = state(1);
23861 st.config.coordination = crate::config::test_coordination();
23862 let mut reader = job(
23863 sh("reader", "echo must-not-run"),
23864 tmp.path(),
23865 Duration::from_secs(30),
23866 );
23867 reader.allow_write = false;
23868 let results = run(
23869 vec![reader.clone()],
23870 Arc::new(Semaphore::new(1)),
23871 None,
23872 &mut st,
23873 )
23874 .await;
23875 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23876 assert_eq!(results[0].1.turns, 0);
23877 assert!(FailClass::of(&results[0].2).is_none());
23878 assert_eq!(
23879 st.resource_block.as_ref().unwrap().kind,
23880 "repository coordination"
23881 );
23882 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23883 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23884 drop(external);
23885 let editor = crate::repo_lock::SeatLease::for_checkout(
23886 tmp.path(),
23887 true,
23888 crate::config::test_coordination().as_ref(),
23889 )
23890 .await
23891 .unwrap();
23892 let pause = Pause::new();
23893 let interrupt = Pause::new();
23894 let prompts = Prompts::default();
23895 let ctx = WaveCtx {
23896 run: "r",
23897 node: "review",
23898 prompts: &prompts,
23899 cache: None,
23900 round: None,
23901 carry_seats: false,
23902 pauses: Some((&pause, &interrupt)),
23903 };
23904 let (results, ()) = tokio::join!(
23905 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23906 async {
23907 tokio::time::sleep(Duration::from_millis(50)).await;
23908 pause.park();
23909 }
23910 );
23911 assert!(
23912 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23913 );
23914 assert_eq!(results[0].1.turns, 0);
23915 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23916 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23917 drop(editor);
23918 }
23919
23920 #[tokio::test]
23921 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23922 crate::run::pin_test_home();
23923 let tmp = tempfile::tempdir().unwrap();
23924 let cache = tmp.path().join("target-b");
23925 let _held = hold(&cache);
23926 let mut st = state(1);
23927 let results = run(
23928 vec![job(
23929 sh("alpha", "echo ok"),
23930 tmp.path(),
23931 Duration::from_secs(1),
23932 )],
23933 Arc::new(Semaphore::new(1)),
23934 Some(&cache),
23935 &mut st,
23936 )
23937 .await;
23938 let (_, seat, out) = &results[0];
23939 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23940 assert!(wave_blocked(&results));
23941 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23942 assert_eq!(seat.turns, 0, "nothing was asked");
23943 assert!(
23944 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23945 "no agent was spawned"
23946 );
23947 assert!(FailClass::of(out).is_none(), "not a failure class");
23948 let block = st.resource_block.as_ref().expect("recorded for the resume");
23949 assert_eq!(block.kind, "shared build cache");
23950 assert!(matches!(
23951 block.cause,
23952 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23953 ));
23954 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23955 assert!(st.active.is_empty());
23956 }
23957
23958 #[tokio::test]
23959 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23960 crate::run::pin_test_home();
23961 let tmp = tempfile::tempdir().unwrap();
23962 let mut st = state(1);
23963 let results = run(
23964 vec![job(
23965 sh("alpha", "echo ok"),
23966 tmp.path(),
23967 Duration::from_secs(1),
23968 )],
23969 Arc::new(Semaphore::new(0)),
23970 None,
23971 &mut st,
23972 )
23973 .await;
23974 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23975 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23976 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23977 }
23978
23979 #[tokio::test]
23980 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23981 crate::run::pin_test_home();
23982 let tmp = tempfile::tempdir().unwrap();
23983 let mut st = state(60);
23984 let results = run(
23985 vec![job(
23986 sh("alpha", "sleep 30"),
23987 tmp.path(),
23988 Duration::from_secs(1),
23989 )],
23990 Arc::new(Semaphore::new(1)),
23991 None,
23992 &mut st,
23993 )
23994 .await;
23995 let out = &results[0].2;
23996 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23997 let class = FailClass::of(out).expect("a failure");
23998 assert_eq!(class, FailClass::Timeout);
23999 assert!(should_hand_over(None, &class), "existing handover policy");
24000 assert!(st.resource_block.is_none());
24001 }
24002}