1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false, None).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true, None).await
1357 }
1358
1359 pub(crate) async fn authorize_base_recovery_control(
1360 home: &Path,
1361 task: &str,
1362 control: &mut crate::base_recovery_control::Transaction,
1363 ) -> Result<()> {
1364 let id = control.request.run.clone();
1365 Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366 }
1367
1368 async fn authorize_base_recovery_request(
1369 home: &Path,
1370 id: &str,
1371 task: &str,
1372 resumption: bool,
1373 mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374 ) -> Result<()> {
1375 let _owner = crate::pr_head::owner_lock(home, id)?;
1376 let mut state = RunState::load_under(id, home)?;
1377 crate::pr_head::ensure_owner(&state)?;
1378 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379 anyhow::ensure!(
1380 queued.execution_runs().last().is_some_and(|run| run == id)
1381 && queued.status != crate::queue::TaskStatus::Done
1382 && !queued.fresh_start
1383 && queued.review_branch.is_none(),
1384 "run task ownership changed"
1385 );
1386 anyhow::ensure!(
1387 state
1388 .origin
1389 .as_ref()
1390 .and_then(|o| o.task.as_deref())
1391 .is_none_or(|owner| owner == task),
1392 "run belongs to another task"
1393 );
1394 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395 anyhow::ensure!(
1396 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397 "run is live or its ownership is unknown"
1398 );
1399 let _execution = crate::driver::maintenance(home, id)?;
1402 anyhow::ensure!(
1403 state.status.resumable()
1404 && state.released_to.is_none()
1405 && state.retired_to.is_none()
1406 && state.pr_taken_over_by.is_none(),
1407 "run ownership is final"
1408 );
1409 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410 anyhow::ensure!(
1411 !state.pr_publication_pending
1412 && !crate::handover::landing_checkpoint(&state)
1413 && state.merge_decision.is_none(),
1414 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415 );
1416 let winner = state.winner().context("run has no retained winner")?;
1417 anyhow::ensure!(
1418 git::is_clean(&winner.worktree).await?,
1419 "retained worktree has uncommitted changes"
1420 );
1421 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422 anyhow::ensure!(
1423 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424 "retained worktree does not match the winner branch"
1425 );
1426 if let Some(control) = control.as_ref() {
1427 anyhow::ensure!(
1428 head.starts_with(&control.request.head),
1429 "retained HEAD changed since the owner request"
1430 );
1431 anyhow::ensure!(
1432 state
1433 .base_sync_recovery
1434 .stop
1435 .as_ref()
1436 .is_none_or(|stop| stop.head == head),
1437 "retained HEAD differs from the recorded stop head; inspect it first"
1438 );
1439 }
1440 state.ensure_base_episode();
1441 state.record_base_sync_stop(&head);
1442 anyhow::ensure!(
1443 state
1444 .base_sync_recovery
1445 .stop
1446 .as_ref()
1447 .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448 "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449 );
1450 let fetched =
1451 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452 anyhow::ensure!(
1453 fetched.ok(),
1454 "cannot authorize base recovery without a fetched base"
1455 );
1456 let approved_tip = git::rev_parse(
1457 &state.repo,
1458 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459 )
1460 .await?;
1461 if let Some(control) = control.as_deref_mut() {
1462 let queue = crate::queue::Queue::at(home.join("queue"));
1463 control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464 if let Some(prior) = state
1465 .base_sync_recovery
1466 .authorizations
1467 .iter()
1468 .find(|a| a.request.as_deref() == Some(&control.key()))
1469 {
1470 anyhow::ensure!(
1471 !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472 "recovery request was consumed or invalidated; it cannot queue again"
1473 );
1474 return control.enqueue(&queue);
1475 }
1476 }
1477 if resumption {
1478 anyhow::ensure!(
1479 crate::daemon::compatible_run_instruction(&state, &queued),
1480 "resumption instruction does not match the retained run"
1481 );
1482 let approval = queued
1483 .resumption
1484 .as_ref()
1485 .context("no operator resumption")?;
1486 anyhow::ensure!(
1487 approval.instruction == queued.instruction,
1488 "resumption approval does not match the current task instruction"
1489 );
1490 state.authorize_base_sync_resumption(task, approval)?;
1491 } else {
1492 state.authorize_base_sync(task)?;
1493 }
1494 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495 state
1496 .base_sync_recovery
1497 .authorizations
1498 .last_mut()
1499 .unwrap()
1500 .scope = Some(crate::base_sync_recovery::Scope {
1501 tip: approved_tip,
1502 instruction: queued.instruction.clone(),
1503 episode,
1504 });
1505 if let Some(control) = control.as_deref_mut() {
1506 let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507 authorization.request = Some(control.key());
1508 authorization.generation = control.generation();
1509 authorization.decision = control.decision();
1510 authorization.question = control
1511 .request
1512 .decision
1513 .as_ref()
1514 .map(|_| control.request.request.clone());
1515 }
1516 state.event(
1517 "base-recovery",
1518 "owner authorized a bounded base synchronization interval; fresh verification required",
1519 );
1520 state.save_under(home)?;
1521 if let Some(control) = control {
1522 control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523 }
1524 Ok(())
1525 }
1526
1527 pub fn resume(id: &str) -> Result<Self> {
1529 let state = RunState::load(id)?;
1530 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531 if state.startup.as_ref().is_some_and(|s| !s.complete)
1532 || (state.status == RunStatus::Prep
1533 && state.instruction.trim().is_empty()
1534 && state.candidates.is_empty()
1535 && state.prep_complete == Some(false))
1536 {
1537 bail!(
1538 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539 );
1540 }
1541 crate::pr_head::ensure_owner(&state)?;
1542 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543 bail!(
1544 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545 );
1546 }
1547 if let Some(to) = &state.released_to {
1548 bail!(
1549 "run {} cannot be resumed: its worktree was released to run {}",
1550 state.short(),
1551 crate::run::short_of(to)
1552 );
1553 }
1554 let roles = state.config.resolve_roles()?;
1555 if let Some(id) = &state.config.graph.selected_implementer
1556 && state.candidates.iter().any(|c| c.agent != *id)
1557 {
1558 bail!(
1559 "replacement implementer pin does not match the saved candidates; task must stay held"
1560 );
1561 }
1562 let max_parallel = state.config.graph.max_parallel.max(1);
1563 Ok(Self {
1564 state,
1565 roles,
1566 sem: Arc::new(Semaphore::new(max_parallel)),
1567 pause: Pause::new(),
1568 interrupt: Pause::new(),
1569 })
1570 }
1571
1572 pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579 Box::pin(self.execute_inner())
1582 }
1583
1584 async fn execute_inner(&mut self) -> Result<()> {
1585 anyhow::ensure!(
1586 self.state.startup.as_ref().is_none_or(|s| s.complete),
1587 "startup incomplete; inspect retained evidence before explicit recovery"
1588 );
1589 if let Some(selection) = &self.state.pr_recovery
1590 && (selection.source_proof.is_some() || selection.source_run.is_some())
1591 {
1592 Box::pin(selection.verify_source(&self.state.repo)).await?;
1593 }
1594 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597 || self.state.land_approval.is_some()
1598 || self.state.merge_decision.is_some()
1599 || !self.state.merge_repairs.is_empty()
1600 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601 {
1602 let claim = FixClaim::acquire(&self.state.dir())?;
1603 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604 let disk = RunState::load(&self.state.id)?;
1605 let checkpoint = |s: &RunState| {
1606 (
1607 s.merge_repairs.len(),
1608 s.merge_repairs
1609 .last()
1610 .map(|r| (r.question.clone(), r.phase)),
1611 s.land_approval
1612 .as_ref()
1613 .map(|a| (a.question.clone(), a.head.clone())),
1614 )
1615 };
1616 if checkpoint(&disk) != checkpoint(&self.state) {
1617 bail!(
1618 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619 );
1620 }
1621 Some(claim)
1622 } else {
1623 None
1624 };
1625 let mut execution = self.claim_driver().await?;
1626 let runner = &mut *execution.runner;
1627 let wait = runner.repository_wait();
1628 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631 let result = match result {
1632 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633 record_repository_block(&mut runner.state, &e);
1634 runner.park_here()?;
1635 Ok(())
1636 }
1637 other => other,
1638 };
1639 execution.driver.finish(&mut runner.state).await?;
1640 let ended = if result.is_err() {
1641 Some({
1642 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643 if let Err(error) = &result {
1644 notice.message = format!(
1645 "Run {} stopped with an error: {}",
1646 runner.state.id,
1647 error.to_string().lines().next().unwrap_or("unknown error")
1648 );
1649 }
1650 notice
1651 })
1652 } else {
1653 crate::notices::run_ended(&runner.state)
1654 };
1655 if let Some(notice) = ended {
1656 crate::notices::raise(notice);
1657 }
1658 result
1659 }
1660
1661 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663 }
1664
1665 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666 let wait = self.repository_wait();
1667 coordinate_operation(&mut self.state, cwd, &wait).await
1668 }
1669
1670 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672 let owner = match crate::pr_head::owner_lock_wait(
1673 &crate::run::home(),
1674 &self.state.id,
1675 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676 )
1677 .await
1678 {
1679 Ok(owner) => owner,
1680 Err(error) => {
1681 let detail = format!("worktree maintenance or ownership update: {error:#}");
1682 self.state.resource_block = Some(crate::run::ResourceBlock {
1686 cause: crate::run::ResourceBlockCause::Other,
1687 node: "start".into(),
1688 kind: "worktree maintenance".into(),
1689 detail: detail.clone(),
1690 at: Timestamp::now(),
1691 });
1692 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693 }
1694 };
1695 let disk = RunState::load(&self.state.id)?;
1696 crate::pr_head::ensure_owner(&disk)?;
1697 let claimed =
1698 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699 anyhow::ensure!(
1704 disk.liveness(claimed) != crate::run::Liveness::Live,
1705 "run already has a live driver; wait for its execution to end"
1706 );
1707 if let Some(to) = &disk.released_to {
1708 bail!(
1709 "run {} cannot continue: its worktree was released to run {}",
1710 self.state.short(),
1711 crate::run::short_of(to)
1712 );
1713 }
1714 anyhow::ensure!(
1715 serde_json::to_value(&disk.base_sync_recovery)?
1716 == serde_json::to_value(&self.state.base_sync_recovery)?,
1717 "base recovery changed since this run was loaded; reload the saved run"
1718 );
1719 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720 let mut execution = DriverExecution {
1721 runner: self,
1722 driver,
1723 };
1724 let _owner = owner;
1727 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728 .await?;
1729 let state = &mut execution.runner.state;
1730 state.parked = false;
1735 state.resource_wait = None;
1736 state.resource_block = None;
1737 state.clear_active();
1744 let pid = std::process::id();
1759 state.driver_pid = Some(pid);
1760 state.driver_started_at = crate::proc::process_started_at(pid);
1761 state.driver_exited = false;
1762 state.driver_claimed = Some(true);
1763 state.begin_pass();
1764 execution.driver.bind(state);
1765 state.save()?;
1766 execution.driver.record(state)?;
1767 Ok(execution)
1768 }
1769
1770 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774 if let Err(error) = &result
1775 && !error.is::<crate::repo_lock::ResourceBlocked>()
1776 && self
1777 .state
1778 .repair_grants
1779 .last()
1780 .is_some_and(|g| g.conflict.is_none())
1781 {
1782 self.state.repair_grants.last_mut().unwrap().phase =
1783 crate::repair_budget::Phase::Complete;
1784 self.state.status = RunStatus::Blocked;
1785 self.state.save()?;
1786 bail!(
1787 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788 self.state.passes.last().map_or(0, |p| p.index)
1789 );
1790 }
1791 result
1792 }
1793
1794 async fn execute_granted_repair(&mut self) -> Result<()> {
1797 use crate::repair_budget::{Kind, Phase};
1798 let index = self.state.repair_grants.len() - 1;
1799 let winner = self
1800 .state
1801 .winner()
1802 .cloned()
1803 .context("repair winner missing")?;
1804 loop {
1805 let grant = self.state.repair_grants[index].clone();
1806 match grant.phase {
1807 Phase::Pending => {
1808 if grant.spent >= grant.receipt.request.rounds {
1809 self.state.repair_grants[index].phase = Phase::Complete;
1810 self.state.status = RunStatus::Blocked;
1811 self.state.save()?;
1812 return Ok(());
1813 }
1814 self.state.repair_grants[index].spent += 1;
1817 self.state.repair_grants[index].phase = Phase::Fixing;
1818 self.state.save()?;
1819 match grant.receipt.request.kind {
1820 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821 Kind::Gate => {
1822 let failed = self.state.gate.clone();
1823 let _ = self.gate_fix_round(&winner, &failed).await?;
1824 }
1825 Kind::Review => {
1826 let mut record = self
1827 .state
1828 .reviews
1829 .last()
1830 .cloned()
1831 .context("review failure missing")?;
1832 let round = record.round;
1833 let step = self.run_fix_round(&winner, &mut record).await?;
1834 self.state.reviews[round - 1] = record;
1835 if matches!(step, FixStep::Recorded { report_lost: true }) {
1836 self.state.repair_grants[index].phase = Phase::Complete;
1837 self.state.status = RunStatus::Blocked;
1838 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839 self.state.save()?;
1840 return Ok(());
1841 }
1842 if matches!(step, FixStep::Resource) {
1843 self.stop_for_resource()?;
1844 return Ok(());
1845 }
1846 self.state.save()?;
1847 }
1848 }
1849 if self.resource_halt()? {
1850 return Ok(());
1851 }
1852 }
1853 Phase::Fixing => {
1854 if grant.receipt.request.kind == Kind::BaseConflict {
1855 self.repair_base_conflict(false).await?;
1856 }
1857 let _coordination = self.coordinate(&winner.worktree).await?;
1860 let rescued = git::rescue_commit(
1861 &winner.worktree,
1862 "magi: preserve interrupted granted repair",
1863 )
1864 .await?;
1865 self.state.note_withheld("repair-grant", &rescued.withheld);
1866 self.state
1869 .verification_history
1870 .push(crate::run::VerificationEvidence {
1871 superseded_by: None,
1872 archived_at: Some(Timestamp::now()),
1873 fixes_spent: Some(self.state.gate_fixes.len()),
1874 gate_head: self.state.gate_head.take(),
1875 gate_ran: std::mem::take(&mut self.state.gate_ran),
1876 gate: std::mem::take(&mut self.state.gate),
1877 e2e: self.state.head_e2e.take(),
1878 });
1879 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880 self.state
1881 .verification_history
1882 .push(crate::run::VerificationEvidence {
1883 superseded_by: None,
1884 archived_at: Some(Timestamp::now()),
1885 fixes_spent: Some(self.state.gate_fixes.len()),
1886 gate_head: None,
1887 gate_ran: false,
1888 gate: Vec::new(),
1889 e2e: Some(e2e),
1890 });
1891 }
1892 self.state.repair_grants[index].review_start = self.state.reviews.len();
1893 self.state.repair_grants[index].head =
1894 git::rev_parse(&winner.worktree, "HEAD").await?;
1895 self.state.repair_grants[index].phase = Phase::Reviewing;
1896 if grant.receipt.request.kind == Kind::BaseConflict {
1897 self.state.base_sync_recovery.stop = None;
1898 if let Some(sync) = &mut self.state.base_sync {
1899 sync.tip = grant
1900 .receipt
1901 .request
1902 .tip
1903 .clone()
1904 .context("repair base tip missing")?;
1905 sync.conflict = None;
1906 sync.behind = 0;
1907 }
1908 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909 }
1910 self.state.save()?;
1911 }
1912 Phase::Reviewing => {
1913 self.review_loop_pass().await?;
1914 if self.resource_halt()? {
1915 return Ok(());
1916 }
1917 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918 let clean = self.state.reviews.len() > grant.review_start
1919 && self.state.reviews.last().is_some_and(|r| {
1920 r.head == head
1921 && r.clean
1922 && r.expected > 0
1923 && r.answered == r.expected
1924 && matches!(
1925 r.e2e_status(),
1926 E2eStatus::Passed | E2eStatus::NotConfigured
1927 )
1928 });
1929 if !clean {
1930 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932 == Kind::Review
1933 && grant.spent < grant.receipt.request.rounds
1934 && self.state.reviews.last().is_some_and(|r| {
1935 r.fix.is_none()
1936 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937 }) {
1938 Phase::Pending
1939 } else {
1940 Phase::Complete
1941 };
1942 self.state.status = RunStatus::Blocked;
1943 self.state.event(
1944 "repair-grant",
1945 "fresh review did not clear the repaired head; no publication",
1946 );
1947 self.state.save()?;
1948 if self.state.repair_grants[index].phase == Phase::Pending {
1949 continue;
1950 }
1951 return Ok(());
1952 }
1953 self.state.repair_grants[index].phase = Phase::Gating;
1954 self.state.save()?;
1955 }
1956 Phase::Gating => {
1957 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958 self.state.reconcile_verification_head(&head);
1959 let wait = self.repository_wait();
1960 if self
1961 .state
1962 .reviews
1963 .last()
1964 .is_none_or(|r| r.head != head || !r.clean)
1965 {
1966 self.state.status = RunStatus::Blocked;
1967 self.state.repair_grants[index].phase = Phase::Complete;
1968 self.state.event(
1969 "repair-grant",
1970 "head changed since fresh review; repair held",
1971 );
1972 self.state.save()?;
1973 return Ok(());
1974 }
1975 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976 return Ok(());
1977 }
1978 let outcomes = self.run_gate(&winner).await?;
1979 if verify_inconclusive(&outcomes) {
1980 return Ok(());
1981 }
1982 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983 if after != head {
1984 self.state.repair_grants[index].review_start = self.state.reviews.len();
1985 self.state.repair_grants[index].head = after;
1986 self.state.repair_grants[index].phase = Phase::Reviewing;
1987 self.state.status = RunStatus::Blocked;
1988 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989 self.state.save()?;
1990 return Ok(());
1991 }
1992 let passed = outcomes.iter().all(CommandOutcome::ok);
1993 if passed
1996 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997 {
1998 return Ok(());
1999 }
2000 self.state.repair_grants[index].succeeded = passed;
2001 self.state.gate = outcomes;
2002 self.state.gate_ran = true;
2003 self.state.status = if passed {
2004 RunStatus::Gating
2005 } else {
2006 RunStatus::Blocked
2007 };
2008 self.state.repair_grants[index].phase = if !passed
2009 && grant.receipt.request.kind == Kind::Gate
2010 && grant.spent < grant.receipt.request.rounds
2011 {
2012 Phase::Pending
2013 } else {
2014 Phase::Complete
2015 };
2016 if passed && self.state.config.merge.mode == MergeMode::Pr {
2017 self.state.pr_publication_pending = true;
2019 } else if passed {
2020 self.state.status = RunStatus::Ready;
2021 }
2022 self.state.save()?;
2023 if !passed {
2024 continue;
2025 }
2026 if self.state.config.merge.mode != MergeMode::Pr {
2030 self.state.status = RunStatus::Ready;
2031 self.state.event(
2032 "repair-grant",
2033 "repair verified locally; grant carries no merge authorization",
2034 );
2035 self.state.save()?;
2036 return Ok(());
2037 }
2038 return self.merge().await;
2039 }
2040 Phase::Complete => return Ok(()),
2041 }
2042 }
2043 }
2044
2045 async fn execute_graph(&mut self) -> Result<()> {
2046 let archive_home = crate::run::home().canonicalize()?;
2050 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051 let repo = self.state.repo.clone();
2052 let _coordination = self.coordinate(&repo).await?;
2053 ensure_setup_config(&mut self.state, &repo).await?;
2054 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055 }
2056 self.state.configure_recovery_resume()?;
2057 if self
2058 .state
2059 .repair_grants
2060 .last()
2061 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062 {
2063 return self.execute_granted_repair().await;
2064 }
2065 if self
2066 .state
2067 .repair_grants
2068 .last()
2069 .is_some_and(|g| !g.succeeded)
2070 {
2071 self.state.status = RunStatus::Blocked;
2072 self.state.save()?;
2073 return Ok(());
2074 }
2075 self.persist_candidates().await?;
2078 if self.state.base_sync_blocked() {
2081 self.sync_to_base().await?;
2082 if !self.state.base_sync_blocked()
2083 && self.state.status != RunStatus::AlreadyInBase
2084 && !self.resource_halt()?
2085 {
2086 self.finish_after_tally().await?;
2087 }
2088 return Ok(());
2089 }
2090 if self
2091 .state
2092 .merge_decision
2093 .as_ref()
2094 .is_some_and(|decision| !decision.closed)
2095 && self.state.status != RunStatus::Landing
2096 {
2097 if crate::merge_decision::resumable_hold(&self.state) {
2098 self.state.merge = Some(MergeOutcome {
2099 mode: MergeMode::Pr,
2100 ok: false,
2101 detail: self
2102 .state
2103 .merge_decision
2104 .as_ref()
2105 .unwrap()
2106 .target
2107 .pr
2108 .clone(),
2109 empty: false,
2110 });
2111 self.run_land().await?;
2112 self.settle_questions();
2113 return Ok(());
2114 }
2115 bail!(
2116 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117 );
2118 }
2119 if self.state.config.graph.selected_implementer.is_some()
2120 && self.state.status == RunStatus::Blocked
2121 && self.state.tally.is_none()
2122 && self.state.candidate_preservation.is_empty()
2123 && self.state.interrupted_implementations.is_empty()
2124 {
2125 bail!(
2126 "selected implementation is held; explicitly release the owning task to retry it"
2127 );
2128 }
2129
2130 if self.state.merge_repairs.last().is_some_and(|r| {
2131 matches!(
2132 r.phase,
2133 crate::run::MergeRepairPhase::Requested
2134 | crate::run::MergeRepairPhase::Fixing
2135 | crate::run::MergeRepairPhase::Reviewing
2136 )
2137 }) {
2138 if self.repair_merge_review().await? {
2139 self.run_land().await?;
2140 }
2141 self.settle_questions();
2142 return Ok(());
2143 }
2144 if self.state.pr_publication_pending {
2147 self.merge().await?;
2148 if self.resource_halt()? {
2149 return Ok(());
2150 }
2151 self.state.save()?;
2152 self.settle_questions();
2153 return Ok(());
2154 }
2155 if self.state.status == RunStatus::Stalled {
2168 if self.recover_stall().await? {
2169 self.finish_after_tally().await?;
2170 } else {
2171 self.state.save()?;
2173 }
2174 return Ok(());
2175 }
2176 if self.state.status == RunStatus::Landing
2186 || crate::handover::landing_checkpoint(&self.state)
2187 {
2188 self.state.land_sync_pending = true;
2189 self.state.save()?;
2190 self.run_land().await?;
2191 self.settle_questions();
2196 return Ok(());
2197 }
2198 if self.state.github_text.is_some()
2203 && self.state.merge.is_none()
2204 && self.state.status == RunStatus::Gating
2205 {
2206 self.merge().await?;
2207 if self.resource_halt()? {
2208 return Ok(());
2209 }
2210 self.state.save()?;
2211 return Ok(());
2212 }
2213 if let Err(e) = self.prep().await {
2214 if e.is::<crate::repo_lock::ResourceBlocked>() {
2215 return Err(e);
2216 }
2217 self.state.status = RunStatus::Blocked;
2218 self.state
2219 .prep_failure
2220 .get_or_insert_with(|| format!("{e:#}"));
2221 self.state
2222 .event("prep", format!("preparation stopped: {e:#}"));
2223 self.state.save()?;
2224 return Err(e);
2225 }
2226 if self.park_here()? {
2227 return Ok(());
2228 }
2229 self.advise().await?;
2230 if self.park_here()? {
2231 return Ok(());
2232 }
2233 self.implement().await?;
2234 if self.park_here()? {
2235 return Ok(());
2236 }
2237 if self.state.status == RunStatus::VerifiedNoop {
2241 return Ok(());
2242 }
2243 self.judge().await?;
2244 if self.park_here()? {
2245 return Ok(());
2246 }
2247 self.deliberate().await?;
2248 if self.park_here()? {
2249 return Ok(());
2250 }
2251 self.vote().await?;
2252 if self.park_here()? {
2253 return Ok(());
2254 }
2255 self.tally()?;
2256 if self.state.status == RunStatus::Stalled {
2261 self.state.save()?;
2265 return Ok(());
2266 }
2267 self.finish_after_tally().await?;
2268 Ok(())
2269 }
2270
2271 fn stop_for_resource(&mut self) -> Result<()> {
2276 if self.pause.parked() || self.interrupt.parked() {
2279 self.park_here()?;
2280 return Ok(());
2281 }
2282 if self.state.resource_block.is_some() {
2283 self.state.status = RunStatus::Blocked;
2284 }
2285 self.state.save()
2286 }
2287
2288 fn resource_halt(&mut self) -> Result<bool> {
2290 if self.state.resource_block.is_none() {
2291 return Ok(false);
2292 }
2293 self.stop_for_resource()?;
2294 Ok(true)
2295 }
2296
2297 fn park_here(&mut self) -> Result<bool> {
2304 let parking = self.pause.parked() || self.interrupt.parked();
2305 if !parking && self.state.resource_block.is_some() {
2306 self.stop_for_resource()?;
2307 return Ok(true);
2308 }
2309 if !self.pause.parked() && !self.interrupt.parked() {
2314 return Ok(false);
2315 }
2316 if self.state.parked {
2317 return Ok(true);
2318 }
2319 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320 if let Some(block) = &mut self.state.resource_block
2321 && matches!(
2322 block.cause,
2323 crate::run::ResourceBlockCause::Cancelled { .. }
2324 )
2325 {
2326 block.cause = crate::run::ResourceBlockCause::Cancelled {
2327 park: cause.clone(),
2328 };
2329 }
2330 let why = match cause.as_ref().map(ParkCause::label) {
2331 Some(reason) => format!(
2332 "parked after `{}` ({reason}) — resume to carry on from here",
2333 self.state.status.as_str()
2334 ),
2335 None => format!(
2336 "parked after `{}` — resume to carry on from here",
2337 self.state.status.as_str()
2338 ),
2339 };
2340 self.state.event("park", why);
2341 self.state
2342 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343 self.state.parked = true;
2344 self.state.save()?;
2345 Ok(true)
2346 }
2347
2348 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350 if self.state.prep_failure.is_none() {
2351 return Ok(());
2352 }
2353 let failed_at = self
2354 .state
2355 .events
2356 .iter()
2357 .rev()
2358 .find(|event| event.node == "prep")
2359 .map_or(self.state.updated_at, |event| event.at);
2360 anyhow::ensure!(
2361 approval_at > failed_at,
2362 "preparation retry requires a new explicit release after the failure"
2363 );
2364 self.state.prep_failure = None;
2365 self.state.prep_complete = Some(false);
2366 self.state
2367 .event("prep-retry", "operator approved retry of preparation");
2368 Ok(())
2369 }
2370
2371 pub fn on_pause(&mut self, pause: Pause) {
2373 self.pause = pause;
2374 }
2375
2376 pub fn watch_interrupt(&mut self, pause: Pause) {
2382 self.interrupt = pause;
2383 }
2384
2385 fn settle_questions(&mut self) {
2405 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407 }
2408 }
2409
2410 fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413 Box::pin(self.finish_after_tally_inner())
2414 }
2415
2416 async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417 self.graft().await?;
2420 if self.resource_halt()? {
2421 return Ok(());
2422 }
2423 self.fold_losers().await?;
2424 self.sync_to_base().await?;
2429 if self.hold_base_sync().await? || self.resource_halt()? {
2430 return Ok(());
2431 }
2432 if self.state.status == RunStatus::AlreadyInBase {
2433 return Ok(());
2434 }
2435 self.review_loop().await?;
2436 if self.resource_halt()? {
2437 return Ok(());
2438 }
2439 self.sync_to_base().await?;
2440 if self.hold_base_sync().await? || self.resource_halt()? {
2441 return Ok(());
2442 }
2443 if self.state.status == RunStatus::AlreadyInBase {
2444 return Ok(());
2445 }
2446 self.gate().await?;
2447 self.merge().await?;
2448 if self.resource_halt()? {
2449 return Ok(());
2450 }
2451 self.state.save()?;
2452 Ok(())
2453 }
2454
2455 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462 let result = match ensure_setup_config(&mut self.state, repo).await {
2463 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464 Err(e) => Err(e),
2465 };
2466 if let Err(e) = result {
2467 git::worktree_remove_clean(repo, wt).await.ok();
2468 if in_prep {
2469 self.rollback_prep(repo).await;
2470 }
2471 self.state.status = RunStatus::Blocked;
2472 self.state.event(
2473 "setup",
2474 format!("worktree setup failed in {}: {e:#}", wt.display()),
2475 );
2476 self.state.save()?;
2477 return Err(e);
2478 }
2479 Ok(())
2480 }
2481
2482 async fn rollback_prep(&mut self, repo: &Path) {
2484 let root = self.state.worktree_root();
2485 for c in std::mem::take(&mut self.state.candidates) {
2486 if matches!(
2487 git::worktree_remove_clean(repo, &c.worktree).await,
2488 Ok(true)
2489 ) {
2490 git::branch_delete(repo, &c.branch).await.ok();
2491 } else {
2492 self.state.candidates.push(c);
2493 }
2494 }
2495 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496 for j in 1..=seats {
2497 remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2498 .await
2499 .ok();
2500 }
2501 for k in 1..=self.state.config.graph.advisors {
2502 remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2503 .await
2504 .ok();
2505 }
2506 }
2507
2508 async fn prep(&mut self) -> Result<()> {
2509 if let Some(reason) = &self.state.prep_failure {
2510 bail!(
2511 "candidate preparation previously failed: {reason}; correct the source before retrying"
2512 );
2513 }
2514 if self
2515 .state
2516 .prep_complete
2517 .unwrap_or(!self.state.candidates.is_empty())
2518 {
2519 return Ok(());
2520 }
2521 let repo = self.state.repo.clone();
2522 let _coordination = self.coordinate(&repo).await?;
2525 let base = self.state.base_commit.clone();
2526 let plan = self.state.sources.plan(&repo, &base).await?;
2527 self.rollback_prep(&repo).await;
2528 if !self.state.candidates.is_empty() {
2529 bail!(
2530 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531 );
2532 }
2533 self.state.prep_complete = Some(false);
2534 self.state.status = RunStatus::Prep;
2535 self.state.save()?;
2536 let repo = self.state.repo.clone();
2537 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538 let root = self.state.worktree_root();
2539 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541 let hooks_dir = self.state.dir().join("hooks");
2544 if self.state.config.blind.commit_msg_hook {
2545 std::fs::create_dir_all(&hooks_dir)
2546 .with_context(|| format!("create {}", hooks_dir.display()))?;
2547 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548 let path = hooks_dir.join("commit-msg");
2549 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550 make_executable(&path)?;
2551 git::acquire_worktree_config(&repo).await?;
2559 self.state.enabled_worktree_config = true;
2560 }
2561
2562 ensure_setup_config(&mut self.state, &repo).await?;
2565
2566 for (index, (spec, label)) in self
2567 .roles
2568 .implementers
2569 .clone()
2570 .into_iter()
2571 .zip(labels)
2572 .enumerate()
2573 {
2574 let branch = self.state.branch_for(label);
2575 let worktree = root.join(format!("cand-{label}"));
2576 self.state.event(
2580 "prep-intent",
2581 serde_json::json!({
2582 "branch": branch,
2583 "worktree": worktree,
2584 "start": start,
2585 "picks": plan.picks,
2586 })
2587 .to_string(),
2588 );
2589 self.state.save()?;
2590 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591 if self.state.config.blind.commit_msg_hook {
2592 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593 }
2594 git::local_exclude(&worktree, "/.magi/").await?;
2595 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596 git::worktree_remove_clean(&repo, &worktree).await.ok();
2599 git::branch_delete(&repo, &branch).await.ok();
2600 self.rollback_prep(&repo).await;
2601 self.state.status = RunStatus::Blocked;
2602 self.state
2603 .event("prep", format!("worktree setup failed: {e:#}"));
2604 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605 self.state.save()?;
2606 return Err(e);
2607 }
2608 self.state.candidates.push(Candidate {
2609 index,
2610 label,
2611 agent: spec.id.clone(),
2612 branch,
2613 worktree,
2614 summary: String::new(),
2615 stat: String::new(),
2616 files: 0,
2617 commits: 0,
2618 empty: false,
2619 failed: None,
2620 verified_noop: None,
2621 duration_ms: 0,
2622 folded: false,
2623 });
2624 self.state.save()?;
2625 for pick in &plan.picks {
2626 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627 {
2628 self.state.status = RunStatus::Blocked;
2629 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630 self.state
2631 .event("prep", format!("cannot apply referenced commit: {e}"));
2632 self.state.save()?;
2633 return Err(e);
2634 }
2635 }
2636 }
2637
2638 for j in 1..=self.roles.judges.len() {
2639 let wt = root.join(format!("judge-{j}"));
2640 if !wt.exists() {
2641 git::worktree_add_detached(&repo, &wt, &base).await?;
2642 self.setup_seat_worktree(&repo, &wt, true).await?;
2643 }
2644 }
2645
2646 if self.state.config.graph.advise {
2654 for k in 1..=self.state.config.graph.advisors {
2655 let wt = root.join(format!("advisor-{k}"));
2656 if !wt.exists() {
2657 git::worktree_add_detached(&repo, &wt, &base).await?;
2658 self.setup_seat_worktree(&repo, &wt, true).await?;
2659 }
2660 }
2661 }
2662
2663 let authors: Vec<&str> = self
2668 .roles
2669 .implementers
2670 .iter()
2671 .map(|a| a.id.as_str())
2672 .collect();
2673 let overlap: Vec<String> = self
2674 .roles
2675 .judges
2676 .iter()
2677 .enumerate()
2678 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680 .collect();
2681 if !overlap.is_empty() {
2682 let note = format!(
2683 "{} also authored a candidate; blind, but the panel is less \
2684 independent than {} distinct agents would be",
2685 overlap.join(", "),
2686 self.roles.judges.len()
2687 );
2688 self.state.event("prep", note);
2689 }
2690
2691 self.state.event(
2692 "prep",
2693 format!(
2694 "{} candidates, {} judges, base {} ({})",
2695 self.state.candidates.len(),
2696 self.roles.judges.len(),
2697 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698 self.state.base_branch
2699 ),
2700 );
2701 self.state.prep_complete = Some(true);
2702 self.state.status = RunStatus::Implementing;
2703 self.state.save()?;
2704 Ok(())
2705 }
2706
2707 async fn advise(&mut self) -> Result<()> {
2740 let implement_untouched = self
2741 .state
2742 .candidates
2743 .iter()
2744 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745 if !self.state.config.graph.advise || self.state.advise_attempted {
2746 return Ok(());
2747 }
2748 if !implement_untouched {
2749 self.state.event(
2750 "advise",
2751 "skipping the design-deliberation stage: at least one \
2752 candidate already shows implementation progress, so this \
2753 run is past the point the stage exists to run before"
2754 .to_owned(),
2755 );
2756 self.state.advise_attempted = true;
2757 self.state.save()?;
2758 return Ok(());
2759 }
2760 let run_id = self.state.id.clone();
2761 let prompts = self.state.config.prompts.clone();
2762 let instruction = self.state.instruction.clone();
2763 let language = self.state.config.graph.language.clone();
2764 let root = self.state.worktree_root();
2765 let n = self.state.config.graph.advisors;
2766 let where_recorded = self.state.dir().join("run.json");
2767
2768 let seats = match self.state.config.advisors() {
2769 Ok(seats) if !seats.is_empty() => seats,
2770 Ok(_) => {
2771 self.state.event(
2772 "advise",
2773 format!(
2774 "[graph] advisors is 0; skipping the design-deliberation \
2775 stage and continuing without a synthesis brief (see {})",
2776 where_recorded.display()
2777 ),
2778 );
2779 self.state.advise_attempted = true;
2780 self.state.save()?;
2781 return Ok(());
2782 }
2783 Err(e) => {
2784 self.state.event(
2785 "advise",
2786 format!(
2787 "could not resolve advisor seats ({e:#}); continuing \
2788 without a design-deliberation brief (see {})",
2789 where_recorded.display()
2790 ),
2791 );
2792 self.state.advise_attempted = true;
2793 self.state.save()?;
2794 return Ok(());
2795 }
2796 };
2797
2798 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799 let artifacts = agent::artifacts_dir(&self.state.dir());
2800 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802 let mut jobs = Vec::new();
2803 for (i, spec) in seats.iter().cloned().enumerate() {
2804 let seat_key = format!("advisor-{}", i + 1);
2805 let seat = self.seat(&seat_key, &spec.id);
2806 jobs.push(SeatJob {
2807 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808 spec,
2809 seat,
2810 cwd: worktrees[i % worktrees.len()].clone(),
2811 timeout,
2812 allow_write: false,
2813 sessions: false,
2814 artifacts: artifacts.clone(),
2815 stem: seat_key,
2816 handover: None,
2817 });
2818 }
2819
2820 self.state.event(
2821 "advise",
2822 format!(
2823 "{} advisor seat(s) sketching a design in parallel",
2824 jobs.len()
2825 ),
2826 );
2827 let mut quota_losses = Vec::new();
2828 let cache = self.state.config.cache_dir();
2829 let ctx = WaveCtx {
2830 carry_seats: false,
2831 pauses: Some((&self.pause, &self.interrupt)),
2832 run: &run_id,
2833 node: "advise",
2834 prompts: &prompts,
2835 cache: cache.as_deref(),
2836 round: None,
2837 };
2838 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839 let results = ask_json_wave::<Proposal>(
2840 jobs,
2841 Arc::clone(&self.sem),
2842 self.state.config.graph.retries,
2843 &advisor_roster,
2844 &ctx,
2845 &mut quota_losses,
2846 &mut self.state,
2847 &|p: &Proposal| p.validate(),
2848 )
2849 .await;
2850 if self.state.resource_block.is_some() {
2851 return self.stop_for_resource();
2852 }
2853 self.state.quota.extend(quota_losses);
2854
2855 let mut records = Vec::with_capacity(results.len());
2856 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857 let agent_id = seat.agent.clone();
2858 self.state.seats.insert(seat.key.clone(), seat);
2859 match res {
2860 Ok((proposal, out)) => {
2861 self.state
2862 .event("advise", format!("advisor-{} proposed a design", i + 1));
2863 records.push(advise::AdvisorRecord::proposed(
2864 i + 1,
2865 agent_id,
2866 proposal,
2867 out.duration_ms,
2868 ));
2869 }
2870 Err(e) => {
2871 self.state.event(
2872 "advise",
2873 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874 );
2875 records.push(advise::AdvisorRecord::failed(
2876 i + 1,
2877 agent_id,
2878 e.to_string(),
2879 ));
2880 }
2881 }
2882 }
2883
2884 let mut advice = advise::Advice {
2885 records,
2886 synthesis: None,
2887 };
2888 if advice.proposals().is_empty() {
2889 self.state.event(
2890 "advise",
2891 "no advisor produced a usable proposal; continuing without a \
2892 synthesis brief"
2893 .to_owned(),
2894 );
2895 } else {
2896 match self
2897 .synthesize_brief(
2898 &advice,
2899 &instruction,
2900 &language,
2901 &worktrees[0],
2902 &artifacts,
2903 &run_id,
2904 &prompts,
2905 cache.as_deref(),
2906 )
2907 .await
2908 {
2909 Ok(Some(text)) => {
2910 self.state.event(
2911 "advise",
2912 "synthesized a design brief for the implementer".to_owned(),
2913 );
2914 advice.synthesis = Some(text);
2915 }
2916 Ok(None) => {
2917 self.state.event(
2918 "advise",
2919 "the synthesis seat produced nothing usable; continuing \
2920 without a design brief"
2921 .to_owned(),
2922 );
2923 }
2924 Err(e) => {
2925 if e.is::<crate::repo_lock::ResourceBlocked>() {
2926 return Err(e);
2927 }
2928 self.state.event(
2929 "advise",
2930 format!("could not synthesize a design brief: {e:#}"),
2931 );
2932 }
2933 }
2934 }
2935 advise::apply_reflection(&mut advice);
2936
2937 self.state.advice = Some(advice);
2938 self.state.advise_attempted = true;
2939 self.state.save()?;
2940 Ok(())
2941 }
2942
2943 #[allow(clippy::too_many_arguments)]
2955 async fn synthesize_brief(
2956 &mut self,
2957 advice: &advise::Advice,
2958 instruction: &str,
2959 language: &str,
2960 cwd: &Path,
2961 artifacts: &Path,
2962 run_id: &str,
2963 prompts: &Prompts,
2964 cache: Option<&Path>,
2965 ) -> Result<Option<String>> {
2966 let chain = agent::pick_chain(
2967 &self.state.config.agents,
2968 self.state.config.roles.synthesizer.as_ref(),
2969 &agent::installed,
2970 "synthesizer",
2971 )?;
2972 let proposals = advice.proposals();
2973 let mut prompt = prompt::with_overlay(
2974 prompt::synthesize_brief(instruction, &proposals, language),
2975 prompts.overlay("advise"),
2976 );
2977 if cache.is_some() {
2978 prompt.push('\n');
2983 prompt.push_str(&prompt::build_cache_note("advise", false));
2984 }
2985 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986 let mut last = None;
2991 for (n, spec) in chain.iter().enumerate() {
2992 if n > 0 {
2993 self.state
2994 .event("advise", format!("synthesis falling back to {}", spec.id));
2995 }
2996 let mut seat = self.seat("advise-synthesis", &spec.id);
2997 let wait = self.repository_wait();
2998 let outcome = wait
2999 .scope(agent::invoke(
3000 spec,
3001 &mut seat,
3002 &Invocation {
3003 coordination: self.state.config.coordination.as_ref(),
3004 fetched_base: None,
3005 cwd,
3006 prompt: &prompt,
3007 timeout,
3008 allow_write: false,
3009 sessions: false,
3010 artifacts,
3011 stem: &if n == 0 {
3012 "advise-synthesis".to_owned()
3013 } else {
3014 format!("advise-synthesis-{}", spec.id)
3015 },
3016 run: run_id,
3017 node: "advise",
3018 cache_dir: None,
3019 attachments: &[],
3020 writable: &[],
3021 },
3022 ))
3023 .await;
3024 if outcome.is_ok() {
3025 self.state.seats.insert(seat.key.clone(), seat);
3026 }
3027 let advance = agent::chain_advances(&outcome);
3028 last = Some(outcome);
3029 if !advance {
3030 break;
3031 }
3032 }
3033 let out = last.expect("a chain holds at least one agent")?;
3036 if !out.usable() {
3037 return Ok(None);
3038 }
3039 let text =
3040 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041 Ok((!text.trim().is_empty()).then_some(text))
3042 }
3043
3044 async fn implement(&mut self) -> Result<()> {
3047 self.persist_candidates().await?;
3048 let run_id = self.state.id.clone();
3053 let prompts = self.state.config.prompts.clone();
3054 let todo: Vec<usize> = self
3055 .state
3056 .candidates
3057 .iter()
3058 .enumerate()
3059 .filter(|(_, c)| {
3060 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061 && c.failed.is_none()
3062 && !c.empty
3063 })
3064 .map(|(i, _)| i)
3065 .collect();
3066 if todo.is_empty() {
3067 return self.after_implement();
3068 }
3069 self.state.status = RunStatus::Implementing;
3070
3071 let language = self.state.config.graph.language.clone();
3072 let github_language = self.state.config.graph.github_language.clone();
3073 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074 let sessions = self.state.config.graph.sessions;
3075 let artifacts = agent::artifacts_dir(&self.state.dir());
3076 let brief = self
3080 .state
3081 .advice
3082 .as_ref()
3083 .and_then(|a| a.synthesis.as_deref())
3084 .map(str::to_owned);
3085 let attachments = self.state.attachments.clone();
3086
3087 let mut jobs = Vec::new();
3088 for &i in &todo {
3089 let (index, label, worktree) = {
3090 let c = &self.state.candidates[i];
3091 (c.index, c.label, c.worktree.clone())
3092 };
3093 let spec = self.roles.implementers[index].clone();
3094 let seat_key = format!("impl-{label}");
3095 let seat = self.seat(&seat_key, &spec.id);
3096 let instruction = seeded_instruction(&self.state);
3097 jobs.push(SeatJob {
3098 spec,
3099 seat,
3100 prompt: prompt::implement_in(
3101 &instruction,
3102 &worktree.to_string_lossy(),
3103 &language,
3104 &github_language,
3105 brief.as_deref(),
3106 &attachments,
3107 ),
3108 cwd: worktree,
3109 timeout,
3110 allow_write: true,
3111 sessions,
3112 artifacts: artifacts.clone(),
3113 stem: format!("impl-{label}"),
3114 handover: None,
3115 });
3116 }
3117
3118 self.state.event(
3119 "implement",
3120 format!("{} candidates in parallel", jobs.len()),
3121 );
3122 let mut sent = jobs.clone();
3128 let cache = self.state.config.cache_dir();
3129 let ctx = WaveCtx {
3130 carry_seats: false,
3131 pauses: Some((&self.pause, &self.interrupt)),
3132 run: &run_id,
3133 node: "implement",
3134 prompts: &prompts,
3135 cache: cache.as_deref(),
3136 round: None,
3137 };
3138 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139 if !wave_blocked(&results) {
3140 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141 .await;
3142 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143 .await?;
3144 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145 .await;
3146 }
3147 if !results.is_empty()
3150 && results
3151 .iter()
3152 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153 {
3154 let losses: Vec<_> = results
3155 .iter()
3156 .map(|(_, seat, out)| {
3157 let AgentOutcome::Quota(o) = out else {
3158 unreachable!()
3159 };
3160 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161 })
3162 .collect();
3163 self.state.quota.extend(losses.clone());
3164 for (_, seat, _) in results {
3165 self.state.seats.insert(seat.key.clone(), seat);
3166 }
3167 for &i in &todo {
3168 let label = self.state.candidates[i].label;
3169 self.state.interrupted_implementations.insert(label);
3170 self.state
3171 .candidate_preservation
3172 .insert(label, "machine outage awaiting persistence".into());
3173 }
3174 block_machine_outage(&mut self.state, "implement", &losses);
3177 self.state.save()?;
3178 self.persist_candidates().await?;
3179 for &i in &todo {
3180 self.state
3181 .interrupted_implementations
3182 .insert(self.state.candidates[i].label);
3183 }
3184 return self.stop_for_resource();
3185 }
3186 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189 && results
3190 .iter()
3191 .any(|(_, _, out)| FailClass::of(out).is_some());
3192 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194 if self
3198 .state
3199 .candidate_preservation
3200 .contains_key(&self.state.candidates[i].label)
3201 {
3202 self.state.seats.insert(seat.key.clone(), seat);
3203 }
3204 continue;
3205 }
3206 let seat_key = seat.key.clone();
3207 let agent = seat.agent.clone();
3216 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217 self.state.seats.insert(seat.key.clone(), seat);
3218 let label = self.state.candidates[i].label;
3219 self.state.interrupted_implementations.remove(&label);
3220
3221 let (summary, duration, failed, verified_claim) = match out {
3222 AgentOutcome::Ok(o) => {
3223 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224 let failed = (!o.usable()).then(|| {
3225 if o.timed_out {
3226 "agent timed out".to_owned()
3227 } else {
3228 format!(
3229 "agent exited with {}",
3230 crate::notices::exit_status(o.exit_code)
3231 )
3232 }
3233 });
3234 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235 (text, o.duration_ms, failed, verified_claim)
3236 }
3237 AgentOutcome::Dropped(o) => {
3243 let why = o
3244 .dropped
3245 .as_ref()
3246 .map(|d| d.why.as_str())
3247 .unwrap_or("the CLI ended the stream without delivering its answer");
3248 (
3249 String::new(),
3250 o.duration_ms,
3251 Some(format!("the CLI dropped the stream ({why})")),
3252 None,
3253 )
3254 }
3255 AgentOutcome::Quota(o) => {
3256 self.state.quota.push(QuotaLoss::from_report(
3257 seat_key,
3258 "implement",
3259 o.quota.as_ref(),
3260 ));
3261 (
3262 String::new(),
3263 o.duration_ms,
3264 Some("rate limited (quota); produced no change".to_owned()),
3265 None,
3266 )
3267 }
3268 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269 (String::new(), 0, Some(e), None)
3270 }
3271 };
3272
3273 let c = &mut self.state.candidates[i];
3276 if !exhausted_the_fallback_chain {
3277 c.agent = agent;
3278 }
3279 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280 c.duration_ms = duration;
3281 c.failed = failed;
3282 c.verified_noop = verified_claim;
3283 self.state
3284 .candidate_preservation
3285 .insert(label, "awaiting persistence".to_owned());
3286 }
3287 self.state.save()?;
3288 self.persist_candidates().await?;
3289
3290 if resource_blocked {
3291 return self.stop_for_resource();
3292 }
3293 if selected_failed {
3294 self.state.status = RunStatus::Blocked;
3295 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296 self.state.save()?;
3297 bail!(
3298 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299 );
3300 }
3301 self.after_implement()
3302 }
3303
3304 async fn persist_candidates(&mut self) -> Result<()> {
3306 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307 for label in labels {
3308 let i = self
3309 .state
3310 .candidates
3311 .iter()
3312 .position(|c| c.label == label)
3313 .context("pending candidate is missing")?;
3314 let c = self.state.candidates[i].clone();
3315 let interrupted = self.state.interrupted_implementations.contains(&label);
3316 let result = async {
3317 let wait = crate::repo_lock::WaitPolicy::new(
3318 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319 || false,
3320 );
3321 let _coordination =
3322 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323 if let Err(e) = git::snapshot_uncommitted(
3326 &c.worktree,
3327 &self
3328 .state
3329 .dir()
3330 .join("artifacts")
3331 .join(format!("candidate-{label}-recovery")),
3332 )
3333 .await
3334 {
3335 self.state.event(
3336 "preserve",
3337 format!("candidate {label} snapshot incomplete: {e:#}"),
3338 );
3339 }
3340 let rescued = git::rescue_commit(
3341 &c.worktree,
3342 &format!("magi: candidate {label} (uncommitted work)"),
3343 )
3344 .await?;
3345 self.state.note_withheld("implement", &rescued.withheld);
3346 let base = &self.state.base_commit;
3347 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352 if c.empty && commits > 0 && !patch.trim().is_empty()
3353 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355 || self.state.tally.is_some())
3356 {
3357 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361 }
3362 let c = &mut self.state.candidates[i];
3363 c.commits = commits;
3364 c.stat = stat;
3365 c.files = files;
3366 let no_changes = commits == 0 || patch.trim().is_empty();
3367 c.empty = no_changes && !interrupted;
3368 if interrupted && no_changes {
3369 c.failed = None;
3370 c.verified_noop = None;
3371 }
3372 if !c.empty {
3373 c.failed = None;
3374 c.verified_noop = None;
3375 }
3376 self.state.event(
3377 "preserve",
3378 format!("candidate {label}: {files} files, {commits} commits"),
3379 );
3380 Ok::<_, anyhow::Error>(())
3381 }
3382 .await;
3383 match result {
3384 Ok(()) => {
3385 self.state.candidate_preservation.remove(&label);
3386 if !interrupted || self.state.candidates[i].files > 0 {
3387 self.state.interrupted_implementations.remove(&label);
3388 }
3389 }
3390 Err(e) => {
3391 let reason = format!("candidate {label} preservation failed: {e:#}");
3392 self.state.event("preserve", &reason);
3393 self.state.candidate_preservation.insert(label, reason);
3394 }
3395 }
3396 if self.state.candidate_preservation.is_empty()
3397 && self.state.status == RunStatus::Blocked
3398 && self.state.candidates.iter().all(|c| c.failed.is_none())
3399 {
3400 self.state.status = RunStatus::Implementing;
3404 self.state.event(
3405 "preserve",
3406 "candidate persistence recovered; implementation can continue",
3407 );
3408 }
3409 self.state.save()?;
3410 }
3411 if !self.state.candidate_preservation.is_empty() {
3412 self.state.status = RunStatus::Blocked;
3413 self.state.save()?;
3414 bail!(
3415 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416 );
3417 }
3418 Ok(())
3419 }
3420
3421 async fn resume_undelivered(
3449 &mut self,
3450 results: &mut [(usize, SeatState, AgentOutcome)],
3451 sent: &[SeatJob],
3452 prompts: &Prompts,
3453 run_id: &str,
3454 ) {
3455 for (wi, seat, out) in results.iter_mut() {
3456 let Some(dropped) = (match &*out {
3457 AgentOutcome::Dropped(o) => o.dropped.clone(),
3458 _ => None,
3459 }) else {
3460 continue;
3461 };
3462 let Some(job) = sent.get(*wi) else { continue };
3463 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465 self.state.event(
3466 "implement",
3467 format!(
3468 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469 work is in the tree",
3470 seat.key, dropped.output_tokens, dropped.why
3471 ),
3472 );
3473 continue;
3474 }
3475 if !has_context(&job.spec, seat, job.sessions) {
3483 self.state.event(
3484 "implement",
3485 format!(
3486 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487 is no session left to resume",
3488 seat.key, dropped.output_tokens, dropped.why
3489 ),
3490 );
3491 continue;
3492 }
3493 self.state.event(
3494 "implement",
3495 format!(
3496 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497 conversation",
3498 seat.key, dropped.output_tokens, dropped.why
3499 ),
3500 );
3501 let mut retry = job.clone();
3502 retry.seat = seat.clone();
3503 retry.prompt = prompt::resume_after_drop(&dropped.why);
3504 retry.timeout = retry_budget(job.timeout, true);
3505 retry.stem = format!("{}-resume", job.stem);
3506 let cache = self.state.config.cache_dir();
3507 let ctx = WaveCtx {
3508 carry_seats: false,
3509 pauses: Some((&self.pause, &self.interrupt)),
3510 run: run_id,
3511 node: "implement",
3512 prompts,
3513 cache: cache.as_deref(),
3514 round: None,
3515 };
3516 let (resumed_seat, resumed) =
3517 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518 *seat = resumed_seat;
3519 *out = resumed;
3520 }
3521 }
3522
3523 async fn resume_seat_handovers(
3588 &mut self,
3589 results: &mut [(usize, SeatState, AgentOutcome)],
3590 sent: &mut [SeatJob],
3591 prompts: &Prompts,
3592 run_id: &str,
3593 ) -> Result<()> {
3594 let instruction = seeded_instruction(&self.state);
3595 let language = self.state.config.graph.language.clone();
3596 let github_language = self.state.config.graph.github_language.clone();
3597 let brief = self
3598 .state
3599 .advice
3600 .as_ref()
3601 .and_then(|a| a.synthesis.as_deref())
3602 .map(str::to_owned);
3603 let attachments = self.state.attachments.clone();
3604 for (wi, seat, out) in results.iter_mut() {
3605 let others: BTreeSet<String> = sent
3608 .iter()
3609 .enumerate()
3610 .filter(|(j, _)| j != wi)
3611 .map(|(_, j)| j.spec.id.clone())
3612 .collect();
3613 let Some(job) = sent.get_mut(*wi) else {
3614 continue;
3615 };
3616 let start = self
3621 .roles
3622 .implementer_roster
3623 .iter()
3624 .position(|s| s.id == job.spec.id)
3625 .unwrap_or(0);
3626 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627 let mut fallback_attempt = 0usize;
3628 let mut prev: Option<FailClass> = None;
3629 while let Some(cur) = FailClass::of(&*out) {
3630 if !should_hand_over(prev.as_ref(), &cur) {
3631 break;
3632 }
3633 let Some(next) =
3634 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635 .cloned()
3636 else {
3637 break;
3638 };
3639 tried.insert(next.id.clone());
3640 fallback_attempt += 1;
3641
3642 let coordination = self.coordinate(&job.cwd).await?;
3643 if let Ok(r) = git::rescue_commit(
3644 &job.cwd,
3645 &format!(
3646 "magi: candidate {} (uncommitted work before {} fallback)",
3647 seat.key,
3648 if cur == FailClass::Quota {
3649 "quota"
3650 } else {
3651 "handover"
3652 }
3653 ),
3654 )
3655 .await
3656 {
3657 self.state.note_withheld("implement", &r.withheld);
3658 }
3659 drop(coordination);
3660
3661 record_handover(
3662 &mut self.state,
3663 "implement",
3664 &seat.key,
3665 &seat.agent,
3666 &next.id,
3667 &cur,
3668 &fail_reason(&*out),
3669 );
3670 prev = Some(cur.clone());
3671
3672 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674 job.spec = next.clone();
3682 let mut retry = job.clone();
3683 retry.seat = new_seat;
3684 retry.prompt = prompt::implement_in(
3685 &instruction,
3686 &job.cwd.to_string_lossy(),
3687 &language,
3688 &github_language,
3689 brief.as_deref(),
3690 &attachments,
3691 );
3692 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693 let cache = self.state.config.cache_dir();
3694 let ctx = WaveCtx {
3695 carry_seats: false,
3696 pauses: Some((&self.pause, &self.interrupt)),
3697 run: run_id,
3698 node: "implement",
3699 prompts,
3700 cache: cache.as_deref(),
3701 round: None,
3702 };
3703 let (fallback_seat, fallback_out) = run_one(
3704 retry,
3705 Arc::clone(&self.sem),
3706 &ctx,
3707 &mut self.state,
3708 fallback_attempt,
3709 )
3710 .await;
3711 *seat = fallback_seat;
3712 *out = fallback_out;
3713 }
3714 }
3715 Ok(())
3716 }
3717
3718 async fn resume_unconfirmed_commands(
3742 &mut self,
3743 results: &mut [(usize, SeatState, AgentOutcome)],
3744 sent: &[SeatJob],
3745 prompts: &Prompts,
3746 run_id: &str,
3747 ) {
3748 for (wi, seat, out) in results.iter_mut() {
3749 let AgentOutcome::Ok(o) = &*out else {
3750 continue;
3751 };
3752 if !has_unconfirmed_command(&o.commands) {
3753 continue;
3754 }
3755 let Some(job) = sent.get(*wi) else { continue };
3756 if !has_context(&job.spec, seat, job.sessions) {
3757 self.state.event(
3758 "implement",
3759 format!(
3760 "{}: the reply named a command whose own CLI never confirmed the exit \
3761 status of, but there is no session left to resume",
3762 seat.key
3763 ),
3764 );
3765 continue;
3766 }
3767 self.state.event(
3768 "implement",
3769 format!(
3770 "{}: the reply named a command whose own CLI never confirmed the exit \
3771 status of; resuming the conversation",
3772 seat.key
3773 ),
3774 );
3775 let mut retry = job.clone();
3776 retry.seat = seat.clone();
3777 retry.prompt = prompt::resume_incomplete(
3778 "a command in your last reply had no confirmed exit status",
3779 );
3780 retry.timeout = retry_budget(job.timeout, true);
3781 retry.stem = format!("{}-confirm", job.stem);
3782 let cache = self.state.config.cache_dir();
3783 let ctx = WaveCtx {
3784 carry_seats: false,
3785 pauses: Some((&self.pause, &self.interrupt)),
3786 run: run_id,
3787 node: "implement",
3788 prompts,
3789 cache: cache.as_deref(),
3790 round: None,
3791 };
3792 let (resumed_seat, resumed) =
3793 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794 *seat = resumed_seat;
3795 *out = resumed;
3796 }
3797 }
3798
3799 async fn continue_fix_report(
3820 &mut self,
3821 mut seat: SeatState,
3822 parse_err: String,
3823 job: &SeatJob,
3824 prompts: &Prompts,
3825 run_id: &str,
3826 round: usize,
3827 ) -> (
3828 SeatState,
3829 Option<FixReport>,
3830 Option<String>,
3831 ContinuationRecord,
3832 ) {
3833 let mut last_err = parse_err;
3834 let mut cumulative_wait_ms = 0u64;
3835 let mut attempts = 0usize;
3836 loop {
3837 if !has_context(&job.spec, &seat, job.sessions) {
3838 self.state.event(
3839 "fix",
3840 format!(
3841 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842 session left to resume into"
3843 ),
3844 );
3845 let outcome = if attempts == 0 {
3846 ContinuationOutcome::NoSession
3847 } else {
3848 ContinuationOutcome::Exhausted
3849 };
3850 return (
3851 seat,
3852 None,
3853 Some(format!("unparsable fix report: {last_err}")),
3854 ContinuationRecord {
3855 attempts,
3856 cumulative_wait_ms,
3857 outcome,
3858 },
3859 );
3860 }
3861 if attempts >= MAX_FIX_CONTINUATIONS {
3862 self.state.event(
3863 "fix",
3864 format!(
3865 "round {round}: fixer's reply still had no adoption report after \
3866 {attempts} continuation(s) ({last_err}); giving up"
3867 ),
3868 );
3869 return (
3870 seat,
3871 None,
3872 Some(format!(
3873 "unparsable fix report after {attempts} continuation(s): {last_err}"
3874 )),
3875 ContinuationRecord {
3876 attempts,
3877 cumulative_wait_ms,
3878 outcome: ContinuationOutcome::Exhausted,
3879 },
3880 );
3881 }
3882 attempts += 1;
3883 self.state.event(
3884 "fix",
3885 format!(
3886 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888 ),
3889 );
3890 let mut retry = job.clone();
3891 retry.seat = seat.clone();
3892 retry.prompt = prompt::resume_incomplete(&last_err);
3893 retry.timeout = retry_budget(job.timeout, true);
3894 retry.stem = format!("{}-continue{attempts}", job.stem);
3895 let cache = self.state.config.cache_dir();
3896 let ctx = WaveCtx {
3897 carry_seats: false,
3898 pauses: Some((&self.pause, &self.interrupt)),
3899 run: run_id,
3900 node: "fix",
3901 prompts,
3902 cache: cache.as_deref(),
3903 round: Some(round),
3904 };
3905 let (resumed_seat, resumed_out) = run_one(
3906 retry,
3907 Arc::clone(&self.sem),
3908 &ctx,
3909 &mut self.state,
3910 attempts,
3911 )
3912 .await;
3913 seat = resumed_seat;
3914 match resumed_out {
3915 AgentOutcome::Ok(o) => {
3916 cumulative_wait_ms += o.duration_ms;
3917 match verdict::extract_json::<FixReport>(&o.text) {
3918 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919 self.state.event(
3920 "fix",
3921 format!(
3922 "round {round}: fixer's adoption report recovered after \
3923 {attempts} continuation(s)"
3924 ),
3925 );
3926 return (
3927 seat,
3928 Some(report),
3929 None,
3930 ContinuationRecord {
3931 attempts,
3932 cumulative_wait_ms,
3933 outcome: ContinuationOutcome::Resumed,
3934 },
3935 );
3936 }
3937 Ok(_) => {
3945 last_err = "the reply parsed, but it reported a command whose own CLI \
3946 never confirmed an exit status"
3947 .to_owned();
3948 }
3949 Err(e) => last_err = e.to_string(),
3950 }
3951 }
3952 AgentOutcome::Quota(o) => {
3953 cumulative_wait_ms += o.duration_ms;
3954 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956 self.state.quota.push(loss);
3957 self.state.event(
3958 "fix",
3959 format!(
3960 "round {round}: continuation rate limited (quota); not retrying now"
3961 ),
3962 );
3963 return (
3964 seat,
3965 None,
3966 Some("rate limited (quota) while recovering the fix report".to_owned()),
3967 ContinuationRecord {
3968 attempts,
3969 cumulative_wait_ms,
3970 outcome: ContinuationOutcome::QuotaLost,
3971 },
3972 );
3973 }
3974 AgentOutcome::Dropped(o) => {
3975 cumulative_wait_ms += o.duration_ms;
3976 let why = o
3977 .dropped
3978 .as_ref()
3979 .map(|d| d.why.as_str())
3980 .unwrap_or("the CLI ended the stream without delivering its answer");
3981 last_err = format!("the CLI dropped the stream ({why})");
3982 }
3983 AgentOutcome::Failed(e) => last_err = e,
3984 AgentOutcome::ResourceBlocked(e) => {
3987 return (
3988 seat,
3989 None,
3990 Some(format!("the continuation was not started: {e}")),
3991 ContinuationRecord {
3992 attempts: attempts.saturating_sub(1),
3993 cumulative_wait_ms,
3994 outcome: ContinuationOutcome::NoSession,
3995 },
3996 );
3997 }
3998 }
3999 }
4000 }
4001
4002 fn after_implement(&mut self) -> Result<()> {
4003 if !self.state.candidate_preservation.is_empty() {
4004 bail!("candidate preservation incomplete; judging is unsafe");
4005 }
4006 if self.state.leaks.is_empty() {
4008 let cfg = self.state.config.blind.clone();
4009 let mut leaks = Vec::new();
4010 for c in &self.state.candidates {
4011 let Some(patch) =
4012 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013 else {
4014 continue;
4015 };
4016 leaks.extend(blind::scan(
4017 &format!("candidate {} patch", c.label),
4018 &patch,
4019 &cfg.vendor_tokens,
4020 ));
4021 }
4022 if !leaks.is_empty() {
4023 let summary = leaks
4024 .iter()
4025 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026 .collect::<Vec<_>>()
4027 .join(", ");
4028 match cfg.on_leak {
4029 LeakPolicy::Fail => {
4030 self.state.status = RunStatus::Failed;
4031 self.state
4032 .event("blind", format!("vendor text in a patch: {summary}"));
4033 self.state.leaks = leaks;
4034 self.state.save()?;
4035 self.settle_questions();
4036 bail!(
4037 "blind.on_leak = \"fail\" and vendor text reached a \
4038 judged patch: {summary}"
4039 );
4040 }
4041 LeakPolicy::Redact => self.state.event(
4042 "blind",
4043 format!("redacting vendor text for judging: {summary}"),
4044 ),
4045 LeakPolicy::Warn => self.state.event(
4046 "blind",
4047 format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048 ),
4049 }
4050 self.state.leaks = leaks;
4051 }
4052 }
4053
4054 if self.state.viable().is_empty() {
4055 if self.state.all_candidates_verified_noop() {
4056 self.state.status = RunStatus::VerifiedNoop;
4067 self.state.save()?;
4068 self.settle_questions();
4069 return Ok(());
4070 }
4071 self.state.status = RunStatus::Failed;
4072 self.state.save()?;
4073 self.settle_questions();
4074 bail!("no candidate produced a change; nothing to judge");
4075 }
4076 self.state.status = RunStatus::Judging;
4077 self.state.save()?;
4078 Ok(())
4079 }
4080
4081 async fn judge(&mut self) -> Result<()> {
4084 let run_id = self.state.id.clone();
4089 let prompts = self.state.config.prompts.clone();
4090 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091 return Ok(());
4092 }
4093 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094 if viable.len() == 1 {
4095 self.state.judge_skipped = true;
4102 self.state.event(
4103 "judge",
4104 format!(
4105 "only candidate {} produced a change; judging skipped",
4106 viable[0].label
4107 ),
4108 );
4109 self.state.save()?;
4110 return Ok(());
4111 }
4112 self.state.status = RunStatus::Judging;
4113
4114 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115 let language = self.state.config.graph.language.clone();
4116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117 let sessions = self.state.config.graph.sessions;
4118 let artifacts = agent::artifacts_dir(&self.state.dir());
4119 let root = self.state.worktree_root();
4120 let base_short = short(&self.state.base_commit);
4121
4122 let mut jobs = Vec::new();
4123 let mut orders = Vec::new();
4124 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128 let seat_key = format!("judge-{}", j + 1);
4129 let seat = self.seat(&seat_key, &spec.id);
4130 jobs.push(SeatJob {
4131 prompt: prompt::judge(
4132 &self.state.instruction,
4133 &views,
4134 self.roles.judges.len(),
4135 &base_short,
4136 &language,
4137 ),
4138 spec,
4139 seat,
4140 cwd: root.join(format!("judge-{}", j + 1)),
4141 timeout,
4142 allow_write: false,
4143 sessions,
4144 artifacts: artifacts.clone(),
4145 stem: format!("judge-{}", j + 1),
4146 handover: None,
4147 });
4148 }
4149
4150 self.state.event(
4151 "judge",
4152 format!(
4153 "{} judges ranking {} candidates blind",
4154 jobs.len(),
4155 viable.len()
4156 ),
4157 );
4158 let labels_for_check = labels.clone();
4159 let mut quota_losses = Vec::new();
4160 let cache = self.state.config.cache_dir();
4161 let ctx = WaveCtx {
4162 carry_seats: false,
4163 pauses: Some((&self.pause, &self.interrupt)),
4164 run: &run_id,
4165 node: "judge",
4166 prompts: &prompts,
4167 cache: cache.as_deref(),
4168 round: None,
4169 };
4170 let results = ask_json_wave::<Ranking>(
4171 jobs,
4172 Arc::clone(&self.sem),
4173 self.state.config.graph.retries,
4174 &self.roles.judge_roster,
4175 &ctx,
4176 &mut quota_losses,
4177 &mut self.state,
4178 &move |r: &Ranking| r.validate(&labels_for_check),
4179 )
4180 .await;
4181 if self.state.resource_block.is_some() {
4182 return self.stop_for_resource();
4183 }
4184 self.state.quota.extend(quota_losses);
4185
4186 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187 let agent_id = seat.agent.clone();
4188 self.state.seats.insert(seat.key.clone(), seat);
4189 let mut record = Judgement {
4190 judge: j + 1,
4191 seat: format!("judge-{}", j + 1),
4192 agent: agent_id,
4193 ranking: Vec::new(),
4194 reasons: BTreeMap::new(),
4195 confidence: None,
4196 strengths: BTreeMap::new(),
4197 order: orders[j].clone(),
4198 failed: None,
4199 duration_ms: 0,
4200 };
4201 match res {
4202 Ok((ranking, out)) => {
4203 record.ranking = ranking.normalized();
4204 record.reasons = ranking.reasons;
4205 record.strengths = ranking.strengths;
4206 record.confidence = ranking.confidence;
4207 record.duration_ms = out.duration_ms;
4208 self.state.event(
4209 "judge",
4210 format!(
4211 "judge {} ranked {}",
4212 j + 1,
4213 record.ranking.iter().collect::<String>()
4214 ),
4215 );
4216 }
4217 Err(e) => {
4218 record.failed = Some(e.to_string());
4219 self.state
4220 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221 }
4222 }
4223 self.state.judgements.push(record);
4224 self.state.save()?;
4225 }
4226 Ok(())
4227 }
4228
4229 async fn deliberate(&mut self) -> Result<()> {
4232 if self.hold_base_sync().await? {
4233 return Ok(());
4234 }
4235 let run_id = self.state.id.clone();
4240 let prompts = self.state.config.prompts.clone();
4241 let rounds = self.state.config.graph.deliberate_rounds;
4242 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244 return Ok(());
4245 }
4246 let tops: Vec<char> = self
4247 .state
4248 .judgements
4249 .iter()
4250 .filter_map(|j| j.ranking.first().copied())
4251 .collect();
4252 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254 self.state.event(
4255 "deliberate",
4256 format!("judges agreed on {} outright; no deliberation", tops[0]),
4257 );
4258 }
4259 self.state.status = RunStatus::Voting;
4260 self.state.save()?;
4261 return Ok(());
4262 }
4263
4264 self.state.status = RunStatus::Deliberating;
4265 self.state.event(
4266 "deliberate",
4267 format!(
4268 "split: first choices were {} — opening {rounds} round(s)",
4269 tops.iter().collect::<String>()
4270 ),
4271 );
4272
4273 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274 let language = self.state.config.graph.language.clone();
4275 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276 let sessions = self.state.config.graph.sessions;
4277 let artifacts = agent::artifacts_dir(&self.state.dir());
4278 let root = self.state.worktree_root();
4279 let base_short = short(&self.state.base_commit);
4280
4281 for round in self.state.deliberation.len() + 1..=rounds {
4285 let mut turns: Vec<DeliberationTurn> = Vec::new();
4286 let mut round_losses = Vec::new();
4287 let mut asked = 0;
4288 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289 if self.state.judgements[j].failed.is_some() {
4290 continue;
4291 }
4292 let seat_key = format!("judge-{}", j + 1);
4293 let spec = self.occupant(&seat_key, spec);
4294 let mut seat = self.seat(&seat_key, &spec.id);
4295 let transcript = self.transcript(&turns, j);
4296 let build = |context: Option<&str>| {
4297 prompt::deliberate(
4298 &self.state.instruction,
4299 context,
4300 &transcript,
4301 round,
4302 rounds,
4303 &language,
4304 )
4305 };
4306 let block = self.candidate_block(&viable, &base_short);
4307 let full = build(Some(&block));
4308 let text = if has_context(&spec, &seat, sessions) {
4309 build(None)
4310 } else {
4311 full.clone()
4312 };
4313 let job = SeatJob {
4314 spec,
4315 seat: seat.clone(),
4316 prompt: text,
4317 cwd: root.join(format!("judge-{}", j + 1)),
4318 timeout,
4319 allow_write: false,
4320 sessions,
4321 artifacts: artifacts.clone(),
4322 stem: format!("delib-{round}-judge-{}", j + 1),
4323 handover: Some(full),
4324 };
4325 let cache = self.state.config.cache_dir();
4326 let ctx = WaveCtx {
4327 carry_seats: false,
4328 pauses: Some((&self.pause, &self.interrupt)),
4329 run: &run_id,
4330 node: "deliberate",
4331 prompts: &prompts,
4332 cache: cache.as_deref(),
4333 round: None,
4334 };
4335 let mut losses = Vec::new();
4339 asked += 1;
4340 let mut results = ask_wave_with::<String>(
4341 vec![job],
4342 Arc::clone(&self.sem),
4343 0,
4344 &self.roles.judge_roster,
4345 &ctx,
4346 &mut losses,
4347 &mut self.state,
4348 &|text: &str| {
4349 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350 },
4351 )
4352 .await;
4353 if self.state.resource_block.is_some() {
4354 return self.stop_for_resource();
4355 }
4356 round_losses.extend(losses.iter().cloned());
4357 self.state.quota.extend(losses);
4358 let (updated, res, _) = results.pop().expect("one job in, one result out");
4359 seat = updated;
4360 let agent_id = seat.agent.clone();
4361 self.state.seats.insert(seat.key.clone(), seat);
4362 let body = match res {
4363 Ok((body, _)) => body,
4364 Err(e) => {
4367 self.state
4368 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369 continue;
4370 }
4371 };
4372 let tentative = verdict::extract_json::<Position>(&body)
4373 .ok()
4374 .and_then(|p| p.tentative)
4375 .and_then(|s| s.trim().chars().next())
4376 .map(|c| c.to_ascii_uppercase());
4377 self.state.event(
4378 "deliberate",
4379 format!(
4380 "round {round}: judge {} now favours {}",
4381 j + 1,
4382 tentative.map_or("—".to_owned(), |c| c.to_string())
4383 ),
4384 );
4385 turns.push(DeliberationTurn {
4386 judge: j + 1,
4387 agent: agent_id,
4388 body: blind::sanitize_prose(&body, &self.state.config.blind),
4389 tentative,
4390 });
4391 }
4392 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394 return self.stop_for_resource();
4395 }
4396 self.state
4397 .deliberation
4398 .push(DeliberationRound { round, turns });
4399 self.state.save()?;
4400 }
4401
4402 self.state.status = RunStatus::Voting;
4403 self.state.save()?;
4404 Ok(())
4405 }
4406
4407 async fn vote(&mut self) -> Result<()> {
4410 if self.hold_base_sync().await? {
4411 return Ok(());
4412 }
4413 let run_id = self.state.id.clone();
4418 let prompts = self.state.config.prompts.clone();
4419 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420 return Ok(());
4421 }
4422 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423 if viable.len() == 1 {
4424 return Ok(());
4425 }
4426 self.state.status = RunStatus::Voting;
4427
4428 let language = self.state.config.graph.language.clone();
4429 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430 let sessions = self.state.config.graph.sessions;
4431 let artifacts = agent::artifacts_dir(&self.state.dir());
4432 let root = self.state.worktree_root();
4433 let base_short = short(&self.state.base_commit);
4434 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436 let mut jobs = Vec::new();
4437 let mut seats_at = Vec::new();
4438 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439 if self
4440 .state
4441 .judgements
4442 .get(j)
4443 .is_some_and(|r| r.failed.is_some())
4444 {
4445 continue;
4446 }
4447 let seat_key = format!("judge-{}", j + 1);
4448 let spec = self.occupant(&seat_key, spec);
4449 let seat = self.seat(&seat_key, &spec.id);
4450 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451 let text = if has_context(&spec, &seat, sessions) {
4452 prompt::final_vote(&viable, &language)
4453 } else {
4454 full.clone()
4455 };
4456 jobs.push(SeatJob {
4457 spec,
4458 seat,
4459 prompt: text,
4460 cwd: root.join(format!("judge-{}", j + 1)),
4461 timeout,
4462 allow_write: false,
4463 sessions,
4464 artifacts: artifacts.clone(),
4465 stem: format!("vote-judge-{}", j + 1),
4466 handover: Some(full),
4467 });
4468 seats_at.push(j);
4469 }
4470
4471 self.state.event(
4472 "vote",
4473 format!(
4474 "collecting {} final votes one by one, privately",
4475 jobs.len()
4476 ),
4477 );
4478 let allowed = viable.clone();
4479 let mut quota_losses = Vec::new();
4480 let cache = self.state.config.cache_dir();
4481 let ctx = WaveCtx {
4482 carry_seats: false,
4483 pauses: Some((&self.pause, &self.interrupt)),
4484 run: &run_id,
4485 node: "vote",
4486 prompts: &prompts,
4487 cache: cache.as_deref(),
4488 round: None,
4489 };
4490 let results = ask_json_wave::<FinalVote>(
4491 jobs,
4492 Arc::clone(&self.sem),
4493 self.state.config.graph.retries,
4494 &self.roles.judge_roster,
4495 &ctx,
4496 &mut quota_losses,
4497 &mut self.state,
4498 &move |v: &FinalVote| match v.label() {
4499 Some(c) if allowed.contains(&c) => Ok(()),
4500 other => bail!("vote {other:?} is not one of {allowed:?}"),
4501 },
4502 )
4503 .await;
4504 if self.state.resource_block.is_some() {
4505 return self.stop_for_resource();
4506 }
4507 self.state.quota.extend(quota_losses);
4508
4509 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510 let agent_id = seat.agent.clone();
4511 self.state.seats.insert(seat.key.clone(), seat);
4512 let initial = self
4513 .state
4514 .judgements
4515 .get(j)
4516 .and_then(|r| r.ranking.first().copied());
4517 let mut record = VoteRecord {
4518 judge: j + 1,
4519 agent: agent_id,
4520 vote: None,
4521 reason: String::new(),
4522 changed: false,
4523 };
4524 match res {
4525 Ok((v, _)) => {
4526 record.vote = v.label();
4527 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529 self.state.event(
4530 "vote",
4531 format!(
4532 "judge {} voted {}{}",
4533 j + 1,
4534 record.vote.unwrap_or('?'),
4535 if record.changed { " (changed)" } else { "" }
4536 ),
4537 );
4538 }
4539 Err(e) => {
4540 self.state
4541 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542 }
4543 }
4544 self.state.votes.push(record);
4545 self.state.save()?;
4546 }
4547 Ok(())
4548 }
4549
4550 fn tally(&mut self) -> Result<()> {
4553 if self.state.tally.is_some() {
4554 return Ok(());
4555 }
4556 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557 let tops: Vec<char> = self
4558 .state
4559 .judgements
4560 .iter()
4561 .filter_map(|j| j.ranking.first().copied())
4562 .collect();
4563 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568 let mut cast: Vec<char> = Vec::new();
4569 for (i, j) in self.state.judgements.iter().enumerate() {
4570 let vote = self
4571 .state
4572 .votes
4573 .iter()
4574 .find(|v| v.judge == i + 1)
4575 .and_then(|v| v.vote)
4576 .or_else(|| j.ranking.first().copied());
4577 if let Some(v) = vote {
4578 *first_choice.entry(v).or_insert(0) += 1;
4579 cast.push(v);
4580 }
4581 }
4582
4583 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584 for j in &self.state.judgements {
4585 let n = j.ranking.len();
4586 for (pos, label) in j.ranking.iter().enumerate() {
4587 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588 }
4589 }
4590
4591 let best = first_choice.values().copied().max().unwrap_or(0);
4592 let mut leaders: Vec<char> = first_choice
4593 .iter()
4594 .filter(|(_, v)| **v == best)
4595 .map(|(k, _)| *k)
4596 .collect();
4597 let mut tie_break = None;
4598 if leaders.len() > 1 {
4599 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600 let borda_leaders: Vec<char> = leaders
4601 .iter()
4602 .copied()
4603 .filter(|l| borda[l] == top_borda)
4604 .collect();
4605 tie_break = Some(if borda_leaders.len() == 1 {
4606 format!(
4607 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608 leaders.len()
4609 )
4610 } else {
4611 format!(
4612 "{} way tie on both first-choice votes and Borda points, broken by label order",
4613 leaders.len()
4614 )
4615 });
4616 leaders = borda_leaders;
4617 leaders.sort_unstable();
4618 }
4619 let winner = *leaders
4620 .first()
4621 .or(viable.first())
4622 .context("no candidate to declare a winner from")?;
4623
4624 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626 let deliberated = !self.state.deliberation.is_empty();
4627
4628 let quota_seats: std::collections::BTreeSet<&str> =
4632 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633 let mut present = 0usize;
4634 for (i, j) in self.state.judgements.iter().enumerate() {
4635 if quota_seats.contains(j.seat.as_str()) {
4636 continue;
4637 }
4638 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639 let voted = self
4640 .state
4641 .votes
4642 .iter()
4643 .any(|v| v.judge == i + 1 && v.vote.is_some());
4644 if ranked || voted {
4645 present += 1;
4646 }
4647 }
4648 let needs_quorum = viable.len() > 1;
4654 let judges_total = if needs_quorum {
4655 self.roles.judges.len()
4656 } else {
4657 0
4658 };
4659 let quorum = if needs_quorum {
4660 judges_total / 2 + 1
4661 } else {
4662 0
4663 };
4664 let met_quorum = !needs_quorum || present >= quorum;
4665 let uncontested = (!needs_quorum).then(|| {
4666 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667 });
4668
4669 self.state.event(
4670 "tally",
4671 match &uncontested {
4672 Some(reason) => format!("winner {winner} — {reason}"),
4673 None => format!(
4674 "winner {winner} — votes {} | initial {} | {} changed | \
4675 {present}/{judges_total} judges{}",
4676 first_choice
4677 .iter()
4678 .map(|(k, v)| format!("{k}:{v}"))
4679 .collect::<Vec<_>>()
4680 .join(" "),
4681 if unanimous_initial {
4682 "unanimous"
4683 } else {
4684 "split"
4685 },
4686 changed_votes,
4687 if met_quorum {
4688 String::new()
4689 } else {
4690 format!(" — below quorum ({quorum} required)")
4691 },
4692 ),
4693 },
4694 );
4695 if !met_quorum {
4696 self.state.event(
4697 "stall",
4698 format!(
4699 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700 the run stops here, resumable"
4701 ),
4702 );
4703 }
4704 self.state.tally = Some(Tally {
4705 first_choice,
4706 borda,
4707 winner,
4708 rankings: tops.len(),
4709 unanimous_initial,
4710 deliberated,
4711 changed_votes,
4712 unanimous_final,
4713 tie_break,
4714 judges: judges_total,
4715 present,
4716 quorum,
4717 met_quorum,
4718 uncontested,
4719 });
4720 self.state.status = if met_quorum {
4721 RunStatus::Reviewing
4722 } else {
4723 RunStatus::Stalled
4724 };
4725 self.state.save()?;
4726 Ok(())
4727 }
4728
4729 #[allow(clippy::too_many_lines)]
4750 async fn recover_stall(&mut self) -> Result<bool> {
4751 let run_id = self.state.id.clone();
4756 let prompts = self.state.config.prompts.clone();
4757 let quota_seats: BTreeSet<&str> =
4762 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763 let absent: Vec<String> = self
4764 .state
4765 .judgements
4766 .iter()
4767 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768 .map(|j| j.seat.clone())
4769 .collect();
4770 if absent.is_empty() {
4771 return Ok(false);
4772 }
4773 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774 if viable.len() <= 1 {
4775 return Ok(false);
4776 }
4777 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778 let language = self.state.config.graph.language.clone();
4779 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780 let sessions = self.state.config.graph.sessions;
4781 let artifacts = agent::artifacts_dir(&self.state.dir());
4782 let root = self.state.worktree_root();
4783 let base_short = short(&self.state.base_commit);
4784 let candidates: Vec<Candidate> = viable.clone();
4785
4786 let mut positions: Vec<usize> = absent
4788 .iter()
4789 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790 .collect();
4791 if positions.is_empty() {
4792 return Ok(false);
4793 }
4794 positions.sort_unstable();
4795 positions.dedup();
4796
4797 let mut judge_jobs = Vec::new();
4799 for &j in &positions {
4800 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802 let seat_key = format!("judge-{}", j + 1);
4803 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804 let seat = self.seat(&seat_key, &spec.id);
4805 judge_jobs.push(SeatJob {
4806 spec,
4807 seat,
4808 prompt: prompt::judge(
4809 &self.state.instruction,
4810 &views,
4811 self.roles.judges.len(),
4812 &base_short,
4813 &language,
4814 ),
4815 cwd: root.join(seat_key),
4816 timeout,
4817 allow_write: false,
4818 sessions,
4819 artifacts: artifacts.clone(),
4820 stem: format!("judge-{}-recover", j + 1),
4821 handover: None,
4822 });
4823 }
4824
4825 let labels_for_check = labels.clone();
4826 let mut judge_losses = Vec::new();
4827 let retries = self.state.config.graph.retries;
4828 let cache = self.state.config.cache_dir();
4829 let ctx = WaveCtx {
4830 carry_seats: false,
4831 pauses: Some((&self.pause, &self.interrupt)),
4832 run: &run_id,
4833 node: "judge",
4834 prompts: &prompts,
4835 cache: cache.as_deref(),
4836 round: None,
4837 };
4838 let results = ask_json_wave::<Ranking>(
4839 judge_jobs,
4840 Arc::clone(&self.sem),
4841 retries,
4842 &self.roles.judge_roster,
4843 &ctx,
4844 &mut judge_losses,
4845 &mut self.state,
4846 &move |r: &Ranking| r.validate(&labels_for_check),
4847 )
4848 .await;
4849 if self.state.resource_block.is_some() {
4850 self.stop_for_resource()?;
4851 return Ok(false);
4852 }
4853
4854 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857 let agent_id = seat.agent.clone();
4858 self.state.seats.insert(seat.key.clone(), seat);
4859 let record = &mut self.state.judgements[j];
4860 match res {
4861 Ok((ranking, out)) => {
4862 record.agent = agent_id;
4863 record.ranking = ranking.normalized();
4864 record.reasons = ranking.reasons;
4865 record.strengths = ranking.strengths;
4866 record.confidence = ranking.confidence;
4867 record.failed = None;
4868 record.duration_ms = out.duration_ms;
4869 recovered.insert(j);
4870 self.state.event(
4871 "recover",
4872 format!("judge {} ranked again after the limit", j + 1),
4873 );
4874 }
4875 Err(e) => {
4876 self.state
4877 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878 }
4879 }
4880 }
4881
4882 let mut vote_jobs = Vec::new();
4884 let mut vote_pos: Vec<usize> = Vec::new();
4885 for &j in &recovered {
4886 let seat_key = format!("judge-{}", j + 1);
4887 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888 let seat = self.seat(&seat_key, &spec.id);
4889 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890 let text = if has_context(&spec, &seat, sessions) {
4891 prompt::final_vote(&labels, &language)
4892 } else {
4893 full.clone()
4894 };
4895 vote_jobs.push(SeatJob {
4896 spec,
4897 seat,
4898 prompt: text,
4899 cwd: root.join(seat_key),
4900 timeout,
4901 allow_write: false,
4902 sessions,
4903 artifacts: artifacts.clone(),
4904 stem: format!("vote-judge-{}-recover", j + 1),
4905 handover: Some(full),
4906 });
4907 vote_pos.push(j);
4908 }
4909 let allowed = labels.clone();
4910 let mut vote_losses = Vec::new();
4911 let vote_retries = self.state.config.graph.retries;
4912 let vote_cache = self.state.config.cache_dir();
4913 let ctx = WaveCtx {
4914 carry_seats: false,
4915 pauses: Some((&self.pause, &self.interrupt)),
4916 run: &run_id,
4917 node: "vote",
4918 prompts: &prompts,
4919 cache: vote_cache.as_deref(),
4920 round: None,
4921 };
4922 let votes = ask_json_wave::<FinalVote>(
4923 vote_jobs,
4924 Arc::clone(&self.sem),
4925 vote_retries,
4926 &self.roles.judge_roster,
4927 &ctx,
4928 &mut vote_losses,
4929 &mut self.state,
4930 &move |v: &FinalVote| match v.label() {
4931 Some(c) if allowed.contains(&c) => Ok(()),
4932 other => bail!("vote {other:?} is not one of {allowed:?}"),
4933 },
4934 )
4935 .await;
4936 if self.state.resource_block.is_some() {
4937 self.stop_for_resource()?;
4938 return Ok(false);
4939 }
4940 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941 let agent_id = seat.agent.clone();
4942 self.state.seats.insert(seat.key.clone(), seat);
4943 match res {
4944 Ok((v, _)) => {
4945 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946 rec.vote = v.label();
4947 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948 } else {
4949 self.state.votes.push(VoteRecord {
4950 judge: j + 1,
4951 agent: agent_id,
4952 vote: v.label(),
4953 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954 changed: false,
4955 });
4956 }
4957 self.state.event(
4958 "recover",
4959 format!("judge {} voted again after the limit", j + 1),
4960 );
4961 }
4962 Err(e) => {
4963 self.state
4964 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965 }
4966 }
4967 }
4968
4969 let recovered_keys: BTreeSet<String> = recovered
4973 .iter()
4974 .map(|&j| format!("judge-{}", j + 1))
4975 .collect();
4976 self.state
4977 .quota
4978 .retain(|q| !recovered_keys.contains(&q.seat));
4979 for loss in judge_losses.into_iter().chain(vote_losses) {
4983 if recovered_keys.contains(&loss.seat) {
4984 continue;
4985 }
4986 self.state.quota.retain(|q| q.seat != loss.seat);
4987 self.state.quota.push(loss);
4988 }
4989
4990 self.state.tally = None;
4992 self.tally()?;
4993 Ok(self
4994 .state
4995 .tally
4996 .as_ref()
4997 .map(|t| t.met_quorum)
4998 .unwrap_or(false))
4999 }
5000
5001 async fn graft(&mut self) -> Result<()> {
5017 let Err(e) = self.graft_inner().await else {
5018 return Ok(());
5019 };
5020 if e.is::<crate::repo_lock::ResourceBlocked>() {
5021 return Err(e);
5022 }
5023 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024 self.state.status = RunStatus::Blocked;
5025 self.state.event("graft", format!("blocked: {e:#}"));
5026 self.state.save()?;
5027 return Err(e);
5028 }
5029 if let Some(rec) = self.state.graft.clone()
5034 && !rec.outcome.settled()
5035 {
5036 if let Some(winner) = self.state.winner().cloned() {
5037 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038 if re.is::<crate::repo_lock::ResourceBlocked>() {
5039 return Err(re);
5040 }
5041 self.state.status = RunStatus::Blocked;
5042 self.state.event("graft", format!("blocked: {re:#}"));
5043 self.state.save()?;
5044 return Err(re);
5045 }
5046 }
5047 if let Some(g) = self.state.graft.as_mut() {
5048 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049 }
5050 }
5051 self.state.event("graft", format!("skipped: {e:#}"));
5052 if let Err(e) = self.state.save() {
5053 tracing::warn!("save after graft: {e:#}");
5054 }
5055 Ok(())
5056 }
5057
5058 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061 let _coordination = self.coordinate(wt).await?;
5062 crate::recovery::restore_repair(wt, pre)
5063 .await
5064 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065 }
5066
5067 fn graft_skipped(&mut self, why: &str) {
5069 let message = format!("skipped: {why}");
5070 if !self
5071 .state
5072 .events
5073 .iter()
5074 .any(|e| e.node == "graft" && e.message == message)
5075 {
5076 self.state.event("graft", message);
5077 }
5078 }
5079
5080 async fn graft_inner(&mut self) -> Result<()> {
5081 if let Some(rec) = self.state.graft.clone()
5084 && !rec.outcome.settled()
5085 {
5086 if self.state.reviews.is_empty()
5090 && let Some(winner) = self.state.winner().cloned()
5091 {
5092 self.graft_restore(&winner.worktree, &rec.pre).await?;
5093 }
5094 if rec.outcome == GraftOutcome::WaitingForRestore {
5095 self.state.graft = None;
5096 self.state.save()?;
5097 } else {
5098 if let Some(g) = self.state.graft.as_mut() {
5099 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100 }
5101 self.state.event(
5102 "graft",
5103 "rolled back: the previous round was interrupted; the winner is untouched",
5104 );
5105 self.state.save()?;
5106 return Ok(());
5107 }
5108 }
5109 let plan = match graft::plan(&self.state) {
5110 Ok(plan) => plan,
5111 Err(skip) => {
5112 if skip != graft::Skip::AlreadyDone {
5113 self.graft_skipped(skip.reason());
5114 self.state.save()?;
5115 }
5116 return Ok(());
5117 }
5118 };
5119 let Some(winner) = self.state.winner().cloned() else {
5120 return Ok(());
5121 };
5122 let wt = winner.worktree.clone();
5123 if !git::is_clean(&wt).await? {
5124 self.graft_skipped("the winner's worktree has uncommitted changes");
5125 self.state.save()?;
5126 return Ok(());
5127 }
5128 let pre = git::rev_parse(&wt, "HEAD").await?;
5129 self.state.graft = Some(GraftRecord {
5132 pre: pre.clone(),
5133 post: None,
5134 lines: None,
5135 limit: None,
5136 agent: None,
5137 outcome: GraftOutcome::Running,
5138 requested: graft::requested(&plan),
5139 adoption: None,
5140 adoption_error: None,
5141 });
5142 self.state.event(
5143 "graft",
5144 format!(
5145 "porting strengths of candidate(s) {} onto {}",
5146 plan.strengths
5147 .iter()
5148 .map(|(l, _, _)| l.to_string())
5149 .collect::<Vec<_>>()
5150 .join(", "),
5151 plan.winner
5152 ),
5153 );
5154 self.state.save()?;
5155
5156 let fix_prompt = prompt::graft(
5157 &self.state.instruction,
5158 plan.winner,
5159 &plan.strengths,
5160 &self.state.config.graph.language,
5161 );
5162 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163 let sessions = self.state.config.graph.sessions;
5164 let artifacts = agent::artifacts_dir(&self.state.dir());
5165 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166 let (_, seat, out) = self
5167 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168 SeatJob {
5169 prompt: fix_prompt.clone(),
5170 spec,
5171 seat,
5172 cwd: wt.clone(),
5173 timeout,
5174 allow_write: true,
5175 sessions,
5176 artifacts: artifacts.clone(),
5177 stem: "graft".to_owned(),
5178 handover: None,
5179 }
5180 })
5181 .await;
5182 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183 self.state.graft.as_mut().expect("graft record").outcome =
5186 GraftOutcome::WaitingForRestore;
5187 return self.stop_for_resource();
5188 }
5189 let mut why: Option<String> = None;
5190 match &out {
5191 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192 why = Some("the fixer produced nothing usable".to_owned());
5193 }
5194 AgentOutcome::Ok(_) => {}
5195 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196 AgentOutcome::Quota(o) => {
5197 self.state.quota.push(QuotaLoss::from_report(
5198 seat.key.clone(),
5199 "graft",
5200 o.quota.as_ref(),
5201 ));
5202 why = Some("rate limited (quota)".to_owned());
5203 }
5204 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205 why = Some(format!(
5206 "the fixer failed: {}",
5207 e.lines()
5208 .next()
5209 .unwrap_or("")
5210 .chars()
5211 .take(160)
5212 .collect::<String>()
5213 ));
5214 }
5215 }
5216 let agent_id = seat.agent.clone();
5217 self.state.seats.insert(seat.key.clone(), seat);
5218
5219 let mut post = None;
5220 let mut lines = None;
5221 let _coordination = self.coordinate(&wt).await?;
5222 if why.is_none() {
5223 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224 if let Ok(r) = &rescued {
5225 self.state.note_withheld("graft", &r.withheld);
5226 }
5227 let head = git::rev_parse(&wt, "HEAD").await?;
5228 let dirty = match &rescued {
5232 Ok(r) => git::status_porcelain(&wt)
5233 .await
5234 .map(|st| {
5235 st.lines().any(|l| {
5236 !r.withheld
5237 .iter()
5238 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239 })
5240 })
5241 .unwrap_or(true),
5242 Err(_) => true,
5243 };
5244 if dirty {
5245 why = Some("the fixer left changes that could not be committed".to_owned());
5246 } else if head == pre {
5247 why = Some("the fixer changed nothing".to_owned());
5248 } else {
5249 lines = git::changed_lines(&wt, &pre, &head).await;
5250 post = Some(head);
5251 }
5252 }
5253 let (adoption, adoption_error) = match &out {
5254 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255 _ => (None, None),
5256 };
5257 let requested = graft::requested(&plan);
5258 let outcome = match why {
5259 None => {
5260 self.state.event(
5261 "graft",
5262 format!(
5263 "applied: {}",
5264 lines.map_or_else(
5265 || "changed lines unavailable".to_owned(),
5266 |n| format!("{n} changed lines")
5267 )
5268 ),
5269 );
5270 GraftOutcome::Applied
5271 }
5272 Some(why) => {
5273 crate::recovery::restore_repair(&wt, &pre)
5276 .await
5277 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278 self.state.event(
5279 "graft",
5280 format!("rolled back: {why}; the winner is untouched"),
5281 );
5282 GraftOutcome::RolledBack(why)
5283 }
5284 };
5285 self.state.graft = Some(GraftRecord {
5286 pre,
5287 post,
5288 lines,
5289 limit: None,
5290 agent: Some(agent_id),
5291 outcome,
5292 requested,
5293 adoption,
5294 adoption_error,
5295 });
5296 self.state.save()?;
5297 Ok(())
5298 }
5299
5300 async fn fold_losers(&mut self) -> Result<()> {
5303 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304 return Ok(());
5305 };
5306 if !self.state.candidate_preservation.is_empty() {
5307 bail!("candidate preservation incomplete; refusing cleanup");
5308 }
5309 let repo = self.state.repo.clone();
5310 let _coordination = self.coordinate(&repo).await?;
5311 let mut folded = Vec::new();
5312 for i in 0..self.state.candidates.len() {
5313 let c = &self.state.candidates[i];
5314 if c.label == winner || c.folded {
5315 continue;
5316 }
5317 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318 remove_preserved_worktree(&repo, &wt).await?;
5319 if git::branch_exists(&repo, &branch).await?
5320 && !git::branch_delete(&repo, &branch).await?
5321 {
5322 bail!("candidate branch removal refused");
5323 }
5324 self.state.candidates[i].folded = true;
5325 self.state.save()?;
5326 folded.push(label.to_string());
5327 }
5328 let root = self.state.worktree_root();
5330 for j in 1..=self.roles.judges.len() {
5331 let wt = root.join(format!("judge-{j}"));
5332 if wt.exists() {
5333 remove_read_seat_worktree(&mut self.state, &wt).await?;
5334 self.state.save()?;
5335 }
5336 }
5337 if self.state.config.graph.advise {
5340 for k in 1..=self.state.config.graph.advisors {
5341 let wt = root.join(format!("advisor-{k}"));
5342 if wt.exists() {
5343 remove_read_seat_worktree(&mut self.state, &wt).await?;
5344 self.state.save()?;
5345 }
5346 }
5347 }
5348 if !folded.is_empty() {
5349 self.state
5350 .event("fold", format!("folded candidates {}", folded.join(", ")));
5351 self.state.save()?;
5352 }
5353 Ok(())
5354 }
5355
5356 async fn hold_base_sync(&mut self) -> Result<bool> {
5359 self.hold_base_sync_as(None).await
5360 }
5361
5362 async fn hold_base_sync_kind(
5363 &mut self,
5364 kind: crate::base_sync_recovery::StopKind,
5365 ) -> Result<()> {
5366 self.hold_base_sync_as(Some(kind)).await?;
5367 Ok(())
5368 }
5369
5370 async fn hold_base_sync_as(
5371 &mut self,
5372 kind: Option<crate::base_sync_recovery::StopKind>,
5373 ) -> Result<bool> {
5374 if !self.state.base_sync_blocked() {
5375 self.state.base_sync_recovery.stop = None;
5376 return Ok(false);
5377 }
5378 let head = match self.state.winner() {
5379 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5380 .await
5381 .unwrap_or_default(),
5382 None => String::new(),
5383 };
5384 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5385 && !authorization.consumed
5386 && authorization.stop.head != head
5387 {
5388 authorization.refuse(format!(
5389 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5390 authorization.stop.head
5391 ));
5392 }
5393 self.state.record_base_sync_stop(&head);
5394 if let Some(kind) = kind {
5395 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5396 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5397 stop.kind = kind;
5398 }
5399 }
5400 self.state.save()?;
5401 Ok(true)
5402 }
5403
5404 async fn sync_to_base(&mut self) -> Result<()> {
5446 if self.state.status == RunStatus::AlreadyInBase {
5447 return Ok(());
5448 }
5449 self.hold_base_sync().await?;
5450 let conflicted = self
5451 .state
5452 .base_sync
5453 .as_ref()
5454 .is_some_and(|s| s.conflict.is_some());
5455 let Some(winner) = self.state.winner().cloned() else {
5456 return Ok(());
5457 };
5458
5459 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5460
5461 let repo = self.state.repo.clone();
5462 let remote = self.state.config.merge.remote.clone();
5463 let base_branch = self.state.base_branch.clone();
5464 let tracking = format!("{remote}/{base_branch}");
5465
5466 let fetched_base =
5467 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5468 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5473 return Ok(());
5474 };
5475
5476 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5477 self.state.ensure_base_episode();
5478 let mut pending = self
5479 .state
5480 .base_sync_recovery
5481 .operations
5482 .last()
5483 .filter(|o| !o.finished)
5484 .cloned();
5485 let scratch = self.state.dir().join("base-sync");
5486 if git::rebase_in_progress(&scratch).await {
5487 if pending.is_none()
5490 && self.state.base_sync_recovery.operations.is_empty()
5491 && !conflicted
5492 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5493 .await
5494 .ok()
5495 .as_deref()
5496 == Some(head.as_str())
5497 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5498 && git::is_clean(&winner.worktree).await?
5499 {
5500 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5501 if self.state.base_sync.is_none() {
5502 self.state.base_sync = Some(BaseSync {
5503 tip: saved_tip.clone(),
5504 behind: 0,
5505 attempts: 0,
5506 conflict: None,
5507 already_in: None,
5508 });
5509 }
5510 self.state.reserve_base_operation(&head, &saved_tip);
5511 self.state.event(
5512 "base-recovery",
5513 "imported standing legacy rebase as a spent synchronization reservation",
5514 );
5515 self.state.save()?;
5516 pending = self.state.base_sync_recovery.operations.last().cloned();
5517 }
5518 let expected = pending.as_ref().map(|op| op.tip.as_str());
5519 if expected.is_none()
5520 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5521 {
5522 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5523 if self.state.base_sync.is_none() {
5524 self.state.base_sync = Some(BaseSync {
5525 tip: tip.clone(),
5526 behind: 0,
5527 attempts: 0,
5528 conflict: None,
5529 already_in: None,
5530 });
5531 }
5532 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5533 self.state.event("land", why);
5534 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535 .await?;
5536 return Ok(());
5537 }
5538 }
5539 if let Some(op) = &pending {
5540 let mut replay_proven =
5542 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5543 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5544 let wait = repository_wait(&self.state, None);
5545 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5546 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5547 Ok(true) => {
5548 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5549 self.state.save()?;
5550 git::worktree_remove(&repo, &scratch).await.ok();
5551 replay_proven = true;
5552 }
5553 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5554 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5555 }
5556 }
5557 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5558 if self.state.foreign_checkout.is_some() {
5559 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5560 .await
5561 .ok()
5562 .as_deref()
5563 == Some(op.head.as_str())
5564 && git::rev_parse(&repo, &format!("{head}^2"))
5565 .await
5566 .ok()
5567 .as_deref()
5568 == Some(op.tip.as_str());
5569 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5570 .await
5571 .is_ok_and(|m| m.is_empty())
5572 && let Ok(added) =
5573 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5574 {
5575 replay_proven = true;
5576 for commit in added {
5577 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5578 replay_proven = false;
5579 break;
5580 }
5581 }
5582 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5583 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5584 }
5585 }
5586 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590 || (head != op.head
5591 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5592 .await?
5593 .is_empty())
5594 || (!git::is_clean(&winner.worktree).await?
5595 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5596 .await
5597 .is_ok_and(|o| o.ok()))
5598 || !replay_proven
5599 {
5600 let why = format!(
5601 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5602 op.id
5603 );
5604 let sync = self
5605 .state
5606 .base_sync
5607 .as_mut()
5608 .context("reserved synchronization has no lifetime accounting")?;
5609 sync.conflict = Some(why.clone());
5610 self.state.event("land", why);
5611 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5612 .await?;
5613 return Ok(());
5614 }
5615 if head != op.head {
5616 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5617 }
5618 tip = op.tip.clone();
5619 }
5620 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5621 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5622 if pending.is_none()
5623 && !conflicted
5624 && fetched_base
5625 && self
5626 .state
5627 .base_sync_recovery
5628 .episodes
5629 .last()
5630 .is_some_and(|e| e.boundary.is_some())
5631 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5632 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5633 && owner.execution_runs().last() == Some(&self.state.id)
5634 && owner.status != crate::queue::TaskStatus::Done
5635 && !owner.fresh_start
5636 && owner.review_branch.is_none()
5637 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5638 && let Some(approval) = owner.resumption.as_ref()
5639 && approval.instruction == owner.instruction
5640 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5641 && git::is_clean(&winner.worktree).await?
5642 && self.state.renew_base_episode(&task, approval, &head, &tip)
5643 {
5644 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5645 self.state.save()?;
5646 }
5647
5648 if (behind > 0 || conflicted || head == tip)
5657 && self
5658 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5659 .await?
5660 {
5661 return Ok(());
5662 }
5663 if conflicted && self.state.base_recovery_pending() {
5664 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5665 let owner =
5666 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5667 let owner_valid = owner.as_ref().is_ok_and(|t| {
5668 t.execution_runs().last() == Some(&self.state.id)
5669 && t.status != crate::queue::TaskStatus::Done
5670 && !t.fresh_start
5671 && t.review_branch.is_none()
5672 });
5673 let invalidation = if !owner_valid {
5674 Some("latest task ownership changed")
5675 } else if !owner.as_ref().is_ok_and(|t| {
5676 authorization
5677 .generation
5678 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5679 }) {
5680 Some("owner approval generation changed")
5681 } else if authorization.question.as_ref().is_some_and(|question| {
5682 !crate::base_recovery_control::decision_current(
5683 &crate::queue::Queue::at(crate::run::home().join("queue")),
5684 &self.state,
5685 &authorization.task,
5686 question,
5687 )
5688 }) {
5689 Some("recovery decision ownership, routing or stop evidence changed")
5690 } else if authorization.request.is_some()
5691 && !owner
5692 .as_ref()
5693 .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5694 {
5695 Some("owner decision changed")
5696 } else if let Some(scope) = &authorization.scope {
5697 if scope.tip != tip {
5698 Some("fetched base changed")
5699 } else if !self
5700 .state
5701 .base_sync_recovery
5702 .episodes
5703 .last()
5704 .is_some_and(|e| e.id == scope.episode)
5705 {
5706 Some("active synchronization interval changed")
5707 } else if !owner
5708 .as_ref()
5709 .is_ok_and(|t| t.instruction == scope.instruction)
5710 {
5711 Some("task instruction changed")
5712 } else {
5713 None
5714 }
5715 } else {
5716 None
5717 };
5718 if let Some(reason) = invalidation {
5719 let authorization = self
5720 .state
5721 .base_sync_recovery
5722 .authorizations
5723 .last_mut()
5724 .unwrap();
5725 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5726 self.state.save()?;
5727 return Ok(());
5728 }
5729 if !fetched_base
5730 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5731 || !git::is_clean(&winner.worktree).await?
5732 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5733 {
5734 return Ok(());
5735 }
5736 }
5737 let recovery = conflicted && self.state.consume_base_recovery(&head);
5738 if conflicted {
5739 self.state.save()?;
5741 if !recovery {
5742 return Ok(());
5743 }
5744 }
5745
5746 if behind == 0 {
5747 if let Some(from) = self
5756 .state
5757 .rebase_fixes
5758 .iter()
5759 .rev()
5760 .find_map(|r| r.from.clone())
5761 && from != head
5762 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5763 .await
5764 .is_ok_and(|o| o.ok())
5765 {
5766 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5767 }
5768 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5774 if let Some(why) = &conflict {
5775 self.state.status = RunStatus::Blocked;
5776 self.state.event("land", why.clone());
5777 }
5778 self.state.base_sync = Some(BaseSync {
5779 tip,
5780 behind: 0,
5781 attempts,
5782 conflict,
5783 already_in: None,
5784 });
5785 if pending.is_some() {
5786 self.state.finish_base_operation(Some(&head));
5787 }
5788 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5789 .await?;
5790 self.state.save()?;
5791 return Ok(());
5792 }
5793
5794 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5795 let why = format!(
5796 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5797 synchronization(s); synchronizing again would only race it",
5798 winner.branch
5799 );
5800 self.state.status = RunStatus::Blocked;
5801 self.state.base_sync = Some(BaseSync {
5802 tip,
5803 behind,
5804 attempts,
5805 conflict: Some(why.clone()),
5806 already_in: None,
5807 });
5808 self.state.event("land", why);
5809 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5810 .await?;
5811 self.state.save()?;
5812 return Ok(());
5813 }
5814
5815 if self.state.foreign_checkout.is_some() {
5816 let wait = repository_wait(&self.state, None);
5819 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5820 if self.state.base_sync.is_none() {
5821 self.state.base_sync = Some(BaseSync {
5822 tip: tip.clone(),
5823 behind,
5824 attempts: 0,
5825 conflict: None,
5826 already_in: None,
5827 });
5828 }
5829 self.state.reserve_base_operation(&head, &tip);
5830 self.state.save()?;
5831 let merged = git::git_raw(
5832 &winner.worktree,
5833 &[
5834 "-c",
5835 "core.hooksPath=/dev/null",
5836 "merge",
5837 "--no-ff",
5838 "--no-autostash",
5839 "--no-edit",
5840 "-m",
5841 &format!("Merge {tracking} before verifying"),
5842 &tip,
5843 ],
5844 )
5845 .await?;
5846 let conflict = if merged.ok() {
5847 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5848 self.state.event(
5849 "land",
5850 format!("merged {tracking} into {} before verifying", winner.branch),
5851 );
5852 None
5853 } else {
5854 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5855 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5856 }
5857 let why = format!(
5858 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5859 winner.branch,
5860 merged.stderr.chars().take(600).collect::<String>()
5861 );
5862 self.state.status = RunStatus::Blocked;
5863 self.state.event("land", &why);
5864 Some(why)
5865 };
5866 let synced = conflict.is_none();
5867 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5868 self.state
5869 .finish_base_operation(synced.then_some(fresh.as_str()));
5870 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5871 self.state.base_sync = Some(BaseSync {
5872 tip,
5873 behind: if synced { 0 } else { behind },
5874 attempts,
5875 conflict,
5876 already_in: None,
5877 });
5878 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5879 .await?;
5880 self.state.save()?;
5881 return Ok(());
5882 }
5883
5884 self.state.event(
5885 "land",
5886 format!(
5887 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5888 winner.branch
5889 ),
5890 );
5891 self.state.save()?;
5892
5893 let branch_tracking = format!("{remote}/{}", winner.branch);
5899 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5900 let remote_tip = if self.state.task_pr_branch().is_none()
5901 && matches!(&fetched_branch, Ok(o) if o.ok())
5902 {
5903 git::rev_parse(&repo, &branch_tracking).await.ok()
5904 } else {
5905 None
5906 };
5907 if let Some(theirs) = &remote_tip
5911 && !git::is_ancestor(&repo, theirs, &head).await
5912 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5913 .await
5914 .is_ok_and(|missing| missing.is_empty())
5915 {
5916 let why = format!(
5917 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5918 them",
5919 short(theirs),
5920 winner.branch
5921 );
5922 self.state.status = RunStatus::Blocked;
5923 self.state.base_sync = Some(BaseSync {
5924 tip,
5925 behind,
5926 attempts,
5927 conflict: Some(why.clone()),
5928 already_in: None,
5929 });
5930 self.state.event("land", why);
5931 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5932 .await?;
5933 self.state.save()?;
5934 return Ok(());
5935 }
5936
5937 if self.state.base_sync.is_none() {
5938 self.state.base_sync = Some(BaseSync {
5939 tip: tip.clone(),
5940 behind,
5941 attempts: 0,
5942 conflict: None,
5943 already_in: None,
5944 });
5945 }
5946 let scratch = self.state.dir().join("base-sync");
5947 if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5948 {
5949 if error.is::<crate::repo_lock::ResourceBlocked>() {
5950 return Ok(());
5951 }
5952 return Err(error);
5953 }
5954 self.state.reserve_base_operation(&head, &tip);
5955 self.state.save()?;
5956 let rebased =
5957 match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
5958 .await
5959 {
5960 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5961 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5962 Err(e) => Err(e),
5963 };
5964 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5965 let stop_kind = match &rebased {
5966 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5967 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5968 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5969 };
5970 match rebased {
5971 Ok(None) => {
5972 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5976 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5977 self.state.finish_base_operation(Some(&fresh));
5978 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5979 let mut conflict = None;
5980 if let Some(pinned) = &remote_tip {
5981 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5982 match pushed {
5983 Ok(o) if o.ok() => self.state.event(
5984 "land",
5985 format!("pushed rebased {} to {remote}", winner.branch),
5986 ),
5987 Ok(o) => {
5988 conflict = Some(format!(
5989 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5990 winner.branch,
5991 short(pinned),
5992 o.stderr.chars().take(600).collect::<String>()
5993 ));
5994 }
5995 Err(e) => {
5996 conflict = Some(format!(
5997 "rebased {} locally but could not push it: {e:#}",
5998 winner.branch
5999 ));
6000 }
6001 }
6002 }
6003 if let Some(why) = &conflict {
6004 self.state.status = RunStatus::Blocked;
6005 self.state.event("land", why.clone());
6006 }
6007 self.state.base_sync = Some(BaseSync {
6008 tip: tip.clone(),
6009 behind: 0,
6010 attempts,
6011 conflict,
6012 already_in: None,
6013 });
6014 self.state
6015 .event("land", format!("rebased {} onto {tracking}", winner.branch));
6016 }
6017 Ok(Some(conflict)) => {
6018 self.state.finish_base_operation(None);
6019 let why = format!(
6020 "{} conflicts with {tracking} and did not rebase: {}",
6021 winner.branch,
6022 conflict.chars().take(600).collect::<String>()
6023 );
6024 self.state.status = RunStatus::Blocked;
6025 self.state.base_sync = Some(BaseSync {
6026 tip,
6027 behind,
6028 attempts,
6029 conflict: Some(why.clone()),
6030 already_in: None,
6031 });
6032 self.state.event("land", why);
6033 }
6034 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6038 self.state.save()?;
6039 return Ok(());
6040 }
6041 Err(e) => {
6042 self.state.finish_base_operation(None);
6043 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6044 self.state.status = RunStatus::Blocked;
6045 self.state.base_sync = Some(BaseSync {
6046 tip,
6047 behind,
6048 attempts,
6049 conflict: Some(why.clone()),
6050 already_in: None,
6051 });
6052 self.state.event("land", why);
6053 }
6054 }
6055 if !self.state.base_sync_blocked() {
6056 self.state.base_sync_recovery.stop = None;
6057 }
6058 self.hold_base_sync_kind(stop_kind).await?;
6059 self.state.save()?;
6060 Ok(())
6061 }
6062
6063 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6069 if self.state.task_pr_branch().is_some() {
6072 return None;
6073 }
6074 let repo = self.state.repo.clone();
6075 let remote = self.state.config.merge.remote.clone();
6076 let fetched = git::fetch(&repo, &remote, branch).await;
6077 if !matches!(&fetched, Ok(o) if o.ok()) {
6078 return None;
6079 }
6080 let branch_tracking = format!("{remote}/{branch}");
6081 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6082 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6083 return None;
6084 }
6085 if !crate::reconcile::origin_missing(&repo, head, &theirs)
6086 .await
6087 .is_ok_and(|missing| missing.is_empty())
6088 {
6089 return Some(format!(
6090 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6091 them",
6092 short(&theirs)
6093 ));
6094 }
6095 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6096 Ok(o) if o.ok() => {
6097 self.state
6098 .event("land", format!("pushed rebased {branch} to {remote}"));
6099 None
6100 }
6101 Ok(o) => Some(format!(
6102 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6103 someone may have pushed): {}",
6104 short(&theirs),
6105 o.stderr.chars().take(600).collect::<String>()
6106 )),
6107 Err(e) => Some(format!(
6108 "{branch} is rebased locally but could not be pushed: {e:#}"
6109 )),
6110 }
6111 }
6112
6113 async fn settle_already_in(
6130 &mut self,
6131 branch: &str,
6132 tip: &str,
6133 head: &str,
6134 attempts: usize,
6135 behind: usize,
6136 ) -> Result<bool> {
6137 let repo = self.state.repo.clone();
6138 let remote = self.state.config.merge.remote.clone();
6139 let start = self.state.base_commit.clone();
6140 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6141 Ok(Some(e)) => e,
6142 Ok(None) => return Ok(false),
6143 Err(e) => {
6144 tracing::warn!("already-in-base check for {branch}: {e:#}");
6145 return Ok(false);
6146 }
6147 };
6148 let mut verified = vec![head.to_owned()];
6149 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6150 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6151 if matches!(&fetched, Ok(o) if o.ok())
6152 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6153 && theirs != head
6154 {
6155 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6156 Ok(Some(_)) => verified.push(theirs),
6157 _ => return Ok(false),
6158 }
6159 }
6160 let base_branch = self.state.base_branch.clone();
6161 let message = format!(
6162 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6163 land",
6164 evidence.names(),
6165 evidence.proof.as_str()
6166 );
6167 let closed =
6168 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6169 .await;
6170 match closed {
6171 Ok(Ok(url)) => self
6172 .state
6173 .event("land", format!("closed {url}: superseded on {base_branch}")),
6174 Ok(Err(why)) => self
6175 .state
6176 .event("land", format!("did not close a pull request: {why}")),
6177 Err(e) => {
6178 let why = format!(
6179 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6180 not be closed ({e:#}); resume to retry"
6181 );
6182 self.state.status = RunStatus::Blocked;
6183 self.state.base_sync = Some(BaseSync {
6184 tip: tip.to_owned(),
6185 behind,
6186 attempts,
6187 conflict: Some(why.clone()),
6188 already_in: None,
6189 });
6190 self.state.event("land", why);
6191 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6192 .await?;
6193 self.state.save()?;
6194 return Ok(true);
6195 }
6196 }
6197 self.state.status = RunStatus::AlreadyInBase;
6198 self.state.base_sync_recovery.stop = None;
6199 self.state.base_sync = Some(BaseSync {
6200 tip: tip.to_owned(),
6201 behind,
6202 attempts,
6203 conflict: None,
6204 already_in: Some(evidence),
6205 });
6206 self.state.event("land", message);
6207 self.state.save()?;
6208 self.settle_questions();
6209 Ok(true)
6210 }
6211
6212 fn landing_base(&self) -> String {
6222 self.state
6223 .base_sync
6224 .as_ref()
6225 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6226 }
6227
6228 pub async fn fix_selected(
6261 &mut self,
6262 ids: &[String],
6263 reason: &str,
6264 allow_stale: bool,
6265 ) -> Result<()> {
6266 let reason = reason.trim();
6267 if reason.is_empty() {
6268 bail!("a fix request needs a reason — that is the operator's own record of why");
6269 }
6270 if ids.is_empty() {
6271 bail!("no finding id given");
6272 }
6273 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6274 bail!(
6275 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6276 has already concluded — can be given a targeted fix. A run still \
6277 in progress should simply be resumed; a `merged` run's branch has \
6278 already landed, so its answer is a fresh `magi review <branch>`, \
6279 not reopening this run's own record",
6280 self.state.id,
6281 self.state.status.as_str()
6282 );
6283 }
6284 let _claim = FixClaim::acquire(&self.state.dir())?;
6287 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6288 let Some(winner) = self.state.winner().cloned() else {
6289 bail!("run {} has no winning candidate to fix", self.state.id);
6290 };
6291 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6292 bail!(
6293 "branch `{}` no longer exists; this run cannot be extended",
6294 winner.branch
6295 );
6296 }
6297 let home = crate::run::home();
6298 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6299 bail!(
6300 "run {} is currently being worked on by another magi process",
6301 self.state.id
6302 );
6303 }
6304 let mut seen = BTreeSet::new();
6308 let mut findings = Vec::new();
6309 let mut missing = Vec::new();
6310 for id in ids {
6311 if !seen.insert(id.clone()) {
6312 continue;
6313 }
6314 match self.state.finding(id) {
6315 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6316 id: f.id.clone(),
6317 severity: f.severity,
6318 reviewer_vote: rec.vote,
6319 round: round.round,
6320 round_head: round.head.clone(),
6321 reviewer: rec.reviewer,
6322 agent: rec.agent.clone(),
6323 file: f.file.clone(),
6324 line: f.line,
6325 title: f.title.clone(),
6326 detail: f.detail.clone(),
6327 outcome: OperatorFixOutcome::Pending,
6328 }),
6329 None => missing.push(id.clone()),
6330 }
6331 }
6332 if !missing.is_empty() {
6333 bail!(
6334 "unknown finding id(s): {}; nothing was changed",
6335 missing.join(", ")
6336 );
6337 }
6338
6339 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6340 let stale_details: Vec<(String, String)> = findings
6341 .iter()
6342 .filter(|f| f.round_head != head_at_request)
6343 .map(|f| (f.id.clone(), f.round_head.clone()))
6344 .collect();
6345 let stale = !stale_details.is_empty();
6346 if stale && !allow_stale {
6347 bail!(
6348 "the branch has moved since some finding(s) were raised — {} — now \
6349 at {}; pass --allow-stale to fix anyway, or re-run review first",
6350 stale_details
6351 .iter()
6352 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6353 .collect::<Vec<_>>()
6354 .join(", "),
6355 short(&head_at_request)
6356 );
6357 }
6358
6359 let request = OperatorFixRequest {
6360 requested_at: Timestamp::now(),
6361 reason: reason.to_owned(),
6362 findings,
6363 head_at_request: head_at_request.clone(),
6364 allow_stale,
6365 stale,
6366 fix: None,
6367 result_head: None,
6368 follow_up_review_run: None,
6369 };
6370 self.state.event(
6371 "fix",
6372 format!(
6373 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6374 request.findings.len(),
6375 request
6376 .findings
6377 .iter()
6378 .map(|f| f.id.as_str())
6379 .collect::<Vec<_>>()
6380 .join(", "),
6381 ),
6382 );
6383 self.state.operator_fixes.push(request);
6390 self.state.save()?;
6391 let request_index = self.state.operator_fixes.len() - 1;
6392
6393 let repo = self.state.repo.clone();
6402 let coordination = self.coordinate(&repo).await?;
6403 if winner.worktree.exists() {
6404 let dirty = git::git(
6407 &winner.worktree,
6408 &["status", "--porcelain", "--untracked-files=all"],
6409 )
6410 .await?;
6411 let only_withheld = dirty.lines().all(|l| {
6412 l.strip_prefix("?? ")
6413 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6414 });
6415 if !only_withheld {
6416 bail!(
6417 "`{}` has uncommitted changes; refusing to touch it — commit or \
6418 discard them first",
6419 winner.worktree.display()
6420 );
6421 }
6422 git::worktree_remove(&self.state.repo, &winner.worktree)
6423 .await
6424 .ok();
6425 }
6426 let fix_worktree = self.state.worktree_root().join("operator-fix");
6427 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6428 git::git(
6429 &self.state.repo,
6430 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6431 )
6432 .await
6433 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6434 if !git::is_clean(&fix_worktree).await? {
6435 git::worktree_remove(&self.state.repo, &fix_worktree)
6436 .await
6437 .ok();
6438 bail!(
6439 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6440 winner.branch
6441 );
6442 }
6443 let repo_for_setup = self.state.repo.clone();
6444 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6445 Ok(()) => {
6446 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6447 }
6448 Err(e) => Err(e),
6449 };
6450 if let Err(e) = setup {
6451 git::worktree_remove(&self.state.repo, &fix_worktree)
6452 .await
6453 .ok();
6454 return Err(e);
6455 }
6456
6457 self.state.resource_block = None;
6460
6461 let run_id = self.state.id.clone();
6462 let prompts = self.state.config.prompts.clone();
6463 let language = self.state.config.graph.language.clone();
6464 let sessions = self.state.config.graph.sessions;
6465 let artifacts = agent::artifacts_dir(&self.state.dir());
6466 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6467 .findings
6468 .iter()
6469 .map(|f| Finding {
6470 id: f.id.clone(),
6471 severity: f.severity,
6472 file: f.file.clone(),
6473 line: f.line,
6474 title: f.title.clone(),
6475 detail: f.detail.clone(),
6476 })
6477 .collect();
6478 let fix_prompt = prompt::operator_fix_in(
6479 &self.state.instruction,
6480 &finding_list,
6481 reason,
6482 &stale_details,
6483 &head_at_request,
6484 &language,
6485 &self.state.config.graph.github_language,
6486 );
6487 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6488 drop(coordination);
6489 let (job, seat, out) = self
6490 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6491 prompt: fix_prompt.clone(),
6492 spec,
6493 seat,
6494 cwd: fix_worktree.clone(),
6495 timeout,
6496 allow_write: true,
6497 sessions,
6498 artifacts: artifacts.clone(),
6499 stem: "operator-fix".to_owned(),
6500 handover: None,
6501 })
6502 .await;
6503 let agent_id = seat.agent.clone();
6504
6505 let mut fix = FixRecord {
6506 agent: agent_id,
6507 addressed: Vec::new(),
6508 rejected: Vec::new(),
6509 notes: String::new(),
6510 committed: false,
6511 failed: None,
6512 duration_ms: 0,
6513 continuation: None,
6514 };
6515 let mut final_seat = seat.clone();
6516 match out {
6517 AgentOutcome::Ok(o) => {
6518 fix.duration_ms = o.duration_ms;
6519 let parsed = verdict::extract_json::<FixReport>(&o.text);
6520 let incomplete_reason = match &parsed {
6521 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6522 "the reply parsed, but it reported a command whose own CLI \
6523 never confirmed an exit status"
6524 .to_owned(),
6525 ),
6526 Ok(_) => None,
6527 Err(e) => Some(e.to_string()),
6528 };
6529 match incomplete_reason {
6530 None => {
6531 let report = parsed.expect("checked Ok above");
6532 fix.addressed = report.addressed;
6533 fix.rejected = report.rejected;
6534 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6535 }
6536 Some(reason) => {
6537 let (resumed_seat, resolved, failure, cont) = self
6538 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6539 .await;
6540 fix.duration_ms += cont.cumulative_wait_ms;
6541 fix.continuation = Some(cont);
6542 final_seat = resumed_seat;
6543 match resolved {
6544 Some(report) => {
6545 fix.addressed = report.addressed;
6546 fix.rejected = report.rejected;
6547 fix.notes =
6548 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6549 }
6550 None => fix.failed = failure,
6551 }
6552 }
6553 }
6554 }
6555 AgentOutcome::Dropped(o) => {
6556 fix.duration_ms = o.duration_ms;
6557 let why = o
6558 .dropped
6559 .as_ref()
6560 .map(|d| d.why.as_str())
6561 .unwrap_or("the CLI ended the stream without delivering its answer");
6562 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6563 }
6564 AgentOutcome::Quota(o) => {
6565 self.state.quota.push(QuotaLoss::from_report(
6566 final_seat.key.clone(),
6567 "fix",
6568 o.quota.as_ref(),
6569 ));
6570 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6571 }
6572 AgentOutcome::Failed(e) => fix.failed = Some(e),
6573 AgentOutcome::ResourceBlocked(e) => {
6574 fix.failed = Some(format!("the fixer was not started: {e}"));
6575 }
6576 }
6577 if fix.continuation.is_none() {
6580 fix.continuation = Some(ContinuationRecord::not_needed());
6581 }
6582 self.state.seats.insert(final_seat.key.clone(), final_seat);
6583
6584 let rescue_message = format!(
6585 "magi: operator-selected fix ({}) (uncommitted work)",
6586 self.state.operator_fixes[request_index]
6587 .findings
6588 .iter()
6589 .map(|f| f.id.as_str())
6590 .collect::<Vec<_>>()
6591 .join(", ")
6592 );
6593 let repo = self.state.repo.clone();
6594 let coordination = self.coordinate(&repo).await?;
6595 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6596 self.state.note_withheld("fix", &r.withheld);
6597 }
6598 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6599 fix.committed = after != head_at_request;
6600 git::worktree_remove(&self.state.repo, &fix_worktree)
6601 .await
6602 .ok();
6603 drop(coordination);
6604
6605 self.state.event(
6606 "fix",
6607 match &fix.failed {
6608 Some(reason) => format!(
6609 "operator fix: adoption report was lost ({reason}); {}",
6610 if fix.committed {
6611 "committed"
6612 } else {
6613 "NO new commit"
6614 }
6615 ),
6616 None => format!(
6617 "operator fix: {} addressed, {} rejected, {}",
6618 fix.addressed.len(),
6619 fix.rejected.len(),
6620 if fix.committed {
6621 "committed"
6622 } else {
6623 "NO new commit"
6624 }
6625 ),
6626 },
6627 );
6628
6629 for f in &mut self.state.operator_fixes[request_index].findings {
6636 f.outcome = if fix.failed.is_some() {
6637 OperatorFixOutcome::Unreported
6638 } else if fix.addressed.contains(&f.id) {
6639 OperatorFixOutcome::Addressed
6640 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6641 OperatorFixOutcome::Rejected { why: r.why.clone() }
6642 } else {
6643 OperatorFixOutcome::Unreported
6644 };
6645 }
6646
6647 let committed = fix.committed;
6648 if committed {
6649 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6650 }
6651 self.state.operator_fixes[request_index].fix = Some(fix);
6652 self.state.save()?;
6655 if self.state.resource_block.is_some() {
6656 return self.stop_for_resource();
6657 }
6658
6659 if committed {
6660 self.state.event(
6661 "fix",
6662 format!(
6663 "operator fix committed {}; opening a follow-up review-only run",
6664 short(&after)
6665 ),
6666 );
6667 let origin =
6670 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6671 match Self::review(
6672 &self.state.repo,
6673 &winner.branch,
6674 self.state.config.clone(),
6675 origin,
6676 )
6677 .await
6678 {
6679 Ok(mut follow_up) => {
6680 follow_up.state.event(
6681 "start",
6682 format!(
6683 "requested by an operator fix on run {} for finding(s) {}",
6684 self.state.id,
6685 self.state.operator_fixes[request_index]
6686 .findings
6687 .iter()
6688 .map(|f| f.id.as_str())
6689 .collect::<Vec<_>>()
6690 .join(", "),
6691 ),
6692 );
6693 follow_up.state.save()?;
6694 let follow_up_id = follow_up.state.id.clone();
6695 let adopted = match crate::direct::adopt(
6698 &follow_up.state,
6699 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6700 ) {
6701 Ok(a) => a,
6702 Err(e) => {
6703 self.state.event(
6706 "fix",
6707 format!(
6708 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6709 ),
6710 );
6711 self.state.save()?;
6712 return Ok(());
6713 }
6714 };
6715 let executed = follow_up.execute().await;
6716 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6717 if let Some(a) = adopted {
6718 a.finish(&follow_up.state, executed);
6719 }
6720 if let Some(e) = failure {
6721 self.state.event(
6722 "fix",
6723 format!(
6724 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6725 ),
6726 );
6727 }
6728 self.state.operator_fixes[request_index].follow_up_review_run =
6729 Some(follow_up_id);
6730 }
6731 Err(e) => {
6732 self.state.event(
6733 "fix",
6734 format!("committed the fix but could not open a follow-up review: {e:#}"),
6735 );
6736 }
6737 }
6738 self.state.save()?;
6739 }
6740
6741 Ok(())
6742 }
6743
6744 async fn ask_fixer(
6764 &mut self,
6765 winner: &Candidate,
6766 node: &'static str,
6767 round: Option<usize>,
6768 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6769 ) -> (SeatJob, SeatState, AgentOutcome) {
6770 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6771 self.ask_fixer_among(attempts, node, round, None, build)
6772 .await
6773 }
6774
6775 async fn ask_fixer_among(
6780 &mut self,
6781 attempts: Vec<(AgentSpec, String)>,
6782 node: &'static str,
6783 round: Option<usize>,
6784 restore: Option<(&Path, &str)>,
6785 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6786 ) -> (SeatJob, SeatState, AgentOutcome) {
6787 let run_id = self.state.id.clone();
6788 let prompts = self.state.config.prompts.clone();
6789 let cache = self.state.config.cache_dir();
6790 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6791 let mut last = None;
6792 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6793 let seat = self.seat(&key, &spec.id);
6794 let mut job = build(spec.clone(), seat);
6795 if i > 0 {
6796 job.stem = format!("{}-{}", job.stem, spec.id);
6797 }
6798 let ctx = WaveCtx {
6799 carry_seats: false,
6800 pauses: Some((&self.pause, &self.interrupt)),
6801 run: &run_id,
6802 node,
6803 prompts: &prompts,
6804 cache: cache.as_deref(),
6805 round,
6806 };
6807 let (seat, out) =
6808 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6809 let advances = match &out {
6812 AgentOutcome::Ok(o) => agent::output_advances(o),
6813 AgentOutcome::ResourceBlocked(_) => false,
6814 _ => true,
6815 };
6816 if let Some(next) = ids.get(i + 1)
6817 && advances
6818 {
6819 self.state.seats.insert(seat.key.clone(), seat.clone());
6820 let class =
6821 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6822 let reason = match &out {
6823 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6824 other => fail_reason(other),
6825 };
6826 if let Some((wt, rev)) = restore
6830 && let Err(e) = self.graft_restore(wt, rev).await
6831 {
6832 let why =
6833 format!("could not restore the worktree before the next fixer: {e:#}");
6834 tracing::warn!("{why}");
6835 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6836 AgentOutcome::ResourceBlocked(why)
6837 } else {
6838 AgentOutcome::Failed(why)
6839 };
6840 last = Some((job, seat, out));
6841 break;
6842 }
6843 record_handover(
6844 &mut self.state,
6845 node,
6846 &seat.key,
6847 &spec.id,
6848 next,
6849 &class,
6850 &reason,
6851 );
6852 continue;
6853 }
6854 last = Some((job, seat, out));
6855 break;
6856 }
6857 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6858 if let AgentOutcome::Quota(output) = &out
6859 && node != "graft"
6860 {
6861 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6866 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6867 self.state.quota.push(loss);
6868 self.state.seats.insert(seat.key.clone(), seat.clone());
6869 self.state.status = RunStatus::Blocked;
6870 return (
6871 job,
6872 seat,
6873 AgentOutcome::ResourceBlocked(
6874 "fixer unavailable; restore agent availability and resume".into(),
6875 ),
6876 );
6877 }
6878 (job, seat, out)
6879 }
6880
6881 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6892 use crate::recovery::{Mode, Repair};
6893 if !self.state.reviews.is_empty() {
6894 return Ok(());
6895 }
6896 let Some(mut rec) = self.state.recovery.clone() else {
6897 return Ok(());
6898 };
6899 if rec.mode != Mode::VerifyRepair {
6900 return Ok(());
6901 }
6902 let wt = winner.worktree.clone();
6903 let mut head = git::rev_parse(&wt, "HEAD").await?;
6904 match rec.repair.clone() {
6905 Repair::Done { .. } => return Ok(()),
6906 Repair::WaitingForRestore { before } => {
6907 self.graft_restore(&wt, &before).await?;
6908 head = before;
6909 rec.repair = Repair::NotStarted;
6910 self.state.recovery = Some(rec.clone());
6911 self.state.save()?;
6912 }
6913 Repair::Started { before } => {
6914 if let Err(e) = self.graft_restore(&wt, &before).await {
6918 if e.is::<crate::repo_lock::ResourceBlocked>() {
6919 return Err(e);
6920 }
6921 return self
6922 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6923 .await;
6924 }
6925 rec.repair = Repair::Done {
6926 before: before.clone(),
6927 after: before,
6928 committed: false,
6929 failed: Some("interrupted; rolled back".to_owned()),
6930 };
6931 self.state
6932 .event("recovery", "the repair was interrupted and rolled back");
6933 self.state.recovery = Some(rec);
6934 self.state.save()?;
6935 return Ok(());
6936 }
6937 Repair::NotStarted => {}
6938 }
6939 if crate::recovery::recheck(&mut rec, &head) {
6940 self.state.event("recovery", rec.summary());
6941 self.state.recovery = Some(rec);
6942 self.state.save()?;
6943 return Ok(());
6944 }
6945 rec.repair = Repair::Started {
6946 before: head.clone(),
6947 };
6948 self.state.recovery = Some(rec.clone());
6949 self.state.save()?;
6950
6951 let language = self.state.config.graph.language.clone();
6952 let sessions = self.state.config.graph.sessions;
6953 let artifacts = agent::artifacts_dir(&self.state.dir());
6954 let fix_prompt = prompt::recovery_fix(
6955 &self.state.instruction,
6956 &rec,
6957 self.state.config.graph.review_rounds,
6958 &language,
6959 );
6960 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6961 self.state.event(
6962 "recovery",
6963 format!(
6964 "repair: the fixer starts from the failed verification on {}",
6965 short(&head)
6966 ),
6967 );
6968 let (_job, seat, out) = self
6969 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6970 prompt: fix_prompt.clone(),
6971 spec,
6972 seat,
6973 cwd: wt.clone(),
6974 timeout,
6975 allow_write: true,
6976 sessions,
6977 artifacts: artifacts.clone(),
6978 stem: "fix-recovery".to_owned(),
6979 handover: None,
6980 })
6981 .await;
6982 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6983 rec.repair = Repair::WaitingForRestore { before: head };
6984 self.state.recovery = Some(rec);
6985 return self.stop_for_resource();
6986 }
6987 let failed = match &out {
6988 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6989 AgentOutcome::Quota(o) => {
6990 self.state.quota.push(QuotaLoss::from_report(
6991 seat.key.clone(),
6992 "fix",
6993 o.quota.as_ref(),
6994 ));
6995 Some("rate limited (quota); the fixer could not run".to_owned())
6996 }
6997 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6998 AgentOutcome::Failed(e) => Some(e.clone()),
6999 };
7000 self.state.seats.insert(seat.key.clone(), seat);
7001 let coordination = self.coordinate(&wt).await?;
7002 let failed = match failed {
7003 Some(why) => {
7005 if let Err(e) = git::restore_to(&wt, &head).await {
7006 return self
7007 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7008 .await;
7009 }
7010 Some(why)
7011 }
7012 None => {
7013 if let Ok(r) = git::rescue_commit(
7014 &wt,
7015 "magi: recovery repair of the failed verification (uncommitted work)",
7016 )
7017 .await
7018 {
7019 self.state.note_withheld("fix", &r.withheld);
7020 }
7021 None
7022 }
7023 };
7024 let after = git::rev_parse(&wt, "HEAD").await?;
7025 drop(coordination);
7026 self.state.event(
7027 "recovery",
7028 match &failed {
7029 Some(why) => format!("repair: nothing kept ({why})"),
7030 None if after != head => {
7031 format!("repair: committed {}", short(&after))
7032 }
7033 None => "repair: the fixer made no commit".to_owned(),
7034 },
7035 );
7036 rec.repair = Repair::Done {
7037 before: head.clone(),
7038 committed: after != head,
7039 after,
7040 failed,
7041 };
7042 self.state.recovery = Some(rec);
7043 self.state.save()?;
7044 Ok(())
7045 }
7046
7047 async fn stop_recovery(
7050 &mut self,
7051 mut rec: crate::recovery::RecoveryRecord,
7052 head: &str,
7053 why: String,
7054 ) -> Result<()> {
7055 rec.repair = crate::recovery::Repair::Started {
7057 before: head.to_owned(),
7058 };
7059 self.state.recovery = Some(rec);
7060 self.state
7061 .event("recovery", format!("repair could not be undone: {why}"));
7062 self.state.status = RunStatus::Blocked;
7063 self.state.save()?;
7064 bail!("recovery repair could not be rolled back: {why}")
7065 }
7066
7067 async fn review_loop(&mut self) -> Result<()> {
7068 if self.hold_base_sync().await? {
7069 return Ok(());
7070 }
7071 loop {
7072 if self
7073 .state
7074 .base_sync
7075 .as_ref()
7076 .is_some_and(|s| s.conflict.is_some())
7077 {
7078 return Ok(());
7079 }
7080 if self.catchup_repairable() {
7081 let record = self.state.reviews.last().expect("repairable round").clone();
7082 let winner = self
7083 .state
7084 .winner()
7085 .cloned()
7086 .context("catch-up repair has no winner")?;
7087 if !self.fix_review_round(&winner, record).await? {
7088 return Ok(());
7089 }
7090 }
7091 self.review_loop_pass().await?;
7092 if !self.catchup_repairable() {
7093 return Ok(());
7094 }
7095 }
7096 }
7097
7098 fn catchup_repairable(&self) -> bool {
7101 self.state.reviews.len() < self.state.config.graph.review_rounds
7102 && self.state.reviews.last().is_some_and(|r| {
7103 !r.clean
7104 && r.blocking == 0
7105 && r.fix.is_none()
7106 && r.e2e_status() == E2eStatus::Failed
7107 })
7108 }
7109
7110 async fn review_loop_pass(&mut self) -> Result<()> {
7111 if self
7116 .state
7117 .base_sync
7118 .as_ref()
7119 .is_some_and(|s| s.conflict.is_some())
7120 {
7121 return Ok(());
7122 }
7123 let run_id = self.state.id.clone();
7128 let prompts = self.state.config.prompts.clone();
7129 let Some(winner) = self.state.winner().cloned() else {
7130 return Ok(());
7131 };
7132 let granted_review = self
7133 .state
7134 .repair_grants
7135 .last()
7136 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7137 .cloned();
7138 let fresh_review = granted_review
7139 .as_ref()
7140 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7141 let max_rounds = granted_review
7142 .as_ref()
7143 .map_or(self.state.config.graph.review_rounds, |g| {
7144 g.review_start.saturating_add(1)
7145 });
7146 if let Some(status) =
7156 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7157 {
7158 if status == RunStatus::Gating {
7161 self.record_contested_handoff();
7162 }
7163 self.state.status = status;
7164 self.state.save()?;
7165 return Ok(());
7166 }
7167 self.state.status = RunStatus::Reviewing;
7168 if !fresh_review
7178 && self.state.reviews.last().is_some_and(|r| {
7179 r.e2e_status() == E2eStatus::ResourceBlocked
7180 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7181 })
7182 {
7183 let shell = self.state.config.shell();
7184 return self
7185 .stop_unresolved_review(
7186 "the last round's own verification never resolved",
7187 &shell,
7188 &winner.worktree,
7189 )
7190 .await;
7191 }
7192
7193 if granted_review.is_none() {
7194 self.recovery_repair(&winner).await?;
7195 }
7196 if self.resource_halt()? {
7197 return Ok(());
7198 }
7199
7200 let repo = self.state.repo.clone();
7201 let root = self.state.worktree_root();
7202 let language = self.state.config.graph.language.clone();
7203 let sessions = self.state.config.graph.sessions;
7204 let artifacts = self.merge_repair_artifacts();
7205 let base = self.landing_base();
7206 let base_short = short(&base);
7207 let reviewers = self.roles.reviewers.clone();
7208 let shell = self.state.config.shell();
7209
7210 for round in (self.state.reviews.len() + 1)..=max_rounds {
7211 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7212 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7213 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7214 let prev_verification = self
7223 .state
7224 .reviews
7225 .last()
7226 .and_then(|r| r.verification_summary(&head));
7227 let prior_section = match (&self.state.recovery, round) {
7230 (Some(rec), 1) if self.state.reviews.is_empty() => {
7231 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7232 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7233 Some(crate::recovery::head_of(
7234 &git::diff(&winner.worktree, reviewed, "HEAD")
7235 .await
7236 .unwrap_or_default(),
7237 crate::recovery::SINCE_DIFF_MAX,
7238 ))
7239 }
7240 _ => None,
7241 };
7242 Some(prompt::recovery_review(rec, since.as_deref()))
7243 }
7244 _ => None,
7245 };
7246
7247 let mut jobs = Vec::new();
7251 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7252 self.coordinate(&repo).await?
7253 } else {
7254 None
7255 };
7256 for (r, spec) in reviewers.iter().cloned().enumerate() {
7257 let wt = root.join(format!("review-{}", r + 1));
7258 if wt.exists() {
7261 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7262 } else {
7263 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7264 None
7265 } else {
7266 self.coordinate(&repo).await?
7267 };
7268 git::worktree_add_detached(&repo, &wt, &head).await?;
7269 }
7270 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7271 None
7272 } else {
7273 self.coordinate(&repo).await?
7274 };
7275 self.setup_seat_worktree(&repo, &wt, false).await?;
7276 let seat_key = format!("review-{}", r + 1);
7277 let spec = pick_start_spec(
7280 &self.roles.reviewer_roster,
7281 spec,
7282 self.state.seat_history.get(&seat_key),
7283 );
7284 let seat = self.seat(&seat_key, &spec.id);
7285 jobs.push(SeatJob {
7286 prompt: prompt::review(&prompt::ReviewCtx {
7287 instruction: &self.state.instruction,
7288 branch: &winner.branch,
7289 base_short: &base_short,
7290 stat: &stat,
7291 patch: &patch,
7292 verification: prev_verification.as_ref(),
7293 prior: prior_section.as_deref(),
7294 reviewers: reviewers.len(),
7295 round,
7296 rounds: max_rounds,
7297 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7300 lens: Lens::for_seat(r),
7301 language: &language,
7302 github_language: &self.state.config.graph.github_language,
7303 }),
7304 spec,
7305 seat,
7306 cwd: wt,
7307 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7308 allow_write: false,
7309 sessions,
7310 artifacts: artifacts.clone(),
7311 stem: format!("review-{round}-{}", r + 1),
7312 handover: None,
7313 });
7314 }
7315
7316 drop(coordination);
7317 self.state.event(
7318 "review",
7319 format!(
7320 "round {round}: {} reviewers on {}",
7321 jobs.len(),
7322 short(&head)
7323 ),
7324 );
7325 let mut quota_losses = Vec::new();
7326 let review_retries = self.state.config.graph.retries;
7327 let review_cache = self.state.config.cache_dir();
7328 let ctx = WaveCtx {
7329 carry_seats: true,
7330 pauses: Some((&self.pause, &self.interrupt)),
7331 run: &run_id,
7332 node: "review",
7333 prompts: &prompts,
7334 cache: review_cache.as_deref(),
7335 round: Some(round),
7336 };
7337 let results = ask_json_wave::<Review>(
7338 jobs,
7339 Arc::clone(&self.sem),
7340 review_retries,
7341 &self.roles.reviewer_roster,
7342 &ctx,
7343 &mut quota_losses,
7344 &mut self.state,
7345 &|_: &Review| Ok(()),
7346 )
7347 .await;
7348 if self.state.resource_block.is_some() {
7349 return self.stop_for_resource();
7350 }
7351 let round_quota_missing = quota_losses.len();
7355 self.state.quota.extend(quota_losses);
7356
7357 let mut records = Vec::new();
7358 let mut all_findings = Vec::new();
7359 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7360 let agent_id = seat.agent.clone();
7361 self.state.seats.insert(seat.key.clone(), seat);
7362 let mut record = ReviewRecord {
7363 reviewer: r + 1,
7364 agent: agent_id,
7365 summary: String::new(),
7366 findings: Vec::new(),
7367 vote: None,
7368 failed: None,
7369 duration_ms: 0,
7370 attempts,
7376 };
7377 match res {
7378 Ok((review, out)) => {
7379 record.summary =
7387 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7388 record.vote = Some(review.vote);
7389 record.duration_ms = out.duration_ms;
7390 for (n, mut f) in review.findings.into_iter().enumerate() {
7391 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7394 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7395 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7396 f.file = f
7402 .file
7403 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7404 all_findings.push(f.clone());
7405 record.findings.push(f);
7406 }
7407 self.state.event(
7408 "review",
7409 format!(
7410 "round {round}: reviewer {} voted {} with {} finding(s)",
7411 r + 1,
7412 review.vote.label(),
7413 record.findings.len()
7414 ),
7415 );
7416 }
7417 Err(e) => {
7418 record.failed = Some(e.to_string());
7419 self.state.event(
7420 "review",
7421 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7422 );
7423 }
7424 }
7425 records.push(record);
7426 }
7427
7428 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7435 let vote_split =
7436 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7437 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7438 if vote_split {
7439 self.state.event(
7440 "review",
7441 format!(
7442 "round {round}: votes split ({}) — one round of reconsideration",
7443 initial_votes
7444 .iter()
7445 .map(|v| v.label())
7446 .collect::<Vec<_>>()
7447 .join(", ")
7448 ),
7449 );
7450 let panel: Vec<ReviewSeatReport<'_>> = records
7453 .iter()
7454 .filter_map(|r| {
7455 r.vote.map(|vote| ReviewSeatReport {
7456 reviewer: r.reviewer,
7457 vote,
7458 summary: &r.summary,
7459 findings: &r.findings,
7460 })
7461 })
7462 .collect();
7463
7464 let mut jobs = Vec::new();
7465 let mut seats_at = Vec::new();
7466 for (r, spec) in reviewers.iter().cloned().enumerate() {
7467 if records[r].vote.is_none() {
7471 continue;
7472 }
7473 let wt = root.join(format!("review-{}", r + 1));
7474 let seat_key = format!("review-{}", r + 1);
7475 let spec = self.occupant(&seat_key, spec);
7476 let seat = self.seat(&seat_key, &spec.id);
7477 let build = |with_patch: bool| {
7485 prompt::review_reconsider(&ReviewReconsiderCtx {
7486 instruction: &self.state.instruction,
7487 reviewer: r + 1,
7488 lens: Lens::for_seat(r),
7489 panel: &panel,
7490 patch: with_patch.then_some(ReviewPatch {
7491 branch: &winner.branch,
7492 base_short: &base_short,
7493 stat: &stat,
7494 patch: &patch,
7495 }),
7496 round,
7497 rounds: max_rounds,
7498 language: &language,
7499 })
7500 };
7501 let full = build(true);
7502 let prompt = if has_context(&spec, &seat, sessions) {
7503 build(false)
7504 } else {
7505 full.clone()
7506 };
7507 jobs.push(SeatJob {
7508 prompt,
7509 spec,
7510 seat,
7511 cwd: wt,
7512 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7513 allow_write: false,
7514 sessions,
7515 artifacts: artifacts.clone(),
7516 stem: format!("review-{round}-reconsider-{}", r + 1),
7517 handover: Some(full),
7518 });
7519 seats_at.push(r);
7520 }
7521
7522 let mut recon_quota_losses = Vec::new();
7523 let recon_cache = self.state.config.cache_dir();
7524 let recon_ctx = WaveCtx {
7525 carry_seats: true,
7526 pauses: Some((&self.pause, &self.interrupt)),
7527 run: &run_id,
7528 node: "review",
7529 prompts: &prompts,
7530 cache: recon_cache.as_deref(),
7531 round: Some(round),
7532 };
7533 let recon_results = ask_json_wave::<ReviewRevote>(
7534 jobs,
7535 Arc::clone(&self.sem),
7536 review_retries,
7537 &self.roles.reviewer_roster,
7538 &recon_ctx,
7539 &mut recon_quota_losses,
7540 &mut self.state,
7541 &|_: &ReviewRevote| Ok(()),
7542 )
7543 .await;
7544 if self.state.resource_block.is_some() {
7545 return self.stop_for_resource();
7546 }
7547 self.state.quota.extend(recon_quota_losses);
7548
7549 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7550 let agent_id = seat.agent.clone();
7551 self.state.seats.insert(seat.key.clone(), seat);
7552 let mut rec = ReviewRevoteRecord {
7553 reviewer: r + 1,
7554 agent: agent_id,
7555 vote: None,
7556 reason: String::new(),
7557 failed: None,
7558 };
7559 match res {
7560 Ok((rv, _)) => {
7561 rec.vote = Some(rv.vote);
7562 rec.reason =
7563 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7564 self.state.event(
7565 "review",
7566 format!(
7567 "round {round}: reviewer {} revoted {}",
7568 r + 1,
7569 rv.vote.label()
7570 ),
7571 );
7572 }
7573 Err(e) => {
7574 rec.failed = Some(e.to_string());
7575 self.state.event(
7576 "review",
7577 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7578 );
7579 }
7580 }
7581 reconsideration.push(rec);
7582 }
7583 } else if initial_votes.len() > 1 {
7584 self.state.event(
7585 "review",
7586 format!(
7587 "round {round}: votes agreed ({}) — no reconsideration",
7588 initial_votes[0].label()
7589 ),
7590 );
7591 }
7592
7593 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7594 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7595 let defer_e2e =
7606 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7607 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7608 let reason =
7609 format!("{blocking} blocking finding(s) already required a fix this round");
7610 self.state.event(
7611 "verify",
7612 format!(
7613 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7614 {}); it will run once a round has none left",
7615 short(&head)
7616 ),
7617 );
7618 (Vec::new(), false, true, Some(reason))
7619 } else {
7620 let e2e_commands = self.state.config.verify.e2e.clone();
7621 let cache_dir = self.state.config.cache_dir();
7622 let context = format!("round {round}");
7623 let wait = self.repository_wait();
7624 let (e2e, verify_retried) = with_cache_lease(
7625 &mut self.state,
7626 cache_dir.as_deref(),
7627 "e2e",
7628 "e2e",
7629 &winner.worktree,
7630 &head,
7631 verify_timeout,
7632 &context,
7633 &wait,
7634 |state, budget| {
7635 let shell = shell.clone();
7636 let e2e_commands = e2e_commands.clone();
7637 let worktree = winner.worktree.clone();
7638 let context = context.clone();
7639 async move {
7640 run_e2e_with_retry(
7641 state,
7642 &shell,
7643 &e2e_commands,
7644 &worktree,
7645 budget,
7646 &context,
7647 )
7648 .await
7649 }
7650 },
7651 )
7652 .await?;
7653 (e2e, verify_retried, false, None)
7654 };
7655
7656 let expected = records.len();
7657 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7658 let incomplete = answered < expected;
7659 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7660 let policy = self.state.config.graph.incomplete_review;
7661 let clean = round_is_clean(
7662 blocking,
7663 e2e_ok,
7664 answered,
7665 expected,
7666 round_quota_missing,
7667 policy,
7668 );
7669
7670 let mut round_record = ReviewRound {
7671 round,
7672 head: head.clone(),
7673 verified_head: None,
7674 verified_at: None,
7675 reviews: records,
7676 e2e,
7677 verify_retried,
7678 e2e_deferred,
7679 e2e_defer_reason,
7680 fix: None,
7681 blocking,
7682 answered,
7683 expected,
7684 clean,
7685 progressed: false,
7686 vote_split,
7687 reconsideration,
7688 verdict: None,
7689 };
7690 round_record.verdict = ReviewVote::worst(
7694 round_record
7695 .final_votes()
7696 .into_iter()
7697 .map(|(_, _, vote)| vote),
7698 );
7699 if !matches!(
7710 round_record.e2e_status(),
7711 E2eStatus::Deferred | E2eStatus::NotConfigured
7712 ) {
7713 round_record.verified_head = Some(head.clone());
7714 round_record.verified_at = Some(Timestamp::now());
7715 }
7716
7717 if incomplete {
7718 let missing: Vec<String> = round_record
7719 .reviews
7720 .iter()
7721 .filter(|r| r.failed.is_some())
7722 .map(|r| format!("review-{}", r.reviewer))
7723 .collect();
7724 self.state.event(
7725 "review",
7726 format!(
7727 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7728 missing.join(", ")
7729 ),
7730 );
7731 }
7732
7733 if clean {
7734 self.state.event(
7735 "review",
7736 if incomplete && policy == IncompleteReviewPolicy::Warn {
7737 format!(
7738 "round {round}: clean (warn policy, incomplete panel) — no \
7739 blocking findings from the seats that answered, verification green"
7740 )
7741 } else if incomplete {
7742 format!(
7743 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7744 quorum) — no blocking findings from the seats that answered, \
7745 verification green",
7746 expected - answered
7747 )
7748 } else {
7749 format!("round {round}: clean — no blocking findings, verification green")
7750 },
7751 );
7752 self.state.reviews.push(round_record);
7753 self.state.status = RunStatus::Gating;
7754 self.state.save()?;
7755 return Ok(());
7756 }
7757
7758 if incomplete && blocking == 0 && e2e_ok {
7766 self.state.reviews.push(round_record);
7767 self.state.save()?;
7768 if round == max_rounds {
7769 self.state.status = RunStatus::Blocked;
7770 self.state.event(
7771 "review",
7772 format!(
7773 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7774 refusing to call it clean",
7775 expected - answered
7776 ),
7777 );
7778 return Ok(());
7779 }
7780 continue;
7781 }
7782
7783 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7795 self.state.reviews.push(round_record);
7796 return self
7797 .stop_unresolved_review(
7798 "the round's own verification could not run",
7799 &shell,
7800 &winner.worktree,
7801 )
7802 .await;
7803 }
7804
7805 if round == max_rounds {
7806 self.state.reviews.push(round_record);
7807 return self
7808 .stop_reviewing(
7809 &format!(
7810 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7811 ),
7812 &shell,
7813 &winner.worktree,
7814 )
7815 .await;
7816 }
7817
7818 if !self.fix_review_round(&winner, round_record).await? {
7819 return Ok(());
7820 }
7821 }
7822 Ok(())
7823 }
7824
7825 async fn fix_review_round(
7828 &mut self,
7829 winner: &Candidate,
7830 mut round_record: ReviewRound,
7831 ) -> Result<bool> {
7832 let round = round_record.round;
7833 let shell = self.state.config.shell();
7834 let step = self.run_fix_round(winner, &mut round_record).await?;
7835 if matches!(step, FixStep::Resource) {
7836 self.stop_for_resource()?;
7837 return Ok(false);
7838 }
7839 if self.state.reviews.len() < round {
7840 self.state.reviews.push(round_record);
7841 } else {
7842 self.state.reviews[round - 1] = round_record;
7843 }
7844 self.state.save()?;
7845
7846 if matches!(step, FixStep::Recorded { report_lost: true }) {
7859 self.stop_reviewing(
7860 "the fixer's adoption report never came back, even after resuming its \
7861 own seat; refusing to start another round against the same worktree \
7862 while that is unresolved",
7863 &shell,
7864 &winner.worktree,
7865 )
7866 .await?;
7867 return Ok(false);
7868 }
7869
7870 let streak = self
7871 .state
7872 .reviews
7873 .iter()
7874 .rev()
7875 .take_while(|r| !r.progressed)
7876 .count();
7877 if streak >= STAGNANT_LIMIT {
7878 self.stop_reviewing(
7879 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7880 &shell,
7881 &winner.worktree,
7882 )
7883 .await?;
7884 return Ok(false);
7885 }
7886 Ok(true)
7887 }
7888
7889 async fn run_fix_round(
7891 &mut self,
7892 winner: &Candidate,
7893 round_record: &mut ReviewRound,
7894 ) -> Result<FixStep> {
7895 let round = round_record.round;
7896 let max_rounds = self.state.config.graph.review_rounds;
7897 let run_id = self.state.id.clone();
7898 let prompts = self.state.config.prompts.clone();
7899 let language = self.state.config.graph.language.clone();
7900 let sessions = self.state.config.graph.sessions;
7901 let artifacts = self.merge_repair_artifacts();
7902 let base = self.landing_base();
7903 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7904 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7905 let this_round_verification = round_record.verification_summary(&head);
7906 let all_findings: Vec<_> = round_record
7907 .reviews
7908 .iter()
7909 .flat_map(|r| r.findings.iter().cloned())
7910 .collect();
7911 let blocking_findings: Vec<_> = all_findings
7914 .iter()
7915 .filter(|f| f.severity.blocks())
7916 .cloned()
7917 .collect();
7918 let fix_prompt = prompt::fix_in(
7919 &self.state.instruction,
7920 &blocking_findings,
7921 this_round_verification.as_ref(),
7922 round,
7923 max_rounds,
7924 &language,
7925 &self.state.config.graph.github_language,
7926 );
7927 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7928 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7929 let (job, seat, out) = self
7930 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7931 prompt: fix_prompt.clone(),
7932 spec,
7933 seat,
7934 cwd: winner.worktree.clone(),
7935 timeout,
7936 allow_write: true,
7937 sessions,
7938 artifacts: artifacts.clone(),
7939 stem: format!("fix-{round}"),
7940 handover: None,
7941 })
7942 .await;
7943 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7946 return Ok(FixStep::Resource);
7947 }
7948 let agent_id = seat.agent.clone();
7949
7950 let mut fix = FixRecord {
7951 agent: agent_id,
7952 addressed: Vec::new(),
7953 rejected: Vec::new(),
7954 notes: String::new(),
7955 committed: false,
7956 failed: None,
7957 duration_ms: 0,
7958 continuation: None,
7959 };
7960 let mut continuation = ContinuationRecord::not_needed();
7961 let mut final_seat = seat.clone();
7962 match out {
7963 AgentOutcome::Ok(o) => {
7964 fix.duration_ms = o.duration_ms;
7965 let parsed = verdict::extract_json::<FixReport>(&o.text);
7966 let incomplete_reason = match &parsed {
7973 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7974 "the reply parsed, but it reported a command whose own CLI never \
7975 confirmed an exit status"
7976 .to_owned(),
7977 ),
7978 Ok(_) => None,
7979 Err(e) => Some(e.to_string()),
7980 };
7981 match incomplete_reason {
7982 None => {
7983 let report = parsed.expect("checked Ok above");
7984 fix.addressed = report.addressed;
7985 fix.rejected = report.rejected;
7986 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7987 }
7988 Some(reason) => {
7989 let (resumed_seat, resolved, failure, cont) = self
7990 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7991 .await;
7992 fix.duration_ms += cont.cumulative_wait_ms;
7993 continuation = cont;
7994 final_seat = resumed_seat;
7995 match resolved {
7996 Some(report) => {
7997 fix.addressed = report.addressed;
7998 fix.rejected = report.rejected;
7999 fix.notes =
8000 blind::sanitize_prose(&report.notes, &self.state.config.blind);
8001 }
8002 None => fix.failed = failure,
8003 }
8004 }
8005 }
8006 }
8007 AgentOutcome::Dropped(o) => {
8009 fix.duration_ms = o.duration_ms;
8010 let why = o
8011 .dropped
8012 .as_ref()
8013 .map(|d| d.why.as_str())
8014 .unwrap_or("the CLI ended the stream without delivering its answer");
8015 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8016 }
8017 AgentOutcome::Quota(o) => {
8018 self.state.quota.push(QuotaLoss::from_report(
8019 final_seat.key.clone(),
8020 "fix",
8021 o.quota.as_ref(),
8022 ));
8023 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8024 }
8025 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8026 }
8027 if self.state.resource_block.is_some() {
8028 self.state.seats.insert(final_seat.key.clone(), final_seat);
8029 return Ok(FixStep::Resource);
8030 }
8031 fix.continuation = Some(continuation);
8032 self.state.seats.insert(final_seat.key.clone(), final_seat);
8033 let coordination = self.coordinate(&winner.worktree).await?;
8034 if let Ok(r) = git::rescue_commit(
8035 &winner.worktree,
8036 &format!("magi: review round {round} fixes (uncommitted work)"),
8037 )
8038 .await
8039 {
8040 self.state.note_withheld("fix", &r.withheld);
8041 }
8042 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8043 fix.committed = after != before;
8044 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8052 drop(coordination);
8053 let progressed = diff_after != patch;
8054 let commit_note = if fix.committed {
8055 "committed"
8056 } else {
8057 "NO new commit"
8058 };
8059 let tree_note = if progressed {
8060 "changed vs base"
8061 } else {
8062 "unchanged vs base"
8063 };
8064 self.state.event(
8065 "fix",
8066 match &fix.failed {
8067 Some(reason) => {
8073 format!(
8074 "round {round}: fixer's adoption report was lost ({reason}); \
8075 {commit_note}, tree {tree_note}"
8076 )
8077 }
8078 None => format!(
8079 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8080 {tree_note}{}",
8081 fix.addressed.len(),
8082 fix.rejected.len(),
8083 if continuation.outcome == ContinuationOutcome::Resumed {
8084 format!(
8085 " (adoption report recovered after {} continuation(s))",
8086 continuation.attempts
8087 )
8088 } else {
8089 String::new()
8090 },
8091 ),
8092 },
8093 );
8094 round_record.fix = Some(fix);
8095 round_record.progressed = progressed;
8096 Ok(FixStep::Recorded {
8097 report_lost: matches!(
8098 continuation.outcome,
8099 ContinuationOutcome::Exhausted
8100 | ContinuationOutcome::QuotaLost
8101 | ContinuationOutcome::NoSession
8102 ),
8103 })
8104 }
8105
8106 fn record_contested_handoff(&mut self) {
8140 if self.state.contested_handoff.is_some() {
8141 return;
8142 }
8143 let Some(contested) = self
8144 .state
8145 .reviews
8146 .last()
8147 .and_then(ReviewRound::contested_handoff)
8148 else {
8149 return;
8150 };
8151 self.state.event(
8152 "review",
8153 format!(
8154 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8155 wait for the owner's approval",
8156 contested.findings.len(),
8157 contested.rejecters.len()
8158 ),
8159 );
8160 self.state.contested_handoff = Some(contested);
8161 }
8162
8163 async fn stop_unresolved_review(
8164 &mut self,
8165 why: &str,
8166 shell: &[String],
8167 worktree: &Path,
8168 ) -> Result<()> {
8169 self.stop_reviewing_inner(why, shell, worktree, true).await
8170 }
8171
8172 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8173 self.stop_reviewing_inner(why, shell, worktree, false).await
8174 }
8175
8176 async fn stop_reviewing_inner(
8177 &mut self,
8178 why: &str,
8179 shell: &[String],
8180 worktree: &Path,
8181 repairable: bool,
8182 ) -> Result<()> {
8183 let round_idx = self.state.reviews.len() - 1;
8184 let needs_catchup_run = matches!(
8192 self.state.reviews[round_idx].e2e_status(),
8193 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8194 );
8195 if needs_catchup_run {
8196 let round = self.state.reviews[round_idx].round;
8197 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8198 let commands = self.state.config.verify.e2e.clone();
8199 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8200 let cache_dir = self.state.config.cache_dir();
8201 let context = format!(
8202 "round {round}: verification unresolved, catching up before the final decision"
8203 );
8204 let wait = self.repository_wait();
8205 let (outcomes, verify_retried) = with_cache_lease(
8206 &mut self.state,
8207 cache_dir.as_deref(),
8208 "e2e",
8209 "e2e",
8210 worktree,
8211 &attempted_head,
8212 timeout,
8213 &context,
8214 &wait,
8215 |state, budget| {
8216 let shell = shell.to_vec();
8217 let commands = commands.clone();
8218 let context = context.clone();
8219 async move {
8220 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8221 .await
8222 }
8223 },
8224 )
8225 .await?;
8226 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8227 self.state.record_base_progress(&attempted_head);
8228 }
8229 let last = &mut self.state.reviews[round_idx];
8230 last.e2e = outcomes;
8231 last.verify_retried = verify_retried;
8232 last.verified_head = Some(attempted_head);
8239 last.verified_at = Some(Timestamp::now());
8240 if verify_inconclusive(&last.e2e) {
8241 self.state.save()?;
8248 return Ok(());
8249 }
8250 last.e2e_deferred = false;
8251 if last.answered == last.expected {
8256 last.clean = round_is_clean(
8257 last.blocking,
8258 last.e2e.iter().all(CommandOutcome::ok),
8259 last.answered,
8260 last.expected,
8261 0,
8262 self.state.config.graph.incomplete_review,
8263 );
8264 }
8265 }
8266 let last = &self.state.reviews[round_idx];
8267 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8268
8269 match last.e2e_status() {
8270 E2eStatus::Failed => {
8271 let red: Vec<String> = last
8272 .e2e
8273 .iter()
8274 .filter(|o| !o.ok())
8275 .map(|o| {
8276 format!(
8277 "`{}` -> {:?}\n{}",
8278 o.command,
8279 o.code,
8280 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8281 )
8282 })
8283 .collect();
8284 let can_repair = last.blocking == 0 && last.fix.is_none();
8285 self.state.reviews[round_idx].clean = false;
8286 self.state
8287 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8288 self.state.status = RunStatus::Blocked;
8289 if repairable
8290 && can_repair
8291 && self.state.reviews.len() < self.state.config.graph.review_rounds
8292 {
8293 self.state.status = RunStatus::Reviewing;
8294 }
8295 }
8296 E2eStatus::ResourceBlocked => {
8301 self.state.event(
8302 "review",
8303 format!(
8304 "{why}; e2e could not run (resource unavailable); not \
8305 deciding yet"
8306 ),
8307 );
8308 }
8309 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8310 self.state.event(
8311 "review",
8312 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8313 );
8314 self.record_contested_handoff();
8315 self.state.status = RunStatus::Gating;
8316 }
8317 }
8318 self.state.save()?;
8319 Ok(())
8320 }
8321
8322 async fn gate(&mut self) -> Result<()> {
8325 if self.hold_base_sync().await? {
8326 return Ok(());
8327 }
8328 if self.state.status == RunStatus::Failed
8340 || self
8341 .state
8342 .base_sync
8343 .as_ref()
8344 .is_some_and(|s| s.conflict.is_some())
8345 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8346 != Some(RunStatus::Gating)
8347 {
8348 return Ok(());
8349 }
8350 let Some(winner) = self.state.winner().cloned() else {
8351 return Ok(());
8352 };
8353 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8354 self.state.reconcile_verification_head(&head);
8355 self.state.save()?;
8356 let wait = self.repository_wait();
8357 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8358 return Ok(());
8359 }
8360 if self.state.gate_ran {
8361 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8372 self.state.status = RunStatus::Blocked;
8373 self.state.save()?;
8374 }
8375 return Ok(());
8376 }
8377 self.state.status = RunStatus::Gating;
8378 let mut outcomes = self.run_gate(&winner).await?;
8379 let mut attempted_head = self.state.gate_head.clone();
8380 loop {
8381 if verify_inconclusive(&outcomes) {
8392 self.state.save()?;
8393 return Ok(());
8394 }
8395 if outcomes.iter().all(CommandOutcome::ok) {
8396 break;
8397 }
8398 let fixes_before = self.state.gate_fixes.len();
8399 match self.gate_fix_round(&winner, &outcomes).await? {
8400 GateFix::Retry => {
8401 outcomes = self.run_gate(&winner).await?;
8402 attempted_head = self.state.gate_head.clone();
8403 }
8404 GateFix::Stop => {
8405 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8406 if self.state.gate_fixes.len() > fixes_before
8407 && attempted_head.as_deref() != Some(current.as_str())
8408 && verify_current_head(&mut self.state, &winner, &wait).await?
8409 {
8410 outcomes = self.run_gate(&winner).await?;
8413 attempted_head = self.state.gate_head.clone();
8414 } else {
8415 break;
8416 }
8417 }
8418 GateFix::Defer => {
8419 self.state.save()?;
8420 return Ok(());
8421 }
8422 }
8423 }
8424 let passed = outcomes.iter().all(CommandOutcome::ok);
8425 self.state.gate = outcomes;
8426 self.state.gate_ran = true;
8427 self.state.gate_head = attempted_head;
8428 if !self.state.gate.is_empty()
8429 && !verify_inconclusive(&self.state.gate)
8430 && let Some(head) = self.state.gate_head.clone()
8431 {
8432 self.state.record_base_progress(&head);
8433 }
8434 self.state.save()?;
8435 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8436 return Ok(());
8437 }
8438 if !passed {
8439 self.state.status = RunStatus::Blocked;
8440 let spent = self.state.gate_fixes.len();
8441 self.state.event(
8442 "gate",
8443 if spent == 0 {
8444 "gate failed; not merging".to_owned()
8445 } else {
8446 format!("gate failed after {spent} gate-fix round(s); not merging")
8447 },
8448 );
8449 }
8450 self.state.save()?;
8451 Ok(())
8452 }
8453
8454 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8464 let commands = state.config.verify.pre_gate.clone();
8465 if commands.is_empty() {
8466 return;
8467 }
8468 let shell = state.config.shell();
8469 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8470 let (outcomes, _) = run_commands(
8471 state,
8472 "pre_gate",
8473 "pre_gate",
8474 0,
8475 &shell,
8476 &commands,
8477 &winner.worktree,
8478 timeout,
8479 )
8480 .await;
8481 for o in &outcomes {
8482 if !o.ok() {
8483 tracing::warn!(
8484 "pre_gate `{}` failed ({:?}); the gate decides",
8485 o.command,
8486 o.code
8487 );
8488 }
8489 state.event(
8490 "pre_gate",
8491 format!(
8492 "`{}` -> {}",
8493 o.command,
8494 if o.ok() {
8495 "pass".to_owned()
8496 } else {
8497 format!(
8498 "FAIL ({:?})\n{}",
8499 o.code,
8500 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8501 )
8502 }
8503 ),
8504 );
8505 }
8506 state.pre_gate = outcomes;
8507 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8508 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8509 Ok(head) => {
8510 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8511 state.pre_gate_commit = Some(head);
8512 }
8513 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8514 },
8515 Ok(false) => {}
8516 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8517 }
8518 if let Err(e) = state.save() {
8519 tracing::warn!("could not persist the pre_gate record: {e:#}");
8520 }
8521 }
8522
8523 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8526 let wait = self.repository_wait();
8527 run_landing_gate(&mut self.state, winner, &wait).await
8528 }
8529
8530 async fn gate_fix_round(
8543 &mut self,
8544 winner: &Candidate,
8545 outcomes: &[CommandOutcome],
8546 ) -> Result<GateFix> {
8547 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8548 g.phase == crate::repair_budget::Phase::Fixing
8549 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8550 }) {
8551 self.state.gate_fixes.len().saturating_add(1)
8552 } else {
8553 self.state.config.graph.gate_fix_rounds
8554 };
8555 let spent = self.state.gate_fixes.len();
8556 if spent >= cap {
8557 if cap > 0 {
8558 self.state.event(
8559 "gate",
8560 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8561 );
8562 }
8563 return Ok(GateFix::Stop);
8564 }
8565 if !gate_fixable(outcomes) {
8566 self.state.event(
8567 "gate",
8568 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8569 command or similar); not spending a fix round on it",
8570 );
8571 return Ok(GateFix::Stop);
8572 }
8573 let min_free = self.state.config.disk.min_free_bytes;
8574 if min_free > 0 {
8575 match crate::disk::free_bytes(&winner.worktree) {
8576 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8577 Ok(free) => {
8578 self.state.event(
8579 "gate",
8580 format!(
8581 "only {free} bytes free ({min_free} required by `[disk] \
8582 min_free_bytes`); not spending a fix round on a failure the disk \
8583 may explain"
8584 ),
8585 );
8586 return Ok(GateFix::Stop);
8587 }
8588 Err(e) => {
8589 self.state.event(
8590 "gate",
8591 format!("free disk space could not be measured ({e:#}); no fix round"),
8592 );
8593 return Ok(GateFix::Stop);
8594 }
8595 }
8596 }
8597
8598 let attempt = spent + 1;
8599 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8600 let base = self.landing_base();
8601 let fix_prompt = prompt::gate_fix_in(
8602 &self.state.instruction,
8603 &failed,
8604 attempt,
8605 cap,
8606 &self.state.config.graph.language,
8607 &self.state.config.graph.github_language,
8608 );
8609 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8610 let sessions = self.state.config.graph.sessions;
8611 let artifacts = self.merge_repair_artifacts();
8612 self.state.event(
8613 "gate",
8614 format!("gate failed; gate-fix round {attempt} of {cap}"),
8615 );
8616 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8617 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8618 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8621 head: before.clone(),
8622 passed: false,
8623 outcomes: Vec::new(),
8624 });
8625 self.state.save()?;
8626 let (_, seat, out) = self
8627 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8628 prompt: fix_prompt.clone(),
8629 spec,
8630 seat,
8631 cwd: winner.worktree.clone(),
8632 timeout,
8633 allow_write: true,
8634 sessions,
8635 artifacts: artifacts.clone(),
8636 stem: format!("gate-fix-{attempt}"),
8637 handover: None,
8638 })
8639 .await;
8640 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8643 return Ok(GateFix::Defer);
8644 }
8645 let mut record = GateFixRecord {
8646 agent: seat.agent.clone(),
8647 failed,
8648 notes: String::new(),
8649 committed: false,
8650 error: None,
8651 };
8652 match out {
8653 AgentOutcome::Ok(o) => {
8654 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8657 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8658 }
8659 }
8660 AgentOutcome::Dropped(_) => {
8661 record.error = Some("the CLI dropped the stream".to_owned());
8662 }
8663 AgentOutcome::Quota(o) => {
8664 self.state.quota.push(QuotaLoss::from_report(
8665 seat.key.clone(),
8666 "gate-fix",
8667 o.quota.as_ref(),
8668 ));
8669 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8670 }
8671 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8672 }
8673 self.state.seats.insert(seat.key.clone(), seat);
8674 let coordination = self.coordinate(&winner.worktree).await?;
8675 if let Ok(r) = git::rescue_commit(
8676 &winner.worktree,
8677 &format!("magi: gate fix {attempt} (uncommitted work)"),
8678 )
8679 .await
8680 {
8681 self.state.note_withheld("gate-fix", &r.withheld);
8682 }
8683 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8684 record.committed = after != before;
8685 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8686 let note = record.error.clone();
8687 self.state.gate_fixes.push(record);
8688 self.state.save()?;
8689 if !changed {
8690 self.state.event(
8691 "gate",
8692 match note {
8693 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8694 None => format!("gate-fix round {attempt}: the tree did not change"),
8695 },
8696 );
8697 return Ok(GateFix::Stop);
8698 }
8699 self.state.event(
8700 "gate",
8701 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8702 );
8703
8704 drop(coordination);
8705 let wait = self.repository_wait();
8706 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8707 }
8708
8709 async fn guarded_pr_message(
8716 &mut self,
8717 winner: &Candidate,
8718 facts: Option<&BranchFacts>,
8719 posting: bool,
8720 ) -> Result<Option<PrMessage>> {
8721 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8722 let head = match final_head.as_deref() {
8723 Some(h) => crate::pr_outcome::FinalHead::At(h),
8724 None => crate::pr_outcome::FinalHead::Unknown,
8725 };
8726 let summarized = if posting {
8728 Some(self.summarize_change(winner, final_head.as_deref()).await)
8729 } else {
8730 None
8731 };
8732 if self.state.resource_block.is_some() {
8737 self.state.event(
8738 "pr-summary",
8739 "a shared resource was busy; nothing was published, resume to summarize again",
8740 );
8741 return Ok(None);
8742 }
8743 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8744 anyhow::bail!(
8745 "the final commit moved during public description generation; retry before publishing"
8746 );
8747 }
8748 let note = match (&summarized, &self.state.change_summary) {
8749 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8750 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8751 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8752 (None, _) => ChangeNote::NotAsked,
8753 };
8754 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8755 if !posting {
8756 let identity = crate::scrub::Identity::current();
8757 return Ok(Some(PrMessage {
8758 title: crate::scrub::scrub(&pr.title, &identity),
8759 body: crate::scrub::scrub(&pr.body, &identity),
8760 }));
8761 }
8762 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8765 if let Some(known) = self
8766 .state
8767 .github_text
8768 .clone()
8769 .filter(|g| g.fingerprint == raw_fp)
8770 {
8771 return self.settle_withheld(winner, &pr, known).await;
8772 }
8773 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8774 let mut violations =
8775 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8776 let identity = crate::scrub::Identity::current();
8777 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8778 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8779 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8780 {
8781 violations.push(crate::github_text::Violation::SensitiveData);
8782 }
8783 let has_language = violations
8787 .iter()
8788 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8789 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8790 self.state.event(
8791 "github-text",
8792 if has_language {
8793 format!("description rejected: {violations:?}; requesting one rewrite")
8794 } else {
8795 format!("description rejected: {violations:?}; asking the owner")
8796 },
8797 );
8798 let mut rewritten = false;
8799 let mut rejected_rewrite = None;
8800 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8801 {
8802 let output = text;
8803 if let Some(title) = summary_title(&output) {
8804 let raw_body = summary_without_title(&output);
8807 let clean_title = crate::scrub::scrub(&title, &identity);
8808 let mut body = crate::scrub::scrub(&raw_body, &identity);
8809 let redacted = clean_title != title || body != raw_body;
8810 let title = clean_title;
8811 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8812 if !complete {
8813 self.state.event(
8814 "github-text",
8815 "description rewrite omitted required findings",
8816 );
8817 }
8818 if !body.trim().is_empty() {
8819 body.push_str("\n\n");
8820 body.push_str(&public_pr_evidence(&self.state, head));
8821 }
8822 let marker = format!("magi:run/{}", self.state.id);
8823 if !body.trim().is_empty() && !body.contains(&marker) {
8824 body.push_str(&format!("\n\n{marker}"));
8825 }
8826 let rewrite = PrMessage {
8827 title: title.clone(),
8828 body: body.clone(),
8829 };
8830 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8831 let mut failed = crate::github_text::check_for(
8832 &self.state.config.graph,
8833 &title,
8834 &body,
8835 redecision,
8836 );
8837 if (!crate::github_text::shareable(&title)
8838 || !crate::github_text::shareable(&body))
8839 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8840 {
8841 failed.push(crate::github_text::Violation::SensitiveData);
8842 }
8843 if complete
8844 && !title.trim().is_empty()
8845 && !body.trim().is_empty()
8846 && !failed.is_empty()
8847 {
8848 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8849 title: title.clone(),
8850 body: body.clone(),
8851 failed: failed
8852 .iter()
8853 .map(|v| crate::github_text::category(*v).to_owned())
8854 .collect(),
8855 });
8856 }
8857 if complete
8858 && !title.trim().is_empty()
8859 && !raw_body.trim().is_empty()
8860 && failed.is_empty()
8861 {
8862 decision = redecision;
8863 pr = PrMessage { title, body };
8864 rewritten = true;
8865 self.state
8866 .event("github-text", "description rewrite passed");
8867 if redacted {
8868 self.state.event(
8869 "github-text",
8870 "sensitive data removed from description rewrite",
8871 );
8872 }
8873 }
8874 }
8875 }
8876 if !rewritten {
8877 self.state.event(
8878 "github-text",
8879 "description rewrite unavailable or rejected; asking the owner",
8880 );
8881 if let Some(w) =
8882 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8883 {
8884 self.retire_github_text_question("the withheld text changed");
8887 let record = crate::run::GithubTextAsk {
8888 rewrite: rejected_rewrite,
8889 fingerprint: raw_fp,
8890 title: w.title,
8891 body: w.body,
8892 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8893 question: None,
8894 asks: 0,
8895 resolved: false,
8896 chosen_title: None,
8897 choice: None,
8898 publication: None,
8899 };
8900 return self.settle_withheld(winner, &pr, record).await;
8901 }
8902 }
8903 }
8904 let (title, body) =
8905 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8906 Ok(Some(PrMessage { title, body }))
8907 }
8908
8909 async fn summarize_change(
8920 &mut self,
8921 winner: &Candidate,
8922 head: Option<&str>,
8923 ) -> std::result::Result<(), String> {
8924 let Some(head) = head else {
8925 return Err("the final commit could not be read".to_owned());
8926 };
8927 let remote = self.state.config.merge.remote.clone();
8928 let tracking = format!("{remote}/{}", self.state.base_branch);
8929 if !git::rev_exists(&winner.worktree, &tracking).await {
8930 return Err(format!("the remote base `{tracking}` could not be read"));
8931 }
8932 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8933 Ok(b) if !b.is_empty() => b,
8934 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8935 };
8936 let evidence = crate::pr_outcome::summary_evidence(
8937 &self.state,
8938 crate::pr_outcome::FinalHead::At(head),
8939 );
8940 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8941 if self
8942 .state
8943 .change_summary
8944 .as_ref()
8945 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8946 {
8947 return Ok(());
8948 }
8949 if self.state.change_summary.take().is_some() {
8950 self.state.event(
8951 "pr-summary",
8952 "the final diff or evidence changed; the earlier change summary was discarded",
8953 );
8954 }
8955 if let Err(e) = write_artifact(
8957 &self.state,
8958 &format!("public-description-{fingerprint}.md"),
8959 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8960 ) {
8961 tracing::warn!("could not save public description evidence: {e:#}");
8962 }
8963 let chain = match agent::pick_chain(
8964 &self.state.config.agents,
8965 self.state.config.roles.synthesizer.as_ref(),
8966 &agent::installed,
8967 "synthesizer",
8968 ) {
8969 Ok(chain) => chain,
8970 Err(e) => {
8971 tracing::warn!("no agent for the change summary: {e:#}");
8972 self.state.event("pr-summary", "no summary agent available");
8973 return Err("no agent was available to summarize it".to_owned());
8974 }
8975 };
8976 let subjects = git::subjects(&winner.worktree, &base, head)
8977 .await
8978 .unwrap_or_default();
8979 let stat = git::diff_stat(&winner.worktree, &base, head)
8980 .await
8981 .unwrap_or_default();
8982 let stat: String = stat.chars().take(6_000).collect();
8983 let mut prompt = crate::prompt::change_summary_in(
8984 &base,
8985 head,
8986 &subjects,
8987 stat.trim(),
8988 &self.state.config.graph.github_language,
8989 );
8990 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8991 let sessions = self.state.config.graph.sessions;
8992 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8993 let prompts = self.state.config.prompts.clone();
8994 let cache = self.state.config.cache_dir();
8995 let run = self.state.id.clone();
8996 for (i, spec) in chain.into_iter().enumerate() {
8997 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8998 let github_spec = match agent::for_github_text(
8999 &spec,
9000 &self.state.config.graph.github_language,
9001 &winner.worktree,
9002 ) {
9003 Ok(spec) => spec,
9004 Err(e) => {
9005 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9006 continue;
9007 }
9008 };
9009 let job = SeatJob {
9010 spec: github_spec,
9011 seat,
9012 cwd: winner.worktree.clone(),
9013 prompt: prompt.clone(),
9014 timeout,
9015 allow_write: false,
9016 sessions,
9017 artifacts: agent::artifacts_dir(&self.state.dir()),
9018 stem: if i == 0 {
9019 "change-summary".to_owned()
9020 } else {
9021 format!("change-summary-{}", spec.id)
9022 },
9023 handover: None,
9024 };
9025 let ctx = WaveCtx {
9026 run: &run,
9027 node: "pr-summary",
9028 prompts: &prompts,
9029 cache: cache.as_deref(),
9030 round: None,
9031 carry_seats: false,
9032 pauses: Some((&self.pause, &self.interrupt)),
9033 };
9034 let (seat, outcome) =
9035 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9036 self.state.seats.insert(seat.key.clone(), seat);
9037 let text = match outcome {
9038 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9039 AgentOutcome::ResourceBlocked(why) => {
9040 if self.state.resource_block.is_none() {
9043 self.state.resource_block = Some(crate::run::ResourceBlock {
9044 cause: crate::run::ResourceBlockCause::Other,
9045 node: "pr-summary".to_owned(),
9046 kind: "shared resource".to_owned(),
9047 detail: why,
9048 at: Timestamp::now(),
9049 });
9050 }
9051 return Err("a shared resource was busy".to_owned());
9052 }
9053 _ => continue,
9054 };
9055 let blind = &self.state.config.blind;
9056 let Some(title) = summary_title(&text) else {
9057 continue;
9058 };
9059 let body: String = summary_without_title(&text)
9060 .lines()
9061 .filter(|l| !l.contains("magi:run/"))
9062 .collect::<Vec<_>>()
9063 .join("\n");
9064 let body = blind::sanitize_prose(body.trim(), blind);
9065 let title = blind::sanitize_prose(&title, blind);
9066 if body.trim().is_empty() || title.trim().is_empty() {
9067 continue;
9068 }
9069 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9070 continue;
9071 }
9072 if git::rev_parse(&winner.worktree, "HEAD")
9074 .await
9075 .ok()
9076 .as_deref()
9077 != Some(head)
9078 {
9079 return Err("the branch moved while the summary was written".to_owned());
9080 }
9081 if git::merge_base(&winner.worktree, &tracking, head)
9082 .await
9083 .ok()
9084 .as_deref()
9085 != Some(base.as_str())
9086 || crate::pr_outcome::summary_evidence(
9087 &self.state,
9088 crate::pr_outcome::FinalHead::At(head),
9089 ) != evidence
9090 {
9091 return Err("the summary evidence changed while it was written".to_owned());
9092 }
9093 self.state.change_summary = Some(crate::run::ChangeSummary {
9094 head: head.to_owned(),
9095 base,
9096 evidence: fingerprint,
9097 title,
9098 body,
9099 });
9100 self.state.event(
9101 "pr-summary",
9102 format!("change summary written by {} for the final diff", spec.id),
9103 );
9104 if let Err(e) = self.state.save() {
9105 tracing::warn!("could not save the change summary: {e:#}");
9106 }
9107 return Ok(());
9108 }
9109 self.state
9110 .event("pr-summary", "no agent produced a change summary");
9111 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9112 }
9113
9114 async fn rewrite_pr_text(
9124 &mut self,
9125 winner: &Candidate,
9126 pr: &PrMessage,
9127 violations: &[crate::github_text::Violation],
9128 ) -> Option<String> {
9129 let prompt = format!(
9130 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9131 pr.title, pr.body
9132 );
9133 let sessions = self.state.config.graph.sessions;
9134 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9135 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9138 let key = format!("impl-{}", winner.label);
9139 if let Some(spec) = self
9140 .state
9141 .config
9142 .agents
9143 .iter()
9144 .find(|a| a.id == winner.agent)
9145 .cloned()
9146 && let Some(existing) = self.state.seats.get(&key)
9147 && existing.agent == spec.id
9148 && agent::has_session(spec.kind, existing, sessions)
9149 {
9150 attempts.push((spec, Some(existing.clone())));
9151 }
9152 match agent::pick_chain(
9153 &self.state.config.agents,
9154 self.state.config.roles.language_judge.as_ref(),
9155 &agent::installed,
9156 "language_judge",
9157 ) {
9158 Ok(chain) => {
9159 for spec in chain {
9160 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9161 continue;
9162 }
9163 attempts.push((spec, None));
9164 }
9165 }
9166 Err(e) if attempts.is_empty() => {
9167 tracing::warn!("no agent for the description rewrite: {e:#}");
9168 self.state
9169 .event("github-text", "no rewrite agent available");
9170 return None;
9171 }
9172 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9173 }
9174 let prompts = self.state.config.prompts.clone();
9175 let cache = self.state.config.cache_dir();
9176 let run = self.state.id.clone();
9177 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9178 let resumed = resumed_seat.is_some();
9179 let seat = resumed_seat.unwrap_or_else(|| {
9180 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9181 });
9182 let github_spec = match agent::for_github_text(
9183 &spec,
9184 &self.state.config.graph.github_language,
9185 &winner.worktree,
9186 ) {
9187 Ok(spec) => spec,
9188 Err(e) => {
9189 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9190 continue;
9191 }
9192 };
9193 let job = SeatJob {
9194 spec: github_spec,
9195 seat,
9196 cwd: winner.worktree.clone(),
9197 prompt: prompt.clone(),
9198 timeout: retry_budget(timeout_full, resumed),
9199 allow_write: false,
9200 sessions,
9201 artifacts: agent::artifacts_dir(&self.state.dir()),
9202 stem: if i == 0 {
9203 "github-text-rewrite".to_owned()
9204 } else {
9205 format!("github-text-rewrite-{}", spec.id)
9206 },
9207 handover: None,
9208 };
9209 let ctx = WaveCtx {
9210 run: &run,
9211 node: "github-text",
9212 prompts: &prompts,
9213 cache: cache.as_deref(),
9214 round: None,
9215 carry_seats: false,
9216 pauses: Some((&self.pause, &self.interrupt)),
9217 };
9218 let (seat, outcome) =
9219 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9220 self.state.seats.insert(seat.key.clone(), seat);
9221 match outcome {
9222 AgentOutcome::Ok(o)
9223 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9224 {
9225 return Some(o.text);
9226 }
9227 AgentOutcome::ResourceBlocked(_) => return None,
9228 _ => {}
9229 }
9230 }
9231 None
9232 }
9233
9234 fn retire_github_text_question(&mut self, why: &str) {
9236 let Some(id) = self
9237 .state
9238 .github_text
9239 .as_ref()
9240 .and_then(|g| g.question.clone())
9241 else {
9242 return;
9243 };
9244 let _ = ask::Questions::open().update(&id, |q| {
9245 q.abandon(why);
9246 Ok(())
9247 });
9248 }
9249
9250 async fn settle_withheld(
9254 &mut self,
9255 winner: &Candidate,
9256 pr: &PrMessage,
9257 mut g: crate::run::GithubTextAsk,
9258 ) -> Result<Option<PrMessage>> {
9259 use crate::github_text as gt;
9260 let store = ask::Questions::open();
9261 let timeout = self.state.config.graph.answer_timeout;
9262 if !g.resolved {
9263 let mut standing = g
9264 .question
9265 .as_ref()
9266 .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9267 if let Some(q) = standing.as_ref()
9268 && q.status.open()
9269 && !store.held_reply(&q.id)
9270 && gt::expired(q, timeout)
9271 {
9272 let why = format!("no answer within {}s of asking", timeout.max(1));
9273 standing = store
9274 .update(&q.id, |q| {
9275 q.abandon(&why);
9276 Ok(())
9277 })
9278 .ok()
9279 .map(|(q, ())| q);
9280 }
9281 let mut ask_again: Option<Option<Vec<String>>> = None;
9282 match standing {
9283 None if g.asks <= 1 => {
9286 g.asks = 0;
9287 ask_again = Some(None);
9288 }
9289 None => {
9290 g.resolved = true;
9291 g.chosen_title = None;
9292 g.choice = Some(GithubTextChoice::Fallback);
9293 }
9294 Some(q) if q.status.open() => {
9295 self.park_on_github_text(&g)?;
9296 return Ok(None);
9297 }
9298 Some(q) => match gt::read_reply(&q) {
9299 gt::Reply::Fallback => {
9300 g.resolved = true;
9301 g.chosen_title = None;
9302 g.choice = Some(GithubTextChoice::Fallback);
9303 }
9304 gt::Reply::Original => {
9305 g.resolved = true;
9306 g.chosen_title = None;
9307 g.choice = Some(GithubTextChoice::Original);
9308 }
9309 gt::Reply::Rewrite => {
9310 g.resolved = true;
9311 g.chosen_title = None;
9312 g.choice = Some(if g.rewrite.is_some() {
9313 GithubTextChoice::Rewrite
9314 } else {
9315 GithubTextChoice::Fallback
9316 });
9317 }
9318 gt::Reply::Title(text) => {
9319 let first = text
9320 .lines()
9321 .map(str::trim)
9322 .find(|l| !l.is_empty())
9323 .unwrap_or("");
9324 let decision = if first.is_empty() {
9325 None
9326 } else {
9327 let candidate = PrMessage {
9328 title: first.to_owned(),
9329 body: String::new(),
9330 };
9331 self.judge_pr_language(&winner.worktree, &candidate).await
9332 };
9333 match gt::vet_title(
9334 &text,
9335 decision,
9336 self.state.config.graph.github_english(),
9337 ) {
9338 Ok(title) => {
9339 g.resolved = true;
9340 g.chosen_title = Some(title);
9341 g.choice = Some(GithubTextChoice::MyTitle);
9342 }
9343 Err(cats) if g.asks < 2 => {
9344 ask_again = Some(Some(
9345 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9346 ))
9347 }
9348 Err(_) => {
9349 g.resolved = true;
9350 g.chosen_title = None;
9351 g.choice = Some(GithubTextChoice::Fallback);
9352 }
9353 }
9354 }
9355 },
9356 }
9357 if let Some(rejected) = ask_again {
9358 let w = gt::Withheld {
9361 title: g.title,
9362 body: g.body,
9363 categories: g.categories.clone(),
9364 };
9365 let artifact = match crate::run::write_artifact(
9368 &self.state,
9369 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9370 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9371 ) {
9372 Ok(p) => Some(p),
9373 Err(e) => {
9374 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9375 None
9376 }
9377 };
9378 let (detail, comparison) = gt::question_content(
9379 &w,
9380 &pr.title,
9381 &pr.body,
9382 &self.state.id,
9383 rejected.as_deref(),
9384 artifact.as_deref(),
9385 g.rewrite.as_ref(),
9386 );
9387 let mut q = ask::Question::new(
9388 self.state.id.clone(),
9389 gt::ASK_NODE.to_owned(),
9390 gt::ASK_SEAT.to_owned(),
9391 gt::question_summary(&self.state.config.graph.language, &w),
9392 detail,
9393 gt::question_choices(&w, g.rewrite.as_ref()),
9394 );
9395 q.posting_comparison = Some(comparison);
9396 q.answer_timeout = timeout;
9397 g.question = Some(q.id.clone());
9400 g.asks += 1;
9401 self.state.github_text = Some(g.clone());
9402 self.state.event(
9403 "github-text",
9404 format!("asking the owner about the withheld text ({})", q.short()),
9405 );
9406 self.state.save()?;
9407 store
9408 .put_executable(&mut q)
9409 .context("file the posting-gate question")?;
9410 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9411 tracing::warn!(
9412 "could not notify about posting-gate question {}: {e:#}",
9413 q.short()
9414 );
9415 }
9416 self.park_on_github_text(&g)?;
9417 return Ok(None);
9418 }
9419 }
9420 let resolved = g.chosen_title.clone();
9421 let original = g.choice == Some(GithubTextChoice::Original);
9422 self.state.event(
9424 "github-text",
9425 if original {
9426 format!(
9427 "posting the original text for: {}",
9428 match (g.title, g.body) {
9429 (true, true) => "title, description",
9430 (true, false) => "title",
9431 _ => "description",
9432 }
9433 )
9434 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9435 "posting the saved rewrite title and description".to_owned()
9436 } else if resolved.is_some() {
9437 "posting the owner's replacement title".to_owned()
9438 } else {
9439 "posting the neutral text for the withheld fields".to_owned()
9440 },
9441 );
9442 self.state.github_text = Some(g.clone());
9443 self.state.save()?;
9444 if g.choice == Some(GithubTextChoice::Rewrite)
9445 && let Some(r) = g.rewrite.as_ref()
9446 {
9447 return Ok(Some(PrMessage {
9448 title: r.title.clone(),
9449 body: r.body.clone(),
9450 }));
9451 }
9452 let w = gt::Withheld {
9453 title: g.title,
9454 body: g.body,
9455 categories: g.categories.clone(),
9456 };
9457 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9458 if let Some(t) = resolved {
9459 title = t;
9460 }
9461 let english = gt::LanguageDecision {
9463 title_english: true,
9464 body_english: true,
9465 body_complete: true,
9466 };
9467 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9468 if original {
9469 if g.title {
9472 title = pr.title.clone();
9473 }
9474 if g.body {
9475 body = pr.body.clone();
9476 }
9477 }
9478 Ok(Some(PrMessage { title, body }))
9479 }
9480
9481 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9482 self.state.github_text = Some(g.clone());
9483 self.state.end_pass_parked(ParkCause::Question);
9484 self.state.parked = true;
9485 self.state.event(
9486 "github-text",
9487 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9488 );
9489 self.state.save()
9490 }
9491
9492 async fn judge_pr_language(
9495 &mut self,
9496 cwd: &Path,
9497 pr: &PrMessage,
9498 ) -> Option<crate::github_text::LanguageDecision> {
9499 self.state.config.roles.language_judge.as_ref()?;
9500 if !self.state.config.graph.language_gate() {
9501 return None;
9502 }
9503 let decision =
9504 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9505 self.state.event(
9506 "github-text",
9507 if decision.is_some() {
9508 "language decided by the language judge"
9509 } else {
9510 "language judge unavailable; using built-in heuristics"
9511 },
9512 );
9513 decision
9514 }
9515
9516 fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9517 Box::pin(self.merge_inner())
9518 }
9519
9520 async fn merge_inner(&mut self) -> Result<()> {
9521 if self.resource_halt()? || self.hold_base_sync().await? {
9522 return Ok(());
9523 }
9524 if self
9539 .state
9540 .base_sync
9541 .as_ref()
9542 .is_some_and(|s| s.conflict.is_some())
9543 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9544 != Some(RunStatus::Gating)
9545 || !self.state.gate_allows_progress()
9553 {
9554 return Ok(());
9555 }
9556 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9565 return Ok(());
9566 }
9567 let Some(winner) = self.state.winner().cloned() else {
9568 return Ok(());
9569 };
9570 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9571 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9572 self.state.status = RunStatus::Blocked;
9573 self.state.event(
9574 "gate",
9575 "winner changed after gate; fresh verification required",
9576 );
9577 self.state.save()?;
9578 return Ok(());
9579 }
9580 let wait = self.repository_wait();
9581 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9582 return Ok(());
9583 }
9584 let repo = self.state.repo.clone();
9585 let base = self.state.base_branch.clone();
9586 let mode = self.state.config.merge.mode;
9587 let style = self.state.config.merge.style;
9588 let facts = if is_review_run(&self.state) {
9589 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9590 let start = review_base(
9591 &repo,
9592 &self.state.config.merge.remote,
9593 &base,
9594 &self.state.base_commit,
9595 &winner.branch,
9596 )
9597 .await;
9598 branch_facts(&repo, &start, &winner.branch).await
9599 } else {
9600 None
9601 };
9602 let leaked = if is_review_run(&self.state) {
9605 leaked_subjects(&self.state, &winner.branch).await
9606 } else {
9607 Some(Vec::new())
9608 };
9609 let Some(pr) = self
9613 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9614 .await?
9615 else {
9616 return Ok(());
9617 };
9618 let message = pr.commit_message();
9619
9620 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9621 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9622 return Ok(());
9623 }
9624 let outcome = match mode {
9625 MergeMode::None => MergeOutcome {
9626 mode,
9627 ok: true,
9628 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9629 empty: false,
9630 },
9631 MergeMode::Pr | MergeMode::Local
9632 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9633 {
9634 MergeOutcome {
9635 mode,
9636 ok: false,
9637 detail: empty_candidate_detail(&self.state, &base),
9638 empty: true,
9639 }
9640 }
9641 MergeMode::Local => {
9642 let on = git::current_branch(&repo).await?;
9643 if on.as_deref() != Some(base.as_str()) {
9644 MergeOutcome {
9645 mode,
9646 ok: false,
9647 detail: format!(
9648 "{} has {} checked out, not the base branch {base}",
9649 repo.display(),
9650 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9651 ),
9652 empty: false,
9653 }
9654 } else if !git::is_clean(&repo).await? {
9655 MergeOutcome {
9656 mode,
9657 ok: false,
9658 detail: format!("{} is dirty; refusing to merge", repo.display()),
9659 empty: false,
9660 }
9661 } else {
9662 let out = match style {
9663 MergeStyle::Merge => {
9664 git::merge_no_ff(&repo, &winner.branch, &message).await?
9665 }
9666 MergeStyle::Squash => {
9667 git::merge_squash(&repo, &winner.branch, &message).await?
9668 }
9669 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9670 };
9671 MergeOutcome {
9672 mode,
9673 ok: out.ok(),
9674 detail: if out.ok() { out.stdout } else { out.stderr },
9675 empty: false,
9676 }
9677 }
9678 }
9679 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9680 self.state.pr_publication_pending = true;
9681 self.state.save()?;
9682 let out =
9683 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9684 self.state.pr_publication_pending = out.is_err();
9685 MergeOutcome {
9686 mode,
9687 ok: out.is_ok(),
9688 detail: match out {
9689 Ok(url) => url,
9690 Err(e) => format!("{e:#}"),
9691 },
9692 empty: false,
9693 }
9694 }
9695 MergeMode::Pr => {
9696 let remote = self.state.config.merge.remote.clone();
9697 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9698 let pushed = if self.state.foreign_checkout.is_some() {
9699 let refspec = format!("{delivery_head}:refs/heads/{public}");
9700 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9701 } else {
9702 git::push(&winner.worktree, &remote, &winner.branch).await?
9703 };
9704 if !pushed.ok() {
9705 MergeOutcome {
9706 mode,
9707 ok: false,
9708 detail: pushed.stderr,
9709 empty: false,
9710 }
9711 } else {
9712 if self.state.foreign_checkout.is_some() {
9713 self.state.pr_pushed_head = Some(delivery_head.clone());
9714 }
9715 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9721 let out = match pr_merge_plan(found) {
9722 PrPlan::Create => {
9723 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9724 .await
9725 }
9726 PrPlan::Adopt { url, title } => {
9727 self.state
9728 .event("merge", format!("Pr: adopted open pull request {url}"));
9729 if self.state.explicit_github_text_selection() {
9730 land::set_pr_text_verbatim(
9733 &winner.worktree,
9734 &url,
9735 &pr.title,
9736 &pr.body,
9737 )
9738 .await
9739 .map(|()| url)
9740 } else {
9741 if title != pr.title
9742 && (!is_review_run(&self.state)
9743 || leaked
9744 .as_deref()
9745 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9746 && let Err(e) =
9747 if self.state.github_text.as_ref().is_some_and(|g| {
9748 g.resolved
9749 && g.title
9750 && g.choice == Some(GithubTextChoice::Original)
9751 }) {
9752 land::set_pr_title_verbatim(
9754 &winner.worktree,
9755 &url,
9756 &pr.title,
9757 )
9758 .await
9759 } else {
9760 land::set_pr_title(
9761 &mut self.state,
9762 &winner.worktree,
9763 &url,
9764 &pr.title,
9765 )
9766 .await
9767 }
9768 {
9769 tracing::warn!("could not refresh title of {url}: {e:#}");
9770 self.state
9771 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9772 }
9773 Ok(url)
9774 }
9775 }
9776 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9777 };
9778 match out {
9779 Ok(url) => MergeOutcome {
9780 mode,
9781 ok: true,
9782 detail: url,
9783 empty: false,
9784 },
9785 Err(e) => MergeOutcome {
9786 mode,
9787 ok: false,
9788 detail: e.to_string(),
9789 empty: false,
9790 },
9791 }
9792 }
9793 }
9794 };
9795
9796 self.state.status = match (mode, outcome.ok) {
9797 (MergeMode::None, _) => RunStatus::Ready,
9798 (_, true) if outcome.empty => RunStatus::Ready,
9799 (MergeMode::Pr, true) => RunStatus::Ready,
9800 (_, true) => RunStatus::Merged,
9801 (_, false) => RunStatus::Blocked,
9802 };
9803 self.state.event(
9804 "merge",
9805 format!(
9806 "{:?}: {}",
9807 mode,
9808 outcome.detail.lines().next().unwrap_or("")
9809 ),
9810 );
9811 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9812 self.state.record_github_text_publication(&outcome.detail);
9813 }
9814 self.state.merge = Some(outcome);
9815 self.state.save()?;
9816
9817 if self.state.config.graph.land
9823 && mode == MergeMode::Pr
9824 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9825 {
9826 self.run_land().await?;
9827 }
9828 self.settle_questions();
9833 Ok(())
9834 }
9835
9836 fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9847 Box::pin(self.run_land_inner())
9848 }
9849
9850 async fn run_land_inner(&mut self) -> Result<()> {
9851 crate::pr_head::ensure_owner(&self.state)?;
9852 let url = self
9853 .state
9854 .pr
9855 .as_ref()
9856 .map(|pr| pr.url.clone())
9857 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9858 .unwrap_or_default();
9859 let url = url.lines().next().unwrap_or("").trim().to_owned();
9860 if !url.starts_with("http") {
9861 return Ok(());
9862 }
9863 loop {
9866 match land::land(&mut self.state, &url).await {
9867 Ok(_)
9868 if self
9869 .state
9870 .merge_repairs
9871 .last()
9872 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9873 {
9874 if self.repair_merge_review().await? {
9875 continue;
9876 }
9877 }
9878 Ok(_) if self.state.resource_block.is_some() => {
9879 return self.stop_for_resource();
9880 }
9881 Ok(pr) if self.state.parked => {
9882 let _ = pr;
9886 }
9887 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9888 self.state.status = RunStatus::Superseded;
9889 self.state.save()?;
9890 }
9891 Ok(pr) => {
9892 self.state.status = match pr.state {
9893 land::PrLifecycle::Merged => RunStatus::Merged,
9894 _ => RunStatus::Blocked,
9895 };
9896 if bump::should_release_bump(self.state.status)
9903 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9904 {
9905 self.state
9911 .event("bump", format!("release bump skipped: {e:#}"));
9912 }
9913 if self.state.status == RunStatus::Merged {
9916 crate::followup::after_merge(&mut self.state, &pr.url).await;
9917 }
9918 self.state.save()?;
9919 }
9920 Err(e) => {
9921 if e.is::<crate::repo_lock::ResourceBlocked>() {
9922 return Err(e);
9923 }
9924 self.state.status = RunStatus::Blocked;
9925 self.state.event("land", format!("gave up: {e}"));
9926 self.state.save()?;
9927 }
9928 }
9929 break;
9930 }
9931 Ok(())
9932 }
9933
9934 async fn repair_merge_review(&mut self) -> Result<bool> {
9937 use crate::run::MergeRepairPhase as Phase;
9938 crate::pr_head::ensure_owner(&self.state)?;
9939 let i = self.state.merge_repairs.len() - 1;
9940 let request = self.state.merge_repairs[i].clone();
9941 if request.phase == Phase::Fixing {
9942 let restored = match self.state.winner().cloned() {
9943 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9944 None => Err(anyhow::anyhow!("repair has no winning branch")),
9945 };
9946 let why = match restored {
9947 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9948 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9949 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9950 };
9951 self.hold_merge_repair(i, why)?;
9952 return Ok(false);
9953 }
9954 if request.phase == Phase::Reviewing && self.state.gate_ran {
9959 let winner = self
9960 .state
9961 .winner()
9962 .context("repair has no winning branch")?;
9963 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9964 if self.state.gate_head.as_deref() != Some(head.as_str())
9965 || !git::is_clean(&winner.worktree).await?
9966 {
9967 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9968 return Ok(false);
9969 }
9970 }
9971 let failure = if i >= self.state.config.graph.land_rounds
9972 || self.state.config.graph.review_rounds == 0
9973 {
9974 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9975 } else {
9976 None
9977 };
9978 if let Some(why) = failure {
9979 self.hold_merge_repair(i, why)?;
9980 return Ok(false);
9981 }
9982 if request.phase == Phase::Requested {
9983 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9984 Ok(pr) => pr,
9985 Err(e) => {
9986 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9987 return Ok(false);
9988 }
9989 };
9990 self.state.merge_repairs[i].phase = Phase::Fixing;
9991 self.state.land_approval = None;
9992 self.state.land_armed_head = None;
9993 self.state.save()?;
9994 let evidence = serde_json::to_string_pretty(&request)?;
9997 let reason = format!(
9998 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9999 );
10000 let budget = self.state.config.graph.land_rounds;
10001 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10002 match result {
10003 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10004 Ok(land::Fixed::Deferred) => {
10005 let worktree = self
10009 .state
10010 .winner()
10011 .context("repair has no winning branch")?
10012 .worktree
10013 .clone();
10014 self.graft_restore(&worktree, &request.head).await?;
10015 self.state.merge_repairs[i].phase = Phase::Requested;
10016 self.stop_for_resource()?;
10017 return Ok(false);
10018 }
10019 Ok(land::Fixed::Failed(why)) => {
10020 self.hold_merge_repair(
10021 i,
10022 format!(
10023 "repair failed: {why}; inspect the branch and run an explicit review"
10024 ),
10025 )?;
10026 return Ok(false);
10027 }
10028 Err(e) => {
10029 self.hold_merge_repair(
10030 i,
10031 format!(
10032 "repair failed: {e:#}; inspect the branch and run an explicit review"
10033 ),
10034 )?;
10035 return Ok(false);
10036 }
10037 }
10038 self.begin_merge_repair_review(i)?;
10042 }
10043 self.review_loop().await?;
10044 self.gate().await?;
10045 if self.resource_halt()? || self.state.parked {
10046 return Ok(false);
10047 }
10048 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10049 != Some(RunStatus::Gating)
10050 || !self.state.gate_allows_progress()
10051 {
10052 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10053 return Ok(false);
10054 }
10055 let winner = self
10058 .state
10059 .winner()
10060 .cloned()
10061 .context("repair has no winning branch")?;
10062 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10063 if self.state.gate_head.as_deref() != Some(after.as_str())
10064 || !git::is_clean(&winner.worktree).await?
10065 {
10066 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10067 return Ok(false);
10068 }
10069 let expected = self
10070 .state
10071 .pr_pushed_head
10072 .as_deref()
10073 .unwrap_or(&request.head)
10074 .to_owned();
10075 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10076 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10079 return Ok(false);
10080 }
10081 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10082 let refspec = format!("{after}:refs/heads/{public}");
10083 let push = git::git_raw(
10084 &winner.worktree,
10085 &["push", &lease, &self.state.config.merge.remote, &refspec],
10086 )
10087 .await;
10088 match push {
10089 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10090 Ok(out) => {
10091 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10092 return Ok(false);
10093 }
10094 Err(e) => {
10095 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10096 return Ok(false);
10097 }
10098 }
10099 self.state.merge_repairs[i].phase = Phase::Complete;
10100 self.state.status = RunStatus::Landing;
10101 self.state.event(
10102 "repair",
10103 "new review and gate completed; fresh owner merge decision required",
10104 );
10105 self.state.save()?;
10106 Ok(true)
10107 }
10108
10109 fn merge_repair_artifacts(&self) -> PathBuf {
10110 let artifacts = agent::artifacts_dir(&self.state.dir());
10111 match self
10112 .state
10113 .merge_repairs
10114 .last()
10115 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10116 {
10117 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10118 None => artifacts,
10119 }
10120 }
10121
10122 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10123 if let Some(e2e) = self.state.head_e2e.take() {
10124 self.state
10125 .verification_history
10126 .push(crate::run::VerificationEvidence {
10127 superseded_by: None,
10128 archived_at: Some(Timestamp::now()),
10129 fixes_spent: Some(self.state.gate_fixes.len()),
10130 gate_head: None,
10131 gate_ran: false,
10132 gate: Vec::new(),
10133 e2e: Some(e2e),
10134 });
10135 }
10136 self.state.reviews.clear();
10137 self.state.gate.clear();
10138 self.state.gate_ran = false;
10139 self.state.gate_head = None;
10140 self.state.gate_fixes.clear();
10141 self.state.gate_repair_e2e = None;
10142 self.state.pre_gate.clear();
10143 self.state.pre_gate_commit = None;
10144 self.state.contested_handoff = None;
10145 self.state.land_reruns = None;
10146 self.state.land_approval = None;
10147 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10148 self.state.status = RunStatus::Reviewing;
10149 self.state.save()
10150 }
10151
10152 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10153 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10154 self.state.merge_repairs[index].failure = Some(why.clone());
10155 self.state.status = RunStatus::Blocked;
10156 self.state.event("repair", why);
10157 self.state.save()
10158 }
10159
10160 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10164 if let Some(existing) = self.state.seats.get(key)
10165 && existing.agent == agent
10166 {
10167 return existing.clone();
10168 }
10169 let fresh = if self.state.seats.contains_key(key) {
10172 handover_seat(key, agent, self.state.next_seat_seed())
10173 } else {
10174 SeatState::new(key, agent, self.state.seed)
10175 };
10176 self.state.seats.insert(key.to_owned(), fresh.clone());
10177 fresh
10178 }
10179
10180 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10186 match self.state.seats.get(key) {
10187 Some(s) if s.agent != spec.id => {
10188 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10189 }
10190 _ => spec,
10191 }
10192 }
10193
10194 fn view(&self, c: &Candidate) -> CandidateView {
10196 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10197 .unwrap_or_default();
10198 let (patch, _) = blind::sanitize_patch(
10199 &format!("candidate {} patch", c.label),
10200 &raw,
10201 &self.state.config.blind,
10202 );
10203 CandidateView {
10204 label: c.label,
10205 branch: c.branch.clone(),
10206 summary: c.summary.clone(),
10207 stat: c.stat.clone(),
10208 patch,
10209 }
10210 }
10211
10212 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10214 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10215 prompt::judge(
10216 "(see above)",
10217 &views,
10218 self.roles.judges.len(),
10219 base_short,
10220 "en",
10221 )
10222 }
10223
10224 fn vote_prompt_full(
10230 &self,
10231 j: usize,
10232 labels: &[char],
10233 language: &str,
10234 candidates: &[Candidate],
10235 base_short: &str,
10236 ) -> String {
10237 let mut text = format!(
10238 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10239 prompt::final_vote(labels, language),
10240 self.state.instruction,
10241 self.candidate_block(candidates, base_short)
10242 );
10243 if let Some(own) = self
10244 .state
10245 .judgements
10246 .get(j)
10247 .filter(|r| !r.ranking.is_empty())
10248 {
10249 let reasons = own
10250 .reasons
10251 .iter()
10252 .map(|(k, v)| format!("- {k}: {v}"))
10253 .collect::<Vec<_>>()
10254 .join("\n");
10255 text.push_str(&format!(
10256 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10257 own.ranking.iter().collect::<String>(),
10258 if reasons.is_empty() {
10259 ""
10260 } else {
10261 ", because:\n"
10262 }
10263 ));
10264 }
10265 if !self.state.deliberation.is_empty() {
10266 text.push_str("\n# What was argued before this vote\n");
10267 for t in self.transcript(&[], j) {
10268 text.push_str(&format!(
10269 "\n## {}{}\n\n{}\n",
10270 t.who,
10271 if t.is_self { " (you)" } else { "" },
10272 t.body.trim()
10273 ));
10274 }
10275 }
10276 text
10277 }
10278
10279 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10286 let mut turns = Vec::new();
10287 for j in &self.state.judgements {
10288 if j.ranking.is_empty() {
10289 continue;
10290 }
10291 let reasons = j
10292 .reasons
10293 .iter()
10294 .map(|(k, v)| format!("- {k}: {v}"))
10295 .collect::<Vec<_>>()
10296 .join("\n");
10297 turns.push(Turn {
10298 who: format!("Judge {} (opening ranking)", j.judge),
10299 is_self: j.judge == self_idx + 1,
10300 body: format!(
10301 "Ranked {}{}{reasons}",
10302 j.ranking.iter().collect::<String>(),
10303 if reasons.is_empty() {
10304 ""
10305 } else {
10306 ", because:\n"
10307 }
10308 ),
10309 });
10310 }
10311 for t in self
10312 .state
10313 .deliberation
10314 .iter()
10315 .flat_map(|r| r.turns.iter())
10316 .chain(current)
10317 {
10318 turns.push(Turn {
10319 who: format!("Judge {}", t.judge),
10320 is_self: t.judge == self_idx + 1,
10321 body: t.body.clone(),
10322 });
10323 }
10324 turns
10325 }
10326}
10327
10328fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10330 agent::has_session(spec.kind, seat, sessions)
10331}
10332
10333fn next_untried_in_roster<'a>(
10354 roster: &'a [AgentSpec],
10355 start: usize,
10356 tried: &BTreeSet<String>,
10357) -> Option<&'a AgentSpec> {
10358 roster
10359 .get(start + 1..)?
10360 .iter()
10361 .find(|s| !tried.contains(&s.id))
10362}
10363
10364fn pick_successor<'a>(
10385 roster: &'a [AgentSpec],
10386 start: usize,
10387 tried: &BTreeSet<String>,
10388 carried: Option<&BTreeSet<String>>,
10389 others: &BTreeSet<String>,
10390) -> Option<&'a AgentSpec> {
10391 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10392 roster
10393 .get(start + 1..)
10394 .and_then(|tail| {
10395 tail.iter()
10396 .filter(free)
10397 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10398 })
10399 .or_else(|| {
10400 carried
10401 .is_some()
10402 .then(|| roster.iter().find(free))
10403 .flatten()
10404 })
10405 .or_else(|| match carried {
10406 Some(c) => next_for_seat(roster, start, tried, c),
10407 None => next_untried_in_roster(roster, start, tried),
10408 })
10409}
10410
10411fn next_for_seat<'a>(
10421 roster: &'a [AgentSpec],
10422 start: usize,
10423 round_tried: &BTreeSet<String>,
10424 carried_failed: &BTreeSet<String>,
10425) -> Option<&'a AgentSpec> {
10426 roster
10427 .get(start + 1..)?
10428 .iter()
10429 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10430 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10431}
10432
10433fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10439 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10440 return spec;
10441 };
10442 let ok = |id: &str| !h.failed.contains(id);
10443 if let Some(last) = h.last_ok.as_deref()
10444 && ok(last)
10445 && let Some(s) = roster.iter().find(|s| s.id == last)
10446 {
10447 return s.clone();
10448 }
10449 if ok(&spec.id) {
10450 return spec;
10451 }
10452 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10453 roster
10454 .get(from + 1..)
10455 .into_iter()
10456 .flatten()
10457 .chain(roster.iter())
10458 .find(|s| ok(&s.id))
10459 .cloned()
10460 .unwrap_or(spec)
10461}
10462
10463pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10467 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10468}
10469
10470const TIMED_OUT: &str = "timed out";
10473
10474impl FailClass {
10475 fn of(out: &AgentOutcome) -> Option<Self> {
10477 match out {
10478 AgentOutcome::Ok(_) => None,
10479 AgentOutcome::Quota(_) => Some(Self::Quota),
10480 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10481 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10482 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10483 AgentOutcome::ResourceBlocked(_) => None,
10486 }
10487 }
10488
10489 fn stem_word(&self) -> &'static str {
10491 match self {
10492 Self::Quota => "quota",
10493 _ => "handover",
10494 }
10495 }
10496}
10497
10498fn failure_signature(msg: &str) -> String {
10502 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10503 if let Some(code) = line
10506 .strip_prefix("process exited with no usable output (exit code ")
10507 .and_then(|s| s.strip_suffix(')'))
10508 .and_then(|s| s.parse::<i32>().ok())
10509 {
10510 line = format!("exited with some({code}) and no usable output");
10511 } else if matches!(
10512 line.as_str(),
10513 "process exited with no usable output (no exit code reported)"
10514 | "process exited with no usable output (exit code unavailable)"
10515 ) {
10516 line = "exited with none and no usable output".to_owned();
10517 }
10518 let mut out = Vec::new();
10519 for word in line.split_whitespace() {
10520 if word.contains('/') || word.contains('\\') {
10521 out.push("<path>".to_owned());
10522 continue;
10523 }
10524 let mut w = String::new();
10525 let mut in_digits = false;
10526 for c in word.chars() {
10527 if c.is_ascii_digit() {
10528 if !in_digits {
10529 w.push('#');
10530 }
10531 in_digits = true;
10532 } else {
10533 in_digits = false;
10534 w.push(c);
10535 }
10536 }
10537 out.push(w);
10538 }
10539 out.join(" ").chars().take(120).collect()
10540}
10541
10542fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10549 match cur {
10550 FailClass::Quota | FailClass::Timeout => true,
10551 FailClass::Other(_) => prev != Some(cur),
10552 }
10553}
10554
10555fn fail_reason(out: &AgentOutcome) -> String {
10557 match out {
10558 AgentOutcome::Ok(_) => String::new(),
10559 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10560 AgentOutcome::Dropped(o) => format!(
10561 "the CLI dropped the stream ({})",
10562 o.dropped
10563 .as_ref()
10564 .map(|d| d.why.as_str())
10565 .unwrap_or("it ended without delivering its answer")
10566 ),
10567 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10568 e.lines().next().unwrap_or("").chars().take(160).collect()
10569 }
10570 }
10571}
10572
10573pub(crate) fn record_handover(
10576 state: &mut RunState,
10577 node: &str,
10578 seat: &str,
10579 from: &str,
10580 to: &str,
10581 class: &FailClass,
10582 reason: &str,
10583) {
10584 let message = if *class == FailClass::Quota {
10585 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10586 } else {
10587 format!("{seat}: handed over {from} -> {to} ({reason})")
10588 };
10589 state.event(node, message);
10590 state.handovers.push(Handover {
10591 at: Timestamp::now(),
10592 node: node.to_owned(),
10593 seat: seat.to_owned(),
10594 from: from.to_owned(),
10595 to: to.to_owned(),
10596 reason: reason.to_owned(),
10597 });
10598}
10599
10600fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10614 commands.iter().any(|c| c.exit_code.is_none())
10615}
10616
10617fn verified_noop_claim(
10630 usable: bool,
10631 commands: &[agent::CommandEvidence],
10632 text: &str,
10633) -> Option<String> {
10634 (usable && !has_unconfirmed_command(commands))
10635 .then(|| verdict::verified_noop(text))
10636 .flatten()
10637}
10638
10639fn short(commit: &str) -> String {
10640 commit.chars().take(7).collect()
10641}
10642
10643fn make_executable(path: &Path) -> Result<()> {
10644 #[cfg(unix)]
10645 {
10646 use std::os::unix::fs::PermissionsExt as _;
10647 let mut perms = std::fs::metadata(path)?.permissions();
10648 perms.set_mode(0o755);
10649 std::fs::set_permissions(path, perms)?;
10650 }
10651 #[cfg(not(unix))]
10652 {
10653 let _ = path;
10654 }
10655 Ok(())
10656}
10657
10658struct WaveCtx<'a> {
10665 run: &'a str,
10668 node: &'a str,
10670 prompts: &'a Prompts,
10671 cache: Option<&'a Path>,
10673 round: Option<usize>,
10676 carry_seats: bool,
10680 pauses: Option<(&'a Pause, &'a Pause)>,
10683}
10684
10685fn repository_wait(
10687 state: &RunState,
10688 pauses: Option<(&Pause, &Pause)>,
10689) -> crate::repo_lock::WaitPolicy {
10690 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10691 let inherited = crate::repo_lock::WaitPolicy::current();
10692 crate::repo_lock::WaitPolicy::new(
10693 Duration::from_secs(state.config.graph.timeout_resource_wait),
10694 move || {
10695 inherited.cancelled()
10696 || pauses
10697 .as_ref()
10698 .is_some_and(|(a, b)| a.parked() || b.parked())
10699 },
10700 )
10701}
10702
10703fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10704 if error.is::<crate::repo_lock::ResourceBlocked>() {
10705 state.resource_block = Some(crate::run::ResourceBlock {
10706 cause: crate::run::ResourceBlockCause::Other,
10707 node: "repository".into(),
10708 kind: if error.to_string().contains("disk pressure") {
10709 "disk pressure"
10710 } else if error.to_string().contains("disk admission") {
10711 "disk admission"
10712 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10713 "incremental cache admission"
10714 } else {
10715 "repository coordination"
10716 }
10717 .into(),
10718 detail: error.to_string(),
10719 at: Timestamp::now(),
10720 });
10721 state.event("resource", error.to_string());
10722 }
10723}
10724
10725async fn coordinate_operation(
10726 state: &mut RunState,
10727 cwd: &Path,
10728 wait: &crate::repo_lock::WaitPolicy,
10729) -> Result<Option<crate::repo_lock::SeatLease>> {
10730 let result =
10731 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10732 .await;
10733 if let Err(e) = &result {
10734 record_repository_block(state, e);
10735 }
10736 result
10737}
10738
10739async fn run_one(
10741 job: SeatJob,
10742 sem: Arc<Semaphore>,
10743 ctx: &WaveCtx<'_>,
10744 state: &mut RunState,
10745 attempt: usize,
10746) -> (SeatState, AgentOutcome) {
10747 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10748 .await
10749 .pop()
10750 .expect("one job in, one result out");
10751 (seat, out)
10752}
10753
10754async fn wave(
10760 jobs: Vec<SeatJob>,
10761 sem: Arc<Semaphore>,
10762 ctx: &WaveCtx<'_>,
10763 state: &mut RunState,
10764 attempt: usize,
10765) -> Vec<(usize, SeatState, AgentOutcome)> {
10766 let WaveCtx {
10767 run,
10768 node,
10769 prompts,
10770 cache,
10771 round,
10772 carry_seats: _,
10773 pauses,
10774 } = *ctx;
10775 for job in &jobs {
10776 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10777 }
10778 if let Err(e) = state.save() {
10779 tracing::warn!("could not persist in-progress seats: {e:#}");
10784 }
10785 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10801 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10805 let cache_guard = if let Some(cache_dir) = cache {
10806 if jobs_had_a_writer {
10807 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10808 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10809 match acquire_cache_lease(
10810 state,
10811 cache_dir,
10812 &owner,
10813 resource_limit,
10814 node,
10815 node,
10816 &cancel,
10817 )
10818 .await
10819 {
10820 Ok(Some(g)) => Some(g),
10821 Ok(None) => {
10824 if let Some(block) = &mut state.resource_block {
10825 block.cause = crate::run::ResourceBlockCause::Cancelled {
10826 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10827 };
10828 }
10829 return block_wave(
10830 jobs,
10831 state,
10832 node,
10833 attempt,
10834 "parked while waiting for the shared build cache",
10835 );
10836 }
10837 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10840 Err(e) => {
10841 let why = format!("{e:#}");
10845 state.event(
10846 "resource",
10847 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10848 );
10849 return block_wave(jobs, state, node, attempt, &why);
10850 }
10851 }
10852 } else {
10853 None
10854 }
10855 } else {
10856 None
10857 };
10858 state.seats_spawning(node, attempt);
10861 if let Err(e) = state.save() {
10862 tracing::warn!("could not persist the start of a wave: {e:#}");
10863 }
10864 let slot_limit = resource_limit
10868 + jobs
10869 .iter()
10870 .map(|j| j.timeout)
10871 .max()
10872 .unwrap_or_default()
10873 .saturating_mul(jobs.len() as u32);
10874 let mut set = tokio::task::JoinSet::new();
10875 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10878 let overlay = prompts.overlay(node);
10879 let repository_wait = repository_wait(state, pauses);
10880 for (i, mut job) in jobs.into_iter().enumerate() {
10881 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10882 if cache.is_some() {
10883 job.prompt.push('\n');
10884 job.prompt
10885 .push_str(&prompt::build_cache_note(node, job.allow_write));
10886 }
10887 let sem = Arc::clone(&sem);
10888 let run = run.to_owned();
10889 let node = node.to_owned();
10890 let attachments = if node == "implement" {
10893 state.attachments.clone()
10894 } else {
10895 Vec::new()
10896 };
10897 let cache = cache
10908 .filter(|_| job.allow_write && cache_guard.is_some())
10909 .map(Path::to_path_buf);
10910 let spawned_tx = spawned_tx.clone();
10911 let repository_wait = repository_wait.clone();
10912 let coordination = state.config.coordination.clone();
10913 let fetched_base = state.fetched_base.clone();
10914 set.spawn(async move {
10915 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10919 Ok(p) => p,
10920 Err(_) => {
10921 return (
10922 i,
10923 job.seat,
10924 AgentOutcome::ResourceBlocked(format!(
10925 "no parallel slot became free within {}s",
10926 slot_limit.as_secs()
10927 )),
10928 false,
10929 );
10930 }
10931 };
10932 spawned_tx.send(job.seat.key.clone()).ok();
10933 let mut seat = job.seat;
10934 let out = repository_wait
10935 .scope(agent::invoke(
10936 &job.spec,
10937 &mut seat,
10938 &Invocation {
10939 coordination: coordination.as_ref(),
10940 fetched_base: fetched_base.as_ref(),
10941 cwd: &job.cwd,
10942 prompt: &job.prompt,
10943 timeout: job.timeout,
10944 allow_write: job.allow_write,
10945 sessions: job.sessions,
10946 artifacts: &job.artifacts,
10947 stem: &job.stem,
10948 run: &run,
10949 node: &node,
10950 cache_dir: cache.as_deref(),
10951 attachments: &attachments,
10952 writable: &[],
10953 },
10954 ))
10955 .await;
10956 let interrupted = out
10957 .as_ref()
10958 .err()
10959 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10960 let out = match out {
10961 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10962 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10963 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10971 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10972 Ok(o) => AgentOutcome::Failed(format!(
10973 "process exited with no usable output ({})",
10974 crate::notices::exit_status(o.exit_code)
10975 )),
10976 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10977 AgentOutcome::ResourceBlocked(e.to_string())
10978 }
10979 Err(e) => AgentOutcome::Failed(e.to_string()),
10980 };
10981 (i, seat, out, interrupted)
10982 });
10983 }
10984 drop(spawned_tx);
10985 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10986 loop {
10987 let joined = tokio::select! {
10988 Some(key) = spawned_rx.recv() => {
10989 state.seat_spawned(&key);
10991 if let Err(e) = state.save() {
10992 tracing::warn!("could not persist a seat's start: {e:#}");
10993 }
10994 continue;
10995 }
10996 joined = set.join_next() => joined,
10997 };
10998 let Some(joined) = joined else {
10999 break;
11000 };
11001 let (i, seat, out, interrupted) = match joined {
11002 Ok(v) => v,
11003 Err(e) => {
11007 tracing::error!("agent task panicked: {e}");
11008 continue;
11009 }
11010 };
11011 state.seat_finished(&seat.key);
11012 if interrupted
11013 && node == "implement"
11014 && let Some(label) = state
11015 .candidates
11016 .iter()
11017 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11018 .map(|candidate| candidate.label)
11019 {
11020 state.candidate_preservation.insert(
11021 label,
11022 "interrupted implementation awaiting persistence".into(),
11023 );
11024 state.interrupted_implementations.insert(label);
11025 }
11026 if let AgentOutcome::ResourceBlocked(why) = &out
11027 && state.resource_block.is_none()
11028 {
11029 state.resource_block = Some(crate::run::ResourceBlock {
11030 cause: crate::run::ResourceBlockCause::Other,
11031 node: node.to_owned(),
11032 kind: if why.contains("disk pressure") {
11033 "disk pressure"
11034 } else if why.contains("disk admission") {
11035 "disk admission"
11036 } else if why.contains(crate::incremental::BLOCKED) {
11037 "incremental cache admission"
11038 } else if why.contains("repository coordination") {
11039 "repository coordination"
11040 } else {
11041 "parallel slot"
11042 }
11043 .to_owned(),
11044 detail: why.clone(),
11045 at: Timestamp::now(),
11046 });
11047 let activity = if interrupted {
11048 "agent execution interrupted"
11049 } else {
11050 "no agent was started"
11051 };
11052 state.event("resource", format!("{node}: {activity} - {why}"));
11053 }
11054 record_jobs(state, node, round, &seat.key, &out);
11055 if let Err(e) = state.save() {
11056 tracing::warn!("could not persist a seat's completion: {e:#}");
11057 }
11058 if collected.len() <= i {
11059 collected.resize_with(i + 1, || None);
11060 }
11061 collected[i] = Some((i, seat, out));
11062 }
11063 if state
11069 .active
11070 .values()
11071 .any(|a| a.node == node && a.attempt == attempt)
11072 {
11073 state
11074 .active
11075 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11076 if let Err(e) = state.save() {
11077 tracing::warn!("could not persist the end of a wave: {e:#}");
11078 }
11079 }
11080 if let Some(cache_dir) = cache
11087 && jobs_had_a_writer
11088 {
11089 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11090 }
11091 if let Some(guard) = cache_guard {
11092 guard.release();
11093 }
11094 collected.into_iter().flatten().collect()
11095}
11096
11097fn record_jobs(
11108 state: &mut RunState,
11109 node: &str,
11110 round: Option<usize>,
11111 seat: &str,
11112 out: &AgentOutcome,
11113) {
11114 let commands: &[agent::CommandEvidence] = match out {
11115 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11116 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11117 };
11118 let checked_at = Timestamp::now();
11119 for c in commands {
11120 state.jobs.push(JobRecord {
11121 node: node.to_owned(),
11122 round,
11123 seat: seat.to_owned(),
11124 id: c.id.clone(),
11125 description: c.description.clone(),
11126 checked_at,
11127 status: match c.exit_code {
11128 Some(0) => JobStatus::Completed,
11129 Some(_) => JobStatus::Failed,
11130 None => JobStatus::Unknown,
11131 },
11132 exit_code: c.exit_code,
11133 result_summary: c.result_summary.clone(),
11134 source: c.source.clone(),
11135 });
11136 }
11137}
11138
11139fn round_is_clean(
11160 blocking: usize,
11161 e2e_ok: bool,
11162 answered: usize,
11163 expected: usize,
11164 quota_missing: usize,
11165 policy: IncompleteReviewPolicy,
11166) -> bool {
11167 if blocking != 0 || !e2e_ok {
11168 return false;
11169 }
11170 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11171 return true;
11172 }
11173 answered > 0 && expected - answered <= quota_missing
11174}
11175
11176pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11200 if max_rounds == 0
11201 || reviews.last().is_some_and(|r| {
11202 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11203 })
11204 {
11205 return Some(RunStatus::Gating);
11206 }
11207 let last = reviews.last()?;
11208 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11209 if reviews.len() < max_rounds && !stagnant {
11210 return None;
11211 }
11212 if last.incomplete() && last.blocking == 0 {
11213 return Some(RunStatus::Blocked);
11214 }
11215 if last.e2e_status() == E2eStatus::ResourceBlocked
11216 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11217 {
11218 return None;
11219 }
11220 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11221 RunStatus::Gating
11222 } else {
11223 RunStatus::Blocked
11224 })
11225}
11226
11227pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11242 if nudged {
11243 (full / 4).max(Duration::from_secs(120)).min(full)
11244 } else {
11245 full
11246 }
11247}
11248
11249#[allow(clippy::too_many_arguments)]
11278async fn ask_json_wave<T>(
11279 jobs: Vec<SeatJob>,
11280 sem: Arc<Semaphore>,
11281 retries: usize,
11282 roster: &[AgentSpec],
11283 ctx: &WaveCtx<'_>,
11284 losses: &mut Vec<QuotaLoss>,
11285 state: &mut RunState,
11286 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11287) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11288where
11289 T: serde::de::DeserializeOwned + Send + 'static,
11290{
11291 ask_wave_with(
11292 jobs,
11293 sem,
11294 retries,
11295 roster,
11296 ctx,
11297 losses,
11298 state,
11299 &|text: &str| {
11300 let v = verdict::extract_json::<T>(text)?;
11301 validate(&v)?;
11302 Ok(v)
11303 },
11304 )
11305 .await
11306}
11307
11308#[allow(clippy::too_many_arguments)]
11317async fn ask_wave_with<T>(
11318 jobs: Vec<SeatJob>,
11319 sem: Arc<Semaphore>,
11320 retries: usize,
11321 roster: &[AgentSpec],
11322 ctx: &WaveCtx<'_>,
11323 losses: &mut Vec<QuotaLoss>,
11324 state: &mut RunState,
11325 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11326) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11327where
11328 T: Send + 'static,
11329{
11330 let n = jobs.len();
11331 let originals: Vec<SeatJob> = jobs;
11332 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11333 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11334 let mut nudges: Vec<usize> = vec![0; n];
11341 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11345 let mut tried: Vec<BTreeSet<String>> = specs
11346 .iter()
11347 .map(|s| BTreeSet::from([s.id.clone()]))
11348 .collect();
11349 let starts: Vec<usize> = specs
11350 .iter()
11351 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11352 .collect();
11353 let carry = ctx.carry_seats && !roster.is_empty();
11354 let carried: Vec<BTreeSet<String>> = originals
11357 .iter()
11358 .map(|j| {
11359 state
11360 .seat_history
11361 .get(&j.seat.key)
11362 .filter(|_| carry)
11363 .map(|h| h.failed.clone())
11364 .unwrap_or_default()
11365 })
11366 .collect();
11367 let mut prev: Vec<Option<FailClass>> = originals
11368 .iter()
11369 .map(|j| {
11370 state
11371 .seat_history
11372 .get(&j.seat.key)
11373 .filter(|_| carry)
11374 .and_then(|h| h.last_fail.clone())
11375 })
11376 .collect();
11377 let next_agent =
11378 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11379 let others: BTreeSet<String> = specs
11380 .iter()
11381 .enumerate()
11382 .filter(|(j, _)| *j != i)
11383 .map(|(_, s)| s.id.clone())
11384 .collect();
11385 pick_successor(
11386 roster,
11387 starts[i],
11388 tried,
11389 carry.then(|| &carried[i]),
11390 &others,
11391 )
11392 .cloned()
11393 };
11394 let mut fresh: Vec<Option<String>> = vec![None; n];
11395 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11396 let mut pending: Vec<usize> = (0..n).collect();
11397
11398 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11403 for round in 0..max_rounds {
11404 if pending.is_empty() {
11405 break;
11406 }
11407 let mut batch = Vec::with_capacity(pending.len());
11408 let mut renudged: Vec<&str> = Vec::new();
11409 for &i in &pending {
11410 let src = &originals[i];
11411 let full: &str = match &src.handover {
11416 Some(h) if specs[i].id != src.spec.id => h,
11417 _ => &src.prompt,
11418 };
11419 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11422 (
11423 full.to_owned(),
11424 src.timeout,
11425 format!("{}-{word}-{}", src.stem, specs[i].id),
11426 )
11427 } else if nudges[i] == 0 {
11428 (full.to_owned(), src.timeout, src.stem.clone())
11429 } else {
11430 renudged.push(src.seat.key.as_str());
11431 let why = done[i]
11432 .as_ref()
11433 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11434 .unwrap_or_else(|| "no parsable answer".to_owned());
11435 let nudge = prompt::nudge(&why);
11436 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11437 let prompt = if nudged {
11438 nudge
11439 } else {
11440 format!("{full}\n\n---\n\n{nudge}")
11441 };
11442 (
11443 prompt,
11444 retry_budget(src.timeout, nudged),
11445 format!("{}-retry{}", src.stem, nudges[i]),
11446 )
11447 };
11448 batch.push(SeatJob {
11449 spec: specs[i].clone(),
11450 seat: seats[i].clone(),
11451 cwd: src.cwd.clone(),
11452 prompt,
11453 timeout,
11454 allow_write: src.allow_write,
11455 sessions: src.sessions,
11456 artifacts: src.artifacts.clone(),
11457 stem,
11458 handover: None,
11459 });
11460 }
11461
11462 if !renudged.is_empty() {
11463 state.event(
11464 ctx.node,
11465 format!("retry {round}: re-asking {}", renudged.join(", ")),
11466 );
11467 }
11468 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11469 let mut still = Vec::new();
11470 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11471 seats[i] = seat;
11472 if let AgentOutcome::ResourceBlocked(why) = &out {
11475 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11476 continue;
11477 }
11478 let class = FailClass::of(&out);
11479 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11482 && nudges[i] < retries
11483 && !roster.is_empty();
11484 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11485 let next = should_hand_over(prev[i].as_ref(), &cur)
11486 .then(|| next_agent(i, &tried[i], &specs))
11487 .flatten();
11488 if carry {
11489 let h = state
11490 .seat_history
11491 .entry(originals[i].seat.key.clone())
11492 .or_default();
11493 h.failed.insert(specs[i].id.clone());
11494 h.last_fail = Some(cur.clone());
11495 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11496 h.last_ok = None;
11497 }
11498 if let Err(e) = state.save() {
11501 tracing::warn!("could not persist a seat's failure history: {e:#}");
11502 }
11503 }
11504 if let Some(next) = next {
11505 record_handover(
11506 state,
11507 ctx.node,
11508 &originals[i].seat.key,
11509 &specs[i].id,
11510 &next.id,
11511 &cur,
11512 &fail_reason(&out),
11513 );
11514 tried[i].insert(next.id.clone());
11515 prev[i] = Some(cur.clone());
11516 seats[i] =
11517 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11518 specs[i] = next;
11519 fresh[i] = Some(cur.stem_word().to_owned());
11520 nudges[i] = 0;
11521 done[i] = Some(Err(anyhow::anyhow!(
11522 "handed over after: {}",
11523 fail_reason(&out)
11524 )));
11525 still.push(i);
11526 continue;
11527 }
11528 }
11529 if carry
11530 && !nudge_first
11531 && let Some(cur) = class.clone()
11532 {
11533 let h = state
11535 .seat_history
11536 .entry(originals[i].seat.key.clone())
11537 .or_default();
11538 h.failed.insert(specs[i].id.clone());
11539 h.last_fail = Some(cur);
11540 }
11541 let parsed = match out {
11542 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11543 AgentOutcome::Quota(o) => {
11544 last_quota[i] = o.quota.clone();
11545 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11546 }
11547 AgentOutcome::Dropped(o) => {
11551 let why = o
11552 .dropped
11553 .as_ref()
11554 .map(|d| d.why.as_str())
11555 .unwrap_or("the CLI ended the stream without delivering its answer");
11556 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11557 }
11558 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11559 Err(anyhow::anyhow!(e))
11560 }
11561 };
11562 let quota = class == Some(FailClass::Quota);
11563 let failed = parsed.is_err();
11564 if !failed && matches!(ctx.node, "judge" | "vote") {
11565 state
11568 .quota
11569 .retain(|loss| loss.seat != originals[i].seat.key);
11570 }
11571 done[i] = Some(parsed);
11572 if carry && !failed {
11573 let h = state
11574 .seat_history
11575 .entry(originals[i].seat.key.clone())
11576 .or_default();
11577 h.failed.remove(&specs[i].id);
11578 h.last_ok = Some(specs[i].id.clone());
11579 h.last_fail = None;
11580 }
11581 let agent_failure = class.is_some()
11589 && !nudge_first
11590 && !roster.is_empty()
11591 && next_agent(i, &tried[i], &specs).is_some();
11592 if failed && !quota && !agent_failure && nudges[i] < retries {
11593 nudges[i] += 1;
11594 still.push(i);
11595 }
11596 }
11597 pending = still;
11598 }
11599
11600 for (i, q) in last_quota.into_iter().enumerate() {
11603 if let Some(quota) = q {
11604 losses.push(QuotaLoss::from_report(
11605 originals[i].seat.key.clone(),
11606 ctx.node,
11607 Some("a),
11608 ));
11609 }
11610 }
11611
11612 if state.status != RunStatus::Stalled
11615 && ctx.node != "deliberate"
11616 && n > 0
11617 && losses.len() == n
11618 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11619 {
11620 block_machine_outage(state, ctx.node, losses);
11621 state.quota.append(losses);
11622 }
11623
11624 seats
11625 .into_iter()
11626 .zip(done)
11627 .zip(nudges)
11628 .map(|((seat, res), attempts)| {
11629 (
11630 seat,
11631 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11632 attempts,
11633 )
11634 })
11635 .collect()
11636}
11637
11638pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11641 let mut auth: Vec<_> = losses
11642 .iter()
11643 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11644 .map(|loss| loss.agent.clone())
11645 .collect();
11646 auth.sort();
11647 auth.dedup();
11648 let (kind, detail) = if auth.is_empty() {
11649 (
11650 "agent usage limit",
11651 "All seats reached their agent usage limits; resume after usage is available"
11652 .to_owned(),
11653 )
11654 } else {
11655 (
11656 "agent authentication",
11657 format!(
11658 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11659 auth.join(", ")
11660 ),
11661 )
11662 };
11663 state.resource_block = Some(crate::run::ResourceBlock {
11664 node: node.to_owned(),
11665 kind: kind.to_owned(),
11666 cause: crate::run::ResourceBlockCause::Other,
11667 detail: detail.clone(),
11668 at: Timestamp::now(),
11669 });
11670 state.event(node, detail);
11671}
11672
11673fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11675 results
11676 .iter()
11677 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11678}
11679
11680fn block_wave(
11684 jobs: Vec<SeatJob>,
11685 state: &mut RunState,
11686 node: &str,
11687 attempt: usize,
11688 why: &str,
11689) -> Vec<(usize, SeatState, AgentOutcome)> {
11690 state.resource_wait = None;
11691 state
11692 .active
11693 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11694 if let Err(e) = state.save() {
11695 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11696 }
11697 jobs.into_iter()
11698 .enumerate()
11699 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11700 .collect()
11701}
11702
11703async fn acquire_cache_lease(
11716 state: &mut RunState,
11717 cache_dir: &Path,
11718 owner: &crate::cache::Owner,
11719 budget: Duration,
11720 node: &str,
11721 context: &str,
11722 cancel: &(dyn Fn() -> bool + Sync),
11723) -> Result<Option<crate::cache::Guard>> {
11724 let home = crate::run::home();
11725 let started = Instant::now();
11726 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11727 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11728 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11729 Err(e) => {
11730 state.event(
11731 "verify",
11732 format!("{context}: could not check the shared build cache: {e:#}"),
11733 );
11734 if let Err(e2) = state.save() {
11735 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11736 }
11737 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11738 }
11739 };
11740 state.event(
11741 "verify",
11742 format!(
11743 "{context}: waiting for the shared build cache at {} ({})",
11744 cache_dir.display(),
11745 busy.describe()
11746 ),
11747 );
11748 state.resource_wait = Some(crate::run::ResourceWait {
11751 node: context.to_owned(),
11752 kind: "shared build cache".to_owned(),
11753 holder: busy.describe(),
11754 since: Timestamp::now(),
11755 limit_secs: budget.as_secs(),
11756 });
11757 if let Err(e) = state.save() {
11758 tracing::warn!("could not persist a cache wait: {e:#}");
11759 }
11760 let remaining = budget.saturating_sub(started.elapsed());
11761 let waited = crate::cache::wait_for_unless(
11762 &home,
11763 cache_dir,
11764 owner,
11765 remaining,
11766 Duration::from_secs(5),
11767 cancel,
11768 )
11769 .await;
11770 state.resource_wait = None;
11771 let cause = match &waited {
11772 Ok(Some(_)) => None,
11773 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11774 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11775 Some(crate::run::ResourceBlockCause::Timeout {
11776 waited_secs: started.elapsed().as_secs(),
11777 limit_secs: budget.as_secs(),
11778 })
11779 }
11780 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11781 };
11782 if let Some(cause) = cause {
11783 let detail = match &waited {
11784 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11785 Err(error) => format!("{error:#}"),
11786 };
11787 state.resource_block = Some(crate::run::ResourceBlock {
11788 cause,
11789 node: node.to_owned(),
11790 kind: "shared build cache".into(),
11791 detail,
11792 at: Timestamp::now(),
11793 });
11794 state.event(
11795 "resource",
11796 state.resource_block.as_ref().unwrap().description(),
11797 );
11798 }
11799 if let Err(error) = state.save() {
11800 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11801 }
11802 waited
11803}
11804
11805#[derive(Debug)]
11810struct CacheCheckError(String);
11811
11812impl std::fmt::Display for CacheCheckError {
11813 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11814 f.write_str(&self.0)
11815 }
11816}
11817
11818impl std::error::Error for CacheCheckError {}
11819
11820fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11822 state
11823 .resource_block
11824 .get_or_insert_with(|| crate::run::ResourceBlock {
11825 cause: crate::run::ResourceBlockCause::Other,
11826 node: node.to_owned(),
11827 kind: "shared build cache".into(),
11828 detail: format!("{error:#}"),
11829 at: Timestamp::now(),
11830 });
11831 if let Err(error) = state.save() {
11832 tracing::warn!("could not persist cache resource block: {error:#}");
11833 }
11834}
11835
11836#[allow(clippy::too_many_arguments)]
11857async fn with_cache_lease<'s, F, Fut>(
11858 state: &'s mut RunState,
11859 cache_dir: Option<&Path>,
11860 node: &str,
11861 seat: &str,
11862 worktree: &Path,
11863 head: &str,
11864 budget: Duration,
11865 context: &str,
11866 wait: &crate::repo_lock::WaitPolicy,
11867 body: F,
11868) -> Result<(Vec<CommandOutcome>, bool)>
11869where
11870 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11871 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11872{
11873 let started = Instant::now();
11876 let home = crate::run::home();
11877 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11878 let guard = if let Some(cache_dir) = cache_dir {
11879 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11880 match acquire_cache_lease(
11881 state,
11882 cache_dir,
11883 &owner,
11884 resource_limit,
11885 node,
11886 context,
11887 &|| wait.cancelled(),
11888 )
11889 .await
11890 .and_then(|g| {
11891 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11892 }) {
11893 Ok(g) => Some(g),
11894 Err(e) => {
11895 record_cache_error(state, node, &e);
11896 return Ok((
11897 vec![CommandOutcome {
11898 command: "(waiting for the shared build cache)".into(),
11899 code: None,
11900 output_tail: e.to_string(),
11901 duration_ms: started.elapsed().as_millis() as u64,
11902 resource_blocked: true,
11903 }],
11904 false,
11905 ));
11906 }
11907 }
11908 } else {
11909 None
11910 };
11911 let _coordination = coordinate_operation(state, worktree, wait).await?;
11912 if let Some(cache_dir) = cache_dir {
11913 let identity = crate::cache::Identity::new(worktree, head);
11914 if let Err(e) = crate::cache::ensure_fresh(
11915 &home,
11916 cache_dir,
11917 &identity,
11918 &state.config.verify.cargo_manifests(),
11919 ) {
11920 record_cache_error(state, node, &e);
11921 state.event(
11922 "verify",
11923 format!(
11924 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11925 worktree.display(),
11926 short(head)
11927 ),
11928 );
11929 if let Some(guard) = guard {
11930 guard.release();
11931 }
11932 return Ok((
11933 vec![CommandOutcome {
11934 command: "(confirming the shared build cache is fresh)".into(),
11935 code: None,
11936 output_tail: e.to_string(),
11937 duration_ms: started.elapsed().as_millis() as u64,
11938 resource_blocked: true,
11939 }],
11940 false,
11941 ));
11942 }
11943 }
11944 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11945 if !timed_out_pids.is_empty() {
11946 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11947 }
11948 drop(_coordination);
11951 if let Some(guard) = guard {
11952 guard.release();
11953 }
11954 Ok((outcomes, retried))
11955}
11956
11957async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11969 wait_for_pids_with(
11970 pids,
11971 crate::proc::pid_alive,
11972 LEASE_RELEASE_POLL,
11973 LEASE_RELEASE_MAX_WAIT,
11974 )
11975 .await;
11976}
11977
11978async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11984 pids: &[u32],
11985 alive: F,
11986 poll: Duration,
11987 max_wait: Duration,
11988) {
11989 let deadline = Instant::now() + max_wait;
11990 loop {
11991 if pids.iter().all(|&pid| !alive(pid)) {
11992 return;
11993 }
11994 if Instant::now() >= deadline {
11995 return;
11996 }
11997 tokio::time::sleep(poll).await;
11998 }
11999}
12000
12001fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12008 outcomes.iter().any(|o| o.resource_blocked)
12009}
12010
12011pub(crate) enum GateFix {
12013 Retry,
12015 Stop,
12018 Defer,
12021}
12022
12023pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12031 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12032 red.peek().is_some()
12033 && red.all(|o| {
12034 !o.resource_blocked
12035 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12036 && !o.output_tail.trim().is_empty()
12037 })
12038}
12039
12040fn e2e_outcome_label(o: &CommandOutcome) -> String {
12044 if o.ok() {
12045 return "pass".to_owned();
12046 }
12047 let reason = if o.build_failed() {
12048 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12049 } else {
12050 format!("FAIL ({:?})", o.code)
12051 };
12052 format!(
12053 "{reason}\n{}",
12054 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12055 )
12056}
12057
12058async fn run_e2e_with_retry(
12066 state: &mut RunState,
12067 shell: &[String],
12068 commands: &[String],
12069 worktree: &Path,
12070 timeout: Duration,
12071 context: &str,
12072) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12073 let (mut e2e, mut timed_out_pids) = run_commands(
12074 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12075 )
12076 .await;
12077 for o in &e2e {
12078 state.event(
12079 "verify",
12080 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12081 );
12082 }
12083 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12087 if verify_retried {
12088 state.event(
12089 "verify",
12090 format!(
12091 "{context}: verify could not build/link, not a test result — retrying once \
12092 before concluding"
12093 ),
12094 );
12095 let retried = run_commands(
12096 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12097 )
12098 .await;
12099 e2e = retried.0;
12100 timed_out_pids.extend(retried.1);
12103 for o in &e2e {
12104 state.event(
12105 "verify",
12106 format!(
12107 "{context}: retry `{}` -> {}",
12108 o.command,
12109 e2e_outcome_label(o)
12110 ),
12111 );
12112 }
12113 }
12114 (e2e, verify_retried, timed_out_pids)
12115}
12116
12117#[allow(clippy::too_many_arguments)]
12133async fn run_commands(
12134 state: &mut RunState,
12135 node: &str,
12136 task: &str,
12137 attempt: usize,
12138 shell: &[String],
12139 commands: &[String],
12140 cwd: &Path,
12141 timeout: Duration,
12142) -> (Vec<CommandOutcome>, Vec<u32>) {
12143 if commands.is_empty() {
12144 return (Vec::new(), Vec::new());
12149 }
12150 let mut out = Vec::new();
12151 let mut timed_out_pids = Vec::new();
12152 let total = commands.len();
12153 for (idx, command) in commands.iter().enumerate() {
12154 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12155 if let Err(e) = state.save() {
12156 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12157 }
12158 let started = Instant::now();
12159 let mut cmd = tokio::process::Command::new(&shell[0]);
12160 cmd.quiet();
12161 for key in crate::worker::INVOCATION_ENV {
12165 cmd.env_remove(key);
12166 }
12167 cmd.args(&shell[1..])
12168 .arg(command)
12169 .current_dir(cwd)
12170 .stdin(std::process::Stdio::null())
12171 .stdout(std::process::Stdio::piped())
12172 .stderr(std::process::Stdio::piped())
12173 .kill_on_drop(true);
12174 let mut outputs = Vec::new();
12175 let protected = (|| -> Result<()> {
12176 crate::incremental::configure(&mut cmd, cwd)?;
12177 let mut paths = vec![cwd.to_owned()];
12178 if let Some(cache) = state.config.cache_dir() {
12179 paths.push(cache);
12180 }
12181 outputs.extend(crate::retention::OutputLease::admit(
12182 &crate::run::home(),
12183 &paths,
12184 state.config.disk.min_free_bytes,
12185 )?);
12186 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12187 outputs.push(guard);
12188 }
12189 if let Some(cache) = state.config.cache_dir() {
12190 outputs.push(crate::retention::OutputLease::cache(
12191 &crate::run::home(),
12192 &cache,
12193 false,
12194 )?);
12195 }
12196 Ok(())
12197 })();
12198 if let Err(e) = protected {
12199 let detail = format!("execution admission unavailable: {e:#}");
12200 state.resource_block = Some(crate::run::ResourceBlock {
12201 cause: crate::run::ResourceBlockCause::Other,
12202 node: node.to_owned(),
12203 kind: if detail.contains("disk admission") {
12204 "disk admission"
12205 } else {
12206 "output ownership"
12207 }
12208 .to_owned(),
12209 detail: detail.clone(),
12210 at: Timestamp::now(),
12211 });
12212 out.push(CommandOutcome {
12213 command: command.clone(),
12214 code: None,
12215 output_tail: detail,
12216 duration_ms: 0,
12217 resource_blocked: true,
12218 });
12219 break;
12220 }
12221 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12222 let (code, body) = match spawned {
12223 Ok(mut running) => {
12224 let pid = running.child().id();
12225 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12226 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12227 let mut paths = vec![cwd.to_owned()];
12228 if let Some(cache) = state.config.cache_dir() {
12229 paths.push(cache);
12230 }
12231 let mut pressure = None;
12232 let status = tokio::select! {
12233 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12234 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12235 pressure = Some(reason); None
12236 }
12237 };
12238 let (code, failure) = match status {
12239 Some(Ok(Ok(status))) => (
12240 status.code(),
12241 status
12242 .code()
12243 .is_none()
12244 .then(|| format!("command terminated: {status}")),
12245 ),
12246 Some(Ok(Err(error))) => {
12247 (None, Some(format!("failed waiting for command: {error}")))
12248 }
12249 Some(Err(_)) => {
12250 if !running.coordinated()
12254 && let Some(pid) = pid
12255 {
12256 timed_out_pids.push(pid);
12257 }
12258 (
12259 None,
12260 Some(format!("timed out after {}s", timeout.as_secs())),
12261 )
12262 }
12263 None => (None, None),
12264 };
12265 running.finish().await;
12266 if let Some(reason) = pressure {
12267 state.resource_block = Some(crate::run::ResourceBlock {
12268 cause: crate::run::ResourceBlockCause::Other,
12269 node: node.to_owned(),
12270 kind: "disk pressure".to_owned(),
12271 detail: reason.clone(),
12272 at: Timestamp::now(),
12273 });
12274 out.push(CommandOutcome {
12275 command: command.clone(),
12276 code: None,
12277 output_tail: reason,
12278 duration_ms: started.elapsed().as_millis() as u64,
12279 resource_blocked: true,
12280 });
12281 break;
12282 }
12283 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12284 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12285 if let Some(failure) = failure {
12286 body.push('\n');
12287 body.push_str(&failure);
12288 }
12289 (code, body)
12290 }
12291 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12292 };
12293 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12294 if !body.is_empty() {
12295 match crate::run::write_verification_artifact(state, task, &body) {
12299 Ok(name) if summary != body => {
12300 summary = format!("[full output: artifacts/{name}]\n{summary}");
12301 }
12302 Ok(_) => {}
12303 Err(e) => {
12304 let warning = format!("could not save the full {task} output: {e:#}");
12305 tracing::warn!("{warning}");
12306 state.event(node, warning);
12307 if summary != body {
12308 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12309 }
12310 }
12311 }
12312 }
12313 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12314 out.push(CommandOutcome {
12315 command: command.clone(),
12316 code,
12317 output_tail: summary,
12318 duration_ms: started.elapsed().as_millis() as u64,
12319 resource_blocked,
12320 });
12321 if resource_blocked {
12322 state.resource_block = Some(crate::run::ResourceBlock {
12323 cause: crate::run::ResourceBlockCause::Other,
12324 node: node.to_owned(),
12325 kind: "incremental cache admission".into(),
12326 detail: body,
12327 at: Timestamp::now(),
12328 });
12329 break;
12330 }
12331 }
12332 state.task_finished(task);
12333 if let Err(e) = state.save() {
12334 tracing::warn!("could not persist the end of {task}: {e:#}");
12335 }
12336 (out, timed_out_pids)
12337}
12338
12339fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12351 let repo = repo.display();
12352 match style {
12353 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12354 MergeStyle::Squash => {
12355 let subject = message
12358 .lines()
12359 .next()
12360 .unwrap_or(branch)
12361 .replace(['\\', '"', '$', '`'], "");
12362 format!(
12363 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12364 )
12365 }
12366 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12367 }
12368}
12369
12370const PR_TITLE_MAX: usize = 240;
12381
12382fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12393 let line = queue::first_line(text)?;
12394 let chars: Vec<char> = line.chars().collect();
12395 let (kept, mark) = if chars.len() <= max {
12396 (line.to_owned(), "")
12397 } else if let Some(end) = sentence_end(&chars, max) {
12398 (chars[..end].iter().collect::<String>(), "")
12399 } else {
12400 let room = max.saturating_sub(3);
12401 let cut = if chars[room].is_whitespace() {
12404 room
12405 } else {
12406 chars[..room]
12407 .iter()
12408 .rposition(|c| c.is_whitespace())
12409 .unwrap_or(room)
12410 };
12411 let head: String = chars[..cut].iter().collect();
12412 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12413 (head.to_owned(), "...")
12414 };
12415 if kept.is_empty() {
12416 return None;
12417 }
12418 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12419 return None;
12420 }
12421 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12422 return None;
12423 }
12424 Some(format!("{kept}{mark}"))
12425}
12426
12427fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12431 const MIN: usize = 20;
12432 for i in MIN..max.min(chars.len()) {
12433 if !matches!(chars[i], '.' | '!' | '?') {
12434 continue;
12435 }
12436 let Some(&next) = chars.get(i + 1) else {
12437 continue;
12438 };
12439 if !next.is_whitespace() {
12440 continue;
12441 }
12442 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12443 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12444 continue;
12445 }
12446 let word: String = chars[..i]
12447 .iter()
12448 .rev()
12449 .take_while(|c| !c.is_whitespace())
12450 .collect::<Vec<_>>()
12451 .into_iter()
12452 .rev()
12453 .collect();
12454 let word = word.to_ascii_lowercase();
12455 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12456 continue;
12457 }
12458 let end = chars[..i]
12459 .iter()
12460 .rposition(|c| !c.is_whitespace())
12461 .map_or(i, |p| p + 1);
12462 return Some(end);
12463 }
12464 None
12465}
12466
12467struct PrMessage {
12472 title: String,
12473 body: String,
12474}
12475
12476impl PrMessage {
12477 fn commit_message(&self) -> String {
12481 format!("{}\n\n{}", self.title, self.body)
12482 }
12483}
12484
12485fn title_marker(line: &str) -> Option<&str> {
12487 let line = line.trim();
12488 let head = line.get(..6)?;
12489 head.eq_ignore_ascii_case("title:")
12490 .then(|| line[6..].trim())
12491}
12492
12493fn summary_title(summary: &str) -> Option<String> {
12498 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12499 let raw = title_marker(first)?;
12500 if raw.is_empty() {
12501 return None;
12502 }
12503 let title = queue::title_from(raw, PR_TITLE_MAX);
12504 let lower = title.to_ascii_lowercase();
12505 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12506 return None;
12507 }
12508 Some(title)
12509}
12510
12511const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12513
12514pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12516
12517pub(crate) fn is_review_run(state: &RunState) -> bool {
12521 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12522}
12523
12524fn review_title(state: &RunState) -> Option<String> {
12529 english_subject(
12530 state.reviewed_commits.as_ref()?.first()?,
12531 state.config.graph.github_english(),
12532 )
12533}
12534
12535fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12538 let raw = raw.trim();
12539 if raw.is_empty() {
12540 return None;
12541 }
12542 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12543 return None;
12544 }
12545 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12546 return None;
12547 }
12548 let title = queue::title_from(raw, PR_TITLE_MAX);
12549 let lower = title.to_ascii_lowercase();
12550 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12551 return None;
12552 }
12553 Some(title)
12554}
12555
12556#[derive(Debug, Clone, PartialEq, Eq)]
12559struct BranchFacts {
12560 commits: Vec<(String, String)>,
12562 stat: String,
12564}
12565
12566const STAT_FILE_LINES: usize = 25;
12568const COMMITS_MAX_CHARS: usize = 20_000;
12570
12571async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12574 let commits = git::commit_log(repo, base, branch).await.ok()?;
12575 if commits.is_empty() {
12576 return None;
12577 }
12578 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12579 let lines: Vec<&str> = stat.lines().collect();
12580 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12581 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12582 let more = format!(" ... {omitted} more file(s)");
12583 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12584 kept.push(&more);
12585 kept.push(lines[lines.len() - 1]);
12586 kept.join("\n")
12587 } else {
12588 lines.join("\n")
12589 };
12590 Some(BranchFacts { commits, stat })
12591}
12592
12593fn markdown_safe(text: &str) -> String {
12596 text.replace("</details>", "</details>")
12597 .replace("\x60\x60\x60", "~~~")
12598}
12599
12600fn neutral_title(state: &RunState, winner: char) -> String {
12601 format!(
12602 "chore: land candidate {} of run {}",
12603 winner.to_ascii_uppercase(),
12604 state.id
12605 )
12606}
12607
12608pub(crate) async fn repair_landing_gate(
12611 state: &mut RunState,
12612 winner: &Candidate,
12613 outcomes: &[CommandOutcome],
12614) -> Result<GateFix> {
12615 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12616 return Ok(GateFix::Stop);
12617 }
12618 let mut runner = Runner {
12619 roles: state.config.resolve_roles()?,
12620 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12621 state: state.clone(),
12622 pause: Pause::new(),
12623 interrupt: Pause::new(),
12624 };
12625 let result = runner.gate_fix_round(winner, outcomes).await;
12626 *state = runner.state;
12627 result
12628}
12629
12630pub(crate) async fn verify_current_head(
12633 state: &mut RunState,
12634 winner: &Candidate,
12635 wait: &crate::repo_lock::WaitPolicy,
12636) -> Result<bool> {
12637 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12638 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12640 state
12641 .verification_history
12642 .push(crate::run::VerificationEvidence {
12643 superseded_by: Some(head.clone()),
12644 archived_at: Some(Timestamp::now()),
12645 fixes_spent: Some(state.gate_fixes.len()),
12646 gate_head: None,
12647 gate_ran: false,
12648 gate: Vec::new(),
12649 e2e: state.head_e2e.take(),
12650 });
12651 }
12652 if let Some(e) = &state.head_e2e
12653 && e.head == head
12654 {
12655 if e.passed {
12656 return Ok(true);
12657 }
12658 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12659 state.status = RunStatus::Blocked;
12660 state.save()?;
12661 return Ok(false);
12662 }
12663 } else if state.reviews.last().is_some_and(|r| {
12664 r.verified_head.as_deref() == Some(head.as_str())
12665 && (r.e2e_status() == E2eStatus::Passed
12666 || (r.e2e_status() == E2eStatus::NotConfigured
12667 && state.config.verify.e2e.is_empty()))
12668 }) || state
12669 .gate_repair_e2e
12670 .as_ref()
12671 .is_some_and(|e| e.head == head && e.passed)
12672 {
12673 return Ok(true);
12674 }
12675 if let Some(e) = &state.gate_repair_e2e
12676 && e.head == head
12677 && !e.passed
12678 && !e.outcomes.is_empty()
12679 && !verify_inconclusive(&e.outcomes)
12680 {
12681 state.status = RunStatus::Blocked;
12682 state.save()?;
12683 return Ok(false);
12684 }
12685 state.head_e2e = Some(crate::run::GateRepairE2e {
12686 head: head.clone(),
12687 passed: false,
12688 outcomes: Vec::new(),
12689 });
12690 state.save()?;
12691 let commands = state.config.verify.e2e.clone();
12692 let shell = state.config.shell();
12693 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12694 let cache_dir = state.config.cache_dir();
12695 let outcomes = if commands.is_empty() {
12696 Vec::new()
12697 } else {
12698 let (outcomes, _) = with_cache_lease(
12699 state,
12700 cache_dir.as_deref(),
12701 "e2e",
12702 "e2e",
12703 &winner.worktree,
12704 &head,
12705 timeout,
12706 "current head e2e",
12707 wait,
12708 |state, budget| async move {
12709 run_e2e_with_retry(
12710 state,
12711 &shell,
12712 &commands,
12713 &winner.worktree,
12714 budget,
12715 "current head e2e",
12716 )
12717 .await
12718 },
12719 )
12720 .await?;
12721 outcomes
12722 };
12723 let unresolved = verify_inconclusive(&outcomes);
12724 let passed = !unresolved
12725 && outcomes.iter().all(CommandOutcome::ok)
12726 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12727 if !outcomes.is_empty() && !unresolved {
12728 state.record_base_progress(&head);
12729 }
12730 state.head_e2e = Some(crate::run::GateRepairE2e {
12731 head: head.clone(),
12732 passed,
12733 outcomes,
12734 });
12735 if !passed && !unresolved {
12736 state.status = RunStatus::Blocked;
12737 state.event(
12738 "e2e",
12739 format!("verify.e2e failed on head {head}; nothing published"),
12740 );
12741 }
12742 state.save()?;
12743 Ok(passed)
12744}
12745
12746pub(crate) async fn check_gate_repair_e2e(
12749 state: &mut RunState,
12750 winner: &Candidate,
12751 wait: &crate::repo_lock::WaitPolicy,
12752) -> Result<GateFix> {
12753 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12754 return Ok(GateFix::Retry);
12755 }
12756 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12757 if state
12758 .gate_repair_e2e
12759 .as_ref()
12760 .is_some_and(|r| r.head == head && r.passed)
12761 {
12762 return Ok(GateFix::Retry);
12763 }
12764 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12765 head: head.clone(),
12766 passed: false,
12767 outcomes: Vec::new(),
12768 });
12769 state.save()?;
12770 let commands = state.config.verify.e2e.clone();
12771 let outcomes = if commands.is_empty() {
12772 Vec::new()
12773 } else {
12774 let shell = state.config.shell();
12775 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12776 let cache_dir = state.config.cache_dir();
12777 let (outcomes, _) = with_cache_lease(
12778 state,
12779 cache_dir.as_deref(),
12780 "e2e",
12781 "e2e",
12782 &winner.worktree,
12783 &head,
12784 timeout,
12785 "gate repair e2e",
12786 wait,
12787 |state, budget| async move {
12788 run_e2e_with_retry(
12789 state,
12790 &shell,
12791 &commands,
12792 &winner.worktree,
12793 budget,
12794 "gate repair e2e",
12795 )
12796 .await
12797 },
12798 )
12799 .await?;
12800 outcomes
12801 };
12802 let deferred = verify_inconclusive(&outcomes);
12803 let passed = outcomes.iter().all(CommandOutcome::ok)
12804 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12805 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12806 head,
12807 passed,
12808 outcomes,
12809 });
12810 state.save()?;
12811 if deferred {
12812 Ok(GateFix::Defer)
12813 } else if !passed {
12814 state.event(
12815 "gate",
12816 "verify.e2e failed after gate repair; nothing published",
12817 );
12818 Ok(GateFix::Stop)
12819 } else {
12820 Ok(GateFix::Retry)
12821 }
12822}
12823
12824pub(crate) async fn run_landing_gate(
12826 state: &mut RunState,
12827 winner: &Candidate,
12828 wait: &crate::repo_lock::WaitPolicy,
12829) -> Result<Vec<CommandOutcome>> {
12830 let shell = state.config.shell();
12831 let gate_commands = state.config.verify.gate.clone();
12832 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12833 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12834 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12835 Vec::new()
12836 } else {
12837 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12838 let cache_dir = state.config.cache_dir();
12839 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12840 let (outcomes, _) = with_cache_lease(
12841 state,
12842 cache_dir.as_deref(),
12843 "gate",
12844 "gate",
12845 &winner.worktree,
12846 &head,
12847 timeout,
12848 "final gate",
12849 wait,
12850 |state, budget| {
12851 let shell = shell.clone();
12852 let gate_commands = gate_commands.clone();
12853 let winner = winner.clone();
12854 async move {
12855 Runner::run_pre_gate(state, &winner).await;
12856 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12858 let (outcomes, timed_out_pids) = run_commands(
12859 state,
12860 "gate",
12861 "gate",
12862 0,
12863 &shell,
12864 &gate_commands,
12865 &winner.worktree,
12866 budget,
12867 )
12868 .await;
12869 (outcomes, false, timed_out_pids)
12870 }
12871 },
12872 )
12873 .await?;
12874 outcomes
12875 };
12876 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12877 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12878 let mut outcomes = outcomes;
12879 outcomes.push(CommandOutcome {
12880 command: "gate head stability".into(), code: None,
12881 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12882 duration_ms: 0, resource_blocked: false,
12883 });
12884 return Ok(outcomes);
12885 }
12886 if outcomes.is_empty() {
12887 state.event("gate", "no gate commands configured; not tested");
12892 }
12893 for o in &outcomes {
12894 state.event(
12895 "gate",
12896 format!(
12897 "`{}` -> {}",
12898 o.command,
12899 if o.ok() {
12900 "pass".to_owned()
12901 } else {
12902 format!(
12903 "FAIL ({:?})\n{}",
12904 o.code,
12905 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12906 )
12907 }
12908 ),
12909 );
12910 }
12911 Ok(outcomes)
12912}
12913
12914pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12920 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12921 ""
12922 } else {
12923 pr_title
12924 }
12925}
12926
12927pub fn landing_subject_source(state: &RunState) -> String {
12931 if is_review_run(state) {
12932 let winner = state.candidates.first().map_or('A', |c| c.label);
12933 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12934 }
12935 state.instruction.clone()
12936}
12937
12938fn summary_without_title(summary: &str) -> String {
12941 let mut lines = summary.trim().lines().peekable();
12942 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12943 lines.next();
12944 }
12945 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12946}
12947
12948#[cfg(test)]
12960fn pr_message(state: &RunState, winner: char) -> PrMessage {
12961 pr_message_with(state, winner, None)
12962}
12963
12964#[cfg(test)]
12967fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12968 let raw = pr_message_raw(
12969 state,
12970 winner,
12971 facts,
12972 crate::pr_outcome::FinalHead::Unreported,
12973 ChangeNote::NotAsked,
12974 );
12975 let id = crate::scrub::Identity::current();
12976 PrMessage {
12977 title: crate::scrub::scrub(&raw.title, &id),
12978 body: crate::scrub::scrub(&raw.body, &id),
12979 }
12980}
12981
12982#[derive(Debug, Clone, Copy)]
12984enum ChangeNote<'a> {
12985 NotAsked,
12987 Summary(&'a crate::run::ChangeSummary),
12989 Unavailable(&'a str),
12991}
12992
12993fn pr_message_raw(
12994 state: &RunState,
12995 winner: char,
12996 facts: Option<&BranchFacts>,
12997 head: crate::pr_outcome::FinalHead<'_>,
12998 change: ChangeNote<'_>,
12999) -> PrMessage {
13000 if !matches!(change, ChangeNote::NotAsked) {
13001 let (title, mut body) = match change {
13002 ChangeNote::Summary(c) => (
13003 c.title.clone(),
13004 format!("## Summary\n\n{}\n\n", c.body.trim()),
13005 ),
13006 ChangeNote::Unavailable(why) => (
13007 neutral_title(state, winner),
13008 format!(
13009 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13010 ),
13011 ),
13012 ChangeNote::NotAsked => unreachable!(),
13013 };
13014 body.push_str(&public_pr_evidence(state, head));
13015 body.push_str(&format!(
13016 "\n---\nmagi:run/{} magi:candidate-{}\n",
13017 state.id,
13018 winner.to_ascii_lowercase()
13019 ));
13020 return PrMessage { title, body };
13021 }
13022 pr_preview(state, winner, facts, head)
13023}
13024
13025fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13028 let mut body = crate::pr_outcome::review_rounds(state);
13029 body.push_str(&crate::pr_outcome::final_verification(state, head));
13030 let open = state.last_round_findings();
13031 if !open.is_empty() {
13032 body.push_str("## Open review findings\n\n");
13033 for f in open {
13034 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13035 }
13036 body.push('\n');
13037 }
13038 if let Some(c) = state.change_summary.as_ref()
13039 && head == crate::pr_outcome::FinalHead::At(&c.head)
13040 {
13041 body.push_str(&format!(
13042 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13043 c.base, c.head
13044 ));
13045 }
13046 body
13047}
13048
13049fn pr_preview(
13051 state: &RunState,
13052 winner: char,
13053 facts: Option<&BranchFacts>,
13054 head: crate::pr_outcome::FinalHead<'_>,
13055) -> PrMessage {
13056 let summary = state
13057 .candidates
13058 .iter()
13059 .find(|c| c.label == winner)
13060 .map(|c| c.summary.as_str())
13061 .unwrap_or_default();
13062 let review = is_review_run(state);
13065 let title = if review {
13066 facts
13067 .and_then(|f| {
13068 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13069 })
13070 .or_else(|| review_title(state))
13071 .or_else(|| {
13072 state
13073 .candidates
13074 .iter()
13075 .find(|c| c.label == winner)
13076 .filter(|c| !c.branch.starts_with("magi/"))
13077 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13078 })
13079 .unwrap_or_else(|| neutral_title(state, winner))
13080 } else {
13081 summary_title(summary).unwrap_or_else(|| {
13082 english_title(
13083 &state.instruction,
13084 PR_TITLE_MAX,
13085 state.config.graph.github_english(),
13086 )
13087 .unwrap_or_else(|| neutral_title(state, winner))
13088 })
13089 };
13090
13091 let mut body = String::new();
13092 let what = summary_without_title(summary);
13093 if !what.is_empty() {
13094 body.push_str("## Summary\n\n");
13095 body.push_str(&what);
13096 body.push_str("\n\n");
13097 }
13098
13099 body.push_str(&crate::pr_outcome::review_rounds(state));
13101 body.push_str(&crate::pr_outcome::final_verification(state, head));
13102 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13103
13104 let open = state.open_findings();
13105 if !open.is_empty() {
13106 body.push_str("## Open review findings\n\n");
13107 for f in &open {
13108 body.push_str(&format!(
13109 "- `{}` [{:?}] {}\n",
13110 f.id, f.severity, "Context retained in local review records."
13111 ));
13112 }
13113 body.push('\n');
13114 }
13115
13116 if let Some(fix) = fix
13117 && !fix.rejected.is_empty()
13118 {
13119 body.push_str("## Declined by the fixer\n\n");
13120 for r in &fix.rejected {
13121 body.push_str(&format!(
13122 "- `{}`: {}\n",
13123 r.id, "Reason retained in local review records."
13124 ));
13125 }
13126 body.push('\n');
13127 }
13128
13129 if review {
13130 body.push_str("## Commits under review\n\n");
13132 if let Some(facts) = facts {
13133 let mut left = COMMITS_MAX_CHARS;
13134 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13135 let mut entry = format!("- {}\n", markdown_safe(subject));
13136 for l in markdown_safe(text).lines() {
13137 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13138 }
13139 if left == 0 {
13140 body.push_str(&format!(
13141 "- ... {} more commit(s)\n",
13142 facts.commits.len() - i
13143 ));
13144 break;
13145 }
13146 if entry.len() > left {
13147 let mut end = left;
13150 while !entry.is_char_boundary(end) {
13151 end -= 1;
13152 }
13153 entry.truncate(end);
13154 entry.push_str("\n ... (truncated)\n");
13155 left = 0;
13156 } else {
13157 left -= entry.len();
13158 }
13159 body.push_str(&entry);
13160 }
13161 if !facts.stat.trim().is_empty() {
13162 body.push_str(&format!(
13163 "\n## Diff stat\n\n```\n{}\n```\n",
13164 markdown_safe(facts.stat.trim())
13165 ));
13166 }
13167 } else {
13168 match &state.reviewed_commits {
13169 Some(subjects) => {
13170 for s in subjects {
13171 body.push_str(&format!("- {}\n", s.trim()));
13172 }
13173 }
13174 None => {
13175 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13178 body.push_str(rest.trim());
13179 body.push('\n');
13180 }
13181 }
13182 }
13183 } else {
13184 let task = state.instruction.trim();
13185 let task = if task.is_empty() {
13186 "(empty task)"
13187 } else {
13188 task
13189 };
13190 body.push_str(&format!(
13191 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13192 task.replace("</details>", "</details>")
13193 ));
13194 }
13195
13196 body.push_str(&format!(
13197 "\n---\nmagi:run/{} magi:candidate-{}\n",
13198 state.id,
13199 winner.to_ascii_lowercase()
13200 ));
13201
13202 PrMessage { title, body }
13203}
13204
13205fn seeded_instruction(state: &RunState) -> String {
13208 let mut inputs = Vec::new();
13209 if let Some(start) = &state.sources.start_from {
13210 inputs.push(format!(
13211 "Start from `{}` at `{}` (including ancestry).",
13212 start.token, start.sha
13213 ));
13214 }
13215 for pick in &state.sources.pick_commits {
13216 inputs.push(format!(
13217 "Pick `{}` at `{}` (single commit).",
13218 pick.token, pick.sha
13219 ));
13220 }
13221 if inputs.is_empty() {
13222 state.instruction.clone()
13223 } else {
13224 format!(
13225 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13226 state.instruction,
13227 inputs.join("\n")
13228 )
13229 }
13230}
13231
13232async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13236 let base = &state.base_branch;
13237 let mut against = base.clone();
13238 if mode == MergeMode::Pr {
13239 let remote = &state.config.merge.remote;
13243 let tracking = format!("{remote}/{base}");
13244 let fetched = git::fetch(repo, remote, base).await;
13245 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13246 against = tracking;
13247 } else {
13248 return false;
13249 }
13250 }
13251 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13252}
13253
13254fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13257 let mut detail = format!(
13258 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13259 nothing to open a pull request for"
13260 );
13261 detail.push_str(
13262 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13263 );
13264 detail
13265}
13266
13267#[derive(Debug, PartialEq, Eq)]
13270enum PrPlan {
13271 Create,
13272 Adopt { url: String, title: String },
13273 Stop(String),
13274}
13275
13276fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13279 match found {
13280 Ok(land::OpenPr::None) => PrPlan::Create,
13281 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13282 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13283 "several open pull requests exist for this branch, not picking one: {}",
13284 urls.join(" ")
13285 )),
13286 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13287 }
13288}
13289
13290pub(crate) async fn gh_pr_create(
13292 cwd: &Path,
13293 base: &str,
13294 head: &str,
13295 title: &str,
13296 body: &str,
13297) -> Result<String> {
13298 let out = tokio::process::Command::new("gh")
13299 .args([
13300 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13301 ])
13302 .current_dir(cwd)
13303 .quiet()
13304 .stdin(std::process::Stdio::null())
13305 .output()
13306 .await
13307 .context("spawn gh")?;
13308 if out.status.success() {
13309 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13310 } else {
13311 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13312 }
13313}
13314
13315async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13317 match std::fs::symlink_metadata(path) {
13318 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13319 git::worktree_remove_clean(repo, path).await?;
13322 return Ok(());
13323 }
13324 Err(e) => return Err(e.into()),
13325 Ok(_) => {}
13326 }
13327 if !git::is_clean(path).await? {
13328 bail!(
13329 "worktree has uncommitted work; refusing removal: {}",
13330 path.display()
13331 );
13332 }
13333 if !git::worktree_remove_clean(repo, path).await? {
13334 bail!("worktree removal refused: {}", path.display());
13335 }
13336 Ok(())
13337}
13338
13339async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13342 if !crate::retention::read_seat(path) || !path.exists() {
13343 return remove_preserved_worktree(&state.repo, path).await;
13344 }
13345 let owned = std::fs::symlink_metadata(path)?.is_dir()
13346 && path.parent().map(Path::canonicalize).transpose()?
13347 == Some(state.worktree_root().canonicalize()?)
13348 && crate::retention::worktrees(state)
13349 .await?
13350 .contains(&path.canonicalize()?);
13351 let common = crate::repo_lock::common_dir(&state.repo).await?;
13352 if !owned
13353 || common.is_none()
13354 || crate::repo_lock::common_dir(path).await? != common
13355 || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13356 .await?
13357 .ok()
13358 || state.candidates.iter().any(|c| c.worktree == path)
13359 {
13360 return remove_preserved_worktree(&state.repo, path).await;
13361 }
13362 let changes = git::status_porcelain(path).await?;
13363 if !git::worktree_remove(&state.repo, path).await? {
13364 bail!(
13365 "read-only seat worktree removal refused: {}",
13366 path.display()
13367 );
13368 }
13369 if !changes.is_empty() {
13370 let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13371 let paths: String = paths.chars().take(1024).collect();
13372 state.event(
13373 "fold",
13374 format!(
13375 "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13376 path.file_name().unwrap_or_default().to_string_lossy(),
13377 changes.lines().count()
13378 ),
13379 );
13380 }
13381 Ok(())
13382}
13383
13384pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13393 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13394 bail!("refusing fold: {reason}");
13395 }
13396 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13397 bail!("candidate preservation incomplete; refusing cleanup");
13398 }
13399 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13403 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13404 let _driver = crate::driver::maintenance(home, &state.id)?;
13405 let mut output_guards = Vec::new();
13406 for entry in std::fs::read_dir(state.worktree_root())
13407 .into_iter()
13408 .flatten()
13409 .flatten()
13410 {
13411 if entry.path().join(".git").is_file() {
13412 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13413 }
13414 }
13415 let repo = state.repo.clone();
13416 let wait = repository_wait(state, None);
13419 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13420 Ok(lease) => lease,
13421 Err(e) => {
13422 state.save_under(home)?;
13423 return Err(e);
13424 }
13425 };
13426 let _source_outputs = if state.status == RunStatus::Merged
13429 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13430 {
13431 verify_merged_fold_sources(state, drop_winner, home).await?
13432 } else {
13433 Vec::new()
13434 };
13435 let root = state.worktree_root();
13436 let winner = state.tally.as_ref().map(|t| t.winner);
13437 let mut removed = Vec::new();
13438
13439 for i in 0..state.candidates.len() {
13440 let c = state.candidates[i].clone();
13441 let is_winner = Some(c.label) == winner;
13442 if is_winner && !drop_winner {
13443 continue;
13444 }
13445 let existed = c.worktree.exists();
13446 remove_preserved_worktree(&repo, &c.worktree).await?;
13449 if existed {
13450 removed.push(c.worktree.to_string_lossy().into_owned());
13451 }
13452 let handed_over = state.released_branches.contains(&c.branch);
13455 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13456 if !git::branch_delete(&repo, &c.branch).await? {
13457 bail!("candidate branch removal refused");
13458 }
13459 removed.push(c.branch.clone());
13460 }
13461 state.candidates[i].folded = true;
13462 }
13463
13464 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13465 let path = name.path();
13466 let keep = !drop_winner
13467 && winner.is_some_and(|w| {
13468 path.file_name()
13469 .is_some_and(|n| n == format!("cand-{w}").as_str())
13470 });
13471 if keep {
13472 continue;
13473 }
13474 remove_read_seat_worktree(state, &path).await?;
13475 removed.push(path.to_string_lossy().into_owned());
13476 }
13477
13478 remove_if_empty(&root);
13487
13488 if state.enabled_worktree_config && drop_winner {
13489 git::release_worktree_config(&repo).await.ok();
13493 state.enabled_worktree_config = false;
13494 }
13495 state.save_under(home)?;
13496 Ok(removed)
13497}
13498
13499async fn verify_merged_fold_sources(
13500 state: &RunState,
13501 drop_winner: bool,
13502 home: &Path,
13503) -> Result<Vec<std::fs::File>> {
13504 let winner = state.tally.as_ref().map(|t| t.winner);
13505 let mut heads = BTreeSet::new();
13506 for candidate in &state.candidates {
13507 if Some(candidate.label) == winner && !drop_winner {
13508 continue;
13509 }
13510 anyhow::ensure!(
13511 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13512 "candidate recovery source is outside the inventoried run root: {}",
13513 candidate.worktree.display()
13514 );
13515 match std::fs::symlink_metadata(&candidate.worktree) {
13516 Ok(_) => {}
13517 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13518 Err(error) => return Err(error.into()),
13519 }
13520 if !state.released_branches.contains(&candidate.branch)
13521 && git::branch_exists(&state.repo, &candidate.branch).await?
13522 {
13523 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13524 }
13525 }
13526 let mut cargo_guards = Vec::new();
13527 let entries = match std::fs::read_dir(state.worktree_root()) {
13528 Ok(entries) => Some(entries),
13529 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13530 Err(error) => return Err(error.into()),
13531 };
13532 for entry in entries.into_iter().flatten() {
13533 let path = entry?.path();
13534 let metadata = std::fs::symlink_metadata(&path)?;
13535 anyhow::ensure!(
13536 metadata.is_dir() && !metadata.file_type().is_symlink(),
13537 "non-directory or symlink recovery source retained: {}",
13538 path.display()
13539 );
13540 if !drop_winner
13541 && winner.is_some_and(|w| {
13542 path.file_name()
13543 .is_some_and(|name| name == format!("cand-{w}").as_str())
13544 })
13545 {
13546 continue;
13547 }
13548 anyhow::ensure!(
13549 path.join(".git").is_file(),
13550 "unregistered recovery source retained: {}",
13551 path.display()
13552 );
13553 anyhow::ensure!(
13554 git::git(
13555 &path,
13556 &[
13557 "status",
13558 "--porcelain",
13559 "--untracked-files=all",
13560 "--ignored",
13561 "--",
13562 ".",
13563 ":(exclude)target"
13564 ]
13565 )
13566 .await?
13567 .is_empty(),
13568 "source changes or ignored recovery files retained: {}",
13569 path.display()
13570 );
13571 match std::fs::symlink_metadata(path.join("target")) {
13572 Ok(_) => {
13573 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13574 }
13575 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13576 Err(error) => return Err(error.into()),
13577 }
13578 for checkpoint in [
13579 "rebase-merge",
13580 "rebase-apply",
13581 "sequencer",
13582 "CHERRY_PICK_HEAD",
13583 "MERGE_HEAD",
13584 "REBASE_HEAD",
13585 "REVERT_HEAD",
13586 "BISECT_LOG",
13587 "AUTO_MERGE",
13588 "MERGE_MSG",
13589 ] {
13590 let location = git::git(
13591 &path,
13592 &[
13593 "rev-parse",
13594 "--path-format=absolute",
13595 "--git-path",
13596 checkpoint,
13597 ],
13598 )
13599 .await?;
13600 anyhow::ensure!(
13601 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13602 if e.kind() == std::io::ErrorKind::NotFound),
13603 "Git recovery checkpoint retained: {checkpoint}"
13604 );
13605 }
13606 heads.insert(git::rev_parse(&path, "HEAD").await?);
13607 }
13608 crate::fold_evidence::preserve(state, home, &heads).await?;
13609
13610 Ok(cargo_guards)
13611}
13612
13613fn remove_if_empty(dir: &Path) {
13624 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13625 std::fs::remove_dir(dir).ok();
13626 }
13627}
13628
13629pub fn worst_open(state: &RunState) -> Option<Severity> {
13631 state
13632 .reviews
13633 .last()?
13634 .reviews
13635 .iter()
13636 .flat_map(|r| r.findings.iter())
13637 .map(|f| f.severity)
13638 .max()
13639}
13640
13641async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13645 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13646 git::acquire_worktree_config(repo).await?;
13647 state.enabled_worktree_config = true;
13648 }
13649 Ok(())
13650}
13651
13652#[cfg(test)]
13653mod tests {
13654 #[tokio::test]
13655 async fn launcher_base_fetch_requires_remote_success() {
13656 let tmp = tempfile::tempdir().unwrap();
13657 let repo = tmp.path().join("repo");
13658 let remote = tmp.path().join("remote");
13659 std::fs::create_dir(&repo).unwrap();
13660 std::fs::create_dir(&remote).unwrap();
13661 git::git(&remote, &["init", "--bare", "-b", "main"])
13662 .await
13663 .unwrap();
13664 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13665 git::git(
13666 &repo,
13667 &[
13668 "-c",
13669 "user.name=Fixture",
13670 "-c",
13671 "user.email=fixture@example.invalid",
13672 "commit",
13673 "--allow-empty",
13674 "-m",
13675 "fixture",
13676 ],
13677 )
13678 .await
13679 .unwrap();
13680 git::git(
13681 &repo,
13682 &["remote", "add", "origin", remote.to_str().unwrap()],
13683 )
13684 .await
13685 .unwrap();
13686 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13687 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13688 assert_eq!(
13689 resolve_base(&repo, "main", "origin", None)
13690 .await
13691 .unwrap()
13692 .commit,
13693 head
13694 );
13695 #[cfg(unix)]
13696 {
13697 let policy: crate::config::Coordination = toml::from_str(
13698 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13699 ).unwrap();
13700 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13701 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13702 let held = crate::repo_lock::SeatLease::operation_with(
13703 &repo,
13704 Some(&policy),
13705 &crate::repo_lock::WaitPolicy::default(),
13706 )
13707 .await
13708 .unwrap();
13709 assert!(held.is_some());
13710 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13711 tokio::pin!(fetch);
13712 assert!(
13713 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13714 .await
13715 .is_err()
13716 );
13717 assert!(
13718 !repo.join(".git/FETCH_HEAD").exists(),
13719 "no fetch before admission"
13720 );
13721 drop(held);
13722 let fetched = fetch.await.unwrap();
13723 assert_eq!(fetched.commit, head);
13724 assert_eq!(fetched.remote, "origin");
13725 assert_eq!(fetched.branch, "main");
13726 }
13727 git::git(
13729 &repo,
13730 &[
13731 "remote",
13732 "set-url",
13733 "origin",
13734 tmp.path().join("missing").to_str().unwrap(),
13735 ],
13736 )
13737 .await
13738 .unwrap();
13739 let error = resolve_base(&repo, "main", "origin", None)
13740 .await
13741 .unwrap_err()
13742 .to_string();
13743 assert!(error.contains("cannot read origin/main"));
13744 assert!(error.contains("refusing to branch off"));
13745 }
13746
13747 #[test]
13748 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13749 let leaked = vec!["chore(deps): update a crate".to_owned()];
13750 let own = "fix(daemon): apply a chosen action";
13751 assert!(should_retitle("", own, &leaked));
13752 assert!(should_retitle(
13753 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13754 own,
13755 &leaked
13756 ));
13757 assert!(should_retitle(
13758 "chore: land candidate A of run 1",
13759 own,
13760 &leaked
13761 ));
13762 assert!(should_retitle(
13763 "magi: candidate A (uncommitted work)",
13764 own,
13765 &leaked
13766 ));
13767 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13768 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13769 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13770 }
13771
13772 #[test]
13773 fn pr_merge_plan_creates_adopts_or_stops() {
13774 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13775 assert_eq!(
13776 pr_merge_plan(Ok(land::OpenPr::One {
13777 url: "u".into(),
13778 title: "t".into()
13779 })),
13780 PrPlan::Adopt {
13781 url: "u".into(),
13782 title: "t".into()
13783 }
13784 );
13785 let PrPlan::Stop(many) =
13786 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13787 else {
13788 panic!("many must stop");
13789 };
13790 assert!(many.contains('a') && many.contains('b'));
13791 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13792 panic!("a failed lookup must stop");
13793 };
13794 assert!(err.contains("bad token"));
13795 }
13796
13797 use super::*;
13798 use crate::run::GateStatus;
13799 use std::collections::BTreeMap;
13800 use std::time::Duration;
13801
13802 fn conductor() -> AgentSpec {
13803 AgentSpec {
13804 id: "conductor".to_owned(),
13805 kind: crate::config::AgentKind::Command,
13806 model: None,
13807 command: vec!["true".to_owned()],
13808 extra_args: Vec::new(),
13809 env: BTreeMap::new(),
13810 prompt_delivery: None,
13811 max_turn_secs: None,
13812 }
13813 }
13814
13815 fn spec(id: &str) -> AgentSpec {
13816 AgentSpec {
13817 id: id.to_owned(),
13818 kind: crate::config::AgentKind::Command,
13819 model: None,
13820 command: vec!["true".to_owned()],
13821 extra_args: Vec::new(),
13822 env: BTreeMap::new(),
13823 prompt_delivery: None,
13824 max_turn_secs: None,
13825 }
13826 }
13827
13828 fn ids(xs: &[&str]) -> BTreeSet<String> {
13829 xs.iter().map(|s| (*s).to_owned()).collect()
13830 }
13831
13832 #[test]
13833 fn pick_successor_skips_an_agent_another_seat_holds() {
13834 let roster = [spec("a"), spec("b"), spec("c")];
13836 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13837 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13838 }
13839
13840 #[test]
13841 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13842 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13844 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13845 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13846 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13847 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13848 }
13849
13850 #[test]
13851 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13852 let roster = [spec("a"), spec("b")];
13853 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13854 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13855 let carried = ids(&["b"]);
13856 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13857 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13858 }
13859
13860 #[test]
13861 fn pick_successor_returns_none_without_any_untried_successor() {
13862 let roster = [spec("a"), spec("b")];
13863 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13864 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13865 let carried = ids(&["a"]);
13866 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13867 }
13868
13869 #[test]
13870 fn pick_successor_rescue_avoids_another_seats_occupant() {
13871 let roster = [spec("a"), spec("b"), spec("c")];
13873 let carried = ids(&["b", "c"]);
13874 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13875 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13876 }
13877
13878 #[test]
13879 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13880 let roster = [spec("a"), spec("b"), spec("c")];
13881 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13882 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13883 }
13884
13885 #[test]
13886 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13887 let roster = [spec("a"), spec("b"), spec("c")];
13888 let failed = ids(&["a", "b", "c"]);
13889 let mut tried = ids(&["b"]);
13891 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13892 assert_eq!(first.id, "a");
13893 tried.insert(first.id.clone());
13894 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13895 assert_eq!(second.id, "c");
13896 tried.insert(second.id.clone());
13897 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13898 }
13899
13900 #[test]
13901 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13902 let roster = [spec("a"), spec("b")];
13903 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13904 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13905 }
13906
13907 #[test]
13908 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13909 let roster = [spec("a"), spec("b"), spec("c")];
13910 let h = SeatHistory {
13911 failed: ids(&["a"]),
13912 last_ok: Some("b".to_owned()),
13913 last_fail: None,
13914 };
13915 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13916 let gone = SeatHistory {
13918 last_ok: Some("zzz".to_owned()),
13919 ..h.clone()
13920 };
13921 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13922 let failed = SeatHistory {
13923 failed: ids(&["a", "b"]),
13924 last_ok: Some("b".to_owned()),
13925 last_fail: None,
13926 };
13927 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13928 }
13929
13930 #[test]
13931 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13932 let first = SeatState::new("review-1", "alpha", 7);
13933 let next = handover_seat("review-1", "gamma", 7);
13934 assert_ne!(first.claude_session, next.claude_session);
13935 }
13936
13937 #[test]
13938 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13939 let mut state = state_with_summary("x", "y");
13940 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13941 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13942 assert_ne!(a.claude_session, b.claude_session);
13943 }
13944
13945 #[test]
13946 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13947 let roster = [spec("a"), spec("b")];
13948 let h = SeatHistory {
13949 failed: ids(&["a", "b"]),
13950 ..SeatHistory::default()
13951 };
13952 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13953 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13954 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13955 }
13956
13957 #[test]
13964 fn failure_signature_ignores_numbers_and_paths() {
13965 assert_eq!(
13966 failure_signature("exited with Some(2) and no usable output"),
13967 failure_signature("exited with Some(137) and no usable output")
13968 );
13969 assert_eq!(
13970 failure_signature("process exited with no usable output (exit code 2)"),
13971 failure_signature("process exited with no usable output (exit code 137)")
13972 );
13973 for (old, new) in [
13974 (
13975 "exited with Some(2) and no usable output",
13976 "process exited with no usable output (exit code 2)",
13977 ),
13978 (
13979 "exited with None and no usable output",
13980 "process exited with no usable output (exit code unavailable)",
13981 ),
13982 ] {
13983 let previous = FailClass::Other(failure_signature(old));
13984 let current = FailClass::Other(failure_signature(new));
13985 assert_eq!(previous, current, "resumption retains the failure class");
13986 assert!(!should_hand_over(Some(&previous), ¤t));
13987 }
13988 assert_eq!(
13989 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13990 failure_signature("cannot open /var/x.txt: denied")
13991 );
13992 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13993 }
13994
13995 #[test]
13996 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13997 let other = FailClass::Other("x".into());
13998 assert!(should_hand_over(None, &FailClass::Quota));
13999 assert!(should_hand_over(Some(&other), &FailClass::Quota));
14000 assert!(should_hand_over(
14001 Some(&FailClass::Timeout),
14002 &FailClass::Timeout
14003 ));
14004 assert!(should_hand_over(None, &other));
14005 assert!(!should_hand_over(Some(&other), &other));
14006 assert!(should_hand_over(
14007 Some(&other),
14008 &FailClass::Other("y".into())
14009 ));
14010 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14012 assert!(should_hand_over(Some(&FailClass::Quota), &other));
14013 }
14014
14015 #[test]
14016 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14017 let a = SeatState::new("judge-1", "alpha", 7);
14018 let b = handover_seat("judge-1", "beta", 7);
14019 assert_ne!(a.claude_session, b.claude_session);
14020 assert_eq!(b.turns, 0);
14021 }
14022
14023 #[test]
14024 fn a_timeout_is_classified_apart_from_other_failures() {
14025 assert_eq!(
14026 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14027 Some(FailClass::Timeout)
14028 );
14029 assert!(matches!(
14030 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14031 Some(FailClass::Other(_))
14032 ));
14033 }
14034
14035 #[test]
14036 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14037 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14038 let tried = BTreeSet::from(["beta".to_owned()]);
14039 let next = next_untried_in_roster(&roster, 1, &tried);
14042 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14043 }
14044
14045 #[test]
14046 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14047 let roster = vec![spec("alpha"), spec("beta")];
14048 let tried = BTreeSet::from(["beta".to_owned()]);
14049 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14053 }
14054
14055 #[test]
14056 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14057 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14058 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14059 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14063 }
14064
14065 #[test]
14066 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14067 let roster = vec![spec("a"), spec("a"), spec("b")];
14068 let tried = BTreeSet::from(["a".to_owned()]);
14069 let next = next_untried_in_roster(&roster, 0, &tried);
14070 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14071 }
14072
14073 #[test]
14074 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14075 let roster = vec![spec("a"), spec("b")];
14076 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14077 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14078 }
14079
14080 #[test]
14081 fn remove_if_empty_only_ever_takes_a_bare_directory() {
14082 let dir = tempfile::tempdir().unwrap();
14083 let bay = dir.path().join("ffff");
14084
14085 remove_if_empty(&bay);
14087 assert!(!bay.exists());
14088
14089 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14092 remove_if_empty(&bay);
14093 assert!(bay.exists(), "non-empty directory must survive");
14094
14095 std::fs::remove_dir(bay.join("cand-A")).unwrap();
14097 remove_if_empty(&bay);
14098 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14099 }
14100
14101 #[test]
14110 fn a_full_panel_that_found_nothing_is_clean() {
14111 assert!(round_is_clean(
14112 0,
14113 true,
14114 2,
14115 2,
14116 0,
14117 IncompleteReviewPolicy::Block
14118 ));
14119 }
14120
14121 #[test]
14122 fn a_missing_seat_is_never_clean_under_the_default_policy() {
14123 assert!(!round_is_clean(
14124 0,
14125 true,
14126 1,
14127 2,
14128 0,
14129 IncompleteReviewPolicy::Block
14130 ));
14131 }
14132
14133 #[test]
14134 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14135 assert!(!round_is_clean(
14136 1,
14137 true,
14138 1,
14139 2,
14140 0,
14141 IncompleteReviewPolicy::Warn
14142 ));
14143 }
14144
14145 #[test]
14146 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14147 assert!(round_is_clean(
14148 0,
14149 true,
14150 1,
14151 2,
14152 0,
14153 IncompleteReviewPolicy::Warn
14154 ));
14155 }
14156
14157 #[test]
14158 fn a_full_panel_with_an_open_finding_is_not_clean() {
14159 assert!(!round_is_clean(
14160 1,
14161 true,
14162 2,
14163 2,
14164 0,
14165 IncompleteReviewPolicy::Block
14166 ));
14167 }
14168
14169 #[test]
14170 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14171 assert!(!round_is_clean(
14172 0,
14173 false,
14174 2,
14175 2,
14176 0,
14177 IncompleteReviewPolicy::Block
14178 ));
14179 }
14180
14181 #[test]
14188 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14189 assert!(round_is_clean(
14192 0,
14193 true,
14194 1,
14195 2,
14196 1,
14197 IncompleteReviewPolicy::Block
14198 ));
14199 }
14200
14201 #[test]
14202 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14203 assert!(!round_is_clean(
14206 0,
14207 true,
14208 1,
14209 2,
14210 0,
14211 IncompleteReviewPolicy::Block
14212 ));
14213 }
14214
14215 #[test]
14216 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14217 assert!(!round_is_clean(
14218 1,
14219 true,
14220 1,
14221 2,
14222 1,
14223 IncompleteReviewPolicy::Block
14224 ));
14225 assert!(!round_is_clean(
14226 0,
14227 false,
14228 1,
14229 2,
14230 1,
14231 IncompleteReviewPolicy::Block
14232 ));
14233 }
14234
14235 #[test]
14236 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14237 assert!(!round_is_clean(
14241 0,
14242 true,
14243 0,
14244 2,
14245 2,
14246 IncompleteReviewPolicy::Block
14247 ));
14248 }
14249
14250 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14251 CommandOutcome {
14252 command: "test".to_owned(),
14253 code,
14254 output_tail: String::new(),
14255 duration_ms: 0,
14256 resource_blocked,
14257 }
14258 }
14259
14260 #[cfg(unix)]
14261 #[tokio::test]
14262 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14263 ask_test_home();
14264 let temp = tempfile::tempdir().unwrap();
14265 let mut runner = runner_at(RunStatus::Gating);
14266 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14267 let commands = vec![
14268 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14269 "touch must-not-run".to_owned(),
14270 ];
14271 let (outcomes, retried, pids) = run_e2e_with_retry(
14272 &mut runner.state,
14273 &shell,
14274 &commands,
14275 temp.path(),
14276 Duration::from_secs(5),
14277 "fixture",
14278 )
14279 .await;
14280 assert_eq!(outcomes.len(), 1);
14281 assert!(outcomes[0].resource_blocked);
14282 assert!(!outcomes[0].build_failed());
14283 assert!(!retried);
14284 assert!(pids.is_empty());
14285 assert!(!temp.path().join("must-not-run").exists());
14286 assert_eq!(
14287 runner.state.resource_block.as_ref().unwrap().kind,
14288 "incremental cache admission"
14289 );
14290 }
14291
14292 #[test]
14293 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14294 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14295 assert!(
14296 !verify_inconclusive(&[outcome(Some(1), false)]),
14297 "an ordinary failure is still evidence about the patch"
14298 );
14299 assert!(verify_inconclusive(&[outcome(None, true)]));
14300 assert!(
14301 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14302 "one inconclusive outcome taints the whole batch"
14303 );
14304 assert!(!verify_inconclusive(&[]));
14305 }
14306
14307 #[tokio::test]
14308 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14309 let calls = std::sync::atomic::AtomicUsize::new(0);
14313 let started = Instant::now();
14314 wait_for_pids_with(
14315 &[123],
14316 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14317 Duration::from_millis(5),
14318 Duration::from_secs(5),
14319 )
14320 .await;
14321 assert!(
14322 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14323 "must keep checking rather than deciding on the first answer"
14324 );
14325 assert!(
14326 started.elapsed() < Duration::from_secs(1),
14327 "must return the moment it is confirmed dead, not wait out the ceiling"
14328 );
14329 }
14330
14331 #[tokio::test]
14332 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14333 let started = Instant::now();
14334 wait_for_pids_with(
14335 &[123],
14336 |_| true, Duration::from_millis(5),
14338 Duration::from_millis(30),
14339 )
14340 .await;
14341 let elapsed = started.elapsed();
14342 assert!(
14343 elapsed >= Duration::from_millis(30),
14344 "must not give up before its own ceiling: {elapsed:?}"
14345 );
14346 assert!(
14347 elapsed < Duration::from_secs(1),
14348 "must not wait past its own ceiling either: {elapsed:?}"
14349 );
14350 }
14351
14352 #[tokio::test]
14353 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14354 let started = Instant::now();
14355 wait_for_pids_with(
14356 &[],
14357 |_| true,
14358 Duration::from_secs(5),
14359 Duration::from_secs(5),
14360 )
14361 .await;
14362 assert!(
14363 started.elapsed() < Duration::from_millis(200),
14364 "an empty pid list has nothing to confirm"
14365 );
14366 }
14367
14368 fn review_round(
14374 clean: bool,
14375 blocking: usize,
14376 answered: usize,
14377 expected: usize,
14378 progressed: bool,
14379 e2e_ok: bool,
14380 ) -> ReviewRound {
14381 ReviewRound {
14382 round: 1,
14383 head: "h".to_owned(),
14384 verified_head: None,
14385 verified_at: None,
14386 reviews: Vec::new(),
14387 e2e: vec![CommandOutcome {
14388 command: "test".to_owned(),
14389 code: Some(if e2e_ok { 0 } else { 1 }),
14390 output_tail: String::new(),
14391 duration_ms: 0,
14392 resource_blocked: false,
14393 }],
14394 verify_retried: false,
14395 e2e_deferred: false,
14396 e2e_defer_reason: None,
14397 fix: None,
14398 blocking,
14399 answered,
14400 expected,
14401 clean,
14402 progressed,
14403 vote_split: false,
14404 reconsideration: Vec::new(),
14405 verdict: None,
14406 }
14407 }
14408
14409 #[test]
14410 fn review_conclusion_is_none_when_nothing_has_run() {
14411 assert_eq!(review_conclusion(&[], 3), None);
14412 }
14413
14414 #[test]
14415 fn review_conclusion_is_none_while_rounds_remain() {
14416 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14417 assert_eq!(review_conclusion(&rounds, 3), None);
14418 }
14419
14420 #[test]
14421 fn review_conclusion_is_gating_once_a_round_is_clean() {
14422 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14423 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14424 }
14425
14426 #[test]
14427 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14428 let rounds = vec![
14429 review_round(false, 1, 2, 2, true, true),
14430 review_round(false, 1, 2, 2, true, true),
14431 ];
14432 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14433 }
14434
14435 #[test]
14436 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14437 let rounds = vec![
14438 review_round(false, 1, 2, 2, true, true),
14439 review_round(false, 1, 2, 2, true, false),
14440 ];
14441 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14442 }
14443
14444 #[test]
14445 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14446 let mut blocked = review_round(false, 1, 2, 2, true, false);
14453 blocked.e2e[0].resource_blocked = true;
14454 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14455 assert_eq!(review_conclusion(&rounds, 2), None);
14456 }
14457
14458 #[test]
14459 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14460 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14462 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14463 }
14464
14465 #[test]
14466 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14467 let rounds = vec![
14468 review_round(false, 1, 2, 2, false, true),
14469 review_round(false, 1, 2, 2, false, true),
14470 ];
14471 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14472 }
14473
14474 fn secs(n: u64) -> Duration {
14475 Duration::from_secs(n)
14476 }
14477
14478 #[tokio::test]
14481 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14482 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14483 if std::env::var_os(CHILD).is_none() {
14484 let output = std::process::Command::new(std::env::current_exe().unwrap())
14488 .args([
14489 "--exact",
14490 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14491 "--nocapture",
14492 ])
14493 .env(CHILD, "1")
14494 .env_remove("MAGI_WORKER_HOME")
14495 .env_remove("MAGI_WORKER_ID")
14496 .output()
14497 .unwrap();
14498 assert!(
14499 output.status.success()
14500 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14501 "{}\n{}",
14502 String::from_utf8_lossy(&output.stdout),
14503 String::from_utf8_lossy(&output.stderr)
14504 );
14505 return;
14506 }
14507 ask_test_home();
14508 let tmp = tempfile::tempdir().unwrap();
14509 let repo = tmp.path().join("repo");
14510 std::fs::create_dir_all(&repo).unwrap();
14511 init_repo(&repo);
14512 let g = |args: &[&str]| {
14513 let out = std::process::Command::new("git")
14514 .args(args)
14515 .current_dir(&repo)
14516 .quiet()
14517 .output()
14518 .unwrap();
14519 assert!(
14520 out.status.success(),
14521 "{}",
14522 String::from_utf8_lossy(&out.stderr)
14523 );
14524 };
14525 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14526 g(&["checkout", "-b", "historical"]);
14527 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14528 g(&["add", "."]);
14529 g(&["commit", "-m", "old alternative"]);
14530 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14531 g(&["checkout", "-b", "magi/task-selected", &base]);
14532 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14533 g(&["add", "."]);
14534 g(&["commit", "-m", "measurement infrastructure"]);
14535 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14536 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14537 g(&["add", "."]);
14538 g(&["commit", "-m", "dependent implementation"]);
14539 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14540 g(&["checkout", "main"]);
14541 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14542 g(&["add", "."]);
14543 g(&["commit", "-m", "unrelated base change"]);
14544 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14545 let origin = tmp.path().join("origin.git");
14546 g(&[
14547 "clone",
14548 "--bare",
14549 repo.to_str().unwrap(),
14550 origin.to_str().unwrap(),
14551 ]);
14552 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14553 g(&["fetch", "origin"]);
14554 let selection = crate::pr_recovery::Selection {
14555 url: "https://example.test/project/pull/1".into(),
14556 branch: "magi/task-selected".into(),
14557 head: head.clone(),
14558 source_head: None,
14559 source_run: None,
14560 source_proof: None,
14561 disposition: Some(crate::pr_recovery::Disposition::Review),
14562 instruction: "preserve measured behavior".into(),
14563 approved_at: jiff::Timestamp::now(),
14564 };
14565 let mut template = runner_at(RunStatus::Reviewing);
14566 template.state.repo = repo.clone();
14567 template.state.base_commit = current.clone();
14568 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14569 template.state.origin = Some(crate::run::Origin::queue("selected"));
14570 template.state.pr_head = Some(selection.branch.clone());
14571 template.state.pr_recovery = Some(selection.clone());
14572 let mut runner =
14573 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14574 .await
14575 .unwrap();
14576 let wt = runner.state.winner().unwrap().worktree.clone();
14577 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14578 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14579 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14580 assert!(runner.state.seeds.is_empty());
14581 assert_eq!(runner.state.instruction, "preserve measured behavior");
14582 runner.sync_to_base().await.unwrap();
14583 assert_eq!(
14584 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14585 "prerequisite"
14586 );
14587 assert_eq!(
14588 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14589 "measured implementation"
14590 );
14591 assert_eq!(
14592 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14593 "current main change"
14594 );
14595 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14596 assert_eq!(
14597 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14598 head
14599 );
14600
14601 let mut partial = runner_at(RunStatus::Reviewing);
14603 partial.state.repo = repo.clone();
14604 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14605 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14606 run: Some("exit 1".into()),
14607 ..Default::default()
14608 }];
14609 let selection = runner.state.pr_recovery.clone().unwrap();
14610 partial.state.pr_recovery = Some(selection.clone());
14611 partial.state.pr_head = Some(selection.branch.clone());
14612 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14613 let id = partial.state.id.clone();
14614 let error = match Runner::open_selected(
14615 &repo,
14616 partial.state,
14617 partial.roles,
14618 1,
14619 runner.state.base_commit.clone(),
14620 selection,
14621 )
14622 .await
14623 {
14624 Ok(_) => panic!("setup must fail"),
14625 Err(e) => e,
14626 };
14627 assert_eq!(
14628 error
14629 .downcast_ref::<crate::run::StartupFailure>()
14630 .unwrap()
14631 .run,
14632 id
14633 );
14634 let failed = RunState::load(&id).unwrap();
14635 assert_eq!(failed.status, RunStatus::Failed);
14636 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14637 assert!(Runner::resume(&id).is_err());
14638 runner.state.status = RunStatus::Ready;
14639 runner.state.driver_exited = true;
14640 runner.state.save().unwrap();
14641 let mut retry = runner_at(RunStatus::Reviewing);
14642 retry.state.repo = repo.clone();
14643 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14644 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14645 reselected.approved_at = jiff::Timestamp::now();
14646 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14647 retry.state.pr_head = Some(reselected.branch.clone());
14648 retry.state.pr_recovery = Some(reselected.clone());
14649 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14650 .await
14651 .unwrap();
14652 let restarted = Runner::open_selected(
14653 &repo,
14654 retry.state,
14655 retry.roles,
14656 1,
14657 runner.state.base_commit.clone(),
14658 reselected,
14659 )
14660 .await
14661 .unwrap();
14662 assert_eq!(
14663 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14664 .await
14665 .unwrap(),
14666 head
14667 );
14668 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14669 assert_eq!(
14670 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14671 head
14672 );
14673 }
14674
14675 #[tokio::test]
14676 #[cfg(unix)]
14677 async fn fallback_and_interrupted_restoration_require_coordination() {
14678 use std::os::fd::AsRawFd as _;
14679 ask_test_home();
14680 let tmp = tempfile::tempdir().unwrap();
14681 let mut runner = registered_runner(tmp.path()).await;
14682 runner.state.config.graph.timeout_resource_wait = 1;
14683 let winner = runner.state.candidates[0].clone();
14684 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14685 let leftover = winner.worktree.join("leftover");
14686 std::fs::write(&leftover, "untrusted edits").unwrap();
14687 let next = tmp.path().join("next-fixer");
14688 let mut failed = spec("first");
14689 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14690 let mut successor = spec("second");
14691 successor.command = vec![
14692 "sh".into(),
14693 "-c".into(),
14694 format!("touch '{}'; echo done", next.display()),
14695 ];
14696 let attempts = vec![
14697 (failed.clone(), "first".into()),
14698 (successor.clone(), "second".into()),
14699 ];
14700 let lock_path = runner.state.repo.join(".git/updater.lock");
14701 let external = std::fs::File::create(&lock_path).unwrap();
14702 assert_eq!(
14703 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14704 0
14705 );
14706 let artifacts = tmp.path().join("artifacts");
14707 let build = |spec, seat| SeatJob {
14708 spec,
14709 seat,
14710 cwd: winner.worktree.clone(),
14711 prompt: "fixture".into(),
14712 timeout: Duration::from_secs(10),
14713 allow_write: true,
14714 sessions: false,
14715 artifacts: artifacts.clone(),
14716 stem: "fix".into(),
14717 handover: None,
14718 };
14719 let (_, _, out) = runner
14720 .ask_fixer_among(
14721 attempts.clone(),
14722 "graft",
14723 None,
14724 Some((&winner.worktree, &head)),
14725 build,
14726 )
14727 .await;
14728 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14729 assert!(leftover.exists(), "contention must prevent clean");
14730 assert!(!next.exists());
14731 assert!(runner.state.handovers.is_empty());
14732 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14734 from_run: "earlier".into(),
14735 generation: 1,
14736 mode: crate::recovery::Mode::VerifyRepair,
14737 reasons: Vec::new(),
14738 reviewed_head: None,
14739 rounds: Vec::new(),
14740 open_findings: Vec::new(),
14741 evidence: None,
14742 criteria_inherited: true,
14743 repair: crate::recovery::Repair::Started {
14744 before: head.clone(),
14745 },
14746 });
14747 assert!(
14748 runner
14749 .recovery_repair(&winner)
14750 .await
14751 .unwrap_err()
14752 .is::<crate::repo_lock::ResourceBlocked>()
14753 );
14754 assert!(matches!(
14755 runner.state.recovery.as_ref().unwrap().repair,
14756 crate::recovery::Repair::Started { .. }
14757 ));
14758 assert!(leftover.exists());
14759 drop(external);
14760 runner.recovery_repair(&winner).await.unwrap();
14761 assert!(!leftover.exists());
14762 assert!(matches!(
14763 runner.state.recovery.as_ref().unwrap().repair,
14764 crate::recovery::Repair::Done { .. }
14765 ));
14766 std::fs::remove_file(&lock_path).unwrap();
14768 std::fs::create_dir(&lock_path).unwrap();
14769 std::fs::write(&leftover, "untrusted edits").unwrap();
14770 let (_, _, out) = runner
14771 .ask_fixer_among(
14772 attempts,
14773 "graft",
14774 None,
14775 Some((&winner.worktree, &head)),
14776 build,
14777 )
14778 .await;
14779 assert!(matches!(out, AgentOutcome::Failed(_)));
14780 assert!(leftover.exists());
14781 assert!(!next.exists());
14782 assert_eq!(
14783 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14784 head
14785 );
14786 }
14787
14788 #[cfg(unix)]
14789 async fn registered_runner(root: &Path) -> Runner {
14790 repository_runner(root, crate::config::test_coordination()).await
14791 }
14792
14793 async fn repository_runner(
14794 root: &Path,
14795 coordination: Option<crate::config::Coordination>,
14796 ) -> Runner {
14797 ask_test_home();
14798 let repo = root.join("repo");
14799 std::fs::create_dir_all(&repo).unwrap();
14800 init_repo(&repo);
14801 if coordination.is_some() {
14802 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14803 }
14804 let mut runner = runner_at(RunStatus::Prep);
14805 runner.state.repo = repo.clone();
14806 runner.state.config.coordination = coordination;
14807 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14808 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14809 runner.state.config.graph.advise = false;
14810 runner.roles.implementers = vec![spec("implement")];
14811 runner.roles.judges = vec![spec("judge")];
14812 runner.prep().await.unwrap();
14813 runner
14814 }
14815
14816 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14817
14818 #[tokio::test]
14819 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14820 let tmp = tempfile::tempdir().unwrap();
14821 let mut runner = repository_runner(tmp.path(), None).await;
14822 let prompts = runner.state.config.prompts.clone();
14823 let run = runner.state.id.clone();
14824 let ctx = WaveCtx {
14825 carry_seats: false,
14826 pauses: None,
14827 run: &run,
14828 node: "advise",
14829 prompts: &prompts,
14830 cache: None,
14831 round: None,
14832 };
14833 let jobs: Vec<_> = ["alpha", "beta"]
14834 .into_iter()
14835 .map(|id| SeatJob {
14836 spec: scripted(id, AUTH_OUTAGE),
14837 seat: SeatState::new(id, id, 7),
14838 prompt: "propose".into(),
14839 cwd: runner.state.candidates[0].worktree.clone(),
14840 timeout: Duration::from_secs(10),
14841 allow_write: false,
14842 sessions: false,
14843 artifacts: agent::artifacts_dir(&runner.state.dir()),
14844 stem: id.into(),
14845 handover: None,
14846 })
14847 .collect();
14848 let mut losses = Vec::new();
14849 let results = ask_wave_with(
14850 jobs.clone(),
14851 runner.sem.clone(),
14852 2,
14853 &[],
14854 &ctx,
14855 &mut losses,
14856 &mut runner.state,
14857 &|_: &str| Ok(()),
14858 )
14859 .await;
14860 assert!(
14861 results
14862 .iter()
14863 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14864 );
14865 assert_eq!(runner.state.quota.len(), 2);
14866 assert!(
14867 runner
14868 .state
14869 .quota
14870 .iter()
14871 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14872 );
14873 assert!(!runner.state.advise_attempted);
14874 runner.stop_for_resource().unwrap();
14875 assert_eq!(runner.state.status, RunStatus::Blocked);
14876 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14877 assert_eq!(
14878 runner.state.resource_block.as_ref().unwrap().kind,
14879 "agent authentication"
14880 );
14881 runner.state.resource_block = None;
14884 runner.state.quota.clear();
14885 let mut mixed = jobs;
14886 mixed[1].spec.command = vec!["false".into()];
14887 let results = ask_wave_with(
14888 mixed,
14889 runner.sem.clone(),
14890 0,
14891 &[],
14892 &ctx,
14893 &mut losses,
14894 &mut runner.state,
14895 &|_: &str| Ok(()),
14896 )
14897 .await;
14898 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14899 assert!(runner.state.resource_block.is_none());
14900 assert_eq!(losses.len(), 1);
14901 }
14902
14903 #[tokio::test]
14904 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14905 let tmp = tempfile::tempdir().unwrap();
14906 let mut runner = repository_runner(tmp.path(), None).await;
14907 let implementer = scripted("implement", AUTH_OUTAGE);
14908 runner.roles.implementers = vec![implementer.clone()];
14909 runner.state.config.agents = vec![implementer];
14910 runner.implement().await.unwrap();
14911 assert_eq!(runner.state.status, RunStatus::Blocked);
14912 assert_eq!(runner.state.quota.len(), 1);
14913 assert_eq!(
14914 runner.state.quota[0].cause,
14915 agent::QuotaCause::Authentication
14916 );
14917 let candidate = &runner.state.candidates[0];
14918 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14919 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14920 }
14921
14922 #[tokio::test]
14923 #[cfg(unix)]
14924 async fn parked_implementer_persists_completed_work() {
14925 for registered in [false, true] {
14926 let tmp = tempfile::tempdir().unwrap();
14927 let mut runner = registered_runner(tmp.path()).await;
14928 if !registered {
14929 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14930 }
14931 let marker = tmp.path().join("started");
14932 let release = tmp.path().join("release");
14933 let mut implementer = spec("implement");
14934 implementer.command = vec![
14935 "sh".into(),
14936 "-c".into(),
14937 format!(
14938 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14939 marker.display(),
14940 release.display()
14941 ),
14942 ];
14943 runner.roles.implementers = vec![implementer.clone()];
14944 runner.state.config.agents = vec![implementer];
14945 let pause = runner.pause.clone();
14946 let (result, ()) = tokio::join!(runner.implement(), async {
14947 tokio::time::timeout(Duration::from_secs(10), async {
14948 while !marker.exists() {
14949 tokio::time::sleep(Duration::from_millis(10)).await;
14950 }
14951 })
14952 .await
14953 .unwrap();
14954 pause.park();
14955 std::fs::write(&release, "go").unwrap();
14956 });
14957 result.unwrap();
14958 let candidate = &runner.state.candidates[0];
14959 assert!(candidate.commits > 0, "{candidate:?}");
14960 assert!(candidate.summary.contains("saved-result"));
14961 assert_eq!(candidate.files, 1);
14962 let before = candidate.commits;
14963 runner.implement().await.unwrap();
14964 assert_eq!(runner.state.candidates[0].commits, before);
14965 }
14966 }
14967
14968 #[tokio::test]
14969 #[cfg(unix)]
14970 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14971 let tmp = tempfile::tempdir().unwrap();
14972 let mut runner = registered_runner(tmp.path()).await;
14973 let winner = runner.state.candidates[0].clone();
14974 let cache = tmp.path().join("cache");
14975 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14976 runner.state.config.graph.timeout_resource_wait = 0;
14977 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14978 let crate::cache::AcquireOutcome::Acquired(held) =
14979 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14980 else {
14981 panic!("cache must be free")
14982 };
14983 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14984 from_run: "earlier".into(),
14985 generation: 1,
14986 mode: crate::recovery::Mode::VerifyRepair,
14987 reasons: Vec::new(),
14988 reviewed_head: None,
14989 rounds: Vec::new(),
14990 open_findings: Vec::new(),
14991 evidence: None,
14992 criteria_inherited: true,
14993 repair: crate::recovery::Repair::NotStarted,
14994 });
14995 runner.recovery_repair(&winner).await.unwrap();
14996 assert!(matches!(
14997 runner.state.recovery.as_ref().unwrap().repair,
14998 crate::recovery::Repair::WaitingForRestore { .. }
14999 ));
15000 drop(held);
15001 runner.state.resource_block = None;
15002 runner.state.config.graph.timeout_resource_wait = 1;
15003 runner.recovery_repair(&winner).await.unwrap();
15004 assert!(matches!(
15005 runner.state.recovery.as_ref().unwrap().repair,
15006 crate::recovery::Repair::Done { .. }
15007 ));
15008 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15009 }
15010
15011 #[tokio::test]
15012 #[cfg(unix)]
15013 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15014 use std::os::fd::AsRawFd as _;
15015 for graft in [false, true] {
15016 let tmp = tempfile::tempdir().unwrap();
15017 let mut runner = registered_runner(tmp.path()).await;
15018 runner.state.config.graph.timeout_resource_wait = 1;
15019 let winner = runner.state.candidates[0].clone();
15020 let marker = tmp.path().join("fixer-started");
15021 let mut fixer = spec("fixer");
15022 fixer.command = vec![
15023 "sh".into(),
15024 "-c".into(),
15025 format!("echo started >> '{}'; echo done", marker.display()),
15026 ];
15027 runner.roles.fixer = Some(vec![fixer]);
15028 if graft {
15029 let mut loser = winner.clone();
15030 loser.label = 'B';
15031 loser.index = 1;
15032 runner.state.candidates.push(loser);
15033 runner.state.config.graph.graft_runner_up = true;
15034 runner.state.tally = Some(Tally {
15035 first_choice: BTreeMap::new(),
15036 borda: BTreeMap::new(),
15037 winner: winner.label,
15038 rankings: 1,
15039 unanimous_initial: true,
15040 deliberated: false,
15041 changed_votes: 0,
15042 unanimous_final: true,
15043 tie_break: None,
15044 judges: 1,
15045 present: 1,
15046 quorum: 1,
15047 met_quorum: true,
15048 uncontested: None,
15049 });
15050 runner.state.judgements.push(crate::run::Judgement {
15051 judge: 1,
15052 seat: "judge-1".into(),
15053 agent: "judge".into(),
15054 ranking: vec![winner.label, 'B'],
15055 reasons: BTreeMap::new(),
15056 confidence: None,
15057 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15058 order: vec![0, 1],
15059 failed: None,
15060 duration_ms: 0,
15061 });
15062 } else {
15063 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15064 from_run: "earlier".into(),
15065 generation: 1,
15066 mode: crate::recovery::Mode::VerifyRepair,
15067 reasons: Vec::new(),
15068 reviewed_head: None,
15069 rounds: Vec::new(),
15070 open_findings: Vec::new(),
15071 evidence: None,
15072 criteria_inherited: true,
15073 repair: crate::recovery::Repair::NotStarted,
15074 });
15075 }
15076 let reader = crate::repo_lock::SeatLease::for_checkout(
15077 &winner.worktree,
15078 false,
15079 crate::config::test_coordination().as_ref(),
15080 )
15081 .await
15082 .unwrap();
15083 let pause = runner.pause.clone();
15084 let (result, ()) = tokio::join!(
15085 async {
15086 if graft {
15087 runner.graft().await
15088 } else {
15089 runner.recovery_repair(&winner).await
15090 }
15091 },
15092 async {
15093 tokio::time::sleep(Duration::from_millis(100)).await;
15094 pause.park();
15095 }
15096 );
15097 result.unwrap();
15098 assert!(!marker.exists(), "a blocked seat must not spawn");
15099 runner.state = RunState::load(&runner.state.id).unwrap();
15101 if graft {
15102 assert_eq!(
15103 runner.state.graft.as_ref().unwrap().outcome,
15104 GraftOutcome::WaitingForRestore
15105 );
15106 } else {
15107 assert!(matches!(
15108 runner.state.recovery.as_ref().unwrap().repair,
15109 crate::recovery::Repair::WaitingForRestore { .. }
15110 ));
15111 }
15112 drop(reader);
15113 runner.pause = Pause::new();
15114 runner.state.resource_block = None;
15115 let external =
15117 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15118 assert_eq!(
15119 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15120 0
15121 );
15122 let leftover = winner.worktree.join("leftover");
15123 std::fs::write(&leftover, "earlier fallback edits").unwrap();
15124 let result = if graft {
15125 runner.graft().await
15126 } else {
15127 runner.recovery_repair(&winner).await
15128 };
15129 assert!(
15130 result
15131 .unwrap_err()
15132 .is::<crate::repo_lock::ResourceBlocked>()
15133 );
15134 assert!(leftover.exists());
15135 assert!(!marker.exists());
15136 drop(external);
15137 runner.state.resource_block = None;
15138 let result = if graft {
15139 runner.graft().await
15140 } else {
15141 runner.recovery_repair(&winner).await
15142 };
15143 result.unwrap();
15144 assert!(!leftover.exists(), "restore must precede retry");
15145 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15146 if graft {
15147 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15148 } else {
15149 assert!(matches!(
15150 runner.state.recovery.as_ref().unwrap().repair,
15151 crate::recovery::Repair::Done { .. }
15152 ));
15153 }
15154 }
15155 }
15156
15157 #[tokio::test]
15158 #[cfg(unix)]
15159 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15160 use std::os::fd::AsRawFd as _;
15161 let tmp = tempfile::tempdir().unwrap();
15162 let mut runner = registered_runner(tmp.path()).await;
15163 runner.state.config.graph.timeout_resource_wait = 1;
15164 let winner = runner.state.candidates[0].clone();
15165 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15166 runner.state.tally = Some(Tally {
15167 first_choice: BTreeMap::new(),
15168 borda: BTreeMap::new(),
15169 winner: winner.label,
15170 rankings: 0,
15171 unanimous_initial: false,
15172 deliberated: false,
15173 changed_votes: 0,
15174 unanimous_final: false,
15175 tie_break: None,
15176 judges: 0,
15177 present: 0,
15178 quorum: 0,
15179 met_quorum: true,
15180 uncontested: Some("fixture".into()),
15181 });
15182 runner.state.graft = Some(GraftRecord {
15183 pre: pre.clone(),
15184 post: None,
15185 lines: None,
15186 limit: None,
15187 agent: None,
15188 outcome: GraftOutcome::Running,
15189 requested: Vec::new(),
15190 adoption: None,
15191 adoption_error: None,
15192 });
15193 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15194 git::commit_all(&winner.worktree, "unfinished graft")
15195 .await
15196 .unwrap();
15197 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15198 let leftover = winner.worktree.join("leftover");
15199 std::fs::write(&leftover, "untrusted edits").unwrap();
15200 let path = runner.state.repo.join(".git/updater.lock");
15201 let external = std::fs::File::create(&path).unwrap();
15202 assert_eq!(
15203 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15204 0
15205 );
15206 assert!(
15207 runner
15208 .graft()
15209 .await
15210 .unwrap_err()
15211 .is::<crate::repo_lock::ResourceBlocked>()
15212 );
15213 assert!(matches!(
15214 runner.state.graft.as_ref().unwrap().outcome,
15215 GraftOutcome::Running
15216 ));
15217 assert_eq!(
15218 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15219 unfinished
15220 );
15221 assert!(leftover.exists());
15222 drop(external);
15223 std::fs::remove_file(&path).unwrap();
15224 std::fs::create_dir(&path).unwrap();
15225 assert!(runner.graft().await.is_err());
15226 assert!(matches!(
15227 runner.state.graft.as_ref().unwrap().outcome,
15228 GraftOutcome::Running
15229 ));
15230 assert_eq!(
15231 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15232 unfinished
15233 );
15234 assert!(leftover.exists());
15235 std::fs::remove_dir(&path).unwrap();
15236 runner.graft().await.unwrap();
15237 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15238 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15239 assert!(!leftover.exists());
15240 }
15241
15242 #[tokio::test]
15243 #[cfg(unix)]
15244 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15245 use std::os::fd::AsRawFd as _;
15246 let tmp = tempfile::tempdir().unwrap();
15247 let mut runner = registered_runner(tmp.path()).await;
15248 runner.state.prep_complete = Some(false);
15249 runner.state.config.graph.timeout_resource_wait = 1;
15250 runner.state.save().unwrap();
15251 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15252 assert_eq!(
15253 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15254 0
15255 );
15256 runner.execute().await.unwrap();
15257 let disk = RunState::load(&runner.state.id).unwrap();
15258 assert!(disk.driver_exited);
15259 assert_eq!(disk.driver_pid, None);
15260 assert_eq!(runner.state.status, RunStatus::Blocked);
15261 assert_eq!(
15262 runner.state.resource_block.as_ref().unwrap().kind,
15263 "repository coordination"
15264 );
15265 assert!(runner.state.prep_failure.is_none());
15266 assert!(runner.state.seats.is_empty());
15267 runner.pause.park();
15268 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15271 .await
15272 .unwrap()
15273 .unwrap();
15274 assert!(runner.state.parked);
15275 assert!(runner.state.prep_failure.is_none());
15276 let disk = RunState::load(&runner.state.id).unwrap();
15277 assert!(disk.driver_exited);
15278 assert_eq!(disk.driver_pid, None);
15279 }
15280
15281 #[tokio::test]
15282 #[cfg(unix)]
15283 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15284 use std::os::fd::AsRawFd as _;
15285 let tmp = tempfile::tempdir().unwrap();
15286 let mut runner = registered_runner(tmp.path()).await;
15287 runner.state.prep_complete = Some(false);
15288 runner.state.config.graph.timeout_resource_wait = 30;
15289 runner.state.save().unwrap();
15290 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15291 assert_eq!(
15292 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15293 0
15294 );
15295 assert!(
15296 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15297 .await
15298 .is_err()
15299 );
15300 let disk = RunState::load(&runner.state.id).unwrap();
15301 assert!(disk.driver_exited);
15302 assert_eq!(disk.driver_pid, None);
15303 assert!(disk.status.resumable());
15304 assert!(runner.state.driver_exited);
15305 assert_eq!(runner.state.driver_pid, None);
15306 runner.state.save().unwrap();
15307 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15308 }
15309
15310 #[tokio::test]
15311 #[cfg(unix)]
15312 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15313 let tmp = tempfile::tempdir().unwrap();
15314 let mut runner = registered_runner(tmp.path()).await;
15315 let winner = runner.state.candidates[0].clone();
15316 let cache = tmp.path().join("cache");
15317 let started = tmp.path().join("pre-gate-started");
15318 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15319 runner.state.config.verify.gate =
15320 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15321 let home = crate::run::home();
15322 let owner =
15323 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15324 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15325 crate::cache::AcquireOutcome::Acquired(g) => g,
15326 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15327 };
15328 let mut gate = Box::pin(runner.run_gate(&winner));
15329 assert!(
15330 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15331 .await
15332 .is_err()
15333 );
15334 assert!(!started.exists(), "pre-gate must wait for the cache too");
15335 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15339 let editor = crate::repo_lock::SeatLease::checkout_with(
15340 &winner.worktree,
15341 true,
15342 crate::config::test_coordination().as_ref(),
15343 &wait,
15344 )
15345 .await
15346 .unwrap();
15347 drop(editor);
15348 guard.release();
15349 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15350 .await
15351 .unwrap()
15352 .unwrap();
15353 drop(gate);
15354 assert!(outcomes.iter().all(CommandOutcome::ok));
15355 assert!(started.exists());
15356 }
15357
15358 #[tokio::test]
15359 #[cfg(unix)]
15360 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15361 use std::os::fd::AsRawFd as _;
15362
15363 ask_test_home();
15364 let tmp = tempfile::tempdir().unwrap();
15365 let repo = tmp.path().join("repo");
15366 std::fs::create_dir_all(&repo).unwrap();
15367 init_repo(&repo);
15368 let common = repo.join(".git");
15369 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15370 let mut runner = runner_at(RunStatus::Prep);
15371 runner.state.repo = repo.clone();
15372 runner.state.config.coordination = crate::config::test_coordination();
15373 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15374 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15375 runner.state.config.graph.advise = false;
15376 runner.roles.implementers = vec![spec("implement")];
15377 runner.roles.judges = vec![spec("judge")];
15378 runner.prep().await.unwrap();
15379 let winner = runner.state.candidates[0].clone();
15380 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15383 let completed = tmp.path().join("seat-completed");
15384 runner.roles.implementers[0].command = vec![
15385 "sh".into(),
15386 "-c".into(),
15387 format!("touch '{}'; echo completed", completed.display()),
15388 ];
15389 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15390 assert_eq!(
15391 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15392 0
15393 );
15394 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15395 let mut implement = Box::pin(runner.implement());
15396 tokio::select! {
15397 result = &mut implement => panic!("rescue must wait: {result:?}"),
15398 result = tokio::time::timeout(Duration::from_secs(10), async {
15399 while !completed.exists() {
15400 tokio::time::sleep(Duration::from_millis(20)).await;
15401 }
15402 }) => result.unwrap(),
15403 }
15404 assert!(
15405 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15406 .await
15407 .is_err()
15408 );
15409 assert_eq!(
15410 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15411 before
15412 );
15413 assert_eq!(
15414 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15415 0
15416 );
15417 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15418 .await
15419 .unwrap()
15420 .unwrap();
15421 drop(implement);
15422 assert_ne!(
15423 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15424 before
15425 );
15426 assert!(git::is_clean(&winner.worktree).await.unwrap());
15427
15428 let started = tmp.path().join("pre-gate-started");
15429 let proceed = tmp.path().join("pre-gate-proceed");
15430 let checked = tmp.path().join("gate-ran");
15431 runner.state.config.verify.pre_gate = vec![format!(
15432 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15433 started.display(),
15434 proceed.display()
15435 )];
15436 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15437 assert_eq!(
15438 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15439 0
15440 );
15441 let mut gate = Box::pin(runner.run_gate(&winner));
15442 assert!(
15443 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15444 .await
15445 .is_err()
15446 );
15447 assert!(!started.exists());
15448 assert!(!checked.exists());
15449 assert!(git::is_clean(&winner.worktree).await.unwrap());
15450 assert_eq!(
15451 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15452 0
15453 );
15454 tokio::select! {
15455 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15456 result = tokio::time::timeout(Duration::from_secs(10), async {
15457 while !started.exists() {
15458 tokio::time::sleep(Duration::from_millis(20)).await;
15459 }
15460 }) => result.unwrap(),
15461 }
15462 assert_eq!(
15463 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15464 -1,
15465 "mechanical edits must hold the real lock"
15466 );
15467 std::fs::write(proceed, "ready").unwrap();
15468 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15469 .await
15470 .unwrap()
15471 .unwrap();
15472 drop(gate);
15473 assert!(outcomes.iter().all(CommandOutcome::ok));
15474 assert!(checked.exists());
15475 assert!(runner.state.pre_gate_commit.is_some());
15476 assert!(git::is_clean(&winner.worktree).await.unwrap());
15477 assert_eq!(
15478 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15479 0,
15480 "gate completion must release coordination"
15481 );
15482 drop(outsider);
15483 std::fs::remove_file(common.join("updater.lock")).unwrap();
15484 std::fs::create_dir(common.join("updater.lock")).unwrap();
15485 std::fs::remove_file(&started).unwrap();
15486 std::fs::remove_file(&checked).unwrap();
15487 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15488 let error = runner.run_gate(&winner).await.unwrap_err();
15489 assert!(
15490 error
15491 .to_string()
15492 .contains("acquire mandatory repository lock")
15493 );
15494 assert!(
15495 !started.exists(),
15496 "failed acquisition must prevent pre-gate"
15497 );
15498 assert!(!checked.exists(), "failed acquisition must prevent gate");
15499 assert!(git::is_clean(&winner.worktree).await.unwrap());
15500 assert_eq!(
15501 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15502 head
15503 );
15504 }
15505
15506 #[tokio::test]
15507 #[cfg(unix)]
15508 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15509 use std::os::fd::AsRawFd as _;
15510 use std::time::Duration;
15511
15512 ask_test_home();
15513 let tmp = tempfile::tempdir().unwrap();
15514 let repo = tmp.path().join("repo");
15515 std::fs::create_dir_all(&repo).unwrap();
15516 init_repo(&repo);
15517 let common = repo.join(".git");
15518 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15519 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15520 assert_eq!(
15521 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15522 0
15523 );
15524 let mut runner = runner_at(RunStatus::Prep);
15525 runner.state.repo = repo.clone();
15526 runner.state.config.coordination = crate::config::test_coordination();
15527 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15528 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15529 runner.state.config.graph.advise = true;
15530 runner.state.config.graph.advisors = 1;
15531 runner.roles.implementers = vec![spec("implement")];
15532 runner.roles.judges = vec![spec("judge")];
15533 let root = runner.state.worktree_root();
15534 let started = tmp.path().join("setup-started");
15535 let proceed = tmp.path().join("setup-proceed");
15536 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15537 run: Some(format!(
15538 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15539 started.display(),
15540 proceed.display()
15541 )),
15542 ..Default::default()
15543 }];
15544 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15545 let config = git::git(&repo, &["config", "--local", "--list"])
15546 .await
15547 .unwrap();
15548 let mut prep = Box::pin(runner.prep());
15549 assert!(
15550 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15551 .await
15552 .is_err()
15553 );
15554 assert!(
15555 !root.exists(),
15556 "contention must prevent even checkout creation"
15557 );
15558 assert!(!started.exists(), "contention must prevent setup");
15559 assert_eq!(
15560 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15561 branches
15562 );
15563 assert_eq!(
15564 git::git(&repo, &["config", "--local", "--list"])
15565 .await
15566 .unwrap(),
15567 config
15568 );
15569 drop(lock);
15570 tokio::select! {
15571 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15572 result = tokio::time::timeout(Duration::from_secs(10), async {
15573 while !started.exists() {
15574 tokio::time::sleep(Duration::from_millis(20)).await;
15575 }
15576 }) => result.unwrap(),
15577 }
15578 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15579 assert_eq!(
15580 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15581 -1
15582 );
15583 assert!(
15584 tokio::time::timeout(
15585 Duration::from_millis(100),
15586 crate::repo_lock::SeatLease::for_checkout(
15587 &repo,
15588 true,
15589 crate::config::test_coordination().as_ref()
15590 )
15591 )
15592 .await
15593 .is_err(),
15594 "preparation must also exclude local editors"
15595 );
15596 std::fs::write(proceed, "ready").unwrap();
15597 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15598 .await
15599 .unwrap()
15600 .unwrap();
15601 drop(prep);
15602 assert_eq!(runner.state.prep_complete, Some(true));
15603 assert!(root.join("judge-1").exists());
15604 assert!(root.join("advisor-1").exists());
15605 assert_eq!(
15606 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15607 0
15608 );
15609 }
15610
15611 #[tokio::test]
15612 #[cfg(unix)]
15613 async fn preparation_lock_failure_prevents_repository_changes() {
15614 ask_test_home();
15615 let tmp = tempfile::tempdir().unwrap();
15616 let repo = tmp.path().join("repo");
15617 std::fs::create_dir_all(&repo).unwrap();
15618 init_repo(&repo);
15619 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15620 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15623 let mut runner = runner_at(RunStatus::Prep);
15624 runner.state.repo = repo.clone();
15625 runner.state.config.coordination = crate::config::test_coordination();
15626 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15627 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15628 runner.roles.implementers = vec![spec("implement")];
15629 let error = runner.prep().await.unwrap_err();
15630 assert!(
15631 error
15632 .to_string()
15633 .contains("acquire mandatory repository lock")
15634 );
15635 assert!(!runner.state.worktree_root().exists());
15636 assert!(runner.state.candidates.is_empty());
15637 assert!(!repo.join(".git/config.worktree").exists());
15638 assert_eq!(
15639 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15640 "* main"
15641 );
15642 }
15643
15644 #[tokio::test]
15645 async fn preparation_records_creation_intent_before_git_can_fail() {
15646 ask_test_home();
15647 let tmp = tempfile::tempdir().unwrap();
15648 let repo = tmp.path().join("repo");
15649 std::fs::create_dir_all(&repo).unwrap();
15650 init_repo(&repo);
15651 let mut runner = runner_at(RunStatus::Prep);
15652 runner.state.repo = repo.clone();
15653 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15654 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15655 runner.state.config.blind.commit_msg_hook = false;
15656 runner.roles.implementers = vec![spec("one")];
15657 let label = blind::assign_labels(1, runner.state.seed)[0];
15658 let branch = runner.state.branch_for(label);
15659 git::git(&repo, &["branch", &branch]).await.unwrap();
15662 assert!(runner.prep().await.is_err());
15663 let saved = RunState::load(&runner.state.id).unwrap();
15664 assert!(saved.candidates.is_empty());
15665 let event = saved
15666 .events
15667 .iter()
15668 .find(|e| e.node == "prep-intent")
15669 .unwrap();
15670 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15671 assert_eq!(intent["branch"], branch);
15672 assert_eq!(intent["start"], runner.state.base_commit);
15673 assert_eq!(
15674 intent["worktree"],
15675 runner
15676 .state
15677 .worktree_root()
15678 .join(format!("cand-{label}"))
15679 .to_string_lossy()
15680 .as_ref()
15681 );
15682 assert_eq!(intent["picks"], serde_json::json!([]));
15683 }
15684
15685 #[tokio::test]
15686 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15687 ask_test_home();
15688 let tmp = tempfile::tempdir().unwrap();
15689 let repo = tmp.path().join("repo");
15690 std::fs::create_dir_all(&repo).unwrap();
15691 init_repo(&repo);
15692 let mut runner = runner_at(RunStatus::Prep);
15693 runner.state.repo = repo.clone();
15694 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15695 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15696 runner.state.sources.pick_commits.push(refs::Input {
15697 token: "missing".into(),
15698 sha: "0".repeat(40),
15699 provenance: refs::Provenance::ExplicitCli,
15700 });
15701 assert!(runner.prep().await.is_err());
15702 assert!(runner.state.candidates.is_empty());
15703 assert!(!runner.state.worktree_root().exists());
15704 }
15705
15706 #[tokio::test]
15707 async fn prose_and_legacy_inference_never_import_into_candidates() {
15708 ask_test_home();
15709 let tmp = tempfile::tempdir().unwrap();
15710 let repo = tmp.path().join("repo");
15711 std::fs::create_dir_all(&repo).unwrap();
15712 init_repo(&repo);
15713 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15714 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15715 .await
15716 .unwrap();
15717 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15718 git::git(&repo, &["add", "."]).await.unwrap();
15719 git::git(&repo, &["commit", "-m", "historical implementation"])
15720 .await
15721 .unwrap();
15722 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15723 git::git(&repo, &["checkout", "main"]).await.unwrap();
15724 for instruction in [
15725 format!("Example: magi/history/A at {source}"),
15726 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15727 format!("Never import magi/history/A or {source}"),
15728 format!("Conversation answer: I mentioned {source} only as background"),
15729 ] {
15730 let mut runner = runner_at(RunStatus::Prep);
15731 runner.state.repo = repo.clone();
15732 runner.state.base_commit = base.clone();
15733 runner.state.instruction = instruction.clone();
15734 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15735 runner.state.config.blind.commit_msg_hook = false;
15736 runner.state.seeds = vec![refs::Seed {
15737 token: "magi/history/A".into(),
15738 sha: source.clone(),
15739 kind: refs::SeedKind::Unmerged,
15740 branch: true,
15741 detail: String::new(),
15742 }];
15743 runner.roles.implementers = vec![spec("one")];
15744 assert_eq!(seeded_instruction(&runner.state), instruction);
15745 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15747 legacy.as_object_mut().unwrap().remove("sources");
15748 runner.state = serde_json::from_value(legacy).unwrap();
15749 assert_eq!(runner.state.sources, refs::Sources::default());
15750 runner.state.save().unwrap();
15751 runner.state = RunState::load(&runner.state.id).unwrap();
15752 runner.prep().await.unwrap();
15753 for candidate in &runner.state.candidates {
15754 assert_eq!(
15755 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15756 base
15757 );
15758 assert!(!candidate.worktree.join("unwanted").exists());
15759 }
15760 }
15761 }
15762
15763 #[tokio::test]
15764 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15765 ask_test_home();
15766 let tmp = tempfile::tempdir().unwrap();
15767 let repo = tmp.path().join("repo");
15768 std::fs::create_dir_all(&repo).unwrap();
15769 init_repo(&repo);
15770 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15771 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15772 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15773 git::git(&repo, &["add", "."]).await.unwrap();
15774 git::git(&repo, &["commit", "-m", "required seed"])
15775 .await
15776 .unwrap();
15777 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15778 git::git(&repo, &["checkout", "main"]).await.unwrap();
15779 let mut runner = runner_at(RunStatus::Prep);
15780 runner.state.repo = repo.clone();
15781 runner.state.base_commit = base.clone();
15782 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15783 runner.state.config.graph.advise = false;
15784 runner.state.config.blind.commit_msg_hook = false;
15785 runner.state.sources =
15786 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15787 .await
15788 .unwrap();
15789 runner.roles.implementers = vec![spec("one"), spec("two")];
15790 runner.roles.judges = vec![spec("judge")];
15791 let label = blind::assign_labels(2, runner.state.seed)[0];
15792 let branch = runner.state.branch_for(label);
15793 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15794 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15795 .await
15796 .unwrap();
15797 runner.state.candidates.push(Candidate {
15798 index: 0,
15799 label,
15800 agent: "one".into(),
15801 branch,
15802 worktree,
15803 summary: String::new(),
15804 stat: String::new(),
15805 files: 0,
15806 commits: 0,
15807 empty: false,
15808 failed: None,
15809 verified_noop: None,
15810 duration_ms: 0,
15811 folded: false,
15812 });
15813 runner.state.save().unwrap();
15814 runner.state = RunState::load(&runner.state.id).unwrap();
15816 assert_eq!(runner.state.prep_complete, Some(false));
15817 runner.prep().await.unwrap();
15818 assert_eq!(runner.state.prep_complete, Some(true));
15819 assert_eq!(runner.state.candidates.len(), 2);
15820 for candidate in &runner.state.candidates {
15821 assert_eq!(
15822 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15823 git::tree_of(&repo, &seed).await.unwrap()
15824 );
15825 assert_eq!(
15826 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15827 "required commit"
15828 );
15829 }
15830 assert!(runner.state.worktree_root().join("judge-1").exists());
15831 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15832 runner.prep().await.unwrap();
15833 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15834
15835 runner.state.prep_complete = Some(false);
15836 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15837 run: Some(format!(
15838 "test -f '{}'",
15839 tmp.path().join("dependency-ready").display()
15840 )),
15841 ..Default::default()
15842 }];
15843 assert!(runner.prep().await.is_err());
15844 assert!(runner.state.prep_failure.is_some());
15845 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15846 runner.state = RunState::load(&runner.state.id).unwrap();
15847 assert!(
15848 runner.prep().await.is_err(),
15849 "correcting the cause alone must not auto-retry"
15850 );
15851 let failed_at = runner.state.events.last().unwrap().at;
15852 assert!(
15853 runner.retry_preparation(failed_at).is_err(),
15854 "old approvals cannot unlock the hold"
15855 );
15856 let mut task = crate::queue::Task::new(
15857 "retry".into(),
15858 "retry setup".into(),
15859 repo.clone(),
15860 crate::queue::Source::Human,
15861 );
15862 task.start(runner.state.id.clone());
15863 task.hold_machine(Some("setup failed".into()));
15864 task.release_operator("cli", None);
15865 runner
15866 .retry_preparation(task.resumption.as_ref().unwrap().at)
15867 .unwrap();
15868 runner.prep().await.unwrap();
15869 assert!(runner.state.prep_failure.is_none());
15870 assert_eq!(runner.state.prep_complete, Some(true));
15871 assert_eq!(runner.state.candidates.len(), 2);
15872 assert!(runner.state.worktree_root().join("judge-1").exists());
15873 for candidate in &runner.state.candidates {
15874 assert_eq!(
15875 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15876 git::tree_of(&repo, &seed).await.unwrap()
15877 );
15878 }
15879 }
15880
15881 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15882 let mut heads = Vec::new();
15883 for candidate in candidates {
15884 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15885 }
15886 heads
15887 }
15888
15889 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15890 ask_test_home();
15891 let tmp = tempfile::tempdir().unwrap();
15892 let repo = tmp.path().join("repo");
15893 std::fs::create_dir(&repo).unwrap();
15894 init_repo(&repo);
15895 let mut runner = runner_at(RunStatus::Reviewing);
15896 runner.state.repo = repo.clone();
15897 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15898 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15899 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15900 let branch = format!("candidate-{label}");
15901 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15902 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15903 .await
15904 .unwrap();
15905 if label == 'B' {
15906 std::fs::write(worktree.join("change"), "winner").unwrap();
15907 git::rescue_commit(&worktree, "candidate change")
15908 .await
15909 .unwrap();
15910 }
15911 runner.state.candidates.push(Candidate {
15912 index,
15913 label,
15914 agent: "alpha".into(),
15915 branch,
15916 worktree,
15917 summary: "completed reply".into(),
15918 stat: String::new(),
15919 files: usize::from(label == 'B'),
15920 commits: usize::from(label == 'B'),
15921 empty: label == 'A',
15922 failed: None,
15923 verified_noop: None,
15924 duration_ms: 1,
15925 folded: false,
15926 });
15927 }
15928 runner.state.judge_skipped = true;
15929 runner.state.tally = Some(Tally {
15930 first_choice: BTreeMap::new(),
15931 borda: BTreeMap::new(),
15932 winner: 'B',
15933 rankings: 0,
15934 unanimous_initial: false,
15935 deliberated: false,
15936 changed_votes: 0,
15937 unanimous_final: false,
15938 tie_break: None,
15939 judges: 0,
15940 present: 0,
15941 quorum: 0,
15942 met_quorum: true,
15943 uncontested: Some("only B produced a change".into()),
15944 });
15945 runner.state.save().unwrap();
15946 (tmp, runner)
15947 }
15948
15949 #[tokio::test]
15950 async fn fold_losers_persists_discard_before_later_teardown_failure() {
15951 let (_tmp, mut runner) = preservation_resume_fixture().await;
15952 runner.state.candidates[0].folded = true;
15953 runner.state.config.graph.advise = true;
15954 runner.state.config.graph.advisors = 2;
15955 let repo = runner.state.repo.clone();
15956 let root = runner.state.worktree_root();
15957 let discarded = root.join("advisor-1");
15958 git::worktree_add_detached(&repo, &discarded, "main")
15959 .await
15960 .unwrap();
15961 std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
15962 let protected = root.join("advisor-2");
15963 git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
15964 .await
15965 .unwrap();
15966 std::fs::write(protected.join("keep"), "uncommitted").unwrap();
15967 runner.state.save().unwrap();
15968
15969 assert!(runner.fold_losers().await.is_err());
15970 assert!(!discarded.exists());
15971 assert!(protected.join("keep").exists());
15972 let saved = RunState::load(&runner.state.id).unwrap();
15973 assert!(saved.events.iter().any(|event| {
15974 event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
15975 }));
15976 }
15977
15978 #[tokio::test]
15979 async fn read_seat_teardown_discards_only_owned_detached_seats() {
15980 let (tmp, mut runner) = preservation_resume_fixture().await;
15981 let repo = runner.state.repo.clone();
15982 let root = runner.state.worktree_root();
15983 for name in ["advisor-3", "judge-1", "review-2"] {
15984 let path = root.join(name);
15985 git::worktree_add_detached(&repo, &path, "main")
15986 .await
15987 .unwrap();
15988 std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
15989 std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
15990 remove_read_seat_worktree(&mut runner.state, &path)
15991 .await
15992 .unwrap();
15993 assert!(!path.exists());
15994 }
15995 let outside = tmp.path().join("advisor-1");
15996 let unknown = root.join("unknown");
15997 let attached = root.join("advisor-4");
15998 git::worktree_add_detached(&repo, &outside, "main")
15999 .await
16000 .unwrap();
16001 git::worktree_add_detached(&repo, &unknown, "main")
16002 .await
16003 .unwrap();
16004 git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16005 .await
16006 .unwrap();
16007 let foreign = tmp.path().join("foreign");
16008 std::fs::create_dir(&foreign).unwrap();
16009 init_repo(&foreign);
16010 let wrong_repo = root.join("judge-2");
16011 git::worktree_add_detached(&foreign, &wrong_repo, "main")
16012 .await
16013 .unwrap();
16014 let candidate = runner.state.candidates[0].worktree.clone();
16015 for path in [outside, unknown, attached, wrong_repo, candidate] {
16016 std::fs::write(path.join("uncommitted"), "keep").unwrap();
16017 assert!(
16018 remove_read_seat_worktree(&mut runner.state, &path)
16019 .await
16020 .is_err()
16021 );
16022 assert_eq!(
16023 std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16024 "keep"
16025 );
16026 }
16027 }
16028
16029 #[tokio::test]
16030 #[cfg(unix)]
16031 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16032 for (solo, failed_checkpoint) in
16033 [(true, false), (false, false), (true, true), (false, true)]
16034 {
16035 let (_tmp, mut runner) = preservation_resume_fixture().await;
16036 if solo {
16037 runner.state.candidates.retain(|c| c.label == 'A');
16038 }
16039 runner.state.candidates[0].empty = false;
16040 runner.state.candidates[0].summary.clear();
16041 runner.state.tally = None;
16042 runner.state.judge_skipped = false;
16043 runner.state.status = RunStatus::Implementing;
16044 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16045 let mut implementer = spec("alpha");
16046 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16047 runner.roles.implementers = vec![implementer];
16048 let patch = runner.state.dir().join("artifacts/cand-A.patch");
16049 if failed_checkpoint {
16050 std::fs::create_dir_all(&patch).unwrap();
16051 }
16052 runner.state.save().unwrap();
16053 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16054 drop(pressure);
16055 runner.state = RunState::load(&runner.state.id).unwrap();
16056 assert_eq!(runner.state.status, RunStatus::Blocked);
16057 assert!(runner.state.interrupted_implementations.contains(&'A'));
16058 assert_eq!(runner.state.candidates[0].commits, 0);
16059 assert!(!runner.state.candidates[0].empty);
16060 assert!(runner.state.candidates[0].failed.is_none());
16061 if failed_checkpoint {
16062 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16063 std::fs::remove_dir(patch).unwrap();
16064 }
16065 runner.state.resource_block = None;
16066 runner.state.config.disk.min_free_bytes = 0;
16067 runner.roles.implementers[0].command = vec![
16068 "sh".into(),
16069 "-c".into(),
16070 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16071 ];
16072 runner.implement().await.unwrap();
16073 assert!(runner.state.interrupted_implementations.is_empty());
16074 assert!(runner.state.candidate_preservation.is_empty());
16075 assert_eq!(runner.state.candidates[0].commits, 1);
16076 assert!(!runner.state.candidates[0].empty);
16077 assert!(
16078 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16079 .unwrap()
16080 .contains("retried empty interrupted seat")
16081 );
16082 if !solo {
16083 assert_eq!(runner.state.candidates[1].commits, 1);
16084 }
16085 }
16086 }
16087
16088 #[tokio::test]
16089 #[cfg(unix)]
16090 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16091 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16092 let (_tmp, mut runner) = preservation_resume_fixture().await;
16093 runner.state.candidates.retain(|c| c.label == 'A');
16094 runner.state.candidates[0].empty = false;
16095 runner.state.candidates[0].summary.clear();
16096 runner.state.tally = None;
16097 runner.state.judge_skipped = false;
16098 runner.state.status = RunStatus::Implementing;
16099 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16102 let wt = runner.state.candidates[0].worktree.clone();
16103 std::fs::write(
16104 wt.join("interrupted-source"),
16105 "retain implementation work\n",
16106 )
16107 .unwrap();
16108 if committed {
16109 git::rescue_commit(&wt, "implementation checkpoint")
16110 .await
16111 .unwrap();
16112 }
16113 let lock = git::git(
16114 &wt,
16115 &[
16116 "rev-parse",
16117 "--path-format=absolute",
16118 "--git-path",
16119 "index.lock",
16120 ],
16121 )
16122 .await
16123 .unwrap();
16124 if block_rescue {
16125 std::fs::write(&lock, "fixture-owned lock").unwrap();
16126 }
16127 let mut implementer = spec("alpha");
16128 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16129 runner.roles.implementers = vec![implementer];
16130 runner.state.save().unwrap();
16131 let result = runner.implement().await;
16132 drop(pressure);
16133 assert_eq!(result.is_err(), block_rescue);
16134 assert_eq!(
16135 runner.state.resource_block.as_ref().unwrap().kind,
16136 "disk pressure"
16137 );
16138 assert!(
16139 runner
16140 .state
16141 .events
16142 .iter()
16143 .any(|e| e.message.contains("agent execution interrupted"))
16144 );
16145 assert!(
16146 runner
16147 .state
16148 .dir()
16149 .join("artifacts/candidate-A-recovery/staged.patch")
16150 .exists()
16151 );
16152 if block_rescue {
16153 let saved = RunState::load(&runner.state.id).unwrap();
16154 assert!(saved.candidate_preservation.contains_key(&'A'));
16155 assert!(!saved.candidates[0].empty);
16156 assert_eq!(
16157 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16158 "retain implementation work\n"
16159 );
16160 std::fs::remove_file(lock).unwrap();
16161 runner.state = saved;
16162 } else {
16163 assert_eq!(runner.state.status, RunStatus::Blocked);
16164 assert_eq!(runner.state.candidates[0].commits, 1);
16165 }
16166 runner.roles.implementers.clear();
16169 runner.state.config.disk.min_free_bytes = 0;
16170 runner.state.resource_block = None;
16171 runner.implement().await.unwrap();
16172 assert!(runner.state.candidate_preservation.is_empty());
16173 assert_eq!(runner.state.candidates[0].commits, 1);
16174 assert!(!runner.state.candidates[0].empty);
16175 assert!(git::is_clean(&wt).await.unwrap());
16176 assert!(
16177 crate::run::read_artifact(&runner.state, "cand-A.patch")
16178 .unwrap()
16179 .contains("retain implementation work")
16180 );
16181 }
16182 }
16183
16184 #[tokio::test]
16185 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16186 let (_tmp, mut runner) = preservation_resume_fixture().await;
16187 runner.implement().await.unwrap();
16189 assert!(runner.state.candidate_preservation.is_empty());
16190 assert!(
16191 !runner
16192 .state
16193 .dir()
16194 .join("artifacts/candidate-A-recovery")
16195 .exists()
16196 );
16197 runner.fold_losers().await.unwrap();
16198 assert!(!runner.state.candidates[0].worktree.exists());
16199 for schema in [14, crate::run::SCHEMA] {
16200 runner.state.schema = schema;
16201 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16202 runner.state.save().unwrap();
16203 runner.state = RunState::load(&runner.state.id).unwrap();
16204 runner.implement().await.unwrap();
16205 assert!(runner.state.candidate_preservation.is_empty());
16206 assert!(runner.state.candidates[0].folded);
16207 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16208 }
16209 }
16210
16211 #[tokio::test]
16212 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16213 let (_tmp, mut runner) = preservation_resume_fixture().await;
16214 runner.state.candidates.retain(|c| c.label == 'A');
16215 runner.state.tally = None;
16216 runner.state.judge_skipped = false;
16217 let candidate = &mut runner.state.candidates[0];
16218 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16219 candidate.failed = Some("deployment validation unavailable".into());
16220 std::fs::write(
16221 candidate.worktree.join("repair"),
16222 "future launcher policy\n",
16223 )
16224 .unwrap();
16225 runner
16226 .state
16227 .candidate_preservation
16228 .insert('A', "awaiting persistence".into());
16229 runner.persist_candidates().await.unwrap();
16230 let candidate = &runner.state.candidates[0];
16231 assert!(!candidate.empty);
16232 assert_eq!(candidate.commits, 1);
16233 assert_eq!(candidate.files, 1);
16234 assert!(candidate.failed.is_none());
16235 assert!(candidate.summary.contains("EPERM"));
16236 assert!(
16237 crate::run::read_artifact(&runner.state, "cand-A.patch")
16238 .unwrap()
16239 .contains("future launcher policy")
16240 );
16241 assert!(runner.state.candidate_preservation.is_empty());
16242 }
16243
16244 #[tokio::test]
16245 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16246 let (_tmp, mut runner) = preservation_resume_fixture().await;
16247 let wt = runner.state.candidates[0].worktree.clone();
16248 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16249 runner.state.schema = 14;
16250 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16251 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16252 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16253 for _ in 0..2 {
16254 assert!(runner.implement().await.is_err());
16255 assert_eq!(runner.state.status, RunStatus::Blocked);
16256 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16257 assert!(
16258 runner.state.candidates[0].empty,
16259 "old verdict must remain unusable"
16260 );
16261 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16262 assert!(runner.state.judge_skipped);
16263 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16264 assert!(
16265 crate::run::read_artifact(&runner.state, "cand-A.patch")
16266 .unwrap()
16267 .contains("owned edits")
16268 );
16269 assert!(wt.exists());
16270 assert_eq!(
16271 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16272 .await
16273 .unwrap(),
16274 1
16275 );
16276 runner.state = RunState::load(&runner.state.id).unwrap();
16277 }
16278 }
16279
16280 #[tokio::test]
16281 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16282 let (_tmp, mut runner) = preservation_resume_fixture().await;
16283 let wt = runner.state.candidates[0].worktree.clone();
16284 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16285 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16286 let index_lock = PathBuf::from(
16287 git::git(
16288 &wt,
16289 &[
16290 "rev-parse",
16291 "--path-format=absolute",
16292 "--git-path",
16293 "index.lock",
16294 ],
16295 )
16296 .await
16297 .unwrap(),
16298 );
16299 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16300 runner.state.schema = 14;
16301 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16302 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16303 assert!(runner.implement().await.is_err());
16304 assert!(runner.after_implement().is_err());
16305 assert!(runner.fold_losers().await.is_err());
16306 assert_eq!(
16307 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16308 "failed seat edits"
16309 );
16310 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16311 runner.state = RunState::load(&runner.state.id).unwrap();
16312 std::fs::remove_file(index_lock).unwrap();
16313 assert!(runner.implement().await.is_err());
16315 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16316 assert!(runner.state.candidates[0].empty);
16317 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16318 assert!(
16319 crate::run::read_artifact(&runner.state, "cand-A.patch")
16320 .unwrap()
16321 .contains("failed seat edits")
16322 );
16323 assert!(wt.exists());
16324 }
16325
16326 #[tokio::test]
16327 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16328 let (_tmp, mut runner) = preservation_resume_fixture().await;
16329 let repo = runner.state.repo.clone();
16330 let common = git::git(
16331 &repo,
16332 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16333 )
16334 .await
16335 .unwrap();
16336 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16337 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16338 assert!(runner.fold_losers().await.is_err());
16339 assert!(!runner.state.candidates[0].worktree.exists());
16340 assert!(!runner.state.candidates[0].folded);
16341 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16342 runner.state = RunState::load(&runner.state.id).unwrap();
16343 std::fs::remove_file(branch_lock).unwrap();
16344 let judge = runner.state.worktree_root().join("judge-1");
16345 git::worktree_add_detached(&repo, &judge, "main")
16346 .await
16347 .unwrap();
16348 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16349 git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16350 .await
16351 .unwrap();
16352 runner.roles.judges = vec![spec("judge")];
16353 assert!(runner.fold_losers().await.is_err());
16354 runner.state = RunState::load(&runner.state.id).unwrap();
16355 assert!(
16356 runner.state.candidates[0].folded,
16357 "candidate progress saved before judge failure"
16358 );
16359 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16360 assert!(judge.join("uncommitted").exists());
16361 git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16362 .await
16363 .unwrap();
16364 runner.fold_losers().await.unwrap();
16365 assert!(!judge.exists());
16366 runner.fold_losers().await.unwrap();
16367 }
16368
16369 #[tokio::test]
16370 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16371 let (_tmp, mut runner) = preservation_resume_fixture().await;
16372 runner.state.candidates.retain(|c| c.label == 'B');
16373 runner.state.tally = None;
16374 runner.state.judge_skipped = false;
16375 runner.state.prep_complete = Some(true);
16376 runner.state.config.graph.advise = false;
16377 runner.state.config.graph.selected_implementer = Some("alpha".into());
16378 let wt = runner.state.candidates[0].worktree.clone();
16379 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16380 runner
16381 .state
16382 .candidate_preservation
16383 .insert('B', "awaiting persistence".into());
16384 let index_lock = git::git(
16385 &wt,
16386 &[
16387 "rev-parse",
16388 "--path-format=absolute",
16389 "--git-path",
16390 "index.lock",
16391 ],
16392 )
16393 .await
16394 .unwrap();
16395 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16396 runner.state.save().unwrap();
16397 assert!(runner.execute_graph().await.is_err());
16398 assert_eq!(runner.state.status, RunStatus::Blocked);
16399 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16400 std::fs::remove_file(index_lock).unwrap();
16401 for _ in 0..2 {
16404 runner.state = RunState::load(&runner.state.id).unwrap();
16405 runner.pause.park();
16406 runner.execute_graph().await.unwrap();
16407 assert_eq!(runner.state.status, RunStatus::Implementing);
16408 assert!(runner.state.candidate_preservation.is_empty());
16409 assert_eq!(runner.state.candidates[0].commits, 2);
16410 assert!(runner.state.tally.is_none());
16411 }
16412 runner.implement().await.unwrap();
16413 assert_eq!(runner.state.status, RunStatus::Judging);
16414 runner.state.status = RunStatus::Blocked;
16416 runner.state.candidates[0].failed = Some("implementation failed".into());
16417 runner.state.save().unwrap();
16418 let error = runner.execute_graph().await.unwrap_err();
16419 assert!(
16420 error
16421 .to_string()
16422 .contains("selected implementation is held")
16423 );
16424 }
16425
16426 #[tokio::test]
16427 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16428 let (_tmp, mut runner) = preservation_resume_fixture().await;
16429 let repo = runner.state.repo.clone();
16430 let candidate = runner.state.candidates[0].clone();
16431 let registered = git::worktree_holding(&repo, &candidate.branch)
16434 .await
16435 .unwrap()
16436 .unwrap();
16437 assert_eq!(
16438 registered.canonicalize().unwrap(),
16439 candidate.worktree.canonicalize().unwrap()
16440 );
16441 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16442 assert_eq!(
16443 git::worktree_holding(&repo, &candidate.branch)
16444 .await
16445 .unwrap(),
16446 Some(registered)
16447 );
16448 assert!(
16449 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16450 "the stale registration initially keeps the branch checked out"
16451 );
16452 runner.fold_losers().await.unwrap();
16453 assert!(runner.state.candidates[0].folded);
16454 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16455 assert_eq!(
16456 git::worktree_holding(&repo, &candidate.branch)
16457 .await
16458 .unwrap(),
16459 None
16460 );
16461 runner.state = RunState::load(&runner.state.id).unwrap();
16462 runner.fold_losers().await.unwrap();
16463 }
16464
16465 #[tokio::test]
16466 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16467 let (_tmp, mut runner) = preservation_resume_fixture().await;
16468 let repo = runner.state.repo.clone();
16469 let candidate = runner.state.candidates[0].clone();
16470 let winner = runner.state.candidates[1].clone();
16471 let registered = git::worktree_holding(&repo, &candidate.branch)
16472 .await
16473 .unwrap()
16474 .unwrap();
16475 assert_eq!(
16476 registered.canonicalize().unwrap(),
16477 candidate.worktree.canonicalize().unwrap()
16478 );
16479 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16480 assert_eq!(
16481 git::worktree_holding(&repo, &candidate.branch)
16482 .await
16483 .unwrap(),
16484 Some(registered)
16485 );
16486 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16487
16488 for _ in 0..2 {
16489 fold_run(&mut runner.state, false, &crate::run::home())
16490 .await
16491 .unwrap();
16492 runner.state = RunState::load(&runner.state.id).unwrap();
16493 assert!(runner.state.candidates[0].folded);
16494 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16495 assert_eq!(
16496 git::worktree_holding(&repo, &candidate.branch)
16497 .await
16498 .unwrap(),
16499 None
16500 );
16501 assert!(!runner.state.candidates[1].folded);
16502 assert!(winner.worktree.exists());
16503 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16504 }
16505 }
16506
16507 #[tokio::test]
16508 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16509 ask_test_home();
16510 let tmp = tempfile::tempdir().unwrap();
16511 let repo = tmp.path().join("repo");
16512 std::fs::create_dir(&repo).unwrap();
16513 init_repo(&repo);
16514 let mut runner = runner_at(RunStatus::Implementing);
16515 runner.state.repo = repo.clone();
16516 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16517 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16518 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16519 let branch = format!("candidate-{label}");
16520 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16521 git::worktree_add_branch(&repo, &wt, &branch, "main")
16522 .await
16523 .unwrap();
16524 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16525 git::git(&wt, &["add", "README.md"]).await.unwrap();
16526 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16527 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16528 runner.state.candidates.push(Candidate {
16529 index,
16530 label,
16531 agent: "alpha".to_owned(),
16532 branch,
16533 worktree: wt,
16534 summary: "completed implementation".to_owned(),
16535 stat: String::new(),
16536 files: 0,
16537 commits: 0,
16538 empty: false,
16539 failed: None,
16540 verified_noop: None,
16541 duration_ms: 42,
16542 folded: false,
16543 });
16544 runner
16545 .state
16546 .candidate_preservation
16547 .insert(label, "awaiting persistence".to_owned());
16548 }
16549 let wt = runner.state.candidates[0].worktree.clone();
16550 let index_lock = git::git(
16551 &wt,
16552 &[
16553 "rev-parse",
16554 "--path-format=absolute",
16555 "--git-path",
16556 "index.lock",
16557 ],
16558 )
16559 .await
16560 .unwrap();
16561 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16562 runner.state.save().unwrap();
16563 assert!(runner.persist_candidates().await.is_err());
16564 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16565 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16566 assert_eq!(
16567 runner.state.candidates[1].commits, 1,
16568 "other results are still collected"
16569 );
16570 assert!(
16571 !runner.state.candidates[0].empty,
16572 "failed rescue is not no change"
16573 );
16574 assert_eq!(
16575 runner.state.candidates[0].summary,
16576 "completed implementation"
16577 );
16578 assert!(runner.after_implement().is_err());
16579 let home = tmp.path().join("fold-home");
16580 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16581 assert!(wt.join("new-file").exists());
16582 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16583 assert!(recovery.join("staged.patch").exists());
16584 assert!(recovery.join("unstaged.patch").exists());
16585 assert_eq!(
16586 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16587 "untracked edits\n"
16588 );
16589 runner.state.begin_pass();
16590 runner.state.end_pass_parked(ParkCause::Upgrade {
16591 from: "1.0".into(),
16592 to: Some("1.1".into()),
16593 });
16594 runner.state.save().unwrap();
16595 runner.state = RunState::load(&runner.state.id).unwrap();
16596 assert!(matches!(
16597 runner.state.passes.last().unwrap().end,
16598 Some(crate::run::PassEnd::Parked {
16599 cause: ParkCause::Upgrade { .. }
16600 })
16601 ));
16602 std::fs::remove_file(index_lock).unwrap();
16603 runner.implement().await.unwrap();
16605 assert!(runner.state.candidate_preservation.is_empty());
16606 assert!(
16607 runner
16608 .state
16609 .candidates
16610 .iter()
16611 .all(|c| c.commits == 1 && !c.empty)
16612 );
16613 assert!(git::is_clean(&wt).await.unwrap());
16614 assert!(
16615 crate::run::read_artifact(&runner.state, "cand-A.patch")
16616 .unwrap()
16617 .contains("later tracked edits")
16618 );
16619 }
16620
16621 #[tokio::test]
16622 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16623 ask_test_home();
16624 let tmp = tempfile::tempdir().unwrap();
16625 let repo = tmp.path().join("repo");
16626 std::fs::create_dir(&repo).unwrap();
16627 init_repo(&repo);
16628 let wt = tmp.path().join("candidate");
16629 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16630 .await
16631 .unwrap();
16632 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16633 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16634 assert!(wt.join("uncommitted").exists());
16635 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16636 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16637 let mut runner = runner_at(RunStatus::Implementing);
16638 runner.state.repo = repo.clone();
16639 runner.state.base_commit = "missing-base".to_owned();
16640 runner.state.candidates.push(Candidate {
16641 index: 0,
16642 label: 'A',
16643 agent: "alpha".into(),
16644 branch: "candidate".into(),
16645 worktree: wt.clone(),
16646 summary: "completed".into(),
16647 stat: String::new(),
16648 files: 0,
16649 commits: 0,
16650 empty: false,
16651 failed: None,
16652 verified_noop: None,
16653 duration_ms: 1,
16654 folded: false,
16655 });
16656 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16657 assert!(
16658 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16659 .await
16660 .is_err()
16661 );
16662 assert!(!runner.state.candidates[0].folded);
16663 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16664 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16665 runner
16666 .state
16667 .candidate_preservation
16668 .insert('A', "pending".into());
16669 assert!(runner.persist_candidates().await.is_err());
16670 assert!(!runner.state.candidates[0].empty);
16671 assert!(!runner.state.candidates[0].folded);
16672 assert!(wt.exists());
16673 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16674 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16675 std::fs::create_dir_all(&patch_path).unwrap();
16676 assert!(
16677 runner.persist_candidates().await.is_err(),
16678 "artifact failure stays pending"
16679 );
16680 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16681 assert!(!runner.state.candidates[0].empty);
16682 std::fs::remove_dir(patch_path).unwrap();
16683 runner.persist_candidates().await.unwrap();
16684 assert!(
16685 runner.state.candidates[0].empty,
16686 "only successful inspection confirms no change"
16687 );
16688 }
16689
16690 fn init_repo(dir: &Path) {
16691 let run = |args: &[&str]| {
16692 let out = std::process::Command::new("git")
16693 .args(args)
16694 .current_dir(dir)
16695 .quiet()
16696 .output()
16697 .expect("spawn git");
16698 assert!(
16699 out.status.success(),
16700 "git {args:?} failed: {}",
16701 String::from_utf8_lossy(&out.stderr)
16702 );
16703 };
16704 run(&["init", "-b", "main"]);
16705 run(&["config", "user.name", "magi test"]);
16706 run(&["config", "user.email", "magi@example.com"]);
16707 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16708 run(&["add", "-A"]);
16709 run(&["commit", "-m", "init"]);
16710 }
16711
16712 fn ask_test_home() {
16720 crate::run::pin_test_home();
16721 }
16722
16723 #[test]
16724 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16725 ask_test_home();
16726 let mut runner = runner_at(RunStatus::Prep);
16727 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16728 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16731 let error = guard
16732 .finish(
16733 &mut runner.state,
16734 Err(
16735 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16736 .into(),
16737 ),
16738 )
16739 .unwrap_err();
16740 assert_eq!(
16741 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16742 std::io::ErrorKind::PermissionDenied
16743 );
16744 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16745 let message = format!("{error:#}");
16746 assert!(message.contains("original refusal"));
16747 assert!(!message.contains("could not save terminal"));
16748 assert_eq!(
16749 error.chain().count(),
16750 2,
16751 "only startup attribution wraps the original error"
16752 );
16753 let saved = RunState::load(&runner.state.id).unwrap();
16754 assert_eq!(saved.status, RunStatus::Blocked);
16755 assert!(!saved.startup.unwrap().complete);
16756 }
16757
16758 #[test]
16759 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16760 ask_test_home();
16761 let mut runner = runner_at(RunStatus::Prep);
16762 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16763 let id = runner.state.id.clone();
16764 drop(guard);
16765 let saved = RunState::load(&id).unwrap();
16766 assert_eq!(saved.status, RunStatus::Blocked);
16767 assert!(!saved.startup.as_ref().unwrap().complete);
16768 assert!(
16769 saved
16770 .startup
16771 .unwrap()
16772 .failure
16773 .unwrap()
16774 .contains("interrupted")
16775 );
16776 assert!(Runner::resume(&id).is_err());
16777 }
16778
16779 #[test]
16780 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16781 ask_test_home();
16782 let mut runner = runner_at(RunStatus::Prep);
16783 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16784 let error = guard
16785 .finish(
16786 &mut runner.state,
16787 Err(
16788 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16789 .into(),
16790 ),
16791 )
16792 .unwrap_err();
16793 assert_eq!(
16794 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16795 std::io::ErrorKind::PermissionDenied
16796 );
16797 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16798 let mut task = crate::queue::Task::new(
16799 "review".into(),
16800 "criteria".into(),
16801 runner.state.repo.clone(),
16802 crate::queue::Source::Human,
16803 );
16804 task.record_startup_failure(failure.clone());
16805 task.record_startup_failure(failure.clone());
16806 assert_eq!(task.startup_failures.len(), 1);
16807 assert!(task.runs.is_empty());
16808 assert_eq!(task.attempts, 0);
16809 }
16810
16811 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16812 ask_test_home();
16813 let tmp = tempfile::tempdir().unwrap();
16814 let repo = tmp.path().join("repo");
16815 std::fs::create_dir(&repo).unwrap();
16816 init_repo(&repo);
16817 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16818 let origin = tmp.path().join("origin.git");
16819 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16820 .await
16821 .unwrap();
16822 git::git(
16823 &repo,
16824 &["remote", "add", "origin", origin.to_str().unwrap()],
16825 )
16826 .await
16827 .unwrap();
16828 git::git(&repo, &["checkout", "-b", "fix/retained"])
16829 .await
16830 .unwrap();
16831 std::fs::write(repo.join("retained"), "retained change").unwrap();
16832 git::git(&repo, &["add", "."]).await.unwrap();
16833 git::git(&repo, &["commit", "-m", "retained implementation"])
16834 .await
16835 .unwrap();
16836 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16837 git::git(&repo, &["checkout", "main"]).await.unwrap();
16838 std::fs::write(repo.join("base-change"), "new base").unwrap();
16839 git::git(&repo, &["add", "."]).await.unwrap();
16840 git::git(&repo, &["commit", "-m", "base moved"])
16841 .await
16842 .unwrap();
16843 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16844 git::git(&repo, &["checkout", "fix/retained"])
16845 .await
16846 .unwrap();
16847 let mut runner = runner_at(RunStatus::Voting);
16848 runner.state.repo = repo.clone();
16849 runner.state.base_commit = base;
16850 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16851 runner.state.config.disk.min_free_bytes = 0;
16852 runner.state.config.graph.review_rounds = 1;
16853 runner.state.config.verify.e2e = vec!["true".into()];
16854 runner.state.config.verify.gate = vec!["true".into()];
16855 runner.state.candidates = vec![Candidate {
16856 index: 0,
16857 label: 'A',
16858 agent: "fixture".into(),
16859 branch: "fix/retained".into(),
16860 worktree: repo.clone(),
16861 summary: String::new(),
16862 stat: String::new(),
16863 files: 1,
16864 commits: 1,
16865 empty: false,
16866 failed: None,
16867 verified_noop: None,
16868 duration_ms: 0,
16869 folded: false,
16870 }];
16871 runner.state.tally = Some(Tally {
16872 first_choice: BTreeMap::new(),
16873 borda: BTreeMap::new(),
16874 winner: 'A',
16875 rankings: 0,
16876 unanimous_initial: true,
16877 deliberated: false,
16878 changed_votes: 0,
16879 unanimous_final: true,
16880 tie_break: None,
16881 judges: 0,
16882 present: 0,
16883 quorum: 0,
16884 met_quorum: true,
16885 uncontested: Some("review-only".into()),
16886 });
16887 let mut task = crate::queue::Task::new(
16888 "fixture".into(),
16889 "fixture".into(),
16890 repo,
16891 crate::queue::Source::Human,
16892 );
16893 task.link_run(&runner.state.id);
16894 crate::queue::Queue::open().put(&mut task).unwrap();
16895 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16896 runner.state.base_sync = Some(BaseSync {
16897 tip: String::new(),
16898 behind: 1,
16899 attempts: BASE_SYNC_ROUNDS,
16900 conflict: None,
16901 already_in: None,
16902 });
16903 runner.sync_to_base().await.unwrap();
16904 assert_eq!(runner.state.status, RunStatus::Blocked);
16905 (tmp, runner, head, task.id)
16906 }
16907
16908 #[tokio::test]
16909 async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
16910 let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
16911 runner.state.status = RunStatus::Voting;
16912 runner.state.base_sync = None;
16913 runner.state.base_sync_recovery = Default::default();
16914 let scratch = runner.state.dir().join("base-sync");
16915 std::fs::create_dir_all(&scratch).unwrap();
16916 std::fs::write(scratch.join("obstruction"), "keep").unwrap();
16917 runner.sync_to_base().await.unwrap();
16918 assert!(runner.state.resource_block.is_some());
16919 assert!(!runner.state.base_sync_blocked());
16920 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
16921 assert_eq!(runner.state.base_episode_used(), 0);
16922 assert!(runner.state.base_sync_recovery.operations.is_empty());
16923 assert!(runner.state.rebase_fixes.is_empty());
16924 assert_eq!(
16925 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16926 head
16927 );
16928 std::fs::remove_dir_all(&scratch).unwrap();
16929 runner.state.resource_block = None;
16930 runner.sync_to_base().await.unwrap();
16931 assert!(!runner.state.base_sync_blocked());
16932 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
16933 assert_eq!(runner.state.base_episode_used(), 1);
16934 assert!(runner.state.base_sync_recovery.operations[0].finished);
16935 assert!(runner.state.rebase_fixes.is_empty());
16936 assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
16937 assert!(git::is_clean(&runner.state.repo).await.unwrap());
16938 }
16939
16940 #[cfg(unix)]
16941 #[tokio::test]
16942 async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16943 for routed in [false, true] {
16944 let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16945 let home = crate::run::home();
16946 let queue = crate::queue::Queue::open();
16947 let task = queue.get(&owner).unwrap();
16948 runner.state.task_instruction = Some(task.instruction.clone());
16949 runner.state.driver_exited = true;
16950 runner.state.save().unwrap();
16951 let questions = crate::ask::Questions::at(home.join("questions"));
16952 let mut q = crate::ask::Question::new(
16953 runner.state.id.clone(),
16954 "base-recovery".into(),
16955 "owner".into(),
16956 "Recover bounded base movement?".into(),
16957 String::new(),
16958 vec!["recover".into()],
16959 );
16960 q.actions.insert(
16961 "recover".into(),
16962 crate::ask::ChoiceAction::RecoverBase {
16963 run: runner.state.id.clone(),
16964 stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16965 episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16966 },
16967 );
16968 q.routed_task = Some(owner.clone());
16969 q.answer(crate::ask::Answer::Choice("recover".into()))
16970 .unwrap();
16971 questions.put(&mut q).unwrap();
16972 crate::base_recovery_control::recover(
16973 &queue,
16974 &home,
16975 crate::base_recovery_control::Request {
16976 run: runner.state.id.clone(),
16977 head: head.clone(),
16978 actor: "decisions".into(),
16979 request: q.id.clone(),
16980 decision: Some((owner.clone(), q.answered_at.unwrap())),
16981 expected_stop: Some((
16982 runner.state.base_sync_recovery.stop.clone().unwrap(),
16983 runner.state.base_sync_recovery.episodes.last().unwrap().id,
16984 )),
16985 },
16986 )
16987 .await
16988 .unwrap();
16989 if routed {
16990 questions
16991 .update(&q.id, |q| {
16992 q.routed_task = Some("another-owner".into());
16993 Ok(())
16994 })
16995 .unwrap();
16996 } else {
16997 let mut task = queue.get(&owner).unwrap();
16998 task.hold_manual(Some("new owner decision".into()));
16999 task.status = crate::queue::TaskStatus::Queued;
17001 queue.put(&mut task).unwrap();
17002 }
17003 runner.state = RunState::load(&runner.state.id).unwrap();
17004 runner.sync_to_base().await.unwrap();
17005 let approval = runner
17006 .state
17007 .base_sync_recovery
17008 .authorizations
17009 .last()
17010 .unwrap();
17011 assert!(approval.consumed);
17012 assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17013 "routing"
17014 } else {
17015 "owner decision"
17016 }));
17017 assert_eq!(
17018 runner.state.base_sync.as_ref().unwrap().attempts,
17019 BASE_SYNC_ROUNDS
17020 );
17021 assert_eq!(
17022 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17023 head
17024 );
17025 }
17026 }
17027
17028 #[cfg(unix)]
17029 #[tokio::test]
17030 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17031 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17032 if std::env::var_os(CHILD).is_none() {
17033 let home = tempfile::tempdir().unwrap();
17034 let output = std::process::Command::new(std::env::current_exe().unwrap())
17035 .args([
17036 "--exact",
17037 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17038 "--nocapture",
17039 ])
17040 .env(CHILD, "1")
17041 .env("MAGI_HOME", home.path())
17042 .env("MAGI_WORKER_HOME", home.path())
17043 .env("MAGI_WORKER_ID", "abcdef")
17044 .output()
17045 .unwrap();
17046 assert!(
17047 output.status.success()
17048 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17049 "{}\n{}",
17050 String::from_utf8_lossy(&output.stdout),
17051 String::from_utf8_lossy(&output.stderr)
17052 );
17053 return;
17054 }
17055 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17056 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17057 let home = crate::run::home();
17058 let queue = crate::queue::Queue::open();
17059 let mut owner = queue.get(&task).unwrap();
17060 owner.release_operator("test-release", None);
17061 owner.start(runner.state.id.clone());
17062 queue.put(&mut owner).unwrap();
17063 runner.state.driver_exited = true;
17064 runner.state.task_instruction = Some(owner.instruction.clone());
17065 runner.state.save().unwrap();
17066 let _claim = queue.claim(&task).unwrap();
17067 let reserve = |id: &str| {
17068 let directory = home.join("workers/jobs").join(id);
17069 std::fs::create_dir_all(&directory).unwrap();
17070 let job = crate::worker::Job {
17071 id: id.into(),
17072 sequence: 1,
17073 kind: "task".into(),
17074 key: format!("task:{task}"),
17075 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17076 generation: "fixture-generation".into(),
17077 version: None,
17078 status: crate::worker::Status::Running,
17079 result: None,
17080 pid: Some(std::process::id()),
17081 created_at: 0,
17082 finished_at: None,
17083 };
17084 std::fs::write(
17085 directory.join("job.json"),
17086 serde_json::to_vec(&job).unwrap(),
17087 )
17088 .unwrap();
17089 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17090 };
17091 let _own = reserve("abcdef");
17092 let other = reserve("fedcba");
17093 assert!(
17094 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17095 .await
17096 .is_err()
17097 );
17098 assert!(
17099 RunState::load(&runner.state.id)
17100 .unwrap()
17101 .base_sync_recovery
17102 .authorizations
17103 .is_empty()
17104 );
17105 drop(other);
17106 assert!(crate::daemon::is_working_on(
17107 &home,
17108 &runner.state.id,
17109 Timestamp::now()
17110 ));
17111 assert!(
17112 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17113 );
17114 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17115 .await
17116 .unwrap();
17117 assert!(
17118 RunState::load(&runner.state.id)
17119 .unwrap()
17120 .base_recovery_pending()
17121 );
17122 }
17123
17124 #[cfg(unix)]
17125 #[tokio::test]
17126 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17127 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17128 let home = crate::run::home();
17129 runner.state.driver_pid = Some(std::process::id());
17130 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17131 runner.state.driver_exited = false;
17132 runner.state.begin_pass();
17133 runner.state.save().unwrap();
17134 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17135 driver.record(&runner.state).unwrap();
17136 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17137 drop(driver);
17139 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17140 assert_eq!(
17141 runner.state.liveness_under(&home, false),
17142 crate::run::Liveness::Dead
17143 );
17144 let queue = crate::queue::Queue::open();
17145 let _claim = queue.claim(&task).unwrap();
17146 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17147 .await
17148 .unwrap();
17149 let disk = RunState::load(&runner.state.id).unwrap();
17150 assert!(!disk.driver_exited);
17151 assert!(disk.base_recovery_pending());
17152 }
17153
17154 #[cfg(unix)]
17155 #[tokio::test]
17156 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17157 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17158 if std::env::var_os(CHILD).is_none() {
17159 use std::os::unix::fs::PermissionsExt;
17160 let tmp = tempfile::tempdir().unwrap();
17161 let gh = tmp.path().join("gh");
17162 std::fs::write(&gh, r#"#!/bin/sh
17163case "$1 $2" in
17164 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17165 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17166 *) exit 2 ;;
17167esac
17168"#).unwrap();
17169 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17170 let mut paths = vec![tmp.path().to_path_buf()];
17171 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17172 let output = std::process::Command::new(std::env::current_exe().unwrap())
17173 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17174 .env(CHILD, "1")
17175 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17176 .env("PATH", std::env::join_paths(paths).unwrap())
17177 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17178 .output().unwrap();
17179 assert!(
17180 output.status.success()
17181 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17182 "{}\n{}",
17183 String::from_utf8_lossy(&output.stdout),
17184 String::from_utf8_lossy(&output.stderr)
17185 );
17186 return;
17187 }
17188 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17189 let repo = runner.state.repo.clone();
17190 git::git(&repo, &["checkout", "main"]).await.unwrap();
17191 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17192 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17193 git::git(&repo, &["checkout", "fix/retained"])
17194 .await
17195 .unwrap();
17196 runner.state.pr = Some(crate::run::PrRecord {
17197 url: "https://example.test/project/pull/1".into(),
17198 number: 1,
17199 state: "open".into(),
17200 checks: "unknown".into(),
17201 round: 0,
17202 rounds: 1,
17203 red_at_merge: Vec::new(),
17204 });
17205 runner.sync_to_base().await.unwrap();
17206 assert_eq!(runner.state.status, RunStatus::Blocked);
17207 assert_eq!(
17208 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17209 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17210 );
17211 assert!(runner.state.base_sync_closure_pending());
17212 assert!(crate::handover::can_resume(&runner.state));
17213 let queue = crate::queue::Queue::open();
17214 assert!(!crate::base_sync_recovery::held_task(
17215 &queue,
17216 &queue.get(&task).unwrap()
17217 ));
17218 assert!(runner.state.authorize_base_sync(&task).is_err());
17219 runner.state.base_sync_recovery.stop = None;
17221 runner.state.status = RunStatus::Voting;
17222 runner.state.gate_ran = true;
17223 runner.state.config.graph.review_rounds = 0;
17224 runner.state.save().unwrap();
17225 assert!(crate::handover::can_resume(&runner.state));
17226 runner.state = RunState::load(&runner.state.id).unwrap();
17227 runner.execute_graph().await.unwrap();
17228 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17229 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17230 assert_eq!(
17231 runner.state.base_sync.as_ref().unwrap().attempts,
17232 BASE_SYNC_ROUNDS
17233 );
17234 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17235 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17236 assert!(runner.state.reviews.is_empty());
17237 assert!(runner.state.gate.is_empty());
17238 }
17239
17240 #[cfg(unix)]
17241 #[tokio::test]
17242 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17243 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17244 runner.state.driver_exited = true;
17245 runner.state.save().unwrap();
17246 let home = crate::run::home();
17247 let queue = crate::queue::Queue::open();
17248 let _claim = queue.claim(&task).unwrap();
17249 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17250 assert!(
17251 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17252 .await
17253 .is_err()
17254 );
17255 drop(lease);
17256 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17257 assert!(
17258 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17259 .await
17260 .is_err()
17261 );
17262 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17263 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17264 .await
17265 .unwrap();
17266 let disk = RunState::load(&runner.state.id).unwrap();
17267 assert!(disk.base_recovery_pending());
17268 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17269 assert!(
17270 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17271 .await
17272 .is_err()
17273 );
17274 assert_eq!(
17275 RunState::load(&runner.state.id)
17276 .unwrap()
17277 .base_sync_recovery
17278 .authorizations
17279 .len(),
17280 1
17281 );
17282 let mut owner = queue.get(&task).unwrap();
17283 owner.runs.push("20260101-000000-ffff".into());
17284 queue.put(&mut owner).unwrap();
17285 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17286 .await
17287 .unwrap_err();
17288 assert!(error.to_string().contains("run task ownership changed"));
17289 }
17290
17291 #[cfg(unix)]
17292 #[tokio::test]
17293 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17294 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17295 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17296 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17297 legacy["status"] = serde_json::json!("voting");
17298 runner.state = serde_json::from_value(legacy).unwrap();
17299 runner.state.save().unwrap();
17300 for _ in 0..3 {
17301 runner.execute_graph().await.unwrap();
17302 assert_eq!(runner.state.status, RunStatus::Blocked);
17303 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17304 assert!(!crate::handover::can_resume(&runner.state));
17305 assert_eq!(
17306 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17307 head
17308 );
17309 assert!(runner.state.reviews.is_empty());
17310 assert!(!runner.state.gate_ran);
17311 }
17312 let notice = crate::notices::run_ended(&runner.state).unwrap();
17313 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17314 runner.state.begin_pass();
17315 runner.state.status = RunStatus::Voting;
17316 runner.deliberate().await.unwrap();
17317 runner.vote().await.unwrap();
17318 runner.finish_after_tally().await.unwrap();
17319 assert_eq!(runner.state.status, RunStatus::Blocked);
17320 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17321 assert_eq!(
17322 crate::notices::run_ended(&runner.state).unwrap().since,
17323 notice.since
17324 );
17325 }
17326
17327 #[cfg(unix)]
17328 #[tokio::test]
17329 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17330 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17331 for detail in [
17332 "fix/retained conflicts with origin/main and did not rebase: conflict",
17333 "rebased fix/retained locally but origin refused the push",
17334 "unknown failure",
17335 ] {
17336 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17337 runner.state.record_base_sync_stop(&head);
17338 assert!(runner.state.authorize_base_sync(&task).is_err());
17339 runner.state.status = RunStatus::Voting;
17340 runner.execute_graph().await.unwrap();
17341 assert_eq!(runner.state.status, RunStatus::Blocked);
17342 assert!(!crate::handover::can_resume(&runner.state));
17343 assert_eq!(
17344 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17345 head
17346 );
17347 }
17348 }
17349
17350 #[cfg(unix)]
17351 #[tokio::test]
17352 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17353 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17354 runner.state.authorize_base_sync(&task).unwrap();
17355 assert!(runner.state.consume_base_recovery(&head));
17356 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17357 runner.state.reserve_base_operation(&head, &tip);
17358 runner.state.save().unwrap();
17359 runner.state = RunState::load(&runner.state.id).unwrap();
17360 assert!(!runner.state.consume_base_recovery(&head));
17361 runner.sync_to_base().await.unwrap();
17362 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17363 assert_eq!(runner.state.base_episode_used(), 1);
17364 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17365 assert!(
17366 runner
17367 .state
17368 .base_sync_recovery
17369 .operations
17370 .last()
17371 .unwrap()
17372 .finished
17373 );
17374 assert!(!runner.state.base_sync_blocked());
17375 runner.sync_to_base().await.unwrap();
17376 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17377 }
17378
17379 #[cfg(unix)]
17380 #[tokio::test]
17381 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17382 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17383 let original = runner
17385 .state
17386 .base_sync
17387 .as_ref()
17388 .unwrap()
17389 .conflict
17390 .clone()
17391 .unwrap();
17392 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17393 original
17394 .replace("synchronization(s)", "rebase(s)")
17395 .replace("synchronizing", "rebasing"),
17396 );
17397 runner.state.base_sync_recovery = Default::default();
17398 runner.state.record_base_sync_stop(&head);
17399 assert_eq!(
17400 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17401 crate::base_sync_recovery::StopKind::Exhausted
17402 );
17403 runner.state.authorize_base_sync(&task).unwrap();
17404 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17405 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17406 .as_object_mut()
17407 .unwrap();
17408 auth.remove("interval");
17409 auth.remove("scope");
17410 runner.state = serde_json::from_value(legacy).unwrap();
17411 assert!(runner.state.consume_base_recovery(&head));
17412 assert_eq!(runner.state.base_episode_used(), 3);
17413 runner.sync_to_base().await.unwrap();
17414 assert_eq!(runner.state.base_episode_used(), 4);
17415 runner.state.resource_block = Some(crate::run::ResourceBlock {
17416 node: "gate".into(),
17417 kind: "cache".into(),
17418 detail: "legacy wait".into(),
17419 at: Timestamp::now(),
17420 cause: crate::run::ResourceBlockCause::Unrecorded,
17421 });
17422 runner.state.save().unwrap();
17423 assert!(
17424 runner
17425 .state
17426 .base_sync_recovery
17427 .episodes
17428 .last()
17429 .unwrap()
17430 .boundary
17431 .is_none()
17432 );
17433 runner.state.resource_block = None;
17434 advance_base_fixture(&runner).await;
17435 runner.sync_to_base().await.unwrap();
17436 assert!(runner.state.base_sync_blocked());
17437 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17438 }
17439
17440 #[cfg(unix)]
17441 async fn advance_base_fixture(runner: &Runner) {
17442 let repo = &runner.state.repo;
17443 for args in [
17444 vec!["checkout", "main"],
17445 vec!["commit", "--allow-empty", "-m", "base advanced"],
17446 vec!["push", "origin", "main"],
17447 vec!["checkout", "fix/retained"],
17448 ] {
17449 git::git(repo, &args).await.unwrap();
17450 }
17451 }
17452
17453 #[cfg(unix)]
17454 fn fresh_base_interval_fixture(runner: &mut Runner) {
17455 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17456 runner.state.base_sync.as_mut().unwrap().conflict = None;
17457 runner.state.base_sync_recovery = Default::default();
17458 runner.state.status = RunStatus::Voting;
17459 }
17460
17461 #[cfg(unix)]
17462 #[tokio::test]
17463 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17464 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17465 fresh_base_interval_fixture(&mut runner);
17466 runner.state.begin_pass();
17467 runner.sync_to_base().await.unwrap();
17468 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17469 runner.state.resource_block = Some(crate::run::ResourceBlock {
17470 node: "gate".into(),
17471 kind: "build cache".into(),
17472 detail: "busy".into(),
17473 at: Timestamp::now(),
17474 cause: crate::run::ResourceBlockCause::Timeout {
17475 waited_secs: 1,
17476 limit_secs: 1,
17477 },
17478 });
17479 runner.state.save().unwrap();
17480 let queue = crate::queue::Queue::open();
17481 let mut owner = queue.get(&task).unwrap();
17482 runner.state.task_instruction = Some(owner.instruction.clone());
17483 owner.release_operator("fixture-owner", None);
17484 queue.put(&mut owner).unwrap();
17485 let approval = owner.resumption.clone().unwrap();
17486 runner.state.resource_block = None;
17487 runner.state.save().unwrap();
17488 runner.state = RunState::load(&runner.state.id).unwrap();
17489 advance_base_fixture(&runner).await;
17490 runner.sync_to_base().await.unwrap();
17491 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17492 assert_eq!(runner.state.base_episode_used(), 1);
17493 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17494 assert!(
17495 runner.state.base_sync_recovery.episodes[0]
17496 .boundary
17497 .as_ref()
17498 .unwrap()
17499 .consumed
17500 );
17501 assert!(
17502 !runner
17503 .state
17504 .renew_base_episode(&task, &approval, &head, "base")
17505 );
17506 for _ in 0..3 {
17507 owner.release_operator("fixture-owner", None);
17508 queue.put(&mut owner).unwrap();
17509 runner.state = RunState::load(&runner.state.id).unwrap();
17510 runner.state.begin_pass();
17511 runner.sync_to_base().await.unwrap();
17512 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17513 assert_eq!(runner.state.base_episode_used(), 1);
17514 }
17515 runner.state.resource_block = Some(crate::run::ResourceBlock {
17516 node: "gate".into(),
17517 kind: "disk pressure".into(),
17518 detail: "free space required".into(),
17519 at: Timestamp::now(),
17520 cause: crate::run::ResourceBlockCause::Other,
17521 });
17522 runner.state.end_pass_parked(ParkCause::Interrupt {
17523 reason: "disk pressure".into(),
17524 });
17525 runner.state.save().unwrap();
17526 owner.release_operator("fixture-owner-after-disk", None);
17527 queue.put(&mut owner).unwrap();
17528 runner.state.resource_block = None;
17529 advance_base_fixture(&runner).await;
17530 runner.sync_to_base().await.unwrap();
17531 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17532 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17533 assert_eq!(runner.state.base_episode_used(), 1);
17534 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17535 }
17536
17537 #[cfg(unix)]
17538 #[tokio::test]
17539 async fn base_recovery_continuous_advancement_stops_at_four() {
17540 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17541 fresh_base_interval_fixture(&mut runner);
17542 for spent in 1..=BASE_SYNC_ROUNDS {
17543 runner.sync_to_base().await.unwrap();
17544 assert!(!runner.state.base_sync_blocked());
17545 assert_eq!(runner.state.base_episode_used(), spent);
17546 advance_base_fixture(&runner).await;
17547 }
17548 runner.sync_to_base().await.unwrap();
17549 assert_eq!(
17550 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17551 crate::base_sync_recovery::StopKind::Exhausted
17552 );
17553 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17554 assert!(stop.unverified_sync);
17555 let action = runner.state.base_sync_next_action();
17556 assert!(action.contains("never reached its gate"));
17557 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17558 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17559 runner.state = RunState::load(&runner.state.id).unwrap();
17560 runner.sync_to_base().await.unwrap();
17561 assert_eq!(runner.state.base_episode_used(), 4);
17562 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17563 }
17564
17565 #[cfg(unix)]
17566 #[tokio::test]
17567 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17568 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17569 runner.state.ensure_base_episode();
17570 runner.state.record_base_progress(&head);
17572 runner.state.base_sync_recovery.stop = None;
17573 runner.state.record_base_sync_stop(&head);
17574 assert!(
17575 !runner
17576 .state
17577 .base_sync_recovery
17578 .stop
17579 .as_ref()
17580 .unwrap()
17581 .unverified_sync
17582 );
17583
17584 runner.state.finish_base_operation(Some(&head));
17585 runner.state.gate_ran = true;
17587 runner.state.gate_head = Some("older-head".into());
17588 runner.state.base_sync_recovery.stop = None;
17589 runner.state.record_base_sync_stop(&head);
17590 runner.state.save().unwrap();
17591 runner.state = RunState::load(&runner.state.id).unwrap();
17592 assert!(
17593 runner
17594 .state
17595 .base_sync_recovery
17596 .stop
17597 .as_ref()
17598 .unwrap()
17599 .unverified_sync
17600 );
17601
17602 runner.state.gate_head = Some(head.clone());
17603 runner.state.base_sync_recovery.stop = None;
17604 runner.state.record_base_sync_stop(&head);
17605 assert!(
17606 !runner
17607 .state
17608 .base_sync_recovery
17609 .stop
17610 .as_ref()
17611 .unwrap()
17612 .unverified_sync
17613 );
17614 assert!(
17615 !runner
17616 .state
17617 .base_sync_next_action()
17618 .contains("never reached its gate")
17619 );
17620
17621 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17623 legacy["base_sync_recovery"]["stop"]
17624 .as_object_mut()
17625 .unwrap()
17626 .remove("unverified_sync");
17627 legacy["base_sync_recovery"]["episodes"][0]
17628 .as_object_mut()
17629 .unwrap()
17630 .remove("last_sync_head");
17631 runner.state = serde_json::from_value(legacy).unwrap();
17632 assert!(
17633 !runner
17634 .state
17635 .base_sync_recovery
17636 .stop
17637 .as_ref()
17638 .unwrap()
17639 .unverified_sync
17640 );
17641 assert!(
17642 runner.state.base_sync_recovery.episodes[0]
17643 .last_sync_head
17644 .is_none()
17645 );
17646 }
17647
17648 #[cfg(unix)]
17649 #[tokio::test]
17650 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17651 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17652 runner.state.authorize_base_sync(&task).unwrap();
17653 assert!(
17654 runner
17655 .state
17656 .base_sync_next_action()
17657 .contains("Approval recorded")
17658 );
17659 assert!(
17660 runner
17661 .state
17662 .base_sync_next_action()
17663 .contains("4/4 in episode 0, 4 lifetime")
17664 );
17665 assert!(runner.state.consume_base_recovery(&head));
17666 assert!(
17667 runner
17668 .state
17669 .base_sync_next_action()
17670 .contains("0/4 in episode 1, 4 lifetime")
17671 );
17672 }
17673
17674 #[cfg(unix)]
17675 #[tokio::test]
17676 async fn base_recovery_changed_head_invalidates_without_spending() {
17677 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17678 runner.state.authorize_base_sync(&task).unwrap();
17679 git::git(
17680 &runner.state.repo,
17681 &["commit", "--allow-empty", "-m", "external head change"],
17682 )
17683 .await
17684 .unwrap();
17685 runner.sync_to_base().await.unwrap();
17686 let a = runner
17687 .state
17688 .base_sync_recovery
17689 .authorizations
17690 .last()
17691 .unwrap();
17692 assert!(a.consumed);
17693 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17694 runner.state = RunState::load(&runner.state.id).unwrap();
17695 assert!(
17696 runner
17697 .state
17698 .base_sync_next_action()
17699 .contains("Approval invalidated")
17700 );
17701 assert!(!runner.state.base_recovery_pending());
17702 assert!(runner.state.base_sync_blocked());
17703 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17704 }
17705
17706 #[cfg(unix)]
17707 #[tokio::test]
17708 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17709 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17710 runner.state.driver_exited = true;
17711 runner.state.save().unwrap();
17712 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17713 .await
17714 .unwrap();
17715 runner.state = RunState::load(&runner.state.id).unwrap();
17716 advance_base_fixture(&runner).await;
17717 runner.sync_to_base().await.unwrap();
17718 let a = runner
17719 .state
17720 .base_sync_recovery
17721 .authorizations
17722 .last()
17723 .unwrap();
17724 assert!(
17725 a.consumed
17726 && a.invalidated
17727 .as_deref()
17728 .unwrap()
17729 .contains("fetched base changed")
17730 );
17731 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17732 assert_eq!(
17733 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17734 head
17735 );
17736 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17737 .await
17738 .unwrap();
17739 runner.state = RunState::load(&runner.state.id).unwrap();
17740 runner.sync_to_base().await.unwrap();
17741 assert!(!runner.state.base_sync_blocked());
17742 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17743 }
17744
17745 #[cfg(unix)]
17746 #[tokio::test]
17747 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17748 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17749 runner.state.authorize_base_sync(&task).unwrap();
17750 assert!(runner.state.consume_base_recovery(&head));
17751 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17752 runner.state.reserve_base_operation(&head, &tip);
17753 runner.state.save().unwrap();
17754 let scratch = runner.state.dir().join("base-sync");
17755 assert_eq!(
17756 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17757 .await
17758 .unwrap(),
17759 crate::rebase::Rebased::Applied
17760 );
17761 runner.state = RunState::load(&runner.state.id).unwrap();
17762 runner.sync_to_base().await.unwrap();
17763 assert!(
17764 runner
17765 .state
17766 .base_sync_recovery
17767 .operations
17768 .last()
17769 .unwrap()
17770 .finished
17771 );
17772 assert_eq!(runner.state.base_episode_used(), 1);
17773 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17774 assert!(!runner.state.base_sync_blocked());
17775 }
17776
17777 #[cfg(unix)]
17778 #[tokio::test]
17779 async fn base_recovery_crash_preserves_untracked_files() {
17780 for ignored in [false, true] {
17781 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17782 runner.state.authorize_base_sync(&task).unwrap();
17783 assert!(runner.state.consume_base_recovery(&head));
17784 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17785 runner.state.reserve_base_operation(&head, &tip);
17786 runner.state.save().unwrap();
17787 let scratch = runner.state.dir().join("base-sync");
17788 assert_eq!(
17789 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17790 .await
17791 .unwrap(),
17792 crate::rebase::Rebased::Applied
17793 );
17794 let repo = &runner.state.repo;
17795 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17796 assert_ne!(result, head);
17797 if ignored {
17798 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17799 .await
17800 .unwrap();
17801 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17802 }
17803 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17804 assert!(
17805 git::git_raw(repo, &["diff", "--quiet", &head])
17806 .await
17807 .unwrap()
17808 .ok()
17809 );
17810 runner.state = RunState::load(&runner.state.id).unwrap();
17812 runner.sync_to_base().await.unwrap();
17813 assert!(runner.state.base_sync_blocked());
17814 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17815 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17816 assert_eq!(
17817 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17818 result
17819 );
17820 assert_eq!(
17821 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17822 "retained work\n"
17823 );
17824 assert!(
17825 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17826 .await
17827 .unwrap()
17828 .ok()
17829 );
17830 }
17831 }
17832
17833 #[cfg(unix)]
17834 #[tokio::test]
17835 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17836 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17837 runner.state.authorize_base_sync(&task).unwrap();
17838 assert!(runner.state.consume_base_recovery(&head));
17839 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17840 runner.state.reserve_base_operation(&head, &tip);
17841 runner.state.save().unwrap();
17842 let scratch = runner.state.dir().join("base-sync");
17843 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17844 .await
17845 .unwrap();
17846 git::sync_to_head(&runner.state.repo).await.unwrap();
17847 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17848 git::git(&runner.state.repo, &["add", "foreign"])
17849 .await
17850 .unwrap();
17851 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17852 .await
17853 .unwrap();
17854 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17855 runner.state = RunState::load(&runner.state.id).unwrap();
17856 runner.sync_to_base().await.unwrap();
17857 assert!(runner.state.base_sync_blocked());
17858 assert!(
17859 !runner
17860 .state
17861 .base_sync_recovery
17862 .operations
17863 .last()
17864 .unwrap()
17865 .finished
17866 );
17867 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17868 assert_eq!(
17869 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17870 foreign_head
17871 );
17872 }
17873
17874 #[cfg(unix)]
17875 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17876 {
17877 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17878 fresh_base_interval_fixture(&mut runner);
17879 let repo = &runner.state.repo;
17880 git::git(repo, &["checkout", "main"]).await.unwrap();
17881 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17882 git::git(repo, &["add", "retained"]).await.unwrap();
17883 git::git(repo, &["commit", "-m", "upstream retained change"])
17884 .await
17885 .unwrap();
17886 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17887 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17888 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17889 let scratch = runner.state.dir().join("base-sync");
17890 assert!(matches!(
17891 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17892 .await
17893 .unwrap(),
17894 git::RebaseStart::Conflicted(_)
17895 ));
17896 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17897 agent: "fixture".into(),
17898 paths: vec!["retained".into()],
17899 from: Some(head.clone()),
17900 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17901 index_outside: None,
17902 finished: false,
17903 error: None,
17904 launcher: None,
17905 infra: None,
17906 outage: None,
17907 });
17908 (tmp, runner, head, tip, scratch)
17909 }
17910
17911 #[cfg(unix)]
17912 #[tokio::test]
17913 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17914 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17915 std::fs::write(
17918 scratch.join("retained"),
17919 "upstream and retained resolution\n",
17920 )
17921 .unwrap();
17922 runner.state.resource_block = Some(crate::run::ResourceBlock {
17923 node: "rebase".into(),
17924 kind: "repository coordination".into(),
17925 detail: "busy".into(),
17926 at: Timestamp::now(),
17927 cause: crate::run::ResourceBlockCause::Other,
17928 });
17929 runner.state.save().unwrap();
17930 let newer = git::git(
17933 &runner.state.repo,
17934 &[
17935 "commit-tree",
17936 "main^{tree}",
17937 "-p",
17938 "main",
17939 "-m",
17940 "base advanced during wait",
17941 ],
17942 )
17943 .await
17944 .unwrap();
17945 git::git(
17946 &runner.state.repo,
17947 &[
17948 "push",
17949 "origin",
17950 &format!("{}:refs/heads/main", newer.trim()),
17951 ],
17952 )
17953 .await
17954 .unwrap();
17955 runner.state = RunState::load(&runner.state.id).unwrap();
17956 runner.state.resource_block = None;
17957 runner.sync_to_base().await.unwrap();
17958 assert!(!runner.state.base_sync_blocked());
17959 let op = &runner.state.base_sync_recovery.operations[0];
17960 assert_eq!(op.head, head);
17961 assert_eq!(op.tip, tip);
17962 assert!(op.finished);
17963 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17964 assert_eq!(runner.state.rebase_fixes.len(), 1);
17965 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17966 .await
17967 .unwrap();
17968 assert_ne!(latest, tip);
17969 runner.state = RunState::load(&runner.state.id).unwrap();
17970 runner.sync_to_base().await.unwrap();
17971 assert!(!runner.state.base_sync_blocked());
17972 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17973 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17974 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17975 }
17976
17977 #[cfg(unix)]
17978 #[tokio::test]
17979 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17980 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17981 runner.state.reserve_base_operation(&head, &tip);
17982 runner.state.save().unwrap();
17983 std::fs::write(
17984 scratch.join("retained"),
17985 "upstream and retained resolution\n",
17986 )
17987 .unwrap();
17988 git::git(&scratch, &["add", "retained"]).await.unwrap();
17989 git::git(
17990 &scratch,
17991 &["-c", "core.editor=true", "rebase", "--continue"],
17992 )
17993 .await
17994 .unwrap();
17995 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17996 assert!(
17997 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17998 .await
17999 .unwrap()
18000 .is_empty()
18001 );
18002 assert_eq!(
18003 crate::rebase::finish(
18004 &mut runner.state,
18005 &scratch,
18006 "fix/retained",
18007 &head,
18008 &tip,
18009 &["retained".into()],
18010 ""
18011 )
18012 .await
18013 .unwrap(),
18014 crate::rebase::Rebased::Applied
18015 );
18016 assert!(!scratch.exists());
18017 runner.state = RunState::load(&runner.state.id).unwrap();
18019 let op = &runner.state.base_sync_recovery.operations[0];
18020 assert!(!op.finished);
18021 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18022 runner.sync_to_base().await.unwrap();
18023 assert!(!runner.state.base_sync_blocked());
18024 assert!(runner.state.base_sync_recovery.operations[0].finished);
18025 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18026 assert_eq!(runner.state.rebase_fixes.len(), 1);
18027 let winner = runner.state.winner().unwrap().clone();
18028 let wait = runner.repository_wait();
18029 assert!(
18030 verify_current_head(&mut runner.state, &winner, &wait)
18031 .await
18032 .unwrap()
18033 );
18034 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18035 }
18036
18037 #[cfg(unix)]
18038 #[tokio::test]
18039 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18040 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18041 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18042 runner.state.reserve_base_operation(&head, &tip);
18043 runner.state.save().unwrap();
18044 std::fs::write(
18045 scratch.join("retained"),
18046 "upstream and retained resolution\n",
18047 )
18048 .unwrap();
18049 git::git(&scratch, &["add", "retained"]).await.unwrap();
18050 git::git(
18051 &scratch,
18052 &["-c", "core.editor=true", "rebase", "--continue"],
18053 )
18054 .await
18055 .unwrap();
18056 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18057 assert!(!git::rebase_in_progress(&scratch).await);
18058 assert!(
18059 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18060 .await
18061 .unwrap()
18062 .is_empty()
18063 );
18064 match damage {
18065 Some("record") => {
18066 let path =
18067 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18068 .await
18069 .unwrap();
18070 std::fs::write(scratch.join(path), "").unwrap();
18071 }
18072 Some("foreign") => {
18073 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18074 git::git(&scratch, &["add", "foreign"]).await.unwrap();
18075 git::git(&scratch, &["commit", "-m", "foreign work"])
18076 .await
18077 .unwrap();
18078 }
18079 Some("dirty") => {
18080 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18081 }
18082 _ => {}
18083 }
18084 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18085 runner.state = RunState::load(&runner.state.id).unwrap();
18087 assert!(
18088 runner.state.base_sync_recovery.operations[0]
18089 .validated_head
18090 .is_none()
18091 );
18092 runner.sync_to_base().await.unwrap();
18093 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18094 assert_eq!(runner.state.base_episode_used(), 1);
18095 assert_eq!(runner.state.rebase_fixes.len(), 1);
18096 assert_eq!(
18097 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18098 retained_head
18099 );
18100 if damage.is_some() {
18101 assert!(runner.state.base_sync_blocked());
18102 assert!(scratch.exists());
18103 assert!(!runner.state.base_sync_recovery.operations[0].finished);
18104 assert!(
18105 runner.state.base_sync_recovery.operations[0]
18106 .validated_head
18107 .is_none()
18108 );
18109 } else {
18110 assert!(!runner.state.base_sync_blocked());
18111 assert!(!scratch.exists());
18112 assert!(runner.state.base_sync_recovery.operations[0].finished);
18113 assert_eq!(
18114 runner.state.base_sync_recovery.operations[0]
18115 .validated_head
18116 .as_deref(),
18117 Some(result.as_str())
18118 );
18119 let winner = runner.state.winner().unwrap().clone();
18120 let wait = runner.repository_wait();
18121 assert!(
18122 verify_current_head(&mut runner.state, &winner, &wait)
18123 .await
18124 .unwrap()
18125 );
18126 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18127 runner.state = RunState::load(&runner.state.id).unwrap();
18128 runner.sync_to_base().await.unwrap();
18129 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18130 }
18131 }
18132 }
18133
18134 #[cfg(unix)]
18135 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18136 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18137 runner.state.prep_complete = Some(true);
18139 runner.state.base_sync = None;
18140 runner.state.base_sync_recovery = Default::default();
18141 runner.state.config.graph.gate_fix_rounds = 1;
18142 let mut review = review_round(true, 0, 1, 1, false, true);
18143 review.head = head.clone();
18144 review.verified_head = Some(head.clone());
18145 runner.state.reviews = vec![review];
18146 runner.state.gate = vec![CommandOutcome {
18147 command: "old gate".into(),
18148 code: Some(1),
18149 output_tail: "old failure".into(),
18150 duration_ms: 0,
18151 resource_blocked: false,
18152 }];
18153 runner.state.gate_head = Some(head.clone());
18154 runner.state.gate_ran = true;
18155 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18156 agent: "fixture".into(),
18157 failed: runner.state.gate.clone(),
18158 notes: "spent".into(),
18159 committed: false,
18160 error: None,
18161 }];
18162 runner.state.land_approval = Some(crate::run::LandApproval {
18163 question: "old approval".into(),
18164 head: head.clone(),
18165 });
18166 runner.state.status = RunStatus::Blocked;
18167 (tmp, runner, head)
18168 }
18169
18170 #[cfg(unix)]
18171 #[tokio::test]
18172 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18173 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18174 let (tmp, mut runner, old) = resumed_gate_fixture().await;
18175 let phases = tmp.path().join("phases");
18176 runner.state.config.verify.e2e = vec![format!(
18177 "echo e2e >> '{}'; {}",
18178 phases.display(),
18179 if e2e_ok {
18180 "true"
18181 } else {
18182 "echo new-e2e-failure; false"
18183 }
18184 )];
18185 runner.state.config.verify.gate = vec![format!(
18186 "echo gate >> '{}'; {}",
18187 phases.display(),
18188 if gate_ok {
18189 "true"
18190 } else {
18191 "echo new-gate-failure; false"
18192 }
18193 )];
18194 runner.state.save().unwrap();
18195 runner.state = RunState::load(&runner.state.id).unwrap();
18196 runner.sync_to_base().await.unwrap();
18197 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18198 assert_ne!(head, old);
18199 runner.state = RunState::load(&runner.state.id).unwrap();
18201 runner.execute_graph().await.unwrap();
18202 assert_eq!(runner.state.gate_fixes.len(), 1);
18203 assert_eq!(runner.state.reviews.len(), 1);
18204 assert_eq!(runner.state.reviews[0].head, old);
18205 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18206 assert_eq!(runner.state.verification_history.len(), 1);
18207 let history = &runner.state.verification_history[0];
18208 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18209 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18210 assert!(history.archived_at.is_some());
18211 assert_eq!(history.fixes_spent, Some(1));
18212 assert_eq!(history.gate[0].output_tail, "old failure");
18213 let archive_events: Vec<_> = runner
18214 .state
18215 .events
18216 .iter()
18217 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18218 .collect();
18219 assert_eq!(archive_events.len(), 1);
18220 assert!(archive_events[0].message.contains(&short(&old)));
18221 assert!(archive_events[0].message.contains(&short(&head)));
18222 assert!(
18223 archive_events[0]
18224 .message
18225 .contains("1 gate-fix round(s) already spent stay spent")
18226 );
18227 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18228 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18229 if e2e_ok {
18230 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18231 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18232 } else {
18233 assert!(!runner.state.gate_ran);
18234 }
18235 assert_eq!(
18236 runner.state.status,
18237 if e2e_ok && gate_ok {
18238 RunStatus::Ready
18239 } else {
18240 RunStatus::Blocked
18241 }
18242 );
18243 if !e2e_ok || !gate_ok {
18244 let cause = runner.state.verification_blocking_cause().unwrap();
18245 assert!(cause.contains(&head));
18246 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18247 runner.state.event("land", "rebased successfully");
18249 let notice = crate::notices::run_ended(&runner.state).unwrap();
18250 assert!(!format!("{notice:?}").contains("rebased successfully"));
18251 assert!(runner.state.merge.is_none());
18252 }
18253 let before = std::fs::read_to_string(&phases).unwrap();
18254 runner.state.save().unwrap();
18255 runner.state = RunState::load(&runner.state.id).unwrap();
18256 runner.review_loop().await.unwrap();
18257 runner.gate().await.unwrap();
18258 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18259 assert_eq!(runner.state.verification_history.len(), 1);
18260 let history = &runner.state.verification_history[0];
18261 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18262 assert!(history.archived_at.is_some());
18263 assert_eq!(history.fixes_spent, Some(1));
18264 }
18265 }
18266
18267 #[cfg(unix)]
18268 #[tokio::test]
18269 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18270 let (tmp, mut runner, head) = resumed_gate_fixture().await;
18271 let marker = tmp.path().join("gate-ran");
18272 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18273 runner.review_loop().await.unwrap();
18274 runner.gate().await.unwrap();
18275 assert_eq!(runner.state.status, RunStatus::Blocked);
18276 assert!(!marker.exists());
18277 assert!(runner.state.verification_history.is_empty());
18278 runner.state.gate_head = None;
18279 runner.gate().await.unwrap();
18280 assert!(marker.exists());
18281 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18282 assert_eq!(runner.state.verification_history.len(), 1);
18283 assert_eq!(runner.state.gate_fixes.len(), 1);
18284 }
18285
18286 #[cfg(unix)]
18287 #[tokio::test]
18288 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18289 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18290 runner.sync_to_base().await.unwrap();
18291 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18292 runner.state.reconcile_verification_head(&head);
18293 for outcomes in [
18295 Vec::new(),
18296 vec![CommandOutcome {
18297 command: "true".into(),
18298 code: None,
18299 output_tail: "cache unavailable".into(),
18300 duration_ms: 0,
18301 resource_blocked: true,
18302 }],
18303 ] {
18304 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18305 head: head.clone(),
18306 passed: false,
18307 outcomes,
18308 });
18309 runner.state.gate_ran = false;
18310 runner.state.gate.clear();
18311 runner.state.save().unwrap();
18312 runner.state = RunState::load(&runner.state.id).unwrap();
18313 runner.merge().await.unwrap();
18314 assert!(runner.state.merge.is_none());
18315 runner.review_loop().await.unwrap();
18316 runner.gate().await.unwrap();
18317 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18318 assert!(runner.state.gate_allows_progress());
18319 assert_eq!(runner.state.gate_fixes.len(), 1);
18320 assert!(runner.state.merge.is_none());
18321 }
18322 git::git(
18324 &runner.state.repo,
18325 &["commit", "--allow-empty", "-m", "external update"],
18326 )
18327 .await
18328 .unwrap();
18329 runner.merge().await.unwrap();
18330 assert!(runner.state.merge.is_none());
18331 assert_eq!(runner.state.status, RunStatus::Blocked);
18332 }
18333
18334 #[cfg(unix)]
18335 #[tokio::test]
18336 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18337 for waiting_on_gate in [false, true] {
18338 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18339 runner.sync_to_base().await.unwrap();
18340 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18341 runner.state.reconcile_verification_head(&head);
18342 if waiting_on_gate {
18343 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18344 head: head.clone(),
18345 passed: true,
18346 outcomes: Vec::new(),
18347 });
18348 }
18349 let cache = tmp.path().join("cache");
18350 runner.state.config.verify.e2e =
18351 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18352 runner.state.config.verify.gate = vec!["true".into()];
18353 runner.state.config.graph.timeout_resource_wait = 1;
18354 let owner =
18355 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18356 let guard =
18357 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18358 crate::cache::AcquireOutcome::Acquired(g) => g,
18359 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18360 };
18361 runner.review_loop().await.unwrap();
18362 runner.gate().await.unwrap();
18363 assert!(!runner.state.gate_ran);
18364 assert_eq!(runner.state.gate_fixes.len(), 1);
18365 assert!(runner.state.resource_block.is_some());
18366 runner.merge().await.unwrap();
18367 assert!(runner.state.merge.is_none());
18368 drop(guard);
18369 runner.state = RunState::load(&runner.state.id).unwrap();
18370 runner.state.resource_block = None;
18372 runner.review_loop().await.unwrap();
18373 runner.gate().await.unwrap();
18374 assert!(runner.state.gate_allows_progress());
18375 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18376 assert_eq!(runner.state.gate_fixes.len(), 1);
18377 }
18378 }
18379
18380 #[cfg(unix)]
18381 #[tokio::test]
18382 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18383 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18384 runner.sync_to_base().await.unwrap();
18385 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18386 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18387 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18389 runner.review_loop().await.unwrap();
18390 runner.gate().await.unwrap();
18391 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18392 assert_ne!(before, after);
18393 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18394 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18395 assert!(!runner.state.gate_allows_progress());
18396 runner.merge().await.unwrap();
18397 assert!(runner.state.merge.is_none());
18398 assert!(
18399 runner
18400 .state
18401 .verification_history
18402 .iter()
18403 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18404 );
18405
18406 runner.state.reviews[0].clean = false;
18408 runner.state.reviews[0].answered = 0;
18409 runner.state.config.verify.e2e = vec!["true".into()];
18410 runner.review_loop().await.unwrap();
18411 runner.gate().await.unwrap();
18412 runner.merge().await.unwrap();
18413 assert_eq!(runner.state.status, RunStatus::Blocked);
18414 assert!(runner.state.merge.is_none());
18415 assert!(
18416 runner
18417 .state
18418 .verification_blocking_cause()
18419 .unwrap()
18420 .contains("review blocked")
18421 );
18422 }
18423
18424 #[cfg(unix)]
18425 #[tokio::test]
18426 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18427 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18428 runner.state.gate_head = None;
18429 runner.state.config.verify.gate =
18430 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18431 runner.review_loop().await.unwrap();
18432 runner.gate().await.unwrap();
18433 assert_ne!(
18434 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18435 old
18436 );
18437 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18438 assert!(!runner.state.gate_allows_progress());
18439 assert_eq!(runner.state.status, RunStatus::Blocked);
18440 runner.merge().await.unwrap();
18441 assert!(runner.state.merge.is_none());
18442 }
18443
18444 #[cfg(unix)]
18445 #[tokio::test]
18446 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18447 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18448 let mut old_review = review_round(true, 0, 1, 1, false, true);
18449 old_review.head = head.clone();
18450 old_review.verified_head = Some(head.clone());
18451 runner.state.reviews = vec![old_review];
18452 runner.state.gate = vec![CommandOutcome {
18453 command: "old gate".into(),
18454 code: None,
18455 output_tail: "timed out after 2400s".into(),
18456 duration_ms: 2_400_000,
18457 resource_blocked: false,
18458 }];
18459 runner.state.gate_ran = true;
18460 runner.state.gate_head = Some(head.clone());
18461 runner.state.land_approval = Some(crate::run::LandApproval {
18462 question: "old-head-approval".into(),
18463 head: head.clone(),
18464 });
18465 runner.state.authorize_base_sync(&task).unwrap();
18466 runner.sync_to_base().await.unwrap();
18467 assert!(!runner.state.base_sync_blocked());
18468 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18469 assert_ne!(head, fresh);
18470 assert_eq!(runner.state.reviews.len(), 1);
18471 assert!(!runner.state.gate_ran);
18472 let saved = &runner.state.base_sync_recovery.authorizations[0];
18473 assert_eq!(saved.reviews[0].head, head);
18474 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18475 assert!(saved.gate_ran);
18476 assert!(!saved.gate[0].ok());
18477 runner.review_loop().await.unwrap();
18478 runner.gate().await.unwrap();
18479 assert_eq!(runner.state.reviews[0].head, head);
18480 assert_eq!(
18481 runner.state.reviews[0].verified_head.as_deref(),
18482 Some(head.as_str())
18483 );
18484 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18485 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18486 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18487 assert!(runner.state.gate_ran);
18488 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18489 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18490 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18491 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18492 assert_eq!(
18493 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18494 "retained change"
18495 );
18496 }
18497
18498 #[cfg(unix)]
18499 #[tokio::test]
18500 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18501 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18502 let queue = crate::queue::Queue::open();
18503 let mut owner = queue.get(&task).unwrap();
18504 owner.release_operator("test-resume", None);
18505 queue.put(&mut owner).unwrap();
18506 let generation = owner.resumption.as_ref().unwrap().generation;
18507 runner.state.task_instruction = Some(owner.instruction.clone());
18508 runner.state.driver_exited = true;
18509 runner.state.save().unwrap();
18510 let home = crate::run::home();
18511 let _claim = queue.claim(&task).unwrap();
18512 owner.resumption.as_mut().unwrap().at =
18513 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18514 queue.put(&mut owner).unwrap();
18515 assert!(
18516 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18517 .await
18518 .is_err(),
18519 "an approval from before the stop grants nothing"
18520 );
18521 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18522 queue.put(&mut owner).unwrap();
18523 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18524 .await
18525 .unwrap();
18526 assert!(
18527 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18528 .await
18529 .is_err()
18530 );
18531 runner.state = RunState::load(&runner.state.id).unwrap();
18532 assert_eq!(
18533 runner.state.base_sync_recovery.authorizations[0].generation,
18534 Some(generation)
18535 );
18536 runner.sync_to_base().await.unwrap();
18537 let repo = runner.state.repo.clone();
18538 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18539 git::git(&repo, &["checkout", "main"]).await.unwrap();
18540 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18541 git::git(&repo, &["add", "."]).await.unwrap();
18542 git::git(&repo, &["commit", "-m", "move base again"])
18543 .await
18544 .unwrap();
18545 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18546 git::git(&repo, &["checkout", "fix/retained"])
18547 .await
18548 .unwrap();
18549 runner.sync_to_base().await.unwrap();
18550 assert!(!runner.state.base_sync_blocked());
18551 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18552 assert_eq!(runner.state.base_episode_used(), 2);
18553 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18554 assert_ne!(fresh, head);
18555 assert!(!runner.state.base_recovery_pending());
18556 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18558 queue.put(&mut owner).unwrap();
18559 assert!(
18560 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18561 .await
18562 .is_err()
18563 );
18564 runner.state = RunState::load(&runner.state.id).unwrap();
18565 runner.sync_to_base().await.unwrap();
18566 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18567 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18568 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18569 }
18570
18571 #[cfg(unix)]
18572 #[tokio::test]
18573 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18574 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18575 runner.state.base_sync = None;
18576 runner.state.status = RunStatus::Blocked;
18577 let mut second = runner.state.candidates[0].clone();
18578 second.label = 'B';
18579 runner.state.candidates.push(second);
18580 assert!(runner.state.votes.is_empty());
18581 assert!(runner.state.deliberation.is_empty());
18582 runner.deliberate().await.unwrap();
18583 runner.vote().await.unwrap();
18584 assert_eq!(runner.state.status, RunStatus::Blocked);
18585 assert!(runner.state.votes.is_empty());
18586 assert!(runner.state.deliberation.is_empty());
18587 }
18588
18589 fn runner_at(status: RunStatus) -> Runner {
18592 let mut state = RunState::new(
18593 PathBuf::from("/nonexistent/repo"),
18594 "main".to_owned(),
18595 "deadbeef".to_owned(),
18596 "task".to_owned(),
18597 Config::default(),
18598 );
18599 state.config.disk.min_free_bytes = 0;
18601 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18604 state.id = format!(
18605 "20000101-000000-{:04x}",
18606 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18607 );
18608 state.status = status;
18609 Runner {
18610 state,
18611 roles: ResolvedRoles {
18612 implementers: Vec::new(),
18613 judges: Vec::new(),
18614 reviewers: Vec::new(),
18615 fixer: None,
18616 conductor: conductor(),
18617 implementer_roster: Vec::new(),
18618 judge_roster: Vec::new(),
18619 reviewer_roster: Vec::new(),
18620 },
18621 sem: Arc::new(Semaphore::new(1)),
18622 pause: Pause::new(),
18623 interrupt: Pause::new(),
18624 }
18625 }
18626
18627 #[tokio::test]
18628 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18629 crate::run::pin_test_home();
18630 let temp = tempfile::tempdir().unwrap();
18631 let root = temp.path().canonicalize().unwrap();
18632 let repo = root.join("repo");
18633 std::fs::create_dir(&repo).unwrap();
18634 init_repo(&repo);
18635 git::git(&repo, &["config", "core.autocrlf", "false"])
18636 .await
18637 .unwrap();
18638 let mut runner = runner_at(RunStatus::Blocked);
18639 runner.state.repo = repo.clone();
18640 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18641 runner.state.config.agents = vec![conductor()];
18642 runner.state.config.graph.selected_implementer = Some("conductor".into());
18643 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18644 let (shell, command, setup_output) = if cfg!(windows) {
18645 (
18646 vec!["cmd".into(), "/C".into()],
18647 "echo ready>seat-ready",
18648 "ready\r\n",
18649 )
18650 } else {
18651 (
18652 vec!["sh".into(), "-c".into()],
18653 "printf ready > seat-ready",
18654 "ready",
18655 )
18656 };
18657 runner.state.config.verify.shell = Some(shell);
18658 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18659 run: Some(command.into()),
18660 ..Default::default()
18661 }];
18662 runner.state.save().unwrap();
18663 let path = runner.state.worktree_root().join("judge-1");
18664 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18665 .await
18666 .unwrap();
18667 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18668 git::git(&path, &["add", "."]).await.unwrap();
18669 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18670 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18671 .await
18672 .unwrap();
18673 git::git(&path, &["checkout", "--detach", &head])
18674 .await
18675 .unwrap();
18676 let evidence = crate::fold_evidence::snapshot(
18677 &runner.state,
18678 &crate::run::home().canonicalize().unwrap(),
18679 &head,
18680 64 * 1024 * 1024,
18681 )
18682 .await
18683 .unwrap();
18684 let mut contents = BTreeMap::new();
18685 contents.insert(
18686 repo.join("logs/HEAD"),
18687 format!("{head} {head} fixture\n").into_bytes(),
18688 );
18689 let record = serde_json::json!({
18690 "version":1, "run":runner.state.id, "source":path, "head":head,
18691 "evidence":evidence, "admin":{}, "admin_contents":contents,
18692 "history":{head.clone():evidence}, "phase":"evicted"
18693 });
18694 std::fs::write(
18695 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18696 serde_json::to_vec(&record).unwrap(),
18697 )
18698 .unwrap();
18699 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18700 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18701 .await
18702 .unwrap();
18703 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18704 assert!(
18705 !git::git_raw(&repo, &["cat-file", "-e", &head])
18706 .await
18707 .unwrap()
18708 .ok()
18709 );
18710 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18711 let error = resumed.execute().await.unwrap_err();
18712 assert!(
18713 error
18714 .to_string()
18715 .contains("selected implementation is held"),
18716 "{error:#}"
18717 );
18718 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18719 assert_eq!(
18720 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18721 setup_output
18722 );
18723 assert_eq!(
18724 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18725 "unrelated seat"
18726 );
18727 let error = resumed.execute().await.unwrap_err();
18728 assert!(
18729 error
18730 .to_string()
18731 .contains("selected implementation is held"),
18732 "{error:#}"
18733 );
18734 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18735 git::git(&path, &["add", "."]).await.unwrap();
18736 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18737 let next = git::git(
18738 &path,
18739 &[
18740 "commit-tree",
18741 &next_tree,
18742 "-p",
18743 &head,
18744 "-m",
18745 "next review checkpoint",
18746 ],
18747 )
18748 .await
18749 .unwrap();
18750 git::git(&path, &["checkout", "--detach", &next])
18751 .await
18752 .unwrap();
18753 resumed.state.reviews.push(
18754 serde_json::from_value(serde_json::json!({
18755 "round":1, "head":next, "reviews":[]
18756 }))
18757 .unwrap(),
18758 );
18759 resumed.state.save().unwrap();
18760 let error = resumed.execute().await.unwrap_err();
18761 assert!(
18762 error
18763 .to_string()
18764 .contains("selected implementation is held"),
18765 "{error:#}"
18766 );
18767 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18768 }
18769
18770 #[tokio::test]
18771 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18772 crate::run::pin_test_home();
18773 let mut runner = runner_at(RunStatus::Gating);
18774 runner.state.save().unwrap();
18775 let id = runner.state.id.clone();
18776 let passes = runner.state.passes.len();
18777 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18778 let mut claiming = Box::pin(runner.claim_driver());
18779 std::future::poll_fn(|cx| {
18780 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18781 std::task::Poll::Ready(())
18782 })
18783 .await;
18784 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18785 drop(owner);
18786 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18787 .await
18788 .unwrap()
18789 .unwrap();
18790 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18791 assert!(execution.runner.state.resource_block.is_none());
18792 assert_eq!(
18793 execution.runner.state.liveness(false),
18794 crate::run::Liveness::Live
18795 );
18796 drop(execution);
18797 assert!(RunState::load(&id).unwrap().driver_exited);
18798 }
18799
18800 #[tokio::test]
18801 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18802 crate::run::pin_test_home();
18803 let mut runner = runner_at(RunStatus::Gating);
18804 runner.state.save().unwrap();
18805 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18808 assert!(runner.claim_driver().await.is_err());
18809 assert!(runner.state.driver_exited);
18810 assert_eq!(runner.state.driver_pid, None);
18811 let disk = RunState::load(&runner.state.id).unwrap();
18812 assert!(disk.driver_exited);
18813 assert_eq!(disk.driver_pid, None);
18814 assert_eq!(disk.status, RunStatus::Gating);
18815 }
18816
18817 #[tokio::test]
18818 async fn an_early_graph_error_releases_its_driver() {
18819 crate::run::pin_test_home();
18820 let mut runner = runner_at(RunStatus::Blocked);
18821 runner.state.config.graph.selected_implementer = Some("selected".into());
18822 runner.state.save().unwrap();
18823 assert!(runner.execute().await.is_err());
18824 let disk = RunState::load(&runner.state.id).unwrap();
18825 assert!(disk.driver_exited);
18826 assert_eq!(disk.driver_pid, None);
18827 assert_eq!(disk.status, RunStatus::Blocked);
18828 }
18829
18830 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18831 crate::run::MergeRepair {
18832 question: "repair-question".into(),
18833 head: "old-head".into(),
18834 pr: "https://github.com/o/r/pull/42".into(),
18835 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18836 gate: vec![outcome(Some(0), false)],
18837 gate_head: Some("old-head".into()),
18838 gate_fixes: Vec::new(),
18839 pre_gate: Vec::new(),
18840 phase,
18841 failure: None,
18842 }
18843 }
18844
18845 #[tokio::test]
18846 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18847 crate::run::pin_test_home();
18848 let mut runner = runner_at(RunStatus::Landing);
18849 runner.state.merge_repairs.push(merge_repair_request(
18850 crate::run::MergeRepairPhase::Requested,
18851 ));
18852 runner.state.save().unwrap();
18853 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18854 assert!(
18855 runner
18856 .execute()
18857 .await
18858 .unwrap_err()
18859 .to_string()
18860 .contains("already running")
18861 );
18862 assert_eq!(
18863 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18864 crate::run::MergeRepairPhase::Requested
18865 );
18866 drop(claim);
18867 let mut disk = runner.state.clone();
18868 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18869 disk.save().unwrap();
18870 assert!(
18871 runner
18872 .execute()
18873 .await
18874 .unwrap_err()
18875 .to_string()
18876 .contains("decision changed")
18877 );
18878 assert_eq!(
18879 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18880 crate::run::MergeRepairPhase::Complete
18881 );
18882 }
18883
18884 #[test]
18885 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18886 crate::run::pin_test_home();
18887 let mut runner = runner_at(RunStatus::Landing);
18888 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18889 runner.state.reviews = request.reviews.clone();
18890 runner.state.gate = request.gate.clone();
18891 runner.state.gate_ran = true;
18892 runner.state.land_approval = Some(crate::run::LandApproval {
18893 question: "old".into(),
18894 head: "old-head".into(),
18895 });
18896 runner.state.merge_repairs.push(request);
18897 runner.begin_merge_repair_review(0).unwrap();
18898 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18899 assert!(!runner.state.gate_status().ok());
18900 assert!(runner.state.land_approval.is_none());
18901 let disk = RunState::load(&runner.state.id).unwrap();
18902 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18903 assert!(disk.merge_repairs[0].gate[0].ok());
18904 assert_eq!(
18905 disk.merge_repairs[0].phase,
18906 crate::run::MergeRepairPhase::Reviewing
18907 );
18908 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18909 assert!(disk.reviews.is_empty());
18910 }
18911
18912 #[tokio::test]
18913 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18914 crate::run::pin_test_home();
18915 for phase in [
18916 crate::run::MergeRepairPhase::Fixing,
18917 crate::run::MergeRepairPhase::Requested,
18918 ] {
18919 let mut runner = runner_at(RunStatus::Landing);
18920 runner.state.config.graph.land_rounds = 0;
18921 runner.state.merge_repairs.push(merge_repair_request(phase));
18922 runner.state.save().unwrap();
18923 assert!(!runner.repair_merge_review().await.unwrap());
18924 assert_eq!(runner.state.status, RunStatus::Blocked);
18925 assert_eq!(
18926 runner.state.merge_repairs[0].phase,
18927 crate::run::MergeRepairPhase::Failed
18928 );
18929 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18930 assert!(
18931 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18932 "interrupted"
18933 } else {
18934 "budget exhausted"
18935 })
18936 );
18937 assert!(runner.state.merge.is_none());
18938 }
18939 }
18940
18941 #[test]
18945 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18946 crate::run::pin_test_home();
18947 let mut runner = runner_at(RunStatus::Implementing);
18948 let interrupt = Pause::new();
18949 runner.watch_interrupt(interrupt.clone());
18950
18951 interrupt.park_because("task a1b2 asked to run first");
18952
18953 assert!(runner.park_here().expect("park_here"));
18954 assert!(runner.state.parked);
18955 let last = runner.state.events.last().expect("a park event");
18956 assert_eq!(last.node, "park");
18957 assert!(
18958 last.message.contains("task a1b2 asked to run first"),
18959 "expected the interrupt reason in {:?}",
18960 last.message
18961 );
18962 }
18963
18964 #[test]
18967 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18968 crate::run::pin_test_home();
18969 let mut runner = runner_at(RunStatus::Implementing);
18970 runner.state.begin_pass();
18971 let shutdown = Pause::new();
18972 runner.on_pause(shutdown.clone());
18973 let upgrade = ParkCause::Upgrade {
18974 from: "0.121.0".to_owned(),
18975 to: Some("0.121.1".to_owned()),
18976 };
18977 shutdown.park_for(upgrade.clone());
18978 shutdown.park_for(ParkCause::Operator);
18979
18980 assert!(runner.park_here().expect("park_here"));
18981 assert_eq!(
18982 runner.state.passes[0].end,
18983 Some(crate::run::PassEnd::Parked { cause: upgrade })
18984 );
18985 let last = runner.state.events.last().expect("a park event");
18986 assert!(last.message.contains("0.121.0"), "{}", last.message);
18987 }
18988
18989 #[tokio::test]
18990 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18991 crate::run::pin_test_home();
18992 let mut runner = runner_at(RunStatus::Reviewing);
18993 runner.state.save().unwrap();
18994 let mut execution = runner.claim_driver().await.expect("first execution");
18995 assert_eq!(execution.runner.state.passes.len(), 1);
18996 execution
18997 .driver
18998 .finish(&mut execution.runner.state)
18999 .await
19000 .unwrap();
19001 }
19002
19003 #[tokio::test]
19004 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19005 crate::run::pin_test_home();
19006 let mut runner = runner_at(RunStatus::Reviewing);
19007 runner.state.config.agents = vec![conductor()];
19008 runner.state.config.graph.timeout_resource_wait = 0;
19009 runner.state.save().unwrap();
19010 let id = runner.state.id.clone();
19011 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19012 assert!(runner.claim_driver().await.is_err());
19013 drop(owner);
19014 drop(runner);
19015 let mut resumed = Runner::resume(&id).unwrap();
19016 assert_eq!(resumed.state.driver_claimed, Some(false));
19017 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19018 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19019 let mut execution = resumed
19020 .claim_driver()
19021 .await
19022 .expect("retry initial claim after reload");
19023 assert_eq!(execution.runner.state.driver_claimed, Some(true));
19024 assert_eq!(execution.runner.state.passes.len(), 1);
19025 execution
19026 .driver
19027 .finish(&mut execution.runner.state)
19028 .await
19029 .unwrap();
19030 }
19031
19032 #[tokio::test]
19033 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19034 crate::run::pin_test_home();
19035 let mut runner = runner_at(RunStatus::Reviewing);
19036 runner.state.config.agents = vec![conductor()];
19037 runner.state.save().unwrap();
19038 let id = runner.state.id.clone();
19039 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19040 let mut resumed = Runner::resume(&id).unwrap();
19041 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19042 assert!(resumed.claim_driver().await.is_err());
19043 assert!(RunState::load(&id).unwrap().passes.is_empty());
19044 drop(lease);
19045 let mut execution = resumed.claim_driver().await.unwrap();
19046 execution
19047 .driver
19048 .finish(&mut execution.runner.state)
19049 .await
19050 .unwrap();
19051 }
19052
19053 #[tokio::test]
19054 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19055 crate::run::pin_test_home();
19056 let mut runner = runner_at(RunStatus::Reviewing);
19057 let state = &mut runner.state;
19058 state.config.agents = vec![conductor()];
19059 state.driver_claimed = None;
19060 state.driver_pid = Some(std::process::id());
19061 state.driver_started_at = None;
19062 state.begin_pass();
19063 state.save().unwrap();
19064 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19065 std::fs::write(
19066 state.dir().join("driver.json"),
19067 serde_json::to_vec(&serde_json::json!({
19068 "pid": state.driver_pid, "started_at": null,
19069 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19070 }))
19071 .unwrap(),
19072 )
19073 .unwrap();
19074 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19075 drop(lease);
19076 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19077 let mut resumed = Runner::resume(&state.id).unwrap();
19078 assert!(!resumed.state.driver_exited);
19079 let mut execution = resumed
19080 .claim_driver()
19081 .await
19082 .expect("lease proves previous execution ended");
19083 assert_eq!(execution.runner.state.passes.len(), 2);
19084 execution
19085 .driver
19086 .finish(&mut execution.runner.state)
19087 .await
19088 .unwrap();
19089 }
19090
19091 #[tokio::test]
19092 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19093 crate::run::pin_test_home();
19094 let mut runner = runner_at(RunStatus::Reviewing);
19095 let mut json = serde_json::to_value(&runner.state).unwrap();
19096 json.as_object_mut().unwrap().remove("driver_claimed");
19097 runner.state = serde_json::from_value(json).unwrap();
19098 assert_eq!(runner.state.driver_claimed, None);
19099 runner.state.save().unwrap();
19100 let mut execution = runner
19101 .claim_driver()
19102 .await
19103 .expect("the execution lease, not a missing marker, excludes a second driver");
19104 assert_eq!(execution.runner.state.passes.len(), 1);
19105 execution
19106 .driver
19107 .finish(&mut execution.runner.state)
19108 .await
19109 .unwrap();
19110 }
19111
19112 #[tokio::test]
19113 async fn driver_admission_refuses_a_proven_live_driver() {
19114 crate::run::pin_test_home();
19115 let mut runner = runner_at(RunStatus::Reviewing);
19116 let pid = std::process::id();
19117 runner.state.driver_pid = Some(pid);
19118 runner.state.driver_started_at = crate::proc::process_started_at(pid);
19119 assert!(runner.state.driver_started_at.is_some());
19120 runner.state.save().unwrap();
19121 let error = runner
19122 .claim_driver()
19123 .await
19124 .err()
19125 .expect("a live driver must refuse");
19126 assert!(error.to_string().contains("live driver"), "{error:#}");
19127 let saved = RunState::load(&runner.state.id).unwrap();
19128 assert_eq!(saved.driver_pid, Some(pid));
19129 assert_eq!(saved.status, RunStatus::Reviewing);
19130 assert!(saved.passes.is_empty());
19131 }
19132
19133 #[test]
19134 fn parking_resource_halt_persists_the_pass_once() {
19135 crate::run::pin_test_home();
19136 let mut runner = runner_at(RunStatus::Reviewing);
19137 runner.state.begin_pass();
19138 runner.state.resource_block = Some(crate::run::ResourceBlock {
19139 node: "fix".into(),
19140 kind: "shared build cache".into(),
19141 detail: "cancelled".into(),
19142 at: Timestamp::now(),
19143 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19144 });
19145 let cause = ParkCause::Upgrade {
19146 from: "old".into(),
19147 to: Some("new".into()),
19148 };
19149 runner.pause.park_for(cause.clone());
19150 runner.stop_for_resource().unwrap();
19153 assert!(runner.resource_halt().unwrap());
19154 assert!(runner.park_here().unwrap());
19155 let saved = RunState::load(&runner.state.id).unwrap();
19156 assert!(saved.parked);
19157 assert_eq!(saved.status, RunStatus::Reviewing);
19158 assert!(saved.passes[0].ended_at.is_some());
19159 assert_eq!(
19160 saved.passes[0].end,
19161 Some(crate::run::PassEnd::Parked {
19162 cause: cause.clone()
19163 })
19164 );
19165 assert_eq!(
19166 saved.resource_block.unwrap().cause,
19167 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19168 );
19169 assert_eq!(
19170 saved
19171 .events
19172 .iter()
19173 .filter(|event| event.node == "park")
19174 .count(),
19175 1
19176 );
19177 }
19178
19179 #[test]
19187 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19188 crate::run::pin_test_home();
19189 let mut runner = runner_at(RunStatus::Implementing);
19190 let shutdown = Pause::new();
19191 runner.on_pause(shutdown.clone());
19192 let interrupt = Pause::new();
19193 runner.watch_interrupt(interrupt.clone());
19194
19195 assert!(!runner.park_here().expect("park_here"));
19197 assert!(!runner.state.parked);
19198
19199 interrupt.park_because("test");
19201 assert!(!shutdown.parked());
19202 assert!(runner.park_here().expect("park_here"));
19203 }
19204
19205 #[tokio::test]
19219 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19220 crate::run::pin_test_home();
19221 let mut runner = runner_at(RunStatus::Implementing);
19222 let interrupt = Pause::new();
19223 runner.watch_interrupt(interrupt.clone());
19224
19225 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19226 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19227
19228 let node = async move {
19232 started_tx.send(()).expect("send started");
19233 finish_rx.await.expect("recv finish");
19234 "node finished"
19235 };
19236
19237 let interrupter = async move {
19238 started_rx.await.expect("recv started");
19239 interrupt.park_because("higher-priority task waiting");
19241 tokio::task::yield_now().await;
19245 finish_tx.send(()).expect("send finish");
19246 };
19247
19248 let (node_result, ()) = tokio::join!(node, interrupter);
19249 assert_eq!(
19250 node_result, "node finished",
19251 "the in-flight call ran to completion"
19252 );
19253
19254 assert!(runner.park_here().expect("park_here"));
19257 assert!(runner.state.parked);
19258 }
19259
19260 #[test]
19266 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19267 crate::run::pin_test_home();
19268 let mut runner = runner_at(RunStatus::Judging);
19269 runner.state.config.agents = vec![conductor()];
19273 runner.state.candidates = vec![Candidate {
19274 index: 0,
19275 label: 'A',
19276 agent: "alpha".to_owned(),
19277 branch: "magi/x/A".to_owned(),
19278 worktree: PathBuf::from("/nonexistent/worktree"),
19279 summary: "did the thing".to_owned(),
19280 stat: "1 file changed".to_owned(),
19281 files: 1,
19282 commits: 1,
19283 empty: false,
19284 failed: None,
19285 verified_noop: None,
19286 duration_ms: 1234,
19287 folded: false,
19288 }];
19289 let run_id = runner.state.id.clone();
19290
19291 let interrupt = Pause::new();
19292 runner.watch_interrupt(interrupt.clone());
19293 interrupt.park_because("task c3d4 asked to run first");
19294 assert!(runner.park_here().expect("park_here"));
19295
19296 let resumed = Runner::resume(&run_id).expect("resume");
19297 assert_eq!(resumed.state.candidates.len(), 1);
19298 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19299 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19300 assert_eq!(resumed.state.status, runner.state.status);
19301 assert!(
19302 resumed.state.parked,
19303 "still parked until `execute` actually walks the graph again"
19304 );
19305 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19306 }
19307
19308 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19310 let mut q = ask::Question::new(
19311 run.to_owned(),
19312 "implement".to_owned(),
19313 "impl-A".to_owned(),
19314 "Which storage backend should the cache use?".to_owned(),
19315 String::new(),
19316 vec!["SQLite".to_owned(), "Redis".to_owned()],
19317 );
19318 store.put(&mut q).unwrap();
19319 q
19320 }
19321
19322 #[test]
19323 fn a_failed_runs_open_question_is_abandoned() {
19324 ask_test_home();
19325 let store = ask::Questions::open();
19326 let mut runner = runner_at(RunStatus::Failed);
19327 let run = runner.state.id.clone();
19328 let q = ask_open_question(&store, &run);
19329
19330 runner.settle_questions();
19331
19332 let back = store.get(&q.id).unwrap();
19333 assert!(
19334 !back.status.open(),
19335 "the seat that asked died with the run; nobody is left to read an answer"
19336 );
19337 assert!(
19338 back.detail.contains(&run) && back.detail.contains("failed"),
19339 "the reason names what the run became, not just that it is gone: {}",
19340 back.detail
19341 );
19342 }
19343
19344 #[test]
19345 fn a_merged_runs_open_question_is_abandoned_too() {
19346 ask_test_home();
19347 let store = ask::Questions::open();
19348 for status in [RunStatus::Merged, RunStatus::Ready] {
19351 let mut runner = runner_at(status);
19352 let run = runner.state.id.clone();
19353 let q = ask_open_question(&store, &run);
19354
19355 runner.settle_questions();
19356
19357 let back = store.get(&q.id).unwrap();
19358 assert!(
19359 !back.status.open(),
19360 "{status:?} run's question must not outlive the run"
19361 );
19362 }
19363 }
19364
19365 #[test]
19366 fn a_still_resumable_runs_open_question_is_left_alone() {
19367 ask_test_home();
19368 let store = ask::Questions::open();
19369 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19375 let mut runner = runner_at(status);
19376 let run = runner.state.id.clone();
19377 let q = ask_open_question(&store, &run);
19378
19379 runner.settle_questions();
19380
19381 let back = store.get(&q.id).unwrap();
19382 assert!(
19383 back.status.open(),
19384 "{status:?} is still alive; the question must still be waiting"
19385 );
19386 }
19387 }
19388
19389 #[test]
19390 fn settle_questions_never_touches_an_already_answered_question() {
19391 ask_test_home();
19392 let store = ask::Questions::open();
19393 let mut runner = runner_at(RunStatus::Failed);
19394 let run = runner.state.id.clone();
19395 let mut q = ask_open_question(&store, &run);
19396 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19397 .unwrap();
19398 store.put(&mut q).unwrap();
19399
19400 runner.settle_questions();
19405 runner.settle_questions();
19406
19407 let back = store.get(&q.id).unwrap();
19408 assert_eq!(
19409 back.status,
19410 ask::QuestionStatus::Answered,
19411 "a real answer is a decision on record, never overwritten by a sweep"
19412 );
19413 }
19414
19415 #[tokio::test]
19426 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19427 crate::run::pin_test_home();
19428 let tmp = tempfile::tempdir().expect("tempdir");
19429 let repo = tmp.path().join("repo");
19430 std::fs::create_dir_all(&repo).unwrap();
19431 init_repo(&repo);
19432
19433 let mut config = Config::default();
19434 config.graph.worktree_root = Some(tmp.path().join("wt"));
19435
19436 let mut state = RunState::new(
19437 repo.clone(),
19438 "main".to_owned(),
19439 "deadbeef".to_owned(),
19440 "task".to_owned(),
19441 config,
19442 );
19443 let root = state.worktree_root();
19444 let wt_a = root.join("cand-A");
19445 let wt_b = root.join("cand-B");
19446 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19447 .await
19448 .expect("worktree A");
19449 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19450 .await
19451 .expect("worktree B");
19452
19453 state.candidates = vec![
19454 Candidate {
19455 index: 0,
19456 label: 'A',
19457 agent: "alpha".to_owned(),
19458 branch: "magi/x/A".to_owned(),
19459 worktree: wt_a.clone(),
19460 summary: String::new(),
19461 stat: String::new(),
19462 files: 0,
19463 commits: 0,
19464 empty: false,
19465 failed: None,
19466 verified_noop: None,
19467 duration_ms: 0,
19468 folded: false,
19469 },
19470 Candidate {
19471 index: 1,
19472 label: 'B',
19473 agent: "beta".to_owned(),
19474 branch: "magi/x/B".to_owned(),
19475 worktree: wt_b.clone(),
19476 summary: String::new(),
19477 stat: String::new(),
19478 files: 0,
19479 commits: 0,
19480 empty: false,
19481 failed: None,
19482 verified_noop: None,
19483 duration_ms: 0,
19484 folded: false,
19485 },
19486 ];
19487 state.tally = Some(Tally {
19488 first_choice: BTreeMap::from([('A', 1)]),
19489 borda: BTreeMap::new(),
19490 winner: 'A',
19491 rankings: 1,
19492 unanimous_initial: true,
19493 deliberated: false,
19494 changed_votes: 0,
19495 unanimous_final: true,
19496 tie_break: None,
19497 judges: 1,
19498 present: 1,
19499 quorum: 1,
19500 met_quorum: true,
19501 uncontested: None,
19502 });
19503 state.status = RunStatus::Ready;
19504
19505 fold_run(&mut state, false, &crate::run::home())
19506 .await
19507 .expect("fold_run");
19508
19509 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19510 assert!(
19511 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19512 "the unmerged winner's branch survives"
19513 );
19514 assert!(
19515 !state.candidates[0].folded,
19516 "the winner is not marked folded"
19517 );
19518
19519 assert!(!wt_b.exists(), "the loser's worktree is removed");
19520 assert!(
19521 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19522 "the loser's branch is removed"
19523 );
19524 assert!(state.candidates[1].folded, "the loser is marked folded");
19525 }
19526
19527 #[tokio::test]
19528 #[cfg(unix)]
19529 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19530 use std::os::fd::AsRawFd as _;
19531 use std::time::Duration;
19532
19533 let tmp = tempfile::tempdir().unwrap();
19534 let repo = tmp.path().join("repo");
19535 std::fs::create_dir_all(&repo).unwrap();
19536 init_repo(&repo);
19537 let common = repo.join(".git");
19538 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19539 let home = tmp.path().join("home");
19540 let mut config = Config::default();
19541 config.coordination = crate::config::test_coordination();
19542 config.graph.worktree_root = Some(tmp.path().join("wt"));
19543 config.graph.timeout_resource_wait = 1;
19544 let mut state = RunState::new(
19545 repo.clone(),
19546 "main".into(),
19547 "deadbeef".into(),
19548 "task".into(),
19549 config,
19550 );
19551 let root = state.worktree_root();
19552 let candidate = root.join("cand-A");
19553 let reviewer = root.join("review-1");
19554 let branch = state.branch_for('A');
19555 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19556 .await
19557 .unwrap();
19558 git::worktree_add_detached(&repo, &reviewer, "main")
19559 .await
19560 .unwrap();
19561 git::acquire_worktree_config(&repo).await.unwrap();
19562 state.enabled_worktree_config = true;
19563 state.status = RunStatus::Ready;
19564 state.candidates.push(Candidate {
19565 index: 0,
19566 label: 'A',
19567 agent: "implement".into(),
19568 branch: branch.clone(),
19569 worktree: candidate.clone(),
19570 summary: String::new(),
19571 stat: String::new(),
19572 files: 0,
19573 commits: 0,
19574 empty: false,
19575 failed: None,
19576 verified_noop: None,
19577 duration_ms: 0,
19578 folded: false,
19579 });
19580 state.save_under(&home).unwrap();
19581 let config_before = std::fs::read(common.join("config")).unwrap();
19582 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19583 .await
19584 .unwrap();
19585 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19588 assert_eq!(
19589 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19590 0
19591 );
19592 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19593 assert!(
19594 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19595 .await
19596 .is_err()
19597 );
19598 assert!(candidate.exists());
19599 assert!(reviewer.exists());
19600 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19601 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19602 assert_eq!(
19603 git::git(&repo, &["worktree", "list", "--porcelain"])
19604 .await
19605 .unwrap(),
19606 worktrees_before
19607 );
19608 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19609 .await
19610 .unwrap()
19611 .unwrap_err();
19612 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19613 drop(fold);
19614 assert!(candidate.exists() && reviewer.exists());
19615 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19616 assert_eq!(
19617 state.resource_block.as_ref().unwrap().kind,
19618 "repository coordination"
19619 );
19620 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19621 drop(external);
19622 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19623 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19624 .await
19625 .unwrap()
19626 .unwrap();
19627 drop(fold);
19628 assert!(removed.contains(&branch));
19629 assert!(!root.exists());
19630 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19631 assert!(state.candidates[0].folded);
19632 assert!(!state.enabled_worktree_config);
19633 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19634 let enabled = git::git_raw(
19635 &repo,
19636 &["config", "--local", "--get", "extensions.worktreeConfig"],
19637 )
19638 .await
19639 .unwrap();
19640 assert!(!enabled.ok(), "the last config reference was released");
19641 }
19642
19643 #[tokio::test]
19646 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19647 let tmp = tempfile::tempdir().expect("tempdir");
19648 let repo = tmp.path().join("repo");
19649 std::fs::create_dir_all(&repo).unwrap();
19650 init_repo(&repo);
19651 let home = tmp.path().join("home");
19652
19653 let mut config = Config::default();
19654 config.graph.worktree_root = Some(tmp.path().join("wt"));
19655 let mut state = RunState::new(
19656 repo.clone(),
19657 "main".to_owned(),
19658 "deadbeef".to_owned(),
19659 "task".to_owned(),
19660 config,
19661 );
19662 git::git(&repo, &["branch", "magi/x/A", "main"])
19664 .await
19665 .expect("branch");
19666 state.candidates = vec![Candidate {
19667 index: 0,
19668 label: 'A',
19669 agent: "alpha".to_owned(),
19670 branch: "magi/x/A".to_owned(),
19671 worktree: state.worktree_root().join("cand-A"),
19672 summary: String::new(),
19673 stat: String::new(),
19674 files: 0,
19675 commits: 0,
19676 empty: false,
19677 failed: None,
19678 verified_noop: None,
19679 duration_ms: 0,
19680 folded: true,
19681 }];
19682 state.released_to = Some("20260901-000000-new1".to_owned());
19683 state.released_branches = vec!["magi/x/A".to_owned()];
19684
19685 fold_run(&mut state, true, &home).await.expect("fold_run");
19686 state.status = RunStatus::Merged;
19687 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19688 state.merge = Some(crate::run::MergeOutcome {
19689 mode: MergeMode::Pr,
19690 ok: true,
19691 detail: "confirmed merged".into(),
19692 empty: false,
19693 });
19694 fold_run(&mut state, true, &home)
19695 .await
19696 .expect("merged fold");
19697
19698 assert!(
19699 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19700 "the handed-over branch survives a fold"
19701 );
19702 }
19703
19704 #[tokio::test]
19708 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19709 let tmp = tempfile::tempdir().expect("tempdir");
19710 let repo = tmp.path().join("repo");
19711 std::fs::create_dir_all(&repo).unwrap();
19712 init_repo(&repo);
19713 let run = |args: &[&str]| {
19714 let out = std::process::Command::new("git")
19715 .quiet()
19716 .args(args)
19717 .current_dir(&repo)
19718 .output()
19719 .expect("spawn git");
19720 assert!(out.status.success(), "git {args:?}");
19721 };
19722 run(&["branch", "magi/x/A"]);
19723 run(&["checkout", "-q", "-b", "magi/x/B"]);
19724 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19725 run(&["add", "-A"]);
19726 run(&["commit", "-q", "-m", "work"]);
19727 run(&["checkout", "-q", "main"]);
19728
19729 let probe = RunState::new(
19733 repo.clone(),
19734 "main".to_owned(),
19735 "deadbeef".to_owned(),
19736 "task".to_owned(),
19737 Config::default(),
19738 );
19739 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19740 let bare = tmp.path().join("origin.git");
19741 let out = std::process::Command::new("git")
19742 .quiet()
19743 .args(["init", "-q", "--bare"])
19744 .arg(&bare)
19745 .output()
19746 .expect("spawn git");
19747 assert!(out.status.success(), "git init --bare");
19748 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19749 run(&["push", "-q", "origin", "main"]);
19750
19751 let mut state = RunState::new(
19752 repo.clone(),
19753 "main".to_owned(),
19754 "deadbeef".to_owned(),
19755 "task".to_owned(),
19756 Config::default(),
19757 );
19758 state.seeds = vec![refs::Seed {
19759 token: "magi/27b2/A".to_owned(),
19760 kind: refs::SeedKind::Unresolved,
19761 sha: String::new(),
19762 branch: true,
19763 detail: "no branch or commit named magi/27b2/A".to_owned(),
19764 }];
19765
19766 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19767 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19768 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19769 let detail = empty_candidate_detail(&state, "main");
19770 assert!(detail.starts_with("empty candidate"), "{detail}");
19771 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19772 assert!(detail.contains("--start-from"), "{detail}");
19773 }
19774
19775 #[tokio::test]
19784 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19785 let tmp = tempfile::tempdir().expect("tempdir");
19786 let repo = tmp.path().join("repo");
19787 std::fs::create_dir_all(&repo).unwrap();
19788 init_repo(&repo);
19789
19790 let mut config = Config::default();
19791 config.merge.mode = MergeMode::Local;
19792
19793 let mut state = RunState::new(
19794 repo.clone(),
19795 "main".to_owned(),
19796 "deadbeef".to_owned(),
19797 "task".to_owned(),
19798 config,
19799 );
19800 state.candidates = vec![Candidate {
19801 index: 0,
19802 label: 'A',
19803 agent: "alpha".to_owned(),
19804 branch: "does-not-exist".to_owned(),
19805 worktree: repo.clone(),
19806 summary: String::new(),
19807 stat: String::new(),
19808 files: 0,
19809 commits: 0,
19810 empty: false,
19811 failed: None,
19812 verified_noop: None,
19813 duration_ms: 0,
19814 folded: false,
19815 }];
19816 state.tally = Some(Tally {
19817 first_choice: BTreeMap::from([('A', 1)]),
19818 borda: BTreeMap::new(),
19819 winner: 'A',
19820 rankings: 1,
19821 unanimous_initial: true,
19822 deliberated: false,
19823 changed_votes: 0,
19824 unanimous_final: true,
19825 tie_break: None,
19826 judges: 0,
19827 present: 0,
19828 quorum: 0,
19829 met_quorum: true,
19830 uncontested: Some("only candidate A produced a change".to_owned()),
19831 });
19832 state.reviews = vec![ReviewRound {
19833 round: 1,
19834 head: "deadbeef".to_owned(),
19835 verified_head: None,
19836 verified_at: None,
19837 reviews: Vec::new(),
19838 e2e: Vec::new(),
19839 fix: None,
19840 blocking: 0,
19841 answered: 0,
19842 expected: 0,
19843 clean: true,
19844 verify_retried: false,
19845 e2e_deferred: false,
19846 e2e_defer_reason: None,
19847 progressed: false,
19848 vote_split: false,
19849 reconsideration: Vec::new(),
19850 verdict: None,
19851 }];
19852 state.gate = vec![CommandOutcome {
19853 command: "test".to_owned(),
19854 code: Some(0),
19855 output_tail: String::new(),
19856 duration_ms: 0,
19857 resource_blocked: false,
19858 }];
19859 state.gate_ran = true;
19860 state.status = RunStatus::Ready;
19865 state.merge = Some(MergeOutcome {
19866 mode: MergeMode::Local,
19867 ok: false,
19868 detail: "already concluded".to_owned(),
19869 empty: false,
19870 });
19871
19872 let mut runner = Runner {
19873 state,
19874 roles: ResolvedRoles {
19875 implementers: Vec::new(),
19876 judges: Vec::new(),
19877 reviewers: Vec::new(),
19878 fixer: None,
19879 conductor: conductor(),
19880 implementer_roster: Vec::new(),
19881 judge_roster: Vec::new(),
19882 reviewer_roster: Vec::new(),
19883 },
19884 sem: Arc::new(Semaphore::new(1)),
19885 pause: Pause::new(),
19886 interrupt: Pause::new(),
19887 };
19888
19889 runner.merge().await.expect("merge");
19890
19891 assert_eq!(
19892 runner.state.status,
19893 RunStatus::Ready,
19894 "a concluded run's status must not change on reentry"
19895 );
19896 assert_eq!(
19897 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19898 Some("already concluded"),
19899 "merge must not run again once the node already recorded an outcome"
19900 );
19901 }
19902
19903 #[tokio::test]
19912 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19913 let tmp = tempfile::tempdir().expect("tempdir");
19914 let repo = tmp.path().join("repo");
19915 std::fs::create_dir_all(&repo).unwrap();
19916 init_repo(&repo);
19917
19918 let mut config = Config::default();
19919 config.merge.mode = MergeMode::Local;
19920
19921 let mut state = RunState::new(
19922 repo.clone(),
19923 "main".to_owned(),
19924 "deadbeef".to_owned(),
19925 "task".to_owned(),
19926 config,
19927 );
19928 state.candidates = vec![Candidate {
19929 index: 0,
19930 label: 'A',
19931 agent: "alpha".to_owned(),
19932 branch: "does-not-exist".to_owned(),
19933 worktree: repo.clone(),
19934 summary: String::new(),
19935 stat: String::new(),
19936 files: 0,
19937 commits: 0,
19938 empty: false,
19939 failed: None,
19940 verified_noop: None,
19941 duration_ms: 0,
19942 folded: false,
19943 }];
19944 state.tally = Some(Tally {
19945 first_choice: BTreeMap::from([('A', 1)]),
19946 borda: BTreeMap::new(),
19947 winner: 'A',
19948 rankings: 1,
19949 unanimous_initial: true,
19950 deliberated: false,
19951 changed_votes: 0,
19952 unanimous_final: true,
19953 tie_break: None,
19954 judges: 0,
19955 present: 0,
19956 quorum: 0,
19957 met_quorum: true,
19958 uncontested: Some("only candidate A produced a change".to_owned()),
19959 });
19960 state.reviews = vec![ReviewRound {
19961 round: 1,
19962 head: "deadbeef".to_owned(),
19963 verified_head: None,
19964 verified_at: None,
19965 reviews: Vec::new(),
19966 e2e: Vec::new(),
19967 fix: None,
19968 blocking: 0,
19969 answered: 0,
19970 expected: 0,
19971 clean: true,
19972 verify_retried: false,
19973 e2e_deferred: false,
19974 e2e_defer_reason: None,
19975 progressed: false,
19976 vote_split: false,
19977 reconsideration: Vec::new(),
19978 verdict: None,
19979 }];
19980 state.gate = Vec::new();
19982 state.gate_ran = false;
19983 state.status = RunStatus::Gating;
19984
19985 let mut runner = Runner {
19986 state,
19987 roles: ResolvedRoles {
19988 implementers: Vec::new(),
19989 judges: Vec::new(),
19990 reviewers: Vec::new(),
19991 fixer: None,
19992 conductor: conductor(),
19993 implementer_roster: Vec::new(),
19994 judge_roster: Vec::new(),
19995 reviewer_roster: Vec::new(),
19996 },
19997 sem: Arc::new(Semaphore::new(1)),
19998 pause: Pause::new(),
19999 interrupt: Pause::new(),
20000 };
20001
20002 runner.merge().await.expect("merge");
20003
20004 assert!(
20005 runner.state.merge.is_none(),
20006 "an empty gate must never be read as a passing one: {:?}",
20007 runner.state.merge
20008 );
20009 }
20010
20011 #[tokio::test]
20018 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20019 let tmp = tempfile::tempdir().expect("tempdir");
20020 let repo = tmp.path().join("repo");
20021 std::fs::create_dir_all(&repo).unwrap();
20022 init_repo(&repo);
20023
20024 let config = Config::default();
20026
20027 let mut state = RunState::new(
20028 repo.clone(),
20029 "main".to_owned(),
20030 "deadbeef".to_owned(),
20031 "task".to_owned(),
20032 config,
20033 );
20034 state.candidates = vec![Candidate {
20035 index: 0,
20036 label: 'A',
20037 agent: "alpha".to_owned(),
20038 branch: "does-not-exist".to_owned(),
20039 worktree: repo.clone(),
20040 summary: String::new(),
20041 stat: String::new(),
20042 files: 0,
20043 commits: 0,
20044 empty: false,
20045 failed: None,
20046 verified_noop: None,
20047 duration_ms: 0,
20048 folded: false,
20049 }];
20050 state.tally = Some(Tally {
20051 first_choice: BTreeMap::from([('A', 1)]),
20052 borda: BTreeMap::new(),
20053 winner: 'A',
20054 rankings: 1,
20055 unanimous_initial: true,
20056 deliberated: false,
20057 changed_votes: 0,
20058 unanimous_final: true,
20059 tie_break: None,
20060 judges: 0,
20061 present: 0,
20062 quorum: 0,
20063 met_quorum: true,
20064 uncontested: Some("only candidate A produced a change".to_owned()),
20065 });
20066 state.reviews = vec![ReviewRound {
20067 round: 1,
20068 head: "deadbeef".to_owned(),
20069 verified_head: None,
20070 verified_at: None,
20071 reviews: Vec::new(),
20072 e2e: Vec::new(),
20073 fix: None,
20074 blocking: 0,
20075 answered: 0,
20076 expected: 0,
20077 clean: true,
20078 verify_retried: false,
20079 e2e_deferred: false,
20080 e2e_defer_reason: None,
20081 progressed: false,
20082 vote_split: false,
20083 reconsideration: Vec::new(),
20084 verdict: None,
20085 }];
20086
20087 let mut runner = Runner {
20088 state,
20089 roles: ResolvedRoles {
20090 implementers: Vec::new(),
20091 judges: Vec::new(),
20092 reviewers: Vec::new(),
20093 fixer: None,
20094 conductor: conductor(),
20095 implementer_roster: Vec::new(),
20096 judge_roster: Vec::new(),
20097 reviewer_roster: Vec::new(),
20098 },
20099 sem: Arc::new(Semaphore::new(1)),
20100 pause: Pause::new(),
20101 interrupt: Pause::new(),
20102 };
20103
20104 runner.gate().await.expect("gate");
20105 assert!(
20106 runner.state.gate_ran,
20107 "zero configured commands is still a real attempt, not an unrun gate"
20108 );
20109 assert!(runner.state.gate.is_empty());
20110 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20111 assert_ne!(
20112 runner.state.status,
20113 RunStatus::Blocked,
20114 "a gate with nothing to check must not read as failed"
20115 );
20116
20117 runner.merge().await.expect("merge");
20118 assert_eq!(
20119 runner.state.status,
20120 RunStatus::Ready,
20121 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20122 );
20123 }
20124
20125 #[tokio::test]
20136 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20137 crate::run::pin_test_home();
20138 let home = crate::run::home();
20139
20140 let tmp = tempfile::tempdir().expect("tempdir");
20141 let repo = tmp.path().join("repo");
20142 std::fs::create_dir_all(&repo).unwrap();
20143 init_repo(&repo);
20144 let cache_dir = tmp.path().join("target");
20147
20148 let mut config = Config::default();
20149 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20150 config.graph.timeout_verify = Some(2);
20154 config.graph.timeout_resource_wait = 0;
20155
20156 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20157 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20158 .expect("no io error acquiring directly")
20159 {
20160 crate::cache::AcquireOutcome::Acquired(g) => g,
20161 crate::cache::AcquireOutcome::Busy(b) => {
20162 panic!("expected the direct acquire to win the lease first: {b:?}")
20163 }
20164 };
20165
20166 let mut state = RunState::new(
20167 repo.clone(),
20168 "main".to_owned(),
20169 "deadbeef".to_owned(),
20170 "task".to_owned(),
20171 config,
20172 );
20173 state.candidates = vec![Candidate {
20174 index: 0,
20175 label: 'A',
20176 agent: "alpha".to_owned(),
20177 branch: "does-not-exist".to_owned(),
20178 worktree: repo.clone(),
20179 summary: String::new(),
20180 stat: String::new(),
20181 files: 0,
20182 commits: 0,
20183 empty: false,
20184 failed: None,
20185 verified_noop: None,
20186 duration_ms: 0,
20187 folded: false,
20188 }];
20189 state.tally = Some(Tally {
20190 first_choice: BTreeMap::from([('A', 1)]),
20191 borda: BTreeMap::new(),
20192 winner: 'A',
20193 rankings: 1,
20194 unanimous_initial: true,
20195 deliberated: false,
20196 changed_votes: 0,
20197 unanimous_final: true,
20198 tie_break: None,
20199 judges: 0,
20200 present: 0,
20201 quorum: 0,
20202 met_quorum: true,
20203 uncontested: Some("only candidate A produced a change".to_owned()),
20204 });
20205 state.reviews = vec![ReviewRound {
20206 round: 1,
20207 head: "deadbeef".to_owned(),
20208 verified_head: None,
20209 verified_at: None,
20210 reviews: Vec::new(),
20211 e2e: Vec::new(),
20212 fix: None,
20213 blocking: 0,
20214 answered: 0,
20215 expected: 0,
20216 clean: true,
20217 verify_retried: false,
20218 e2e_deferred: false,
20219 e2e_defer_reason: None,
20220 progressed: false,
20221 vote_split: false,
20222 reconsideration: Vec::new(),
20223 verdict: None,
20224 }];
20225
20226 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20227 state.reviews[0].e2e = vec![CommandOutcome {
20228 command: state.config.verify.e2e[0].clone(),
20229 code: Some(0),
20230 output_tail: String::new(),
20231 duration_ms: 0,
20232 resource_blocked: false,
20233 }];
20234 let mut runner = Runner {
20235 state,
20236 roles: ResolvedRoles {
20237 implementers: Vec::new(),
20238 judges: Vec::new(),
20239 reviewers: Vec::new(),
20240 fixer: None,
20241 conductor: conductor(),
20242 implementer_roster: Vec::new(),
20243 judge_roster: Vec::new(),
20244 reviewer_roster: Vec::new(),
20245 },
20246 sem: Arc::new(Semaphore::new(1)),
20247 pause: Pause::new(),
20248 interrupt: Pause::new(),
20249 };
20250
20251 runner.gate().await.expect("gate");
20252 assert!(runner.state.resource_block.is_none());
20253 assert!(
20254 !runner
20255 .state
20256 .events
20257 .iter()
20258 .any(|event| { event.message.contains("waiting for the shared build cache") }),
20259 "a gate with nothing to run must never request the held cache lease"
20260 );
20261 assert!(
20262 runner.state.gate_ran,
20263 "zero commands is still a real, immediate attempt"
20264 );
20265 assert!(runner.state.gate.is_empty());
20266 assert_ne!(
20267 runner.state.status,
20268 RunStatus::Blocked,
20269 "must not read as resource-blocked on a lease it never asked for"
20270 );
20271 }
20272
20273 #[cfg(unix)]
20276 #[tokio::test]
20277 async fn verification_is_standalone_inside_a_live_worker() {
20278 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20279 if std::env::var_os(CHILD).is_none() {
20280 let home = tempfile::tempdir().unwrap();
20281 let generation = crate::worker::pin_current(home.path()).unwrap();
20282 let output = std::process::Command::new(std::env::current_exe().unwrap())
20283 .args([
20284 "--exact",
20285 "graph::tests::verification_is_standalone_inside_a_live_worker",
20286 "--nocapture",
20287 ])
20288 .env(CHILD, "1")
20289 .env("MAGI_HOME", home.path())
20290 .env("MAGI_WORKER_HOME", home.path())
20291 .env("MAGI_WORKER_ID", "abcdef")
20292 .env("MAGI_WORKER_GENERATION", &generation.id)
20293 .env("MAGI_WORKER_EXE", &generation.executable)
20294 .env("MAGI_RUN", "verification-parent")
20295 .env("MAGI_NODE", "gate")
20296 .env("MAGI_SEAT", "worker-fixture")
20297 .env("MAGI_PROCESS_OWNER", "fixture-owner")
20298 .env("MAGI_NO_AUTOUPDATE", "0")
20299 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20300 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20301 .env(
20302 "VERIFY_EXPECT_TOOLCHAIN",
20303 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20304 )
20305 .env(
20306 "VERIFY_EXPECT_CACHE",
20307 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20308 )
20309 .output()
20310 .unwrap();
20311 assert!(
20312 output.status.success()
20313 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20314 "{}\n{}",
20315 String::from_utf8_lossy(&output.stdout),
20316 String::from_utf8_lossy(&output.stderr)
20317 );
20318 return;
20319 }
20320 let home = crate::run::home();
20321 let directory = home.join("workers/jobs/abcdef");
20322 std::fs::create_dir_all(&directory).unwrap();
20323 let job = crate::worker::Job {
20324 id: "abcdef".into(),
20325 sequence: 1,
20326 kind: "task".into(),
20327 key: "task:verification".into(),
20328 payload: serde_json::json!({}),
20329 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20330 version: Some(env!("CARGO_PKG_VERSION").into()),
20331 status: crate::worker::Status::Running,
20332 result: None,
20333 pid: Some(std::process::id()),
20334 created_at: 0,
20335 finished_at: None,
20336 };
20337 std::fs::write(
20338 directory.join("job.json"),
20339 serde_json::to_vec(&job).unwrap(),
20340 )
20341 .unwrap();
20342 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20343 let assert_worker = || {
20344 assert_eq!(
20345 crate::worker::current_job(&home).unwrap().unwrap().id,
20346 job.id
20347 );
20348 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20349 assert_eq!(
20350 crate::worker::helper_executable(&home).unwrap(),
20351 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20352 );
20353 assert!(
20354 crate::updater::Checker::new(&crate::config::Update {
20355 mode: crate::config::UpdateMode::Notify,
20356 interval: None,
20357 auto_restart: false,
20358 })
20359 .is_none(),
20360 "the driver remains pinned, even while tests are standalone"
20361 );
20362 };
20363 assert_worker();
20364 let repo = home.join("repo");
20365 std::fs::create_dir_all(&repo).unwrap();
20366 init_repo(&repo);
20367 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20370 test -z "${MAGI_WORKER_GENERATION+x}" &&
20371 test -z "${MAGI_WORKER_HOME+x}" &&
20372 test -z "${MAGI_WORKER_EXE+x}" &&
20373 test "$MAGI_HOME/repo" -ef . &&
20374 test "$MAGI_RUN" = verification-parent &&
20375 test "$MAGI_NODE" = gate &&
20376 test "$MAGI_SEAT" = worker-fixture &&
20377 test -n "$MAGI_PROCESS_OWNER" &&
20378 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20379 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20380 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20381 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20382 assert!(
20384 !std::process::Command::new("sh")
20385 .args(["-c", probe])
20386 .current_dir(&repo)
20387 .status()
20388 .unwrap()
20389 .success()
20390 );
20391 let unrelated_home = tempfile::tempdir().unwrap();
20392 assert!(
20393 crate::worker::current_job(unrelated_home.path()).is_err(),
20394 "an inherited worker id cannot be resolved in a unit fixture's home"
20395 );
20396 let mut runner = runner_at(RunStatus::Gating);
20397 runner.state.repo = repo.clone();
20398 runner.state.config.disk.min_free_bytes = 0;
20399 runner.state.config.coordination = crate::config::test_coordination();
20400 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20401 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20402 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20403 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20404 let winner = Candidate {
20405 index: 0,
20406 label: 'A',
20407 agent: "fixture".into(),
20408 branch: "main".into(),
20409 worktree: repo.clone(),
20410 summary: String::new(),
20411 stat: String::new(),
20412 files: 0,
20413 commits: 0,
20414 empty: false,
20415 failed: None,
20416 verified_noop: None,
20417 duration_ms: 0,
20418 folded: false,
20419 };
20420 let (e2e, retried, pids) = run_e2e_with_retry(
20421 &mut runner.state,
20422 &["sh".into(), "-c".into()],
20423 &[format!("{probe} && printf initial")],
20424 &repo,
20425 Duration::from_secs(10),
20426 "initial",
20427 )
20428 .await;
20429 assert!(!retried && pids.is_empty());
20430 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20431 assert_eq!(e2e[0].output_tail, "initial");
20432 let gate = runner.run_gate(&winner).await.unwrap();
20433 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20434 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20437 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20438 passed: false,
20439 outcomes: Vec::new(),
20440 });
20441 runner.state.save().unwrap();
20442 let mut resumed = RunState::load(&runner.state.id).unwrap();
20443 let wait = crate::repo_lock::WaitPolicy::default();
20444 assert!(matches!(
20445 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20446 .await
20447 .unwrap(),
20448 GateFix::Retry
20449 ));
20450 let saved = RunState::load(&resumed.id).unwrap();
20451 let repair = saved.gate_repair_e2e.unwrap();
20452 assert!(repair.passed);
20453 assert_eq!(repair.outcomes.len(), 1);
20454 assert!(repair.outcomes[0].ok());
20455 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20456 .await
20457 .unwrap();
20458 assert_eq!(landing.len(), 1);
20459 assert!(landing[0].ok(), "{landing:?}");
20460 assert_eq!(
20461 std::fs::read_to_string(repo.join("phases")).unwrap(),
20462 "pre\ngate\ne2e\npre\ngate\n"
20463 );
20464 assert_worker();
20465 }
20466
20467 #[cfg(unix)]
20468 #[tokio::test]
20469 async fn verification_keeps_the_panic_and_the_full_output() {
20470 crate::run::pin_test_home();
20471 let tmp = tempfile::tempdir().unwrap();
20472 let mut config = Config::default();
20473 config.disk.min_free_bytes = 0;
20474 let mut state = RunState::new(
20475 tmp.path().to_owned(),
20476 "main".into(),
20477 "base".into(),
20478 "verification evidence".into(),
20479 config,
20480 );
20481 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20482 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20483 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20484 let mut names = Vec::new();
20485 for _ in 0..2 {
20486 let (outcomes, _) = run_commands(
20487 &mut state,
20488 "gate",
20489 "gate",
20490 0,
20491 &["sh".into(), "-c".into()],
20492 &[script.into()],
20493 tmp.path(),
20494 Duration::from_secs(60),
20495 )
20496 .await;
20497 let tail = &outcomes[0].output_tail;
20498 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20499 assert!(tail.contains("boom assertion"));
20500 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20501 let line = tail.lines().next().unwrap();
20502 names.push(line.to_owned());
20503 }
20504 assert_ne!(names[0], names[1], "the second run must not overwrite");
20505 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20506 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20507 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20508
20509 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20511 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20512 let (outcomes, _) = run_commands(
20513 &mut state,
20514 "gate",
20515 "gate",
20516 0,
20517 &["sh".into(), "-c".into()],
20518 &["echo failure-output; exit 7".into()],
20519 tmp.path(),
20520 Duration::from_secs(10),
20521 )
20522 .await;
20523 assert_eq!(outcomes[0].code, Some(7));
20524 assert!(state.events.iter().any(|e| {
20525 e.node == "gate" && e.message.contains("could not save the full gate output:")
20526 }));
20527 }
20528
20529 #[cfg(unix)]
20530 #[tokio::test]
20531 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20532 crate::run::pin_test_home();
20533 let tmp = tempfile::tempdir().unwrap();
20534 let mut config = Config::default();
20535 config.disk.min_free_bytes = 0;
20536 let mut state = RunState::new(
20537 tmp.path().to_owned(),
20538 "main".into(),
20539 "base".into(),
20540 "verification evidence".into(),
20541 config,
20542 );
20543 let (outcomes, pids) = run_commands(
20544 &mut state,
20545 "gate",
20546 "gate",
20547 1,
20548 &["sh".into(), "-c".into()],
20549 &[
20550 "printf before-timeout; exec sleep 30".into(),
20551 "kill -TERM $$".into(),
20552 "exit 7".into(),
20553 "printf success".into(),
20554 ],
20555 tmp.path(),
20556 Duration::from_secs(1),
20557 )
20558 .await;
20559 wait_for_timed_out_children_to_die(&pids).await;
20560 assert_eq!(
20561 pids.len(),
20562 1,
20563 "unleased timeout retains its death-check PID"
20564 );
20565 assert_eq!(outcomes.len(), 4);
20566 assert_eq!(outcomes[0].code, None);
20567 assert!(outcomes[0].output_tail.contains("before-timeout"));
20568 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20569 assert_eq!(outcomes[1].code, None);
20570 assert!(outcomes[1].output_tail.contains("command terminated:"));
20571 assert!(!outcomes[1].output_tail.contains("timed out"));
20572 assert_eq!(outcomes[2].code, Some(7));
20573 assert!(!outcomes[2].output_tail.contains("timed out"));
20574 assert_eq!(outcomes[3].code, Some(0));
20575 assert_eq!(outcomes[3].output_tail, "success");
20576 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20577 }
20578
20579 #[tokio::test]
20589 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20590 crate::run::pin_test_home();
20591
20592 let tmp = tempfile::tempdir().expect("tempdir");
20593 let repo = tmp.path().join("repo");
20594 std::fs::create_dir_all(&repo).unwrap();
20595 init_repo(&repo);
20596
20597 let mut config = Config::default();
20598 config.disk.min_free_bytes = 0;
20599 config.verify.gate = vec![
20600 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20601 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20602 .to_owned(),
20603 ];
20604
20605 let mut state = RunState::new(
20606 repo.clone(),
20607 "main".to_owned(),
20608 "deadbeef".to_owned(),
20609 "task".to_owned(),
20610 config,
20611 );
20612 let run_id = state.id.clone();
20613 state.candidates = vec![Candidate {
20614 index: 0,
20615 label: 'A',
20616 agent: "alpha".to_owned(),
20617 branch: "does-not-exist".to_owned(),
20618 worktree: repo.clone(),
20619 summary: String::new(),
20620 stat: String::new(),
20621 files: 0,
20622 commits: 0,
20623 empty: false,
20624 failed: None,
20625 verified_noop: None,
20626 duration_ms: 0,
20627 folded: false,
20628 }];
20629 state.tally = Some(Tally {
20630 first_choice: BTreeMap::from([('A', 1)]),
20631 borda: BTreeMap::new(),
20632 winner: 'A',
20633 rankings: 1,
20634 unanimous_initial: true,
20635 deliberated: false,
20636 changed_votes: 0,
20637 unanimous_final: true,
20638 tie_break: None,
20639 judges: 0,
20640 present: 0,
20641 quorum: 0,
20642 met_quorum: true,
20643 uncontested: Some("only candidate A produced a change".to_owned()),
20644 });
20645 state.reviews = vec![ReviewRound {
20646 round: 1,
20647 head: "deadbeef".to_owned(),
20648 verified_head: None,
20649 verified_at: None,
20650 reviews: Vec::new(),
20651 e2e: Vec::new(),
20652 fix: None,
20653 blocking: 0,
20654 answered: 0,
20655 expected: 0,
20656 clean: true,
20657 verify_retried: false,
20658 e2e_deferred: false,
20659 e2e_defer_reason: None,
20660 progressed: false,
20661 vote_split: false,
20662 reconsideration: Vec::new(),
20663 verdict: None,
20664 }];
20665
20666 let mut runner = Runner {
20667 state,
20668 roles: ResolvedRoles {
20669 implementers: Vec::new(),
20670 judges: Vec::new(),
20671 reviewers: Vec::new(),
20672 fixer: None,
20673 conductor: conductor(),
20674 implementer_roster: Vec::new(),
20675 judge_roster: Vec::new(),
20676 reviewer_roster: Vec::new(),
20677 },
20678 sem: Arc::new(Semaphore::new(1)),
20679 pause: Pause::new(),
20680 interrupt: Pause::new(),
20681 };
20682
20683 let started_marker = repo.join("started.marker");
20684 let release_marker = repo.join("release.marker");
20685 let poller = tokio::spawn(async move {
20686 for _ in 0..100 {
20691 if started_marker.exists()
20692 && let Ok(s) = crate::run::RunState::load(&run_id)
20693 && let Some(a) = s.active.get("gate")
20694 {
20695 std::fs::write(&release_marker, b"go").expect("release marker");
20696 return Some(a.clone());
20697 }
20698 tokio::time::sleep(Duration::from_millis(50)).await;
20699 }
20700 None
20701 });
20702
20703 runner.gate().await.expect("gate");
20704 let captured = poller.await.expect("poller task");
20705 let captured = captured.expect(
20706 "the poller never saw a `gate` task entry in run.json while the command was \
20707 still blocked on its own release marker",
20708 );
20709
20710 assert_eq!(captured.task.as_deref(), Some("gate"));
20711 assert_eq!(captured.node, "gate");
20712 assert_eq!(captured.index, Some(1));
20713 assert_eq!(captured.total, Some(1));
20714 assert!(
20715 captured
20716 .command
20717 .as_deref()
20718 .is_some_and(|c| c.contains("started.marker")),
20719 "{captured:?}"
20720 );
20721
20722 assert!(
20723 runner.state.active.is_empty(),
20724 "the entry must be cleared once the command actually finished: {:?}",
20725 runner.state.active
20726 );
20727 assert!(runner.state.gate_ran);
20728 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20729 }
20730
20731 #[tokio::test]
20734 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20735 use crate::verdict::{Finding, ReviewVote, Severity};
20736 crate::run::pin_test_home();
20737 let tmp = tempfile::tempdir().expect("tempdir");
20738 let repo = tmp.path().join("repo");
20739 std::fs::create_dir_all(&repo).unwrap();
20740 init_repo(&repo);
20741
20742 for (severity, vote, expect) in [
20743 (Severity::Major, ReviewVote::Reject, true),
20744 (Severity::Minor, ReviewVote::Reject, false),
20745 (Severity::Major, ReviewVote::Approve, false),
20746 ] {
20747 let mut round = review_round(false, 1, 1, 1, false, true);
20748 round.reviews = vec![ReviewRecord {
20749 reviewer: 1,
20750 agent: "alpha".to_owned(),
20751 summary: String::new(),
20752 findings: vec![Finding {
20753 id: "R1-1-1".to_owned(),
20754 severity,
20755 file: None,
20756 line: None,
20757 title: "t".to_owned(),
20758 detail: String::new(),
20759 }],
20760 vote: Some(vote),
20761 failed: None,
20762 duration_ms: 0,
20763 attempts: 0,
20764 }];
20765 let mut state = RunState::new(
20766 repo.clone(),
20767 "main".to_owned(),
20768 "deadbeef".to_owned(),
20769 "task".to_owned(),
20770 Config::default(),
20771 );
20772 state.reviews = vec![round];
20773 let mut runner = Runner {
20774 state,
20775 roles: ResolvedRoles {
20776 implementers: Vec::new(),
20777 judges: Vec::new(),
20778 reviewers: Vec::new(),
20779 fixer: None,
20780 conductor: conductor(),
20781 implementer_roster: Vec::new(),
20782 judge_roster: Vec::new(),
20783 reviewer_roster: Vec::new(),
20784 },
20785 sem: Arc::new(Semaphore::new(1)),
20786 pause: Pause::new(),
20787 interrupt: Pause::new(),
20788 };
20789 let shell = runner.state.config.shell();
20790 runner
20791 .stop_reviewing("round budget spent", &shell, &repo)
20792 .await
20793 .expect("stop_reviewing");
20794 assert_eq!(runner.state.status, RunStatus::Gating);
20795 assert_eq!(
20796 runner.state.contested_handoff.is_some(),
20797 expect,
20798 "{severity:?} + {vote:?}"
20799 );
20800 }
20801 }
20802
20803 #[tokio::test]
20816 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20817 crate::run::pin_test_home();
20818 let home = crate::run::home();
20819
20820 let tmp = tempfile::tempdir().expect("tempdir");
20821 let repo = tmp.path().join("repo");
20822 std::fs::create_dir_all(&repo).unwrap();
20823 init_repo(&repo);
20824 let head = crate::git::rev_parse(&repo, "HEAD")
20825 .await
20826 .expect("rev-parse");
20827 let cache_dir = tmp.path().join("target");
20830
20831 let mut config = Config::default();
20832 config.verify.e2e = vec![format!(
20833 "CARGO_TARGET_DIR='{}' test -f README.md",
20834 cache_dir.display()
20835 )];
20836 config.graph.review_rounds = 1;
20837 config.graph.timeout_verify = Some(2);
20840 config.graph.timeout_resource_wait = 2;
20841
20842 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20843 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20844 .expect("no io error acquiring directly")
20845 {
20846 crate::cache::AcquireOutcome::Acquired(g) => g,
20847 crate::cache::AcquireOutcome::Busy(b) => {
20848 panic!("expected the direct acquire to win the lease first: {b:?}")
20849 }
20850 };
20851
20852 let mut state = RunState::new(
20853 repo.clone(),
20854 "main".to_owned(),
20855 head.clone(),
20856 "task".to_owned(),
20857 config,
20858 );
20859 state.candidates = vec![Candidate {
20860 index: 0,
20861 label: 'A',
20862 agent: "alpha".to_owned(),
20863 branch: "does-not-exist".to_owned(),
20864 worktree: repo.clone(),
20865 summary: String::new(),
20866 stat: String::new(),
20867 files: 0,
20868 commits: 0,
20869 empty: false,
20870 failed: None,
20871 verified_noop: None,
20872 duration_ms: 0,
20873 folded: false,
20874 }];
20875 state.tally = Some(Tally {
20876 first_choice: BTreeMap::from([('A', 1)]),
20877 borda: BTreeMap::new(),
20878 winner: 'A',
20879 rankings: 1,
20880 unanimous_initial: true,
20881 deliberated: false,
20882 changed_votes: 0,
20883 unanimous_final: true,
20884 tie_break: None,
20885 judges: 0,
20886 present: 0,
20887 quorum: 0,
20888 met_quorum: true,
20889 uncontested: Some("only candidate A produced a change".to_owned()),
20890 });
20891 state.reviews = vec![ReviewRound {
20895 round: 1,
20896 head: head.clone(),
20897 verified_head: None,
20898 verified_at: None,
20899 reviews: Vec::new(),
20900 e2e: Vec::new(),
20901 fix: None,
20902 blocking: 1,
20903 answered: 1,
20904 expected: 1,
20905 clean: false,
20906 verify_retried: false,
20907 e2e_deferred: true,
20908 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20909 progressed: false,
20910 vote_split: false,
20911 reconsideration: Vec::new(),
20912 verdict: None,
20913 }];
20914
20915 let mut runner = Runner {
20916 state,
20917 roles: ResolvedRoles {
20918 implementers: Vec::new(),
20919 judges: Vec::new(),
20920 reviewers: Vec::new(),
20921 fixer: None,
20922 conductor: conductor(),
20923 implementer_roster: Vec::new(),
20924 judge_roster: Vec::new(),
20925 reviewer_roster: Vec::new(),
20926 },
20927 sem: Arc::new(Semaphore::new(1)),
20928 pause: Pause::new(),
20929 interrupt: Pause::new(),
20930 };
20931
20932 let shell = runner.state.config.shell();
20933 runner
20934 .stop_reviewing("round budget spent", &shell, &repo)
20935 .await
20936 .expect("stop_reviewing");
20937
20938 let last = runner.state.reviews.last().expect("round record");
20939 assert_eq!(
20940 last.e2e_status(),
20941 E2eStatus::ResourceBlocked,
20942 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20943 failed: {last:?}"
20944 );
20945 assert_eq!(
20946 last.verified_head.as_deref(),
20947 Some(head.as_str()),
20948 "which commit this attempt targeted is known even though nothing finished checking \
20949 it"
20950 );
20951 let first_attempt_at = last
20952 .verified_at
20953 .expect("when this attempt ran is known too");
20954 assert_ne!(
20955 runner.state.status,
20956 RunStatus::Blocked,
20957 "contention is evidence about the machine, not the patch — it must not settle the \
20958 run as blocked: {:?}",
20959 runner.state.status
20960 );
20961 assert!(
20962 !runner
20963 .state
20964 .events
20965 .iter()
20966 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20967 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20968 runner.state.events
20969 );
20970
20971 runner
20976 .stop_reviewing("round budget spent", &shell, &repo)
20977 .await
20978 .expect("stop_reviewing retry");
20979 assert_eq!(
20980 runner.state.reviews.len(),
20981 1,
20982 "no new round was started: {:?}",
20983 runner.state.reviews
20984 );
20985 let last = runner.state.reviews.last().expect("round record");
20986 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20987 assert!(
20988 last.verified_at.expect("still known") > first_attempt_at,
20989 "a second reentry must be a fresh attempt, not a stale copy of the first"
20990 );
20991 assert_ne!(runner.state.status, RunStatus::Blocked);
20992
20993 held.release();
20994 }
20995
20996 #[cfg(unix)]
20997 #[tokio::test]
20998 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20999 ask_test_home();
21000 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21001 let tmp = tempfile::tempdir().unwrap();
21002 let repo = tmp.path().join("repo");
21003 std::fs::create_dir(&repo).unwrap();
21004 init_repo(&repo);
21005 let run_git = |args: &[&str]| {
21006 let out = std::process::Command::new("git")
21007 .args(args)
21008 .current_dir(&repo)
21009 .output()
21010 .unwrap();
21011 assert!(
21012 out.status.success(),
21013 "{args:?}: {}",
21014 String::from_utf8_lossy(&out.stderr)
21015 );
21016 };
21017 let origin = tmp.path().join("origin.git");
21018 run_git(&["init", "--bare", origin.to_str().unwrap()]);
21019 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21020 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21021 run_git(&["add", "spec.rs"]);
21022 run_git(&["commit", "-m", "initial spec"]);
21023 run_git(&["push", "origin", "main"]);
21024 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21025 run_git(&["checkout", "-b", "fix/review"]);
21026 std::fs::write(
21027 repo.join("check.rs"),
21028 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21029 )
21030 .unwrap();
21031 run_git(&["add", "check.rs"]);
21032 run_git(&["commit", "-m", "use spec"]);
21033 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21034 run_git(&["checkout", "main"]);
21035 std::fs::write(
21036 repo.join("spec.rs"),
21037 "struct AgentSpec { max_turn_secs: u64 }\n",
21038 )
21039 .unwrap();
21040 run_git(&["add", "spec.rs"]);
21041 run_git(&["commit", "-m", "extend spec"]);
21042 run_git(&["push", "origin", "main"]);
21043 run_git(&["checkout", "fix/review"]);
21044
21045 let mut runner = runner_at(RunStatus::Reviewing);
21046 runner.state.repo = repo.clone();
21047 runner.state.base_commit = base;
21048 runner.state.pr_head = Some("fix/review".into());
21049 runner.state.config.graph.review_rounds = max_rounds;
21050 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21051 runner.state.config.graph.sessions = false;
21052 runner.state.config.disk.min_free_bytes = 0;
21053 runner.state.config.verify.e2e =
21054 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21055 runner.state.candidates = vec![Candidate {
21056 index: 0,
21057 label: 'A',
21058 agent: "fixture".into(),
21059 branch: "fix/review".into(),
21060 worktree: repo.clone(),
21061 summary: String::new(),
21062 stat: String::new(),
21063 files: 1,
21064 commits: 1,
21065 empty: false,
21066 failed: None,
21067 verified_noop: None,
21068 duration_ms: 0,
21069 folded: false,
21070 }];
21071 runner.state.tally = Some(Tally {
21072 first_choice: BTreeMap::new(),
21073 borda: BTreeMap::new(),
21074 winner: 'A',
21075 rankings: 0,
21076 unanimous_initial: true,
21077 deliberated: false,
21078 changed_votes: 0,
21079 unanimous_final: true,
21080 tie_break: None,
21081 judges: 0,
21082 present: 0,
21083 quorum: 0,
21084 met_quorum: true,
21085 uncontested: Some("review-only".into()),
21086 });
21087 let mut last = review_round(false, 0, 1, 1, false, false);
21088 last.head = reviewed.clone();
21089 last.e2e_deferred = true;
21090 last.e2e[0].resource_blocked = true;
21091 last.e2e[0].output_tail = "waiting for the shared build cache".into();
21092 if max_rounds == 1 {
21093 last.e2e.clear();
21094 }
21095 runner.state.reviews = vec![last];
21096 let calls = tmp.path().join("fix-calls");
21097 let mut fixer = spec("repair");
21098 fixer
21099 .env
21100 .insert("FIX_CALLS".into(), calls.display().to_string());
21101 fixer.command = vec!["sh".into(), "-c".into(), r#"
21102 echo fix >> "$FIX_CALLS"
21103 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21104 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21105 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21106 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21107 "#.into()];
21108 let mut reviewer = spec("reviewer");
21109 reviewer.command = vec![
21110 "sh".into(),
21111 "-c".into(),
21112 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21113 .into(),
21114 ];
21115 runner.roles.fixer = Some(vec![fixer]);
21116 runner.roles.reviewers = vec![reviewer];
21117
21118 runner.sync_to_base().await.unwrap();
21119 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21120 assert_ne!(rebased, reviewed);
21121 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21122 if reentry {
21123 let shell = runner.state.config.shell();
21126 runner
21127 .stop_unresolved_review("deferred verification", &shell, &repo)
21128 .await
21129 .unwrap();
21130 assert!(runner.catchup_repairable());
21131 assert!(!calls.exists());
21132 runner.state = RunState::load(&runner.state.id).unwrap();
21133 assert!(runner.catchup_repairable());
21134 }
21135 runner.review_loop().await.unwrap();
21136 let first = &runner.state.reviews[0];
21137 assert_eq!(first.head, reviewed);
21138 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21139 assert!(
21140 first.verify_retried,
21141 "a reproduced build failure gets exactly one retry: {:?}",
21142 first.e2e
21143 );
21144 assert!(first.e2e[0].output_tail.contains("E0063"));
21145 assert!(!runner.catchup_repairable());
21146 if max_rounds == 2 {
21147 assert_eq!(
21148 runner.state.status,
21149 RunStatus::Gating,
21150 "{}",
21151 crate::report::run(&runner.state)
21152 );
21153 assert_eq!(runner.state.reviews.len(), 2);
21154 assert!(first.fix.as_ref().unwrap().committed);
21155 assert!(runner.state.reviews[1].clean);
21156 assert_eq!(
21157 runner
21158 .state
21159 .reviews
21160 .iter()
21161 .filter(|r| r.fix.is_some())
21162 .count(),
21163 1
21164 );
21165 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21166 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21167 } else {
21168 assert_eq!(runner.state.status, RunStatus::Blocked);
21169 assert!(first.fix.is_none());
21170 assert!(!calls.exists());
21171 let recovery = crate::recovery::assess(
21172 "prior",
21173 Some(&runner.state),
21174 &runner.landing_base(),
21175 crate::recovery::Relation::Diverged,
21176 );
21177 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21178 let reasons = recovery.reasons.join("\n");
21179 assert!(
21180 reasons.contains(&reviewed) && reasons.contains(&rebased),
21181 "{reasons}"
21182 );
21183 assert!(
21184 reasons.contains("base sync to")
21185 && reasons.contains("additional review is required"),
21186 "{reasons}"
21187 );
21188 }
21189 }
21190 }
21191
21192 #[tokio::test]
21204 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21205 crate::run::pin_test_home();
21206 let home = crate::run::home();
21207
21208 let tmp = tempfile::tempdir().expect("tempdir");
21209 let repo = tmp.path().join("repo");
21210 std::fs::create_dir_all(&repo).unwrap();
21211 init_repo(&repo);
21212 let head = crate::git::rev_parse(&repo, "HEAD")
21213 .await
21214 .expect("rev-parse");
21215 let cache_dir = tmp.path().join("target");
21216
21217 let mut config = Config::default();
21218 config.verify.e2e = vec![format!(
21219 "CARGO_TARGET_DIR='{}' test -f README.md",
21220 cache_dir.display()
21221 )];
21222 config.graph.review_rounds = 1;
21223 config.graph.timeout_verify = Some(2);
21224 config.graph.timeout_resource_wait = 2;
21225
21226 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21227 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21228 .expect("no io error acquiring directly")
21229 {
21230 crate::cache::AcquireOutcome::Acquired(g) => g,
21231 crate::cache::AcquireOutcome::Busy(b) => {
21232 panic!("expected the direct acquire to win the lease first: {b:?}")
21233 }
21234 };
21235
21236 let mut state = RunState::new(
21237 repo.clone(),
21238 "main".to_owned(),
21239 head.clone(),
21240 "task".to_owned(),
21241 config,
21242 );
21243 state.candidates = vec![Candidate {
21244 index: 0,
21245 label: 'A',
21246 agent: "alpha".to_owned(),
21247 branch: "does-not-exist".to_owned(),
21248 worktree: repo.clone(),
21249 summary: String::new(),
21250 stat: String::new(),
21251 files: 0,
21252 commits: 0,
21253 empty: false,
21254 failed: None,
21255 verified_noop: None,
21256 duration_ms: 0,
21257 folded: false,
21258 }];
21259 state.tally = Some(Tally {
21260 first_choice: BTreeMap::from([('A', 1)]),
21261 borda: BTreeMap::new(),
21262 winner: 'A',
21263 rankings: 1,
21264 unanimous_initial: true,
21265 deliberated: false,
21266 changed_votes: 0,
21267 unanimous_final: true,
21268 tie_break: None,
21269 judges: 0,
21270 present: 0,
21271 quorum: 0,
21272 met_quorum: true,
21273 uncontested: Some("only candidate A produced a change".to_owned()),
21274 });
21275 state.reviews = vec![ReviewRound {
21279 round: 1,
21280 head: head.clone(),
21281 verified_head: Some(head.clone()),
21282 verified_at: Some(jiff::Timestamp::now()),
21283 reviews: Vec::new(),
21284 e2e: vec![CommandOutcome {
21285 command: format!(
21286 "CARGO_TARGET_DIR='{}' test -f README.md",
21287 cache_dir.display()
21288 ),
21289 code: None,
21290 output_tail: "waiting for the shared build cache".to_owned(),
21291 duration_ms: 0,
21292 resource_blocked: true,
21293 }],
21294 fix: None,
21295 blocking: 1,
21296 answered: 1,
21297 expected: 1,
21298 clean: false,
21299 verify_retried: false,
21300 e2e_deferred: false,
21301 e2e_defer_reason: None,
21302 progressed: false,
21303 vote_split: false,
21304 reconsideration: Vec::new(),
21305 verdict: None,
21306 }];
21307
21308 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21309 let mut runner = Runner {
21310 state,
21311 roles: ResolvedRoles {
21312 implementers: Vec::new(),
21313 judges: Vec::new(),
21314 reviewers: Vec::new(),
21315 fixer: None,
21316 conductor: conductor(),
21317 implementer_roster: Vec::new(),
21318 judge_roster: Vec::new(),
21319 reviewer_roster: Vec::new(),
21320 },
21321 sem: Arc::new(Semaphore::new(1)),
21322 pause: Pause::new(),
21323 interrupt: Pause::new(),
21324 };
21325
21326 runner.review_loop().await.expect("review_loop");
21331
21332 assert_eq!(
21333 runner.state.reviews.len(),
21334 1,
21335 "no new round was started on top of the unresolved one: {:?}",
21336 runner.state.reviews
21337 );
21338 let last = &runner.state.reviews[0];
21339 assert_eq!(
21340 last.e2e_status(),
21341 E2eStatus::ResourceBlocked,
21342 "still contended: {last:?}"
21343 );
21344 assert!(
21345 last.verified_at.expect("still known") > first_attempt_at,
21346 "review_loop must have actually retried the check, not left it exactly as found"
21347 );
21348 assert_ne!(
21349 runner.state.status,
21350 RunStatus::Blocked,
21351 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21352 runner.state.status
21353 );
21354
21355 held.release();
21356 }
21357
21358 #[tokio::test]
21359 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21360 crate::run::pin_test_home();
21361 for status in [RunStatus::Blocked, RunStatus::Gating] {
21362 let mut runner = runner_at(status);
21363 runner.state.config.graph.land = true;
21364 runner.state.gate_ran = true;
21365 runner.state.merge = Some(MergeOutcome {
21366 mode: MergeMode::Pr,
21367 ok: false,
21368 detail: "the fixer produced no commit".into(),
21369 empty: false,
21370 });
21371 let url = "https://example.invalid/o/r/pull/1";
21372 runner.state.pr = Some(crate::run::PrRecord {
21373 url: url.into(),
21374 number: 1,
21375 state: "open".into(),
21376 checks: "red".into(),
21377 round: 1,
21378 rounds: 1,
21379 red_at_merge: Vec::new(),
21380 });
21381 runner.state.save().unwrap();
21382 runner.execute_graph().await.unwrap();
21386 assert_eq!(runner.state.status, RunStatus::Blocked);
21387 assert!(runner.state.land_sync_pending);
21388 assert!(runner.state.events.iter().any(|event|
21389 event.node == "land" && event.message == format!("watching {url}")));
21390 assert_eq!(
21391 runner.state.merge.as_ref().unwrap().detail,
21392 "the fixer produced no commit"
21393 );
21394 }
21395 }
21396
21397 #[tokio::test]
21398 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21399 crate::run::pin_test_home();
21400 let tmp = tempfile::tempdir().expect("tempdir");
21401 let repo = tmp.path().join("repo");
21402 std::fs::create_dir_all(&repo).unwrap();
21403 init_repo(&repo);
21404
21405 let mut config = Config::default();
21406 config.merge.mode = MergeMode::Pr;
21407 config.graph.land = true;
21408 config.graph.land_approval = false;
21409
21410 let mut state = RunState::new(
21411 repo.clone(),
21412 "main".to_owned(),
21413 "deadbeef".to_owned(),
21414 "task".to_owned(),
21415 config,
21416 );
21417 state.candidates = vec![Candidate {
21418 index: 0,
21419 label: 'A',
21420 agent: "alpha".to_owned(),
21421 branch: "does-not-exist".to_owned(),
21422 worktree: repo.clone(),
21423 summary: String::new(),
21424 stat: String::new(),
21425 files: 0,
21426 commits: 0,
21427 empty: false,
21428 failed: None,
21429 verified_noop: None,
21430 duration_ms: 0,
21431 folded: false,
21432 }];
21433 state.tally = Some(Tally {
21434 first_choice: BTreeMap::from([('A', 1)]),
21435 borda: BTreeMap::new(),
21436 winner: 'A',
21437 rankings: 1,
21438 unanimous_initial: true,
21439 deliberated: false,
21440 changed_votes: 0,
21441 unanimous_final: true,
21442 tie_break: None,
21443 judges: 0,
21444 present: 0,
21445 quorum: 0,
21446 met_quorum: true,
21447 uncontested: Some("only candidate A produced a change".to_owned()),
21448 });
21449 state.reviews = vec![ReviewRound {
21450 round: 1,
21451 head: "deadbeef".to_owned(),
21452 verified_head: None,
21453 verified_at: None,
21454 reviews: Vec::new(),
21455 e2e: Vec::new(),
21456 fix: None,
21457 blocking: 0,
21458 answered: 0,
21459 expected: 0,
21460 clean: true,
21461 verify_retried: false,
21462 e2e_deferred: false,
21463 e2e_defer_reason: None,
21464 progressed: false,
21465 vote_split: false,
21466 reconsideration: Vec::new(),
21467 verdict: None,
21468 }];
21469 state.gate = vec![CommandOutcome {
21470 command: "test".to_owned(),
21471 code: Some(0),
21472 output_tail: String::new(),
21473 duration_ms: 0,
21474 resource_blocked: false,
21475 }];
21476 state.gate_ran = true;
21477 state.status = RunStatus::Landing;
21481 state.merge = Some(MergeOutcome {
21482 mode: MergeMode::Pr,
21483 ok: true,
21484 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21485 empty: false,
21486 });
21487
21488 ask_test_home();
21492 state.save().expect("save landing run");
21495 let store = ask::Questions::open();
21496 let q = ask_open_question(&store, &state.id);
21497
21498 let mut runner = Runner {
21499 state,
21500 roles: ResolvedRoles {
21501 implementers: Vec::new(),
21502 judges: Vec::new(),
21503 reviewers: Vec::new(),
21504 fixer: None,
21505 conductor: conductor(),
21506 implementer_roster: Vec::new(),
21507 judge_roster: Vec::new(),
21508 reviewer_roster: Vec::new(),
21509 },
21510 sem: Arc::new(Semaphore::new(1)),
21511 pause: Pause::new(),
21512 interrupt: Pause::new(),
21513 };
21514
21515 runner.execute().await.expect("execute");
21520
21521 assert_eq!(
21522 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21523 Some("https://example.invalid/x/y/pull/1"),
21524 "reentry must not push again or open a second pull request over the \
21525 one `land` is already watching"
21526 );
21527 assert_ne!(
21528 runner.state.status,
21529 RunStatus::Landing,
21530 "land could not actually reach the fake pull request, so it must \
21531 have given up rather than left the run silently parked forever"
21532 );
21533 assert_eq!(runner.state.status, RunStatus::Blocked);
21537 assert!(
21538 store.get(&q.id).unwrap().status.open(),
21539 "Blocked is still alive; settle_questions must have been a no-op here"
21540 );
21541 }
21542
21543 fn state_with_round(round: ReviewRound) -> RunState {
21544 let mut s = RunState::new(
21545 PathBuf::from("/repo"),
21546 "main".to_owned(),
21547 "abc1234".to_owned(),
21548 "add retries".to_owned(),
21549 Config::default(),
21550 );
21551 s.reviews = vec![round];
21552 s
21553 }
21554
21555 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21556 crate::verdict::Finding {
21557 id: id.to_owned(),
21558 severity,
21559 file: None,
21560 line: None,
21561 title: title.to_owned(),
21562 detail: String::new(),
21563 }
21564 }
21565
21566 #[test]
21567 fn pr_body_names_open_findings_and_declined_ones() {
21568 let round = ReviewRound {
21569 round: 2,
21570 head: "deadbee".to_owned(),
21571 verified_head: None,
21572 verified_at: None,
21573 reviews: vec![ReviewRecord {
21574 attempts: 0,
21575 reviewer: 1,
21576 agent: "alpha".to_owned(),
21577 summary: String::new(),
21578 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21579 vote: None,
21580 failed: None,
21581 duration_ms: 0,
21582 }],
21583 e2e: vec![CommandOutcome {
21584 command: "cargo test".to_owned(),
21585 code: Some(0),
21586 output_tail: String::new(),
21587 duration_ms: 0,
21588 resource_blocked: false,
21589 }],
21590 verify_retried: false,
21591 e2e_deferred: false,
21592 e2e_defer_reason: None,
21593 fix: Some(FixRecord {
21594 agent: "alpha".to_owned(),
21595 addressed: Vec::new(),
21596 rejected: vec![crate::verdict::Rejection {
21597 id: "R1-1-1".to_owned(),
21598 why: "not reachable from any caller".to_owned(),
21599 }],
21600 notes: String::new(),
21601 committed: true,
21602 failed: None,
21603 duration_ms: 0,
21604 continuation: None,
21605 }),
21606 blocking: 0,
21607 answered: 1,
21608 expected: 1,
21609 clean: false,
21610 progressed: true,
21611 vote_split: false,
21612 reconsideration: Vec::new(),
21613 verdict: None,
21614 };
21615 let state = state_with_round(round);
21616 let body = pr_message(&state, 'A').body;
21617
21618 assert!(body.contains("add retries"), "the task must still be there");
21619 assert!(body.contains("R2-1-1"), "{body}");
21620 assert!(!body.contains("unused import"), "{body}");
21621 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21622 assert!(
21623 !body.contains("not reachable from any caller"),
21624 "the reason it was declined: {body}"
21625 );
21626 }
21627
21628 #[test]
21629 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21630 let round = ReviewRound {
21631 round: 1,
21632 head: "deadbee".to_owned(),
21633 verified_head: None,
21634 verified_at: None,
21635 reviews: vec![ReviewRecord {
21636 attempts: 0,
21637 reviewer: 1,
21638 agent: "alpha".to_owned(),
21639 summary: String::new(),
21640 findings: Vec::new(),
21641 vote: None,
21642 failed: None,
21643 duration_ms: 0,
21644 }],
21645 e2e: Vec::new(),
21646 verify_retried: false,
21647 e2e_deferred: false,
21648 e2e_defer_reason: None,
21649 fix: None,
21650 blocking: 0,
21651 answered: 1,
21652 expected: 1,
21653 clean: true,
21654 progressed: false,
21655 vote_split: false,
21656 reconsideration: Vec::new(),
21657 verdict: None,
21658 };
21659 let state = state_with_round(round);
21660 let body = pr_message(&state, 'A').body;
21661 assert!(!body.contains("Open review findings"), "{body}");
21662 assert!(!body.contains("Declined"), "{body}");
21663 }
21664
21665 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21666 let mut state = RunState::new(
21667 PathBuf::from("/repo"),
21668 "main".to_owned(),
21669 "abc1234".to_owned(),
21670 instruction.to_owned(),
21671 Config::default(),
21672 );
21673 state.config.disk.min_free_bytes = 0;
21675 state.candidates.push(Candidate {
21676 index: 0,
21677 label: 'A',
21678 agent: "alpha".to_owned(),
21679 branch: "magi/x/A".to_owned(),
21680 worktree: PathBuf::from("/wt"),
21681 summary: summary.to_owned(),
21682 stat: String::new(),
21683 files: 1,
21684 commits: 1,
21685 empty: false,
21686 failed: None,
21687 verified_noop: None,
21688 folded: false,
21689 duration_ms: 0,
21690 });
21691 state
21692 }
21693
21694 fn review_state(subjects: &[&str]) -> RunState {
21695 let mut state = state_with_summary(
21696 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21697 "",
21698 );
21699 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21700 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21701 state
21702 }
21703
21704 #[tokio::test]
21707 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21708 ask_test_home();
21709 let tmp = tempfile::tempdir().unwrap();
21710 let repo = tmp.path().join("repo");
21711 std::fs::create_dir_all(&repo).unwrap();
21712 init_repo(&repo);
21713 let origin = tmp.path().join("origin.git");
21714 let g = |dir: &Path, args: &[&str]| {
21715 let out = std::process::Command::new("git")
21716 .args(args)
21717 .current_dir(dir)
21718 .quiet()
21719 .output()
21720 .expect("spawn git");
21721 assert!(
21722 out.status.success(),
21723 "git {args:?}: {}",
21724 String::from_utf8_lossy(&out.stderr)
21725 );
21726 };
21727 g(
21728 tmp.path(),
21729 &[
21730 "clone",
21731 "--bare",
21732 "-q",
21733 repo.to_str().unwrap(),
21734 origin.to_str().unwrap(),
21735 ],
21736 );
21737 g(
21738 &repo,
21739 &["remote", "add", "origin", origin.to_str().unwrap()],
21740 );
21741 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21742
21743 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21745 g(&repo, &["add", "-A"]);
21746 g(
21747 &repo,
21748 &["commit", "-q", "-m", "chore(deps): update a crate"],
21749 );
21750 g(&repo, &["push", "-q", "origin", "main"]);
21751 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21752 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21753 g(&repo, &["add", "-A"]);
21754 g(
21755 &repo,
21756 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21757 );
21758 g(&repo, &["checkout", "-q", "main"]);
21759
21760 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21761 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21762 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21764 assert_eq!(fallback, c1);
21765
21766 let mut state = review_state(&[
21767 "chore(deps): update a crate",
21768 "fix(daemon): apply a chosen action",
21769 ]);
21770 state.repo = repo.clone();
21771 state.base_branch = "main".to_owned();
21772 state.base_commit = c1;
21773 refresh_reviewed_commits(&mut state, "feat/own").await;
21774 assert_eq!(
21775 state.reviewed_commits,
21776 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21777 );
21778 assert_eq!(
21779 review_title(&state).as_deref(),
21780 Some("fix(daemon): apply a chosen action")
21781 );
21782 assert_eq!(
21783 leaked_subjects(&state, "feat/own").await,
21784 Some(vec!["chore(deps): update a crate".to_owned()])
21785 );
21786 state.config.merge.remote = "nowhere".to_owned();
21789 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21790 }
21791
21792 struct SummaryFixture {
21796 _tmp: tempfile::TempDir,
21797 repo: PathBuf,
21798 reply: PathBuf,
21799 runner: Runner,
21800 }
21801
21802 fn summary_fixture() -> SummaryFixture {
21803 ask_test_home();
21804 let tmp = tempfile::tempdir().unwrap();
21805 let repo = tmp.path().join("repo");
21806 std::fs::create_dir_all(&repo).unwrap();
21807 init_repo(&repo);
21808 let origin = tmp.path().join("origin.git");
21809 let g = |args: &[&str]| {
21810 let out = std::process::Command::new("git")
21811 .args(args)
21812 .current_dir(&repo)
21813 .quiet()
21814 .output()
21815 .expect("spawn git");
21816 assert!(
21817 out.status.success(),
21818 "git {args:?}: {}",
21819 String::from_utf8_lossy(&out.stderr)
21820 );
21821 };
21822 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21823 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21824 g(&["push", "-q", "origin", "main"]);
21825 g(&["checkout", "-q", "-b", "feat/own"]);
21826 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21827 g(&["add", "-A"]);
21828 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21829 let reply = tmp.path().join("reply.txt");
21830 std::fs::write(
21831 &reply,
21832 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21833 )
21834 .unwrap();
21835 let mut runner = runner_at(RunStatus::Gating);
21836 let mut state = review_state(&["feat(own): add the own file"]);
21837 state.repo = repo.clone();
21838 state.base_branch = "main".to_owned();
21839 state.candidates[0].branch = "feat/own".to_owned();
21840 state.candidates[0].worktree = repo.clone();
21841 let mut author = spec("alpha");
21842 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21843 author
21844 .env
21845 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21846 state.config.agents = vec![author];
21847 runner.state = state;
21848 SummaryFixture {
21849 _tmp: tmp,
21850 repo,
21851 reply,
21852 runner,
21853 }
21854 }
21855
21856 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21857 let winner = f.runner.state.candidates[0].clone();
21858 f.runner
21859 .guarded_pr_message(&winner, None, true)
21860 .await
21861 .unwrap()
21862 .expect("an English description never parks")
21863 }
21864
21865 fn clean_round_with_minor_findings() -> ReviewRound {
21866 let mut round = passing_round(2, None);
21867 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21868 round.reviews = vec![ReviewRecord {
21869 attempts: 0,
21870 reviewer: 1,
21871 agent: "alpha".into(),
21872 summary: "内部レビュー".into(),
21873 findings: vec![
21874 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21875 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21876 ],
21877 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21878 failed: None,
21879 duration_ms: 0,
21880 }];
21881 round
21882 }
21883
21884 #[tokio::test]
21885 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21886 for review_only in [false, true] {
21887 let mut f = summary_fixture();
21888 if !review_only {
21889 f.runner.state.candidates[0].agent = "alpha".into();
21890 }
21891 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21892 assert!(f.runner.state.reviews[0].clean);
21893 assert!(f.runner.state.reviews[0].fix.is_none());
21894 assert!(f.runner.state.open_findings().is_empty());
21895 assert_eq!(
21896 crate::pr_outcome::required_finding_ids(&f.runner.state),
21897 ["R2-1-1", "R2-1-2"]
21898 );
21899 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21900 let evidence = crate::pr_outcome::summary_evidence(
21901 &f.runner.state,
21902 crate::pr_outcome::FinalHead::At(&head),
21903 );
21904 assert!(
21905 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21906 );
21907 let omitted = posted(&mut f).await;
21909 assert!(omitted.body.contains("No description"));
21910 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21911 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21912 assert!(f.runner.state.change_summary.is_none());
21913 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21914 let described = posted(&mut f).await;
21915 assert!(described.body.contains("improve documentation for callers"));
21916 assert!(described.body.contains("remaining edge case"));
21917 assert!(described.body.contains("## Open review findings"));
21918 assert!(!described.body.contains("未修正"));
21919 }
21920 }
21921
21922 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21923 let mut f = summary_fixture();
21924 if !review_only {
21925 f.runner.state.candidates[0].agent = "alpha".into();
21926 }
21927 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21928 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21929 fixed
21930 .fix
21931 .as_mut()
21932 .unwrap()
21933 .rejected
21934 .push(crate::verdict::Rejection {
21935 id: "R1-1-1".into(),
21936 why: "呼び出されないコードのため辞退".into(),
21937 });
21938 let mut clean = clean_round_with_minor_findings();
21939 clean.verified_head = Some(head.clone());
21940 f.runner.state.reviews = vec![fixed, clean];
21941 ok_gate(&mut f.runner.state, Some(&head));
21942 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21943 f.runner.state.config.agents.push(scripted("beta", reply));
21944 f.runner.state.config.roles.language_judge =
21945 Some(crate::config::AgentChoice::One("beta".into()));
21946 f
21947 }
21948
21949 #[tokio::test]
21950 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21951 for review_only in [false, true] {
21952 let mut f = rewrite_fixture_with_findings(review_only,
21953 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21954 let m = posted(&mut f).await;
21955 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21956 let expected =
21957 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21958 assert!(m.body.contains(&expected));
21959 assert!(m.body.contains("## Review rounds"));
21960 assert!(
21961 m.body
21962 .contains("Gate (`verify.gate`): passed on the final commit")
21963 );
21964 assert!(
21965 m.body
21966 .contains("Tests (`verify.e2e`): passed on the final commit")
21967 );
21968 let c = f.runner.state.change_summary.as_ref().unwrap();
21969 assert!(
21970 m.body
21971 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21972 );
21973 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21974 assert!(crate::pr_outcome::covers_required_findings(
21975 &f.runner.state,
21976 &m.body
21977 ));
21978 assert!(m.body.contains("documentation remains incomplete"));
21979 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21980 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21981 assert!(f.runner.state.github_text.is_none());
21982 }
21983 }
21984
21985 #[tokio::test]
21986 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21987 for review_only in [false, true] {
21988 for reply in [
21989 "TITLE: fix: public description\nA short English description.",
21990 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21991 ] {
21992 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21993 let winner = f.runner.state.candidates[0].clone();
21994 let posted = f
21995 .runner
21996 .guarded_pr_message(&winner, None, true)
21997 .await
21998 .unwrap();
21999 assert!(posted.is_none());
22000 assert!(f.runner.state.parked);
22001 let withheld = f.runner.state.github_text.as_ref().unwrap();
22002 assert!(
22003 withheld.rewrite.is_none(),
22004 "an incomplete rewrite must not be selectable"
22005 );
22006 let q = ask::Questions::open()
22007 .get(withheld.question.as_ref().unwrap())
22008 .unwrap();
22009 assert!(
22010 !q.choices
22011 .contains(&crate::github_text::USE_REWRITE.to_owned())
22012 );
22013 assert!(
22014 q.detail.contains("R1-1-1")
22015 && q.detail.contains("R2-1-1")
22016 && q.detail.contains("R2-1-2")
22017 );
22018 }
22019 }
22020 }
22021
22022 #[tokio::test]
22023 async fn japanese_reports_stay_local_in_both_public_run_modes() {
22024 for review_only in [false, true] {
22025 let mut f = summary_fixture();
22026 if !review_only {
22027 f.runner.state.candidates[0].agent = "alpha".into();
22028 }
22029 f.runner.state.instruction = "内部タスクの説明".into();
22030 f.runner.state.candidates[0].summary = "内部の候補報告".into();
22031 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22032 f.runner.state.reviews = vec![
22033 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22034 round_with_notes(2, Some("二回目の内部修正報告です。")),
22035 passing_round(3, Some(&head)),
22036 ];
22037 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22038 ok_gate(&mut f.runner.state, Some(&head));
22039 let captured = f.repo.join("public-prompt.txt");
22040 let author = &mut f.runner.state.config.agents[0];
22041 author.command = vec![
22042 "sh".into(),
22043 "-c".into(),
22044 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22045 ];
22046 author.env.insert(
22047 "PROMPT_FILE".into(),
22048 captured.to_string_lossy().into_owned(),
22049 );
22050 let m = posted(&mut f).await;
22051 assert!(m.body.contains("Adds `own.txt`"));
22052 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22053 assert!(m.body.contains("R1-1-1"));
22054 assert!(
22055 m.body
22056 .contains("Gate (`verify.gate`): passed on the final commit")
22057 );
22058 assert!(
22059 m.body
22060 .contains("Tests (`verify.e2e`): passed on the final commit")
22061 );
22062 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22063 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22064 let prompt = std::fs::read_to_string(captured).unwrap();
22065 assert!(prompt.contains(&format!(
22066 "git diff {} {head}",
22067 f.runner.state.change_summary.as_ref().unwrap().base
22068 )));
22069 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22070 assert_eq!(
22071 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22072 "二回目の内部修正報告です。"
22073 );
22074 }
22075 }
22076
22077 #[tokio::test]
22078 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22079 for review_only in [false, true] {
22080 let mut f = summary_fixture();
22081 if !review_only {
22082 f.runner.state.candidates[0].agent = "alpha".into();
22083 }
22084 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22085 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22086 std::fs::write(&f.reply, "no usable description").unwrap();
22087 let failed = posted(&mut f).await;
22088 assert!(
22089 failed
22090 .body
22091 .contains("No description of what this change does could be generated")
22092 );
22093 assert!(failed.body.is_ascii());
22094 assert!(f.runner.state.github_text.is_none());
22095 assert!(f.runner.state.change_summary.is_none());
22096 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22097 let evidence = crate::pr_outcome::summary_evidence(
22098 &f.runner.state,
22099 crate::pr_outcome::FinalHead::At(&head),
22100 );
22101 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22102 assert!(
22103 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22104 .exists()
22105 );
22106 f.runner.state =
22107 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22108 std::fs::write(
22109 &f.reply,
22110 "TITLE: feat(own): add the own file\nAdds the committed own file.",
22111 )
22112 .unwrap();
22113 let retried = posted(&mut f).await;
22114 assert!(retried.body.contains("Adds the committed own file."));
22115 assert_eq!(
22116 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22117 "日本語の修正報告"
22118 );
22119 }
22120 }
22121
22122 #[tokio::test]
22123 async fn same_head_verification_and_findings_invalidate_public_prose() {
22124 let mut f = summary_fixture();
22125 posted(&mut f).await;
22126 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22127 let original = f
22128 .runner
22129 .state
22130 .change_summary
22131 .as_ref()
22132 .unwrap()
22133 .evidence
22134 .clone();
22135 ok_gate(&mut f.runner.state, Some(&head));
22136 std::fs::write(
22137 &f.reply,
22138 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22139 )
22140 .unwrap();
22141 let verified = posted(&mut f).await;
22142 assert!(verified.body.contains("now verified"));
22143 assert_ne!(
22144 f.runner.state.change_summary.as_ref().unwrap().evidence,
22145 original
22146 );
22147 let mut round = round_with_notes(1, Some("内部報告"));
22148 round.reviews.push(ReviewRecord {
22149 attempts: 0,
22150 reviewer: 1,
22151 agent: "alpha".into(),
22152 summary: "内部レビュー".into(),
22153 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22154 vote: None,
22155 failed: None,
22156 duration_ms: 0,
22157 });
22158 round
22159 .fix
22160 .as_mut()
22161 .unwrap()
22162 .rejected
22163 .push(crate::verdict::Rejection {
22164 id: "R1-1-1".into(),
22165 why: "呼び出し元から到達できない".into(),
22166 });
22167 f.runner.state.reviews = vec![round];
22168 let omitted = posted(&mut f).await;
22170 assert!(omitted.body.contains("No description"));
22171 assert!(omitted.body.contains("R1-1-1"));
22172 assert!(!omitted.body.contains("到達"));
22173 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22174 let described = posted(&mut f).await;
22175 assert!(described.body.contains("unreachable from callers"));
22176 assert!(described.body.contains("R1-1-1"));
22177 assert!(!described.body.contains("到達"));
22178 let evidence = f
22179 .runner
22180 .state
22181 .change_summary
22182 .as_ref()
22183 .unwrap()
22184 .evidence
22185 .clone();
22186 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22187 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22188 assert!(posted(&mut f).await.body.contains("rationale updated"));
22189 assert_ne!(
22190 f.runner.state.change_summary.as_ref().unwrap().evidence,
22191 evidence
22192 );
22193 f.runner
22195 .state
22196 .change_summary
22197 .as_mut()
22198 .unwrap()
22199 .evidence
22200 .clear();
22201 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22202 assert!(posted(&mut f).await.body.contains("current specification"));
22203 }
22204
22205 #[tokio::test]
22206 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22207 let mut f = summary_fixture();
22208 let m = posted(&mut f).await;
22209 assert_eq!(m.title, "feat(own): add the own file");
22210 assert!(
22211 m.body
22212 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22213 "{}",
22214 m.body
22215 );
22216 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22217 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22218 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22219 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22220 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22221 let kept = f.runner.state.change_summary.as_ref().unwrap();
22222 assert_eq!(
22223 (kept.head.as_str(), kept.base.as_str()),
22224 (head.as_str(), base.as_str())
22225 );
22226 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22227 }
22228
22229 #[tokio::test]
22230 async fn review_only_summary_receives_the_configured_github_language() {
22231 let mut f = summary_fixture();
22232 f.runner.state.config.graph.github_language = "ja".into();
22233 let captured = f.repo.join("summary-prompt.txt");
22234 let author = &mut f.runner.state.config.agents[0];
22235 author.command = vec![
22236 "sh".into(),
22237 "-c".into(),
22238 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22239 ];
22240 author.env.insert(
22241 "PROMPT_FILE".into(),
22242 captured.to_string_lossy().into_owned(),
22243 );
22244 std::fs::write(
22245 &f.reply,
22246 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22247 )
22248 .unwrap();
22249 let m = posted(&mut f).await;
22250 let prompt = std::fs::read_to_string(captured).unwrap();
22251 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22252 assert!(!prompt.contains("in English"));
22253 assert_eq!(m.title, "feat(own): ファイルを追加");
22254 assert!(m.body.contains("own.txt を追加します。"));
22255 }
22256
22257 #[tokio::test]
22258 async fn review_fixes_come_after_the_whole_change_summary() {
22259 let mut f = summary_fixture();
22260 f.runner.state.reviews = vec![
22261 round_with_notes(1, Some("handled the PATH case")),
22262 round_with_notes(2, None),
22263 ];
22264 let m = posted(&mut f).await;
22265 let summary = m.body.find("## Summary").expect("summary");
22266 let fixes = m.body.find("## Review rounds").expect("review rounds");
22267 assert!(summary < fixes, "{}", m.body);
22268 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22269 assert!(m.body.contains("## Final verification"), "{}", m.body);
22270 }
22271
22272 #[tokio::test]
22273 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22274 let mut f = summary_fixture();
22275 let first = posted(&mut f).await;
22276 let first_head = f.runner.state.change_summary.clone().unwrap().head;
22277 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22279 let again = posted(&mut f).await;
22280 assert_eq!((again.title, again.body), (first.title, first.body));
22281
22282 let g = |args: &[&str]| {
22283 let out = std::process::Command::new("git")
22284 .args(args)
22285 .current_dir(&f.repo)
22286 .quiet()
22287 .output()
22288 .expect("spawn git");
22289 assert!(out.status.success(), "git {args:?}");
22290 };
22291 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22292 g(&["add", "-A"]);
22293 g(&["commit", "-q", "-m", "fix(own): handle more"]);
22294 let second = posted(&mut f).await;
22295 let kept = f.runner.state.change_summary.clone().unwrap();
22296 assert_ne!(kept.head, first_head);
22297 assert!(second.body.contains("Not read yet."), "{}", second.body);
22298 assert!(!second.body.contains("so the first summary is read"));
22299 assert_eq!(second.title, "feat(own): other");
22300 }
22301
22302 #[tokio::test]
22303 async fn an_unavailable_summary_is_stated_and_never_invented() {
22304 let mut f = summary_fixture();
22305 std::fs::write(&f.reply, "no title line here").unwrap();
22306 let m = posted(&mut f).await;
22307 assert!(
22308 m.body
22309 .contains("No description of what this change does could be generated"),
22310 "{}",
22311 m.body
22312 );
22313 assert!(m.body.contains("internal reports remain in local records"));
22314 assert!(f.runner.state.change_summary.is_none());
22315 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22316 let winner = f.runner.state.candidates[0].clone();
22318 let preview = f
22319 .runner
22320 .guarded_pr_message(&winner, None, false)
22321 .await
22322 .unwrap()
22323 .unwrap();
22324 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22325 }
22326
22327 #[test]
22328 fn pr_message_review_single_commit_uses_its_subject() {
22329 let state = review_state(&["feat(nats): per-role user"]);
22330 let m = pr_message(&state, 'A');
22331 assert_eq!(m.title, "feat(nats): per-role user");
22332 assert!(!m.body.contains("Review the work already"), "{}", m.body);
22333 assert!(m.body.contains("## Commits under review"), "{}", m.body);
22334 }
22335
22336 #[test]
22337 fn pr_message_review_multi_commit_takes_the_oldest() {
22338 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22339 let m = pr_message(&state, 'A');
22340 assert_eq!(m.title, "feat: the change");
22341 for s in ["feat: the change", "fix: typo", "fix: again"] {
22342 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22343 }
22344 }
22345
22346 #[test]
22347 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22348 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22349 let mut state = review_state(&[first, "fix: later fixup"]);
22350 state.candidates[0].branch = "機能/ログイン".to_owned();
22351 let m = pr_message(&state, 'A');
22352 assert!(
22353 m.title.starts_with("chore: land candidate A of run"),
22354 "{}",
22355 m.title
22356 );
22357 }
22358 }
22359
22360 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22361 BranchFacts {
22362 commits: commits
22363 .iter()
22364 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22365 .collect(),
22366 stat: stat.to_owned(),
22367 }
22368 }
22369
22370 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22371 let mut r = review_round(true, 0, 1, 1, true, true);
22372 r.round = round;
22373 r.fix = notes.map(|n| FixRecord {
22374 agent: "fixer".to_owned(),
22375 addressed: Vec::new(),
22376 rejected: Vec::new(),
22377 notes: n.to_owned(),
22378 committed: true,
22379 failed: None,
22380 duration_ms: 0,
22381 continuation: None,
22382 });
22383 r
22384 }
22385
22386 #[test]
22387 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22388 let state = review_state(&["ignored"]);
22389 let f = facts(
22390 &[
22391 (
22392 "fix(login): resolve PATH on macOS",
22393 "Login shells skip rc files.",
22394 ),
22395 ("fix: address review", ""),
22396 ],
22397 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22398 );
22399 let m = pr_message_with(&state, 'A', Some(&f));
22400 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22401 assert!(
22402 m.body
22403 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22404 "{}",
22405 m.body
22406 );
22407 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22408 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22409 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22410 for banned in [
22411 "Review the work already",
22412 "no task statement",
22413 "Original task",
22414 ] {
22415 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22416 }
22417 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22418 }
22419
22420 #[test]
22421 fn pr_message_review_truncates_a_huge_first_commit_body() {
22422 let state = review_state(&["ignored"]);
22423 let f = facts(
22424 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22425 "s",
22426 );
22427 let m = pr_message_with(&state, 'A', Some(&f));
22428 assert!(m.body.len() < 30_000, "{}", m.body.len());
22429 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22430 assert!(
22431 m.body.contains("- ... 1 more commit(s)"),
22432 "{}",
22433 m.body.len()
22434 );
22435 assert!(m.body.ends_with("magi:candidate-a\n"));
22436 }
22437
22438 #[test]
22439 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22440 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22441 assert_eq!(m.title, "feat: x");
22442 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22443 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22444 assert!(!m.body.contains("no task statement"), "{}", m.body);
22445 }
22446
22447 #[test]
22448 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22449 let mut state = review_state(&["日本語の件名"]);
22450 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22451 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22452 state.candidates[0].branch = "機能/ログイン".to_owned();
22453 assert!(
22454 pr_message(&state, 'A')
22455 .title
22456 .starts_with("chore: land candidate A")
22457 );
22458 }
22459
22460 #[test]
22461 fn pr_message_review_fixes_survive_a_clean_final_round() {
22462 let mut state = review_state(&["feat: x"]);
22463 state.reviews = vec![
22464 round_with_notes(1, Some("handled the PATH case")),
22465 round_with_notes(2, None),
22466 ];
22467 let body = pr_message(&state, 'A').body;
22468 assert!(
22469 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22470 "{body}"
22471 );
22472 assert!(body.contains("### Round 1"), "{body}");
22473
22474 state.reviews = vec![
22475 round_with_notes(1, Some("first fix")),
22476 round_with_notes(2, Some("")),
22477 round_with_notes(3, Some("second fix")),
22478 round_with_notes(4, None),
22479 ];
22480 let body = pr_message(&state, 'A').body;
22481 assert!(
22482 body.contains("### Round 1") && !body.contains("first fix"),
22483 "{body}"
22484 );
22485 assert!(
22486 body.contains("### Round 3") && !body.contains("second fix"),
22487 "{body}"
22488 );
22489 assert!(body.contains("### Round 2"), "{body}");
22490 }
22491
22492 const FINAL: &str = "abcdef1234567890";
22493
22494 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22495 let mut r = round_with_notes(round, None);
22496 r.verified_head = verified.map(str::to_owned);
22497 r
22498 }
22499
22500 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22501 state.gate = vec![CommandOutcome {
22502 command: "gate".to_owned(),
22503 code: Some(0),
22504 output_tail: String::new(),
22505 duration_ms: 0,
22506 resource_blocked: false,
22507 }];
22508 state.gate_ran = true;
22509 state.gate_head = head.map(str::to_owned);
22510 }
22511
22512 fn body_at(state: &RunState) -> String {
22513 pr_message_raw(
22514 state,
22515 'A',
22516 None,
22517 crate::pr_outcome::FinalHead::At(FINAL),
22518 ChangeNote::NotAsked,
22519 )
22520 .body
22521 }
22522
22523 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22524
22525 #[test]
22526 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22527 let mut state = review_state(&["feat: x"]);
22528 state.reviews = vec![
22529 round_with_notes(1, Some(SANDBOXED)),
22530 passing_round(2, Some(FINAL)),
22531 ];
22532 ok_gate(&mut state, Some(FINAL));
22533 let body = body_at(&state);
22534 assert!(!body.contains("Handled the PATH case."), "{body}");
22535 assert!(!body.contains("could not commit"), "{body}");
22536 assert!(!body.contains("could not run"), "{body}");
22537 assert!(
22538 body.contains("The fix is committed on the branch"),
22539 "{body}"
22540 );
22541 assert!(!body.contains("sandbox"), "{body}");
22542 assert!(
22543 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22544 "{body}"
22545 );
22546 assert!(
22547 body.contains("Gate (`verify.gate`): passed on the final commit"),
22548 "{body}"
22549 );
22550 assert!(
22552 state.reviews[0]
22553 .fix
22554 .as_ref()
22555 .unwrap()
22556 .notes
22557 .contains("could not commit")
22558 );
22559 }
22560
22561 #[test]
22562 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22563 let mut state = review_state(&["feat: x"]);
22564 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22565 r1.fix.as_mut().unwrap().committed = false;
22566 let mut r2 = passing_round(2, Some(FINAL));
22567 r2.e2e[0].code = Some(1);
22568 state.reviews = vec![r1, r2];
22569 let body = body_at(&state);
22570 assert!(!body.contains("could not commit"), "{body}");
22571 assert!(!body.contains("could not run"), "{body}");
22572 assert!(body.contains("This round produced no commit"), "{body}");
22573 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22574 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22575 assert!(!body.contains("passed"), "{body}");
22576 }
22577
22578 #[test]
22579 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22580 let mut state = review_state(&["feat: x"]);
22581 state.reviews = vec![
22582 round_with_notes(1, Some(SANDBOXED)),
22583 passing_round(2, Some("0123456789abcdef")),
22584 ];
22585 ok_gate(&mut state, Some("0123456789abcdef"));
22586 let body = body_at(&state);
22587 assert!(!body.contains("HTTP tests could not run"), "{body}");
22588 assert!(!body.contains("could not commit"), "{body}");
22590 assert!(
22591 body.contains("passed on an earlier commit (`0123456`)"),
22592 "{body}"
22593 );
22594 assert!(!body.contains("passed on the final commit"), "{body}");
22595
22596 ok_gate(&mut state, None);
22598 let body = body_at(&state);
22599 assert!(
22600 body.contains("passed on a commit that was not recorded"),
22601 "{body}"
22602 );
22603 }
22604
22605 #[test]
22606 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22607 let state = state_with_summary(
22608 "add retries to the client",
22609 "TITLE: feat: retries\n\nDid it.",
22610 );
22611 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22612 assert_eq!(m.title, "feat: retries");
22613 assert!(
22614 m.body.contains("<summary>Original task</summary>"),
22615 "{}",
22616 m.body
22617 );
22618 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22619 assert!(
22620 m.body
22621 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22622 "{}",
22623 m.body
22624 );
22625 }
22626
22627 #[test]
22628 fn pr_message_review_bounds_a_long_english_subject() {
22629 let long = format!("feat: {}", "word ".repeat(100));
22630 let m = pr_message(&review_state(&[&long]), 'A');
22631 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22632 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22633 }
22634
22635 #[test]
22636 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22637 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22638 let m = pr_message(&state, 'A');
22639 assert_eq!(m.title, "add retries");
22640 assert!(m.body.contains("<summary>Original task</summary>"));
22641 assert!(!m.body.contains("Commits under review"));
22642 assert_eq!(landing_subject_source(&state), state.instruction);
22643 }
22644
22645 #[test]
22646 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22647 let state = review_state(&["feat: the change", "fix: typo"]);
22648 let source = landing_subject_source(&state);
22649 assert_eq!(land::merge_subject("", &source), "feat: the change");
22650 assert_eq!(
22651 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22652 "feat: the change"
22653 );
22654 assert_eq!(
22656 land::merge_subject("feat: renamed by hand", &source),
22657 "feat: renamed by hand"
22658 );
22659 let blank = review_state(&["日本語"]);
22660 assert!(
22661 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22662 );
22663 }
22664
22665 #[test]
22666 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22667 let state = review_state(&["feat: the change"]);
22668 let source = landing_subject_source(&state);
22669 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22670 assert_eq!(
22671 land::merge_subject(landing_title(&state, old), &source),
22672 "feat: the change"
22673 );
22674 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22675 let task = state_with_summary("add retries", "");
22676 assert_eq!(landing_title(&task, old), old);
22677 }
22678
22679 #[test]
22680 fn pr_message_describes_the_change_not_the_task() {
22681 let state = state_with_summary(
22682 "今回やってほしいこと: results projector を直す",
22683 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22684 );
22685 let m = pr_message(&state, 'A');
22686 assert_eq!(m.title, "fix(web): batch the runs list reads");
22687 assert!(
22688 m.body.starts_with("## Summary\n\n- reads run.json once"),
22689 "{}",
22690 m.body
22691 );
22692 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22693 let task_at = m.body.find("今回やってほしいこと").unwrap();
22694 let details_at = m.body.find("<details>").unwrap();
22695 assert!(
22696 details_at < task_at,
22697 "the task lives inside <details>: {}",
22698 m.body
22699 );
22700 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22701 assert!(m.body.contains("magi:candidate-a"));
22702 }
22703
22704 #[test]
22705 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22706 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22707 let m = pr_message(&state, 'A');
22708 assert_eq!(m.title, "add retries");
22709 assert!(
22710 m.body.contains("## Summary\n\n- did some things"),
22711 "{}",
22712 m.body
22713 );
22714
22715 let none = RunState::new(
22716 PathBuf::from("/repo"),
22717 "main".to_owned(),
22718 "abc1234".to_owned(),
22719 "add retries".to_owned(),
22720 Config::default(),
22721 );
22722 let m = pr_message(&none, 'A');
22723 assert_eq!(m.title, "add retries");
22724 assert!(!m.body.contains("## Summary"), "{}", m.body);
22725 }
22726
22727 #[test]
22728 fn pr_message_refuses_the_candidate_commit_subject() {
22729 for bad in [
22730 "TITLE: magi: candidate A (uncommitted work)",
22731 "TITLE: chore: stuff (uncommitted work)",
22732 "TITLE: ",
22733 ] {
22734 let state = state_with_summary("add retries", bad);
22735 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22736 }
22737 }
22738
22739 #[test]
22740 fn pr_message_bounds_a_very_long_task_and_title() {
22741 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22742 let state = state_with_summary(&long, "- nothing");
22743 let m = pr_message(&state, 'A');
22744 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22745 assert!(!m.title.contains('\n'));
22746
22747 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22748 let m = pr_message(&state, 'A');
22749 assert!(m.title.starts_with("feat: "));
22750 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22751 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22752 }
22753
22754 fn long_title_of(instruction: &str) -> String {
22755 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22756 }
22757
22758 #[test]
22759 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22760 let first = "Make the landing path keep a readable title for long tasks";
22761 let line = format!(
22762 "{first}. {}",
22763 "And then keep going with more words ".repeat(20)
22764 );
22765 let t = long_title_of(&line);
22766 assert_eq!(t, first);
22767 assert!(!t.starts_with("chore: land"));
22768 }
22769
22770 #[test]
22771 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22772 let line = "word ".repeat(200);
22773 let t = long_title_of(&line);
22774 assert!(t.ends_with("word..."), "{t}");
22775 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22776 }
22777
22778 #[test]
22779 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22780 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22781 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22782 }
22783 }
22784
22785 #[test]
22786 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22787 let mut state = state_with_summary("再試行を追加する", "- nothing");
22788 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22789 state.config.graph.github_language = "ja".into();
22790 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22791 }
22792
22793 #[test]
22794 fn pr_message_title_limit_is_exact() {
22795 let at = "a".repeat(PR_TITLE_MAX);
22796 assert_eq!(long_title_of(&at), at);
22797 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22798 assert!(over.ends_with("..."), "{over}");
22799 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22800 }
22801
22802 #[test]
22803 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22804 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22805 let t = long_title_of(&line);
22806 assert!(t.ends_with("..."), "{t}");
22807 assert!(t.is_ascii(), "{t}");
22808 }
22809
22810 #[test]
22811 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22812 let line = format!(
22813 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22814 "plus more filler words ".repeat(20)
22815 );
22816 let t = long_title_of(&line);
22817 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22818 }
22819
22820 #[test]
22821 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22822 let line = format!("\n\n# {}", "title words ".repeat(40));
22823 let state = state_with_summary(&line, "- nothing");
22824 let m = pr_message(&state, 'A');
22825 assert!(m.title.starts_with("title words"), "{}", m.title);
22826 assert_eq!(
22827 land::merge_subject(&m.title, &landing_subject_source(&state)),
22828 m.title
22829 );
22830 assert_eq!(
22832 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22833 "feat: renamed by hand"
22834 );
22835 }
22836
22837 #[tokio::test]
22838 async fn github_text_rewrite_is_bounded_and_falls_back() {
22839 crate::run::pin_test_home();
22840 for (reply, accepted) in [
22841 (
22842 "TITLE: fix: retries\nAdd retries for failed requests.",
22843 true,
22844 ),
22845 (
22846 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22847 false,
22848 ),
22849 (
22850 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22851 true,
22852 ),
22853 (
22854 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22855 true,
22856 ),
22857 (
22858 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22859 true,
22860 ),
22861 (
22862 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22863 false,
22864 ),
22865 ("TITLE: fix: retries\n", false),
22866 ] {
22867 let dir = tempfile::tempdir().unwrap();
22868 let mut runner = runner_at(RunStatus::Gating);
22869 runner.state = state_with_summary(
22870 "add retries",
22871 "TITLE: fix: retries\n日本語の説明を書きます。",
22872 );
22873 runner.state.repo = dir.path().to_owned();
22874 runner.state.candidates[0].worktree = dir.path().to_owned();
22875 let mut author = spec("alpha");
22876 author.command = vec![
22877 "sh".into(),
22878 "-c".into(),
22879 "printf '%s' \"$REWRITE_REPLY\"".into(),
22880 ];
22881 author.env.insert("REWRITE_REPLY".into(), reply.into());
22882 runner.state.config.agents = vec![author];
22883 seed_public_summary(&mut runner);
22884 let winner = runner.state.candidates[0].clone();
22885 let first = runner
22886 .guarded_pr_message(&winner, None, true)
22887 .await
22888 .unwrap();
22889 let message = if accepted {
22890 first.expect("a passing rewrite never asks")
22891 } else {
22892 assert!(first.is_none());
22894 assert!(runner.state.parked);
22895 let store = ask::Questions::open();
22896 let asked: Vec<_> = store
22897 .list()
22898 .into_iter()
22899 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22900 .collect();
22901 assert_eq!(asked.len(), 1);
22902 assert!(!asked[0].detail.contains("secret"));
22903 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22904 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22905 assert_eq!(
22906 asked[0]
22907 .choices
22908 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22909 rewrite.is_some()
22910 );
22911 store
22914 .update(&asked[0].id, |q| {
22915 q.abandon("test");
22916 Ok(())
22917 })
22918 .unwrap();
22919 let again = runner
22920 .guarded_pr_message(&winner, None, true)
22921 .await
22922 .unwrap()
22923 .expect("an abandoned question falls back");
22924 let n = store
22925 .list()
22926 .into_iter()
22927 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22928 .count();
22929 assert_eq!(n, 1, "asked once per rejected text");
22930 again
22931 };
22932 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22933 assert!(!message.body.contains("secret"));
22934 assert!(!message.body.contains("prefix"));
22935 assert!(!message.title.contains("secret"));
22936 assert_eq!(
22937 message.body.contains("Add retries for failed requests."),
22938 accepted
22939 );
22940 assert_eq!(
22941 runner.state.seats["github-text-rewrite"].turns, 1,
22942 "only one rewrite invocation"
22943 );
22944 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22945 if accepted {
22946 assert!(runner.state.github_text.is_none());
22947 assert!(!runner.state.parked);
22948 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22949 assert!(message.body.contains("## Final verification"));
22950 assert!(message.body.contains("Evidence: final diff"));
22951 assert!(
22952 message
22953 .body
22954 .contains(&format!("magi:run/{}", runner.state.id))
22955 );
22956 }
22957 if !accepted {
22958 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22959 assert!(
22960 message
22961 .body
22962 .contains(&format!("magi:run/{}", runner.state.id))
22963 );
22964 }
22965 }
22966 }
22967
22968 fn scripted(id: &str, reply: &str) -> AgentSpec {
22969 let mut a = spec(id);
22970 a.command = vec![
22971 "sh".into(),
22972 "-c".into(),
22973 "printf '%s' \"$REWRITE_REPLY\"".into(),
22974 ];
22975 a.env.insert("REWRITE_REPLY".into(), reply.into());
22976 a
22977 }
22978
22979 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22980 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22981
22982 fn seed_public_summary(runner: &mut Runner) {
22984 let repo = &runner.state.repo;
22985 let g = |args: &[&str]| {
22986 let out = std::process::Command::new("git")
22987 .args(args)
22988 .current_dir(repo)
22989 .quiet()
22990 .output()
22991 .unwrap();
22992 assert!(out.status.success(), "git {args:?}");
22993 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22994 };
22995 g(&["init", "-q", "-b", "main"]);
22996 g(&[
22997 "-c",
22998 "user.name=Test",
22999 "-c",
23000 "user.email=test@example.invalid",
23001 "commit",
23002 "-q",
23003 "--allow-empty",
23004 "-m",
23005 "feat: fixture",
23006 ]);
23007 let head = g(&["rev-parse", "HEAD"]);
23008 g(&["update-ref", "refs/remotes/origin/main", &head]);
23009 let evidence = crate::pr_outcome::summary_evidence(
23010 &runner.state,
23011 crate::pr_outcome::FinalHead::At(&head),
23012 );
23013 let generated = &runner.state.candidates[0].summary;
23014 runner.state.change_summary = Some(crate::run::ChangeSummary {
23015 base: head.clone(),
23016 head,
23017 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23018 title: summary_title(generated).unwrap(),
23019 body: summary_without_title(generated),
23020 });
23021 }
23022
23023 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23024 let mut runner = runner_at(RunStatus::Gating);
23025 runner.state = state_with_summary("add retries", JA_SUMMARY);
23026 if review_only {
23027 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23028 }
23029 runner.state.repo = dir.to_owned();
23030 runner.state.candidates[0].worktree = dir.to_owned();
23031 runner.state.config.agents = agents;
23032 seed_public_summary(&mut runner);
23033 runner
23034 }
23035
23036 fn gate_questions(runner: &Runner) -> usize {
23037 ask::Questions::open()
23038 .list()
23039 .into_iter()
23040 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23041 .count()
23042 }
23043
23044 #[tokio::test]
23045 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23046 crate::run::pin_test_home();
23047 let dir = tempfile::tempdir().unwrap();
23048 let mut runner = rewrite_runner(
23049 dir.path(),
23050 true,
23051 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23052 );
23053 runner.state.config.roles.language_judge =
23054 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23055 let winner = runner.state.candidates[0].clone();
23056 let msg = runner
23057 .guarded_pr_message(&winner, None, true)
23058 .await
23059 .unwrap()
23060 .expect("an English rewrite posts without asking");
23061 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23062 assert!(msg.body.contains("<details>kept</details>"));
23063 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23064 assert_eq!(gate_questions(&runner), 0);
23065 assert!(runner.state.github_text.is_none());
23066 assert!(!runner.state.seats.contains_key("impl-A"));
23067 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23068 }
23069
23070 #[tokio::test]
23071 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23072 crate::run::pin_test_home();
23073 let dir = tempfile::tempdir().unwrap();
23074 let mut runner = rewrite_runner(
23075 dir.path(),
23076 false,
23077 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23078 );
23079 runner.state.config.roles.language_judge =
23080 Some(crate::config::AgentChoice::One("beta".into()));
23081 runner.state.seats.insert(
23082 "impl-A".into(),
23083 SeatState::new("impl-A", "alpha", runner.state.seed),
23084 );
23085 let winner = runner.state.candidates[0].clone();
23086 let msg = runner
23087 .guarded_pr_message(&winner, None, true)
23088 .await
23089 .unwrap()
23090 .expect("rewritten");
23091 assert!(msg.body.contains("<details>kept</details>"));
23092 assert_eq!(runner.state.seats["impl-A"].turns, 0);
23093 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23094 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23095 }
23096
23097 #[tokio::test]
23098 async fn rewrite_reuses_a_resumable_implementer_session() {
23099 crate::run::pin_test_home();
23100 let dir = tempfile::tempdir().unwrap();
23101 let mut runner = rewrite_runner(
23102 dir.path(),
23103 false,
23104 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23105 );
23106 runner.state.config.roles.language_judge =
23107 Some(crate::config::AgentChoice::One("beta".into()));
23108 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23109 seat.turns = 1;
23110 runner.state.seats.insert("impl-A".into(), seat);
23111 let winner = runner.state.candidates[0].clone();
23112 runner
23113 .guarded_pr_message(&winner, None, true)
23114 .await
23115 .unwrap()
23116 .expect("rewritten");
23117 assert_eq!(runner.state.seats["impl-A"].turns, 2);
23118 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23119 }
23120
23121 #[tokio::test]
23122 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23123 crate::run::pin_test_home();
23126 let dir = tempfile::tempdir().unwrap();
23127 let mut exhausted = spec("alpha");
23128 exhausted.command = vec!["false".into()];
23129 let mut runner = rewrite_runner(
23130 dir.path(),
23131 false,
23132 vec![exhausted, scripted("beta", EN_REWRITE)],
23133 );
23134 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23136 "alpha".into(),
23137 "beta".into(),
23138 ]));
23139 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23140 seat.turns = 1;
23141 runner.state.seats.insert("impl-A".into(), seat);
23142 let winner = runner.state.candidates[0].clone();
23143 let msg = runner
23144 .guarded_pr_message(&winner, None, true)
23145 .await
23146 .unwrap()
23147 .expect("the role rewrites after the implementer fails");
23148 assert!(msg.body.contains("`src/net.rs`"));
23149 assert_eq!(
23150 runner.state.seats["impl-A"].turns, 2,
23151 "the implementer was tried once"
23152 );
23153 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23154 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23155 }
23156
23157 #[tokio::test]
23158 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23159 crate::run::pin_test_home();
23160 let dir = tempfile::tempdir().unwrap();
23161 let mut broken = spec("bad");
23162 broken.command = vec!["false".into()];
23163 let mut runner = rewrite_runner(
23164 dir.path(),
23165 true,
23166 vec![
23167 broken,
23168 scripted("noformat", "Add retries."),
23169 scripted("good", EN_REWRITE),
23170 ],
23171 );
23172 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23173 "bad".into(),
23174 "noformat".into(),
23175 "good".into(),
23176 ]));
23177 let winner = runner.state.candidates[0].clone();
23178 let msg = runner
23179 .guarded_pr_message(&winner, None, true)
23180 .await
23181 .unwrap()
23182 .expect("the third agent rewrites");
23183 assert!(msg.body.contains("`src/net.rs`"));
23184 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23185 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23186 }
23187
23188 #[tokio::test]
23189 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23190 crate::run::pin_test_home();
23191 for (agents, role, saved) in [
23193 (
23194 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23195 Some("beta"),
23196 true,
23197 ),
23198 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23199 (Vec::new(), None, false),
23200 ] {
23201 let dir = tempfile::tempdir().unwrap();
23202 let mut runner = rewrite_runner(dir.path(), true, agents);
23203 runner.state.config.roles.language_judge =
23204 role.map(|r| crate::config::AgentChoice::One(r.into()));
23205 let winner = runner.state.candidates[0].clone();
23206 let first = runner
23207 .guarded_pr_message(&winner, None, true)
23208 .await
23209 .unwrap();
23210 assert!(first.is_none(), "withheld text parks the run");
23211 assert!(runner.state.parked);
23212 assert_eq!(gate_questions(&runner), 1);
23213 let g = runner.state.github_text.as_ref().unwrap();
23214 assert_eq!(g.rewrite.is_some(), saved);
23215 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23216 }
23217 }
23218
23219 #[tokio::test]
23220 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23221 crate::run::pin_test_home();
23222 let dir = tempfile::tempdir().unwrap();
23223 let mut runner = runner_at(RunStatus::Gating);
23224 runner.state =
23225 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23226 runner.state.repo = dir.path().to_owned();
23227 runner.state.candidates[0].worktree = dir.path().to_owned();
23228 let mut author = spec("alpha");
23229 author.command = vec![
23230 "sh".into(),
23231 "-c".into(),
23232 "printf '%s' \"$REWRITE_REPLY\"".into(),
23233 ];
23234 author.env.insert(
23235 "REWRITE_REPLY".into(),
23236 format!(
23237 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23238 "English sentence. ".repeat(600)
23239 ),
23240 );
23241 runner.state.config.agents = vec![author];
23242 seed_public_summary(&mut runner);
23243 let winner = runner.state.candidates[0].clone();
23244 assert!(
23245 runner
23246 .guarded_pr_message(&winner, None, true)
23247 .await
23248 .unwrap()
23249 .is_none()
23250 );
23251 let record = runner.state.github_text.clone().unwrap();
23252 let rewrite = record.rewrite.clone().unwrap();
23253 assert_eq!(rewrite.failed, ["body-language"]);
23254 assert!(
23255 rewrite
23256 .body
23257 .ends_with(&format!("magi:run/{}", runner.state.id))
23258 );
23259 let store = ask::Questions::open();
23260 let q = store.get(record.question.as_ref().unwrap()).unwrap();
23261 assert!(
23262 q.choices
23263 .contains(&crate::github_text::USE_REWRITE.to_owned())
23264 );
23265 for label in [
23266 "ORIGINAL",
23267 "FALLBACK",
23268 "REWRITE",
23269 "Rules still failed: body-language",
23270 "truncated",
23271 ] {
23272 assert!(q.detail.contains(label), "{label}");
23273 }
23274 let artifact = std::fs::read_to_string(crate::run::artifact_path(
23275 &runner.state,
23276 &crate::github_text::artifact_name(&record.fingerprint),
23277 ))
23278 .unwrap();
23279 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23280 runner.state =
23282 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23283 store
23284 .update(&q.id, |q| {
23285 q.say("まだ日本語")?;
23286 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23287 })
23288 .unwrap();
23289 assert!(
23290 runner
23291 .guarded_pr_message(&winner, None, true)
23292 .await
23293 .unwrap()
23294 .is_none()
23295 );
23296 let retry = runner.state.github_text.as_ref().unwrap();
23297 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23298 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23299 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23300 assert!(
23301 q.posting_comparison
23302 .as_ref()
23303 .unwrap()
23304 .matches_detail(&q.detail)
23305 );
23306 store
23307 .update(&q.id, |q| {
23308 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23309 })
23310 .unwrap();
23311 let message = runner
23312 .guarded_pr_message(&winner, None, true)
23313 .await
23314 .unwrap()
23315 .unwrap();
23316 assert_eq!(message.title, rewrite.title);
23317 assert_eq!(message.body, rewrite.body);
23318 assert_eq!(
23319 runner.state.github_text.as_ref().unwrap().choice,
23320 Some(GithubTextChoice::Rewrite)
23321 );
23322 let again = runner
23323 .guarded_pr_message(&winner, None, true)
23324 .await
23325 .unwrap()
23326 .unwrap();
23327 assert_eq!(again.title, message.title);
23328 assert_eq!(again.body, message.body);
23329 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23330 assert!(
23331 !runner
23332 .state
23333 .events
23334 .iter()
23335 .any(|e| e.message.contains(&rewrite.title)
23336 || e.message.contains("English sentence."))
23337 );
23338 }
23339
23340 #[tokio::test]
23341 async fn historical_posting_reply_never_expires_into_fallback() {
23342 crate::run::pin_test_home();
23343 let dir = tempfile::tempdir().unwrap();
23344 let mut runner = runner_at(RunStatus::Gating);
23345 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23346 runner.state.repo = dir.path().to_owned();
23347 runner.state.candidates[0].worktree = dir.path().to_owned();
23348 seed_public_summary(&mut runner);
23349 let winner = runner.state.candidates[0].clone();
23350 assert!(
23351 runner
23352 .guarded_pr_message(&winner, None, true)
23353 .await
23354 .unwrap()
23355 .is_none()
23356 );
23357 let id = runner
23358 .state
23359 .github_text
23360 .as_ref()
23361 .unwrap()
23362 .question
23363 .clone()
23364 .unwrap();
23365 let store = ask::Questions::open();
23366 let mut q = store.get(&id).unwrap();
23367 q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23368 q.answer(ask::Answer::Choice(
23369 crate::github_text::POST_ORIGINAL.into(),
23370 ))
23371 .unwrap();
23372 q.answer_delivered = true;
23373 store.put(&mut q).unwrap();
23374 let source = std::fs::read(store.path_of(&id)).unwrap();
23375 let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23376 std::fs::create_dir_all(&migration).unwrap();
23377 std::fs::write(
23378 migration.join("state.json"),
23379 serde_json::to_vec(&serde_json::json!({
23380 "schema": 1, "sources": [id], "pending": [id]
23381 }))
23382 .unwrap(),
23383 )
23384 .unwrap();
23385 for _ in 0..2 {
23386 assert!(
23387 runner
23388 .guarded_pr_message(&winner, None, true)
23389 .await
23390 .unwrap()
23391 .is_none()
23392 );
23393 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23394 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23395 }
23396 let home = crate::run::home();
23397 crate::task_chat::sweep(
23398 &crate::queue::Queue::at(home.join("queue")),
23399 &crate::talk::Talks::at(home.join("talks")),
23400 &store,
23401 &home,
23402 );
23403 let outbox = crate::task_inquiry::Outbox::at(&home);
23404 outbox
23405 .update_decision(&id, |d| {
23406 d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23407 d.source_answer_at = q.answered_at;
23408 Ok(true)
23409 })
23410 .unwrap();
23411 outbox
23412 .settle(
23413 &id,
23414 "owner-chat",
23415 "Owner confirms no publication remains necessary",
23416 None,
23417 )
23418 .unwrap();
23419 assert!(
23420 runner
23421 .guarded_pr_message(&winner, None, true)
23422 .await
23423 .unwrap()
23424 .is_none()
23425 );
23426 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23427 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23428 }
23429
23430 #[tokio::test]
23431 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23432 crate::run::pin_test_home();
23433 let dir = tempfile::tempdir().unwrap();
23434 let mut runner = runner_at(RunStatus::Gating);
23435 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23436 runner.state.repo = dir.path().to_owned();
23437 runner.state.candidates[0].worktree = dir.path().to_owned();
23438 seed_public_summary(&mut runner);
23439 let winner = runner.state.candidates[0].clone();
23440 assert!(
23441 runner
23442 .guarded_pr_message(&winner, None, true)
23443 .await
23444 .unwrap()
23445 .is_none()
23446 );
23447 let record = runner.state.github_text.as_ref().unwrap();
23448 assert!(record.rewrite.is_none());
23449 let mut old = serde_json::to_value(record).unwrap();
23451 old.as_object_mut().unwrap().remove("rewrite");
23452 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23453 assert!(old.rewrite.is_none());
23454 let id = record.question.clone().unwrap();
23455 ask::Questions::open()
23456 .update(&id, |q| {
23457 q.choices.push(crate::github_text::USE_REWRITE.into());
23458 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23459 })
23460 .unwrap();
23461 let message = runner
23462 .guarded_pr_message(&winner, None, true)
23463 .await
23464 .unwrap()
23465 .unwrap();
23466 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23467 assert_eq!(
23468 runner.state.github_text.as_ref().unwrap().choice,
23469 Some(GithubTextChoice::Fallback)
23470 );
23471 }
23472
23473 #[tokio::test]
23474 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23475 crate::run::pin_test_home();
23476 let dir = tempfile::tempdir().unwrap();
23477 let mut runner = runner_at(RunStatus::Gating);
23478 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23479 runner.state.repo = dir.path().to_owned();
23480 runner.state.candidates[0].worktree = dir.path().to_owned();
23481 seed_public_summary(&mut runner);
23482 let winner = runner.state.candidates[0].clone();
23483 assert!(
23484 runner
23485 .guarded_pr_message(&winner, None, true)
23486 .await
23487 .unwrap()
23488 .is_none()
23489 );
23490 let store = ask::Questions::open();
23491 let find = |run: &str| {
23492 store
23493 .list()
23494 .into_iter()
23495 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23496 .unwrap()
23497 };
23498 let run = runner.state.id.clone();
23499 let q = find(&run);
23500 let fp = runner
23501 .state
23502 .github_text
23503 .as_ref()
23504 .unwrap()
23505 .fingerprint
23506 .clone();
23507 let name = crate::github_text::artifact_name(&fp);
23508 let path = crate::run::artifact_path(&runner.state, &name);
23509 assert!(q.detail.contains(&path.display().to_string()));
23510 assert!(q.detail.contains("title-language: title"));
23511 let saved = std::fs::read_to_string(&path).expect("artifact written");
23512 assert!(saved.contains("日本語のタイトル"));
23513 assert!(q.cwd.is_none());
23514 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23515 let say = |id: &str, text: &str, choice: &str| {
23517 store
23518 .update(id, |q| {
23519 q.thread.push(ask::Turn {
23520 who: ask::Who::Operator,
23521 body: text.to_owned(),
23522 at: jiff::Timestamp::now(),
23523 note: None,
23524 });
23525 q.answer(ask::Answer::Choice(choice.to_owned()))
23526 })
23527 .unwrap();
23528 };
23529 say(&q.id, "まだ日本語", "use my text");
23530 assert!(
23531 runner
23532 .guarded_pr_message(&winner, None, true)
23533 .await
23534 .unwrap()
23535 .is_none()
23536 );
23537 let second = find(&run);
23538 assert_ne!(second.id, q.id);
23539 assert!(second.detail.contains("did not pass"));
23540 assert!(second.detail.contains("日本語のタイトル"));
23543 assert!(!second.detail.contains("まだ日本語"));
23544 assert!(second.detail.contains(&path.display().to_string()));
23545 say(&second.id, "fix: retry failed requests", "use my text");
23546 let message = runner
23547 .guarded_pr_message(&winner, None, true)
23548 .await
23549 .unwrap()
23550 .expect("a vetted replacement is posted");
23551 assert_eq!(message.title, "fix: retry failed requests");
23552 assert_eq!(
23553 runner
23554 .state
23555 .github_text
23556 .as_ref()
23557 .unwrap()
23558 .chosen_title
23559 .as_deref(),
23560 Some("fix: retry failed requests")
23561 );
23562 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23563 let again = runner
23565 .guarded_pr_message(&winner, None, true)
23566 .await
23567 .unwrap()
23568 .unwrap();
23569 assert_eq!(again.title, message.title);
23570 assert_eq!(
23571 store
23572 .list()
23573 .into_iter()
23574 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23575 .count(),
23576 2
23577 );
23578 }
23579
23580 #[test]
23581 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23582 let mut state = state_with_summary(
23586 "add retries",
23587 "TITLE: fix(web): batch reads\n- reads run.json once",
23588 );
23589 state.config.graph.language = "ja".to_owned();
23590 let m = pr_message(&state, 'A');
23591 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23592 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23593
23594 let task = "今回やってほしいこと: results projector を直す";
23597 let mut state = state_with_summary(task, "- no title line");
23598 state.config.graph.language = "ja".to_owned();
23599 let m = pr_message(&state, 'A');
23600 assert_eq!(
23601 m.title,
23602 format!("chore: land candidate A of run {}", state.id)
23603 );
23604 assert!(
23605 m.body.contains(&format!(
23606 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23607 )),
23608 "{}",
23609 m.body
23610 );
23611 }
23612
23613 #[test]
23614 fn pr_message_scrubs_home_paths_and_addresses() {
23615 let state = state_with_summary(
23616 "fix it in /Users/someone/src/x",
23617 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23618 );
23619 let m = pr_message(&state, 'A');
23620 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23621 assert!(!m.body.contains(leak), "{}", m.body);
23622 }
23623 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23624 }
23625
23626 #[test]
23627 fn pr_message_survives_a_task_that_closes_details() {
23628 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23629 let m = pr_message(&state, 'A');
23630 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23631 }
23632
23633 #[test]
23634 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23635 let cmd = manual_merge_command(
23636 MergeStyle::Squash,
23637 Path::new("/repo"),
23638 "b",
23639 "fix: \"quoted\" $(x) `y`\n\nbody",
23640 );
23641 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23642 }
23643
23644 #[test]
23645 fn manual_merge_command_matches_the_configured_style() {
23646 let repo = Path::new("/repo");
23647 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23648
23649 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23650 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23651
23652 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23653 assert_eq!(
23654 squash,
23655 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23656 \"Merge magi run 0832 (candidate A)\""
23657 );
23658
23659 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23660 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23661 }
23662
23663 #[test]
23664 fn a_nudge_gets_a_quarter_of_the_budget() {
23665 assert_eq!(retry_budget(secs(1200), true), secs(300));
23667 assert_eq!(retry_budget(secs(3600), true), secs(900));
23668 }
23669
23670 #[test]
23671 fn a_resent_prompt_keeps_the_whole_budget() {
23672 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23675 assert_eq!(retry_budget(secs(60), false), secs(60));
23676 }
23677
23678 #[test]
23679 fn the_floor_never_exceeds_the_original_budget() {
23680 assert_eq!(retry_budget(secs(60), true), secs(60));
23684 assert_eq!(retry_budget(secs(480), true), secs(120));
23685 assert_eq!(retry_budget(secs(0), true), secs(0));
23686 }
23687
23688 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23689 agent::CommandEvidence {
23690 id: "item1".to_owned(),
23691 description: "cargo test".to_owned(),
23692 exit_code,
23693 result_summary: String::new(),
23694 source: "codex".to_owned(),
23695 }
23696 }
23697
23698 #[test]
23699 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23700 assert!(!has_unconfirmed_command(&[]));
23704 }
23705
23706 #[test]
23707 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23708 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23712 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23713 assert!(!has_unconfirmed_command(&[
23714 evidence(Some(0)),
23715 evidence(Some(101))
23716 ]));
23717 }
23718
23719 #[test]
23720 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23721 assert!(has_unconfirmed_command(&[
23722 evidence(Some(0)),
23723 evidence(None)
23724 ]));
23725 }
23726
23727 #[test]
23728 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23729 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23730 assert_eq!(
23731 verified_noop_claim(true, &[], text).as_deref(),
23732 Some("already fixed by b32cfc4, on main.")
23733 );
23734 }
23735
23736 #[test]
23737 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23738 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23741 assert!(verified_noop_claim(false, &[], text).is_none());
23742 }
23743
23744 #[test]
23745 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23746 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23747 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23748 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23750 }
23751
23752 #[test]
23753 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23754 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23755 }
23756
23757 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23760 runner.state.candidates = shape
23761 .iter()
23762 .enumerate()
23763 .map(|(i, &(empty, verified))| Candidate {
23764 index: i,
23765 label: (b'A' + i as u8) as char,
23766 agent: "sonnet".to_owned(),
23767 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23768 worktree: PathBuf::from(format!("/wt/{i}")),
23769 summary: String::new(),
23770 stat: String::new(),
23771 files: 0,
23772 commits: 0,
23773 empty,
23774 failed: None,
23775 verified_noop: verified.map(str::to_owned),
23776 duration_ms: 0,
23777 folded: false,
23778 })
23779 .collect();
23780 }
23781
23782 #[test]
23783 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23784 ask_test_home();
23785 let mut runner = runner_at(RunStatus::Implementing);
23786 set_candidates(
23787 &mut runner,
23788 &[
23789 (true, Some("already on main at b32cfc4")),
23790 (true, Some("same fix, see the existing test")),
23791 ],
23792 );
23793
23794 runner
23795 .after_implement()
23796 .expect("a verified no-op is not an error");
23797
23798 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23799 }
23800
23801 #[test]
23802 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23803 ask_test_home();
23804 let mut runner = runner_at(RunStatus::Implementing);
23805 set_candidates(
23809 &mut runner,
23810 &[(true, Some("already on main at b32cfc4")), (true, None)],
23811 );
23812
23813 let err = runner
23814 .after_implement()
23815 .expect_err("an unverified empty candidate must still fail the run");
23816
23817 assert!(
23818 err.to_string().contains("no candidate produced a change"),
23819 "{err}"
23820 );
23821 assert_eq!(runner.state.status, RunStatus::Failed);
23822 }
23823
23824 #[test]
23825 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23826 ask_test_home();
23827 let mut runner = runner_at(RunStatus::Implementing);
23828 set_candidates(&mut runner, &[(true, None), (true, None)]);
23829
23830 let err = runner
23831 .after_implement()
23832 .expect_err("no candidate declared anything; this is an ordinary failure");
23833
23834 assert!(
23835 err.to_string().contains("no candidate produced a change"),
23836 "{err}"
23837 );
23838 assert_eq!(runner.state.status, RunStatus::Failed);
23839 }
23840}
23841
23842#[derive(Debug)]
23845struct GraftRestoreFailed(String);
23846
23847impl std::fmt::Display for GraftRestoreFailed {
23848 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23849 write!(
23850 f,
23851 "could not restore the winner to its pre-graft commit: {}",
23852 self.0
23853 )
23854 }
23855}
23856
23857impl std::error::Error for GraftRestoreFailed {}
23858
23859#[cfg(test)]
23863mod resource_wait_tests {
23864 use super::*;
23865 use std::collections::BTreeMap;
23866
23867 fn sh(id: &str, script: &str) -> AgentSpec {
23868 AgentSpec {
23869 id: id.to_owned(),
23870 kind: crate::config::AgentKind::Command,
23871 model: None,
23872 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23873 extra_args: Vec::new(),
23874 env: BTreeMap::new(),
23875 prompt_delivery: None,
23876 max_turn_secs: None,
23877 }
23878 }
23879
23880 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23881 SeatJob {
23882 seat: SeatState::new("impl-A", &spec.id, 1),
23883 spec,
23884 cwd: dir.to_path_buf(),
23885 prompt: "do it".to_owned(),
23886 timeout,
23887 allow_write: true,
23888 sessions: false,
23889 artifacts: dir.join("art"),
23890 stem: "impl-A".to_owned(),
23891 handover: None,
23892 }
23893 }
23894
23895 fn state(resource_wait_secs: u64) -> RunState {
23896 let mut config = Config::default();
23897 config.graph.timeout_resource_wait = resource_wait_secs;
23898 RunState::new(
23899 PathBuf::from("/repo"),
23900 "main".to_owned(),
23901 "abc1234".to_owned(),
23902 "task".to_owned(),
23903 config,
23904 )
23905 }
23906
23907 fn hold(cache: &Path) -> crate::cache::Guard {
23908 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23909 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23910 crate::cache::AcquireOutcome::Acquired(g) => g,
23911 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23912 }
23913 }
23914
23915 async fn run(
23916 jobs: Vec<SeatJob>,
23917 sem: Arc<Semaphore>,
23918 cache: Option<&Path>,
23919 st: &mut RunState,
23920 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23921 let prompts = Prompts::default();
23922 let ctx = WaveCtx {
23923 run: "r",
23924 node: "implement",
23925 prompts: &prompts,
23926 cache,
23927 round: None,
23928 carry_seats: false,
23929 pauses: None,
23930 };
23931 wave(jobs, sem, &ctx, st, 0).await
23932 }
23933
23934 #[tokio::test]
23935 async fn verification_cache_wait_preserves_the_command_budget() {
23936 crate::run::pin_test_home();
23937 let tmp = tempfile::tempdir().unwrap();
23938 let cache = tmp.path().join("target");
23939 let guard = hold(&cache);
23940 let release = tokio::spawn(async move {
23941 tokio::time::sleep(Duration::from_millis(100)).await;
23942 guard.release();
23943 });
23944 let mut st = state(10);
23945 let budget = Duration::from_millis(50);
23946 let wait = repository_wait(&st, None);
23947 let (outcomes, _) = with_cache_lease(
23948 &mut st,
23949 Some(&cache),
23950 "gate",
23951 "gate",
23952 tmp.path(),
23953 "abc1234",
23954 budget,
23955 "test gate",
23956 &wait,
23957 |_, available| async move {
23958 assert_eq!(
23959 available, budget,
23960 "contention must not shorten verification"
23961 );
23962 (Vec::new(), false, Vec::new())
23963 },
23964 )
23965 .await
23966 .unwrap();
23967 release.await.unwrap();
23968 assert!(outcomes.is_empty(), "{outcomes:?}");
23969 assert!(
23970 st.events
23971 .iter()
23972 .any(|event| event.message.contains("waiting for"))
23973 );
23974 assert!(st.resource_wait.is_none());
23975 hold(&cache).release();
23977 }
23978
23979 #[tokio::test]
23980 async fn verification_cache_wait_expires_without_starting_commands() {
23981 crate::run::pin_test_home();
23982 let tmp = tempfile::tempdir().unwrap();
23983 let cache = tmp.path().join("target");
23984 let _held = hold(&cache);
23985 let mut st = state(0);
23986 let wait = repository_wait(&st, None);
23987 let (outcomes, _) = with_cache_lease(
23988 &mut st,
23989 Some(&cache),
23990 "gate",
23991 "gate",
23992 tmp.path(),
23993 "abc1234",
23994 Duration::from_secs(60),
23995 "test gate",
23996 &wait,
23997 |_, _| async { panic!("commands must not start without the cache lease") },
23998 )
23999 .await
24000 .unwrap();
24001 assert_eq!(outcomes.len(), 1);
24002 assert!(outcomes[0].resource_blocked);
24003 assert!(st.resource_wait.is_none());
24004 assert!(matches!(
24005 st.resource_block.unwrap().cause,
24006 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24007 ));
24008 }
24009
24010 #[tokio::test]
24011 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24012 crate::run::pin_test_home();
24013 let tmp = tempfile::tempdir().unwrap();
24014 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24015 let cache = tmp.path().join("cache");
24016 let mut state = state(21600);
24017 let wait = repository_wait(&state, None);
24018 let (outcomes, _) = with_cache_lease(
24019 &mut state,
24020 Some(&cache),
24021 "gate",
24022 "gate",
24023 tmp.path(),
24024 "head",
24025 Duration::from_secs(60),
24026 "gate",
24027 &wait,
24028 |_, _| async { panic!("freshness failure must not start verification") },
24029 )
24030 .await
24031 .unwrap();
24032 assert_eq!(outcomes.len(), 1);
24033 assert!(outcomes[0].resource_blocked);
24034 let block = state.resource_block.unwrap();
24035 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24036 assert!(!block.description().contains("timed out"));
24037 }
24038
24039 #[tokio::test]
24040 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24041 crate::run::pin_test_home();
24042 let tmp = tempfile::tempdir().unwrap();
24043 let cache = tmp.path().join("cache");
24044 let _held = hold(&cache);
24045 let mut state = state(21600);
24046 let pause = Pause::new();
24047 let interrupt = Pause::new();
24048 let cause = ParkCause::Operator;
24049 pause.park_for(cause.clone());
24050 let prompts = Prompts::default();
24051 let ctx = WaveCtx {
24052 run: "test",
24053 node: "fix",
24054 prompts: &prompts,
24055 cache: Some(&cache),
24056 round: None,
24057 carry_seats: false,
24058 pauses: Some((&pause, &interrupt)),
24059 };
24060 let result = wave(
24061 vec![job(
24062 sh("writer", "exit 99"),
24063 tmp.path(),
24064 Duration::from_secs(1),
24065 )],
24066 Arc::new(Semaphore::new(1)),
24067 &ctx,
24068 &mut state,
24069 0,
24070 )
24071 .await;
24072 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24073 assert_eq!(result[0].1.turns, 0);
24074 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24075 assert_eq!(
24076 state.resource_block.unwrap().cause,
24077 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24078 );
24079 assert!(state.active.is_empty());
24080 }
24081
24082 #[tokio::test]
24083 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24084 crate::run::pin_test_home();
24085 let tmp = tempfile::tempdir().unwrap();
24086 let cache = tmp.path().join("target-a");
24087 let guard = hold(&cache);
24088 tokio::spawn(async move {
24090 tokio::time::sleep(Duration::from_millis(1500)).await;
24091 guard.release();
24092 });
24093 let mut st = state(60);
24094 let results = run(
24095 vec![job(
24096 sh("alpha", "echo ok"),
24097 tmp.path(),
24098 Duration::from_secs(1),
24099 )],
24100 Arc::new(Semaphore::new(1)),
24101 Some(&cache),
24102 &mut st,
24103 )
24104 .await;
24105 assert!(
24106 matches!(results[0].2, AgentOutcome::Ok(_)),
24107 "the wait must not be charged to the agent"
24108 );
24109 assert!(st.resource_block.is_none());
24110 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24111 assert!(st.resource_wait.is_none() && st.active.is_empty());
24112 }
24113
24114 #[tokio::test]
24115 #[cfg(unix)]
24116 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24117 use std::os::fd::AsRawFd as _;
24118 crate::run::pin_test_home();
24119 let tmp = tempfile::tempdir().unwrap();
24120 crate::git::git(tmp.path(), &["init"]).await.unwrap();
24121 let common = tmp.path().join(".git");
24122 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24123 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24124 assert_eq!(
24125 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24126 0
24127 );
24128 let mut st = state(1);
24129 st.config.coordination = crate::config::test_coordination();
24130 let mut reader = job(
24131 sh("reader", "echo must-not-run"),
24132 tmp.path(),
24133 Duration::from_secs(30),
24134 );
24135 reader.allow_write = false;
24136 let results = run(
24137 vec![reader.clone()],
24138 Arc::new(Semaphore::new(1)),
24139 None,
24140 &mut st,
24141 )
24142 .await;
24143 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24144 assert_eq!(results[0].1.turns, 0);
24145 assert!(FailClass::of(&results[0].2).is_none());
24146 assert_eq!(
24147 st.resource_block.as_ref().unwrap().kind,
24148 "repository coordination"
24149 );
24150 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24151 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24152 drop(external);
24153 let editor = crate::repo_lock::SeatLease::for_checkout(
24154 tmp.path(),
24155 true,
24156 crate::config::test_coordination().as_ref(),
24157 )
24158 .await
24159 .unwrap();
24160 let pause = Pause::new();
24161 let interrupt = Pause::new();
24162 let prompts = Prompts::default();
24163 let ctx = WaveCtx {
24164 run: "r",
24165 node: "review",
24166 prompts: &prompts,
24167 cache: None,
24168 round: None,
24169 carry_seats: false,
24170 pauses: Some((&pause, &interrupt)),
24171 };
24172 let (results, ()) = tokio::join!(
24173 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24174 async {
24175 tokio::time::sleep(Duration::from_millis(50)).await;
24176 pause.park();
24177 }
24178 );
24179 assert!(
24180 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24181 );
24182 assert_eq!(results[0].1.turns, 0);
24183 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24184 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24185 drop(editor);
24186 }
24187
24188 #[tokio::test]
24189 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24190 crate::run::pin_test_home();
24191 let tmp = tempfile::tempdir().unwrap();
24192 let cache = tmp.path().join("target-b");
24193 let _held = hold(&cache);
24194 let mut st = state(1);
24195 let results = run(
24196 vec![job(
24197 sh("alpha", "echo ok"),
24198 tmp.path(),
24199 Duration::from_secs(1),
24200 )],
24201 Arc::new(Semaphore::new(1)),
24202 Some(&cache),
24203 &mut st,
24204 )
24205 .await;
24206 let (_, seat, out) = &results[0];
24207 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24208 assert!(wave_blocked(&results));
24209 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24210 assert_eq!(seat.turns, 0, "nothing was asked");
24211 assert!(
24212 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24213 "no agent was spawned"
24214 );
24215 assert!(FailClass::of(out).is_none(), "not a failure class");
24216 let block = st.resource_block.as_ref().expect("recorded for the resume");
24217 assert_eq!(block.kind, "shared build cache");
24218 assert!(matches!(
24219 block.cause,
24220 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24221 ));
24222 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24223 assert!(st.active.is_empty());
24224 }
24225
24226 #[tokio::test]
24227 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24228 crate::run::pin_test_home();
24229 let tmp = tempfile::tempdir().unwrap();
24230 let mut st = state(1);
24231 let results = run(
24232 vec![job(
24233 sh("alpha", "echo ok"),
24234 tmp.path(),
24235 Duration::from_secs(1),
24236 )],
24237 Arc::new(Semaphore::new(0)),
24238 None,
24239 &mut st,
24240 )
24241 .await;
24242 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24243 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24244 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24245 }
24246
24247 #[tokio::test]
24248 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24249 crate::run::pin_test_home();
24250 let tmp = tempfile::tempdir().unwrap();
24251 let mut st = state(60);
24252 let results = run(
24253 vec![job(
24254 sh("alpha", "sleep 30"),
24255 tmp.path(),
24256 Duration::from_secs(1),
24257 )],
24258 Arc::new(Semaphore::new(1)),
24259 None,
24260 &mut st,
24261 )
24262 .await;
24263 let out = &results[0].2;
24264 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24265 let class = FailClass::of(out).expect("a failure");
24266 assert_eq!(class, FailClass::Timeout);
24267 assert!(should_hand_over(None, &class), "existing handover policy");
24268 assert!(st.resource_block.is_none());
24269 }
24270}