1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false, None).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true, None).await
1357 }
1358
1359 pub(crate) async fn authorize_base_recovery_control(
1360 home: &Path,
1361 task: &str,
1362 control: &mut crate::base_recovery_control::Transaction,
1363 ) -> Result<()> {
1364 let id = control.request.run.clone();
1365 Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366 }
1367
1368 async fn authorize_base_recovery_request(
1369 home: &Path,
1370 id: &str,
1371 task: &str,
1372 resumption: bool,
1373 mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374 ) -> Result<()> {
1375 let _owner = crate::pr_head::owner_lock(home, id)?;
1376 let mut state = RunState::load_under(id, home)?;
1377 crate::pr_head::ensure_owner(&state)?;
1378 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379 anyhow::ensure!(
1380 queued.execution_runs().last().is_some_and(|run| run == id)
1381 && queued.status != crate::queue::TaskStatus::Done
1382 && !queued.fresh_start
1383 && queued.review_branch.is_none(),
1384 "run task ownership changed"
1385 );
1386 anyhow::ensure!(
1387 state
1388 .origin
1389 .as_ref()
1390 .and_then(|o| o.task.as_deref())
1391 .is_none_or(|owner| owner == task),
1392 "run belongs to another task"
1393 );
1394 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395 anyhow::ensure!(
1396 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397 "run is live or its ownership is unknown"
1398 );
1399 let _execution = crate::driver::maintenance(home, id)?;
1402 anyhow::ensure!(
1403 state.status.resumable()
1404 && state.released_to.is_none()
1405 && state.retired_to.is_none()
1406 && state.pr_taken_over_by.is_none(),
1407 "run ownership is final"
1408 );
1409 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410 anyhow::ensure!(
1411 !state.pr_publication_pending
1412 && !crate::handover::landing_checkpoint(&state)
1413 && state.merge_decision.is_none(),
1414 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415 );
1416 let winner = state.winner().context("run has no retained winner")?;
1417 anyhow::ensure!(
1418 git::is_clean(&winner.worktree).await?,
1419 "retained worktree has uncommitted changes"
1420 );
1421 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422 anyhow::ensure!(
1423 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424 "retained worktree does not match the winner branch"
1425 );
1426 if let Some(control) = control.as_ref() {
1427 anyhow::ensure!(
1428 head.starts_with(&control.request.head),
1429 "retained HEAD changed since the owner request"
1430 );
1431 anyhow::ensure!(
1432 state
1433 .base_sync_recovery
1434 .stop
1435 .as_ref()
1436 .is_none_or(|stop| stop.head == head),
1437 "retained HEAD differs from the recorded stop head; inspect it first"
1438 );
1439 }
1440 state.ensure_base_episode();
1441 state.record_base_sync_stop(&head);
1442 anyhow::ensure!(
1443 state
1444 .base_sync_recovery
1445 .stop
1446 .as_ref()
1447 .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448 "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449 );
1450 let fetched =
1451 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452 anyhow::ensure!(
1453 fetched.ok(),
1454 "cannot authorize base recovery without a fetched base"
1455 );
1456 let approved_tip = git::rev_parse(
1457 &state.repo,
1458 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459 )
1460 .await?;
1461 if let Some(control) = control.as_deref_mut() {
1462 let queue = crate::queue::Queue::at(home.join("queue"));
1463 control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464 if let Some(prior) = state
1465 .base_sync_recovery
1466 .authorizations
1467 .iter()
1468 .find(|a| a.request.as_deref() == Some(&control.key()))
1469 {
1470 anyhow::ensure!(
1471 !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472 "recovery request was consumed or invalidated; it cannot queue again"
1473 );
1474 return control.enqueue(&queue);
1475 }
1476 }
1477 if resumption {
1478 anyhow::ensure!(
1479 crate::daemon::compatible_run_instruction(&state, &queued),
1480 "resumption instruction does not match the retained run"
1481 );
1482 let approval = queued
1483 .resumption
1484 .as_ref()
1485 .context("no operator resumption")?;
1486 anyhow::ensure!(
1487 approval.instruction == queued.instruction,
1488 "resumption approval does not match the current task instruction"
1489 );
1490 state.authorize_base_sync_resumption(task, approval)?;
1491 } else {
1492 state.authorize_base_sync(task)?;
1493 }
1494 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495 state
1496 .base_sync_recovery
1497 .authorizations
1498 .last_mut()
1499 .unwrap()
1500 .scope = Some(crate::base_sync_recovery::Scope {
1501 tip: approved_tip,
1502 instruction: queued.instruction.clone(),
1503 episode,
1504 });
1505 if let Some(control) = control.as_deref_mut() {
1506 let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507 authorization.request = Some(control.key());
1508 authorization.generation = control.generation();
1509 authorization.decision = control.decision();
1510 authorization.question = control
1511 .request
1512 .decision
1513 .as_ref()
1514 .map(|_| control.request.request.clone());
1515 }
1516 state.event(
1517 "base-recovery",
1518 "owner authorized a bounded base synchronization interval; fresh verification required",
1519 );
1520 state.save_under(home)?;
1521 if let Some(control) = control {
1522 control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523 }
1524 Ok(())
1525 }
1526
1527 pub fn resume(id: &str) -> Result<Self> {
1529 let state = RunState::load(id)?;
1530 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531 if state.startup.as_ref().is_some_and(|s| !s.complete)
1532 || (state.status == RunStatus::Prep
1533 && state.instruction.trim().is_empty()
1534 && state.candidates.is_empty()
1535 && state.prep_complete == Some(false))
1536 {
1537 bail!(
1538 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539 );
1540 }
1541 crate::pr_head::ensure_owner(&state)?;
1542 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543 bail!(
1544 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545 );
1546 }
1547 if let Some(to) = &state.released_to {
1548 bail!(
1549 "run {} cannot be resumed: its worktree was released to run {}",
1550 state.short(),
1551 crate::run::short_of(to)
1552 );
1553 }
1554 let roles = state.config.resolve_roles()?;
1555 if let Some(id) = &state.config.graph.selected_implementer
1556 && state.candidates.iter().any(|c| c.agent != *id)
1557 {
1558 bail!(
1559 "replacement implementer pin does not match the saved candidates; task must stay held"
1560 );
1561 }
1562 let max_parallel = state.config.graph.max_parallel.max(1);
1563 Ok(Self {
1564 state,
1565 roles,
1566 sem: Arc::new(Semaphore::new(max_parallel)),
1567 pause: Pause::new(),
1568 interrupt: Pause::new(),
1569 })
1570 }
1571
1572 pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579 Box::pin(self.execute_inner())
1582 }
1583
1584 async fn execute_inner(&mut self) -> Result<()> {
1585 anyhow::ensure!(
1586 self.state.startup.as_ref().is_none_or(|s| s.complete),
1587 "startup incomplete; inspect retained evidence before explicit recovery"
1588 );
1589 if let Some(selection) = &self.state.pr_recovery
1590 && (selection.source_proof.is_some() || selection.source_run.is_some())
1591 {
1592 Box::pin(selection.verify_source(&self.state.repo)).await?;
1593 }
1594 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597 || self.state.land_approval.is_some()
1598 || self.state.merge_decision.is_some()
1599 || !self.state.merge_repairs.is_empty()
1600 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601 {
1602 let claim = FixClaim::acquire(&self.state.dir())?;
1603 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604 let disk = RunState::load(&self.state.id)?;
1605 let checkpoint = |s: &RunState| {
1606 (
1607 s.merge_repairs.len(),
1608 s.merge_repairs
1609 .last()
1610 .map(|r| (r.question.clone(), r.phase)),
1611 s.land_approval
1612 .as_ref()
1613 .map(|a| (a.question.clone(), a.head.clone())),
1614 )
1615 };
1616 if checkpoint(&disk) != checkpoint(&self.state) {
1617 bail!(
1618 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619 );
1620 }
1621 Some(claim)
1622 } else {
1623 None
1624 };
1625 let mut execution = self.claim_driver().await?;
1626 let runner = &mut *execution.runner;
1627 let wait = runner.repository_wait();
1628 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631 let result = match result {
1632 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633 record_repository_block(&mut runner.state, &e);
1634 runner.park_here()?;
1635 Ok(())
1636 }
1637 other => other,
1638 };
1639 execution.driver.finish(&mut runner.state).await?;
1640 let ended = if result.is_err() {
1641 Some({
1642 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643 if let Err(error) = &result {
1644 notice.message = format!(
1645 "Run {} stopped with an error: {}",
1646 runner.state.id,
1647 error.to_string().lines().next().unwrap_or("unknown error")
1648 );
1649 }
1650 notice
1651 })
1652 } else {
1653 crate::notices::run_ended(&runner.state)
1654 };
1655 if let Some(notice) = ended {
1656 crate::notices::raise(notice);
1657 }
1658 result
1659 }
1660
1661 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663 }
1664
1665 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666 let wait = self.repository_wait();
1667 coordinate_operation(&mut self.state, cwd, &wait).await
1668 }
1669
1670 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672 let owner = match crate::pr_head::owner_lock_wait(
1673 &crate::run::home(),
1674 &self.state.id,
1675 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676 )
1677 .await
1678 {
1679 Ok(owner) => owner,
1680 Err(error) => {
1681 let detail = format!("worktree maintenance or ownership update: {error:#}");
1682 self.state.resource_block = Some(crate::run::ResourceBlock {
1686 cause: crate::run::ResourceBlockCause::Other,
1687 node: "start".into(),
1688 kind: "worktree maintenance".into(),
1689 detail: detail.clone(),
1690 at: Timestamp::now(),
1691 });
1692 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693 }
1694 };
1695 let disk = RunState::load(&self.state.id)?;
1696 crate::pr_head::ensure_owner(&disk)?;
1697 let claimed =
1698 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699 anyhow::ensure!(
1704 disk.liveness(claimed) != crate::run::Liveness::Live,
1705 "run already has a live driver; wait for its execution to end"
1706 );
1707 if let Some(to) = &disk.released_to {
1708 bail!(
1709 "run {} cannot continue: its worktree was released to run {}",
1710 self.state.short(),
1711 crate::run::short_of(to)
1712 );
1713 }
1714 anyhow::ensure!(
1715 serde_json::to_value(&disk.base_sync_recovery)?
1716 == serde_json::to_value(&self.state.base_sync_recovery)?,
1717 "base recovery changed since this run was loaded; reload the saved run"
1718 );
1719 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720 let mut execution = DriverExecution {
1721 runner: self,
1722 driver,
1723 };
1724 let _owner = owner;
1727 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728 .await?;
1729 let state = &mut execution.runner.state;
1730 state.parked = false;
1735 state.resource_wait = None;
1736 state.resource_block = None;
1737 state.clear_active();
1744 let pid = std::process::id();
1759 state.driver_pid = Some(pid);
1760 state.driver_started_at = crate::proc::process_started_at(pid);
1761 state.driver_exited = false;
1762 state.driver_claimed = Some(true);
1763 state.begin_pass();
1764 execution.driver.bind(state);
1765 state.save()?;
1766 execution.driver.record(state)?;
1767 Ok(execution)
1768 }
1769
1770 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774 if let Err(error) = &result
1775 && !error.is::<crate::repo_lock::ResourceBlocked>()
1776 && self
1777 .state
1778 .repair_grants
1779 .last()
1780 .is_some_and(|g| g.conflict.is_none())
1781 {
1782 self.state.repair_grants.last_mut().unwrap().phase =
1783 crate::repair_budget::Phase::Complete;
1784 self.state.status = RunStatus::Blocked;
1785 self.state.save()?;
1786 bail!(
1787 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788 self.state.passes.last().map_or(0, |p| p.index)
1789 );
1790 }
1791 result
1792 }
1793
1794 async fn execute_granted_repair(&mut self) -> Result<()> {
1797 use crate::repair_budget::{Kind, Phase};
1798 let index = self.state.repair_grants.len() - 1;
1799 let winner = self
1800 .state
1801 .winner()
1802 .cloned()
1803 .context("repair winner missing")?;
1804 loop {
1805 let grant = self.state.repair_grants[index].clone();
1806 match grant.phase {
1807 Phase::Pending => {
1808 if grant.spent >= grant.receipt.request.rounds {
1809 self.state.repair_grants[index].phase = Phase::Complete;
1810 self.state.status = RunStatus::Blocked;
1811 self.state.save()?;
1812 return Ok(());
1813 }
1814 self.state.repair_grants[index].spent += 1;
1817 self.state.repair_grants[index].phase = Phase::Fixing;
1818 self.state.save()?;
1819 match grant.receipt.request.kind {
1820 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821 Kind::Gate => {
1822 let failed = self.state.gate.clone();
1823 let _ = self.gate_fix_round(&winner, &failed).await?;
1824 }
1825 Kind::Review => {
1826 let mut record = self
1827 .state
1828 .reviews
1829 .last()
1830 .cloned()
1831 .context("review failure missing")?;
1832 let round = record.round;
1833 let step = self.run_fix_round(&winner, &mut record).await?;
1834 self.state.reviews[round - 1] = record;
1835 if matches!(step, FixStep::Recorded { report_lost: true }) {
1836 self.state.repair_grants[index].phase = Phase::Complete;
1837 self.state.status = RunStatus::Blocked;
1838 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839 self.state.save()?;
1840 return Ok(());
1841 }
1842 if matches!(step, FixStep::Resource) {
1843 self.stop_for_resource()?;
1844 return Ok(());
1845 }
1846 self.state.save()?;
1847 }
1848 }
1849 if self.resource_halt()? {
1850 return Ok(());
1851 }
1852 }
1853 Phase::Fixing => {
1854 if grant.receipt.request.kind == Kind::BaseConflict {
1855 self.repair_base_conflict(false).await?;
1856 }
1857 let _coordination = self.coordinate(&winner.worktree).await?;
1860 let rescued = git::rescue_commit(
1861 &winner.worktree,
1862 "magi: preserve interrupted granted repair",
1863 )
1864 .await?;
1865 self.state.note_withheld("repair-grant", &rescued.withheld);
1866 self.state
1869 .verification_history
1870 .push(crate::run::VerificationEvidence {
1871 superseded_by: None,
1872 archived_at: Some(Timestamp::now()),
1873 fixes_spent: Some(self.state.gate_fixes.len()),
1874 gate_head: self.state.gate_head.take(),
1875 gate_ran: std::mem::take(&mut self.state.gate_ran),
1876 gate: std::mem::take(&mut self.state.gate),
1877 e2e: self.state.head_e2e.take(),
1878 });
1879 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880 self.state
1881 .verification_history
1882 .push(crate::run::VerificationEvidence {
1883 superseded_by: None,
1884 archived_at: Some(Timestamp::now()),
1885 fixes_spent: Some(self.state.gate_fixes.len()),
1886 gate_head: None,
1887 gate_ran: false,
1888 gate: Vec::new(),
1889 e2e: Some(e2e),
1890 });
1891 }
1892 self.state.repair_grants[index].review_start = self.state.reviews.len();
1893 self.state.repair_grants[index].head =
1894 git::rev_parse(&winner.worktree, "HEAD").await?;
1895 self.state.repair_grants[index].phase = Phase::Reviewing;
1896 if grant.receipt.request.kind == Kind::BaseConflict {
1897 self.state.base_sync_recovery.stop = None;
1898 if let Some(sync) = &mut self.state.base_sync {
1899 sync.tip = grant
1900 .receipt
1901 .request
1902 .tip
1903 .clone()
1904 .context("repair base tip missing")?;
1905 sync.conflict = None;
1906 sync.behind = 0;
1907 }
1908 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909 }
1910 self.state.save()?;
1911 }
1912 Phase::Reviewing => {
1913 self.review_loop_pass().await?;
1914 if self.resource_halt()? {
1915 return Ok(());
1916 }
1917 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918 let clean = self.state.reviews.len() > grant.review_start
1919 && self.state.reviews.last().is_some_and(|r| {
1920 r.head == head
1921 && r.clean
1922 && r.expected > 0
1923 && r.answered == r.expected
1924 && matches!(
1925 r.e2e_status(),
1926 E2eStatus::Passed | E2eStatus::NotConfigured
1927 )
1928 });
1929 if !clean {
1930 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932 == Kind::Review
1933 && grant.spent < grant.receipt.request.rounds
1934 && self.state.reviews.last().is_some_and(|r| {
1935 r.fix.is_none()
1936 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937 }) {
1938 Phase::Pending
1939 } else {
1940 Phase::Complete
1941 };
1942 self.state.status = RunStatus::Blocked;
1943 self.state.event(
1944 "repair-grant",
1945 "fresh review did not clear the repaired head; no publication",
1946 );
1947 self.state.save()?;
1948 if self.state.repair_grants[index].phase == Phase::Pending {
1949 continue;
1950 }
1951 return Ok(());
1952 }
1953 self.state.repair_grants[index].phase = Phase::Gating;
1954 self.state.save()?;
1955 }
1956 Phase::Gating => {
1957 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958 self.state.reconcile_verification_head(&head);
1959 let wait = self.repository_wait();
1960 if self
1961 .state
1962 .reviews
1963 .last()
1964 .is_none_or(|r| r.head != head || !r.clean)
1965 {
1966 self.state.status = RunStatus::Blocked;
1967 self.state.repair_grants[index].phase = Phase::Complete;
1968 self.state.event(
1969 "repair-grant",
1970 "head changed since fresh review; repair held",
1971 );
1972 self.state.save()?;
1973 return Ok(());
1974 }
1975 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976 return Ok(());
1977 }
1978 let outcomes = self.run_gate(&winner).await?;
1979 if verify_inconclusive(&outcomes) {
1980 return Ok(());
1981 }
1982 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983 if after != head {
1984 self.state.repair_grants[index].review_start = self.state.reviews.len();
1985 self.state.repair_grants[index].head = after;
1986 self.state.repair_grants[index].phase = Phase::Reviewing;
1987 self.state.status = RunStatus::Blocked;
1988 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989 self.state.save()?;
1990 return Ok(());
1991 }
1992 let passed = outcomes.iter().all(CommandOutcome::ok);
1993 if passed
1996 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997 {
1998 return Ok(());
1999 }
2000 self.state.repair_grants[index].succeeded = passed;
2001 self.state.gate = outcomes;
2002 self.state.gate_ran = true;
2003 self.state.status = if passed {
2004 RunStatus::Gating
2005 } else {
2006 RunStatus::Blocked
2007 };
2008 self.state.repair_grants[index].phase = if !passed
2009 && grant.receipt.request.kind == Kind::Gate
2010 && grant.spent < grant.receipt.request.rounds
2011 {
2012 Phase::Pending
2013 } else {
2014 Phase::Complete
2015 };
2016 if passed && self.state.config.merge.mode == MergeMode::Pr {
2017 self.state.pr_publication_pending = true;
2019 } else if passed {
2020 self.state.status = RunStatus::Ready;
2021 }
2022 self.state.save()?;
2023 if !passed {
2024 continue;
2025 }
2026 if self.state.config.merge.mode != MergeMode::Pr {
2030 self.state.status = RunStatus::Ready;
2031 self.state.event(
2032 "repair-grant",
2033 "repair verified locally; grant carries no merge authorization",
2034 );
2035 self.state.save()?;
2036 return Ok(());
2037 }
2038 return self.merge().await;
2039 }
2040 Phase::Complete => return Ok(()),
2041 }
2042 }
2043 }
2044
2045 async fn execute_graph(&mut self) -> Result<()> {
2046 let archive_home = crate::run::home().canonicalize()?;
2050 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051 let repo = self.state.repo.clone();
2052 let _coordination = self.coordinate(&repo).await?;
2053 ensure_setup_config(&mut self.state, &repo).await?;
2054 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055 }
2056 self.state.configure_recovery_resume()?;
2057 if self
2058 .state
2059 .repair_grants
2060 .last()
2061 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062 {
2063 return self.execute_granted_repair().await;
2064 }
2065 if self
2066 .state
2067 .repair_grants
2068 .last()
2069 .is_some_and(|g| !g.succeeded)
2070 {
2071 self.state.status = RunStatus::Blocked;
2072 self.state.save()?;
2073 return Ok(());
2074 }
2075 self.persist_candidates().await?;
2078 if self.state.base_sync_blocked() {
2081 self.sync_to_base().await?;
2082 if !self.state.base_sync_blocked()
2083 && self.state.status != RunStatus::AlreadyInBase
2084 && !self.resource_halt()?
2085 {
2086 self.finish_after_tally().await?;
2087 }
2088 return Ok(());
2089 }
2090 if self
2091 .state
2092 .merge_decision
2093 .as_ref()
2094 .is_some_and(|decision| !decision.closed)
2095 && self.state.status != RunStatus::Landing
2096 {
2097 if crate::merge_decision::resumable_hold(&self.state) {
2098 self.state.merge = Some(MergeOutcome {
2099 mode: MergeMode::Pr,
2100 ok: false,
2101 detail: self
2102 .state
2103 .merge_decision
2104 .as_ref()
2105 .unwrap()
2106 .target
2107 .pr
2108 .clone(),
2109 empty: false,
2110 });
2111 self.run_land().await?;
2112 self.settle_questions();
2113 return Ok(());
2114 }
2115 bail!(
2116 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117 );
2118 }
2119 if self.state.config.graph.selected_implementer.is_some()
2120 && self.state.status == RunStatus::Blocked
2121 && self.state.tally.is_none()
2122 && self.state.candidate_preservation.is_empty()
2123 && self.state.interrupted_implementations.is_empty()
2124 {
2125 bail!(
2126 "selected implementation is held; explicitly release the owning task to retry it"
2127 );
2128 }
2129
2130 if self.state.merge_repairs.last().is_some_and(|r| {
2131 matches!(
2132 r.phase,
2133 crate::run::MergeRepairPhase::Requested
2134 | crate::run::MergeRepairPhase::Fixing
2135 | crate::run::MergeRepairPhase::Reviewing
2136 )
2137 }) {
2138 if self.repair_merge_review().await? {
2139 self.run_land().await?;
2140 }
2141 self.settle_questions();
2142 return Ok(());
2143 }
2144 if self.state.pr_publication_pending {
2147 self.merge().await?;
2148 if self.resource_halt()? {
2149 return Ok(());
2150 }
2151 self.state.save()?;
2152 self.settle_questions();
2153 return Ok(());
2154 }
2155 if self.state.status == RunStatus::Stalled {
2168 if self.recover_stall().await? {
2169 self.finish_after_tally().await?;
2170 } else {
2171 self.state.save()?;
2173 }
2174 return Ok(());
2175 }
2176 if self.state.status == RunStatus::Landing
2186 || crate::handover::landing_checkpoint(&self.state)
2187 {
2188 self.state.land_sync_pending = true;
2189 self.state.save()?;
2190 self.run_land().await?;
2191 self.settle_questions();
2196 return Ok(());
2197 }
2198 if self.state.github_text.is_some()
2203 && self.state.merge.is_none()
2204 && self.state.status == RunStatus::Gating
2205 {
2206 self.merge().await?;
2207 if self.resource_halt()? {
2208 return Ok(());
2209 }
2210 self.state.save()?;
2211 return Ok(());
2212 }
2213 if let Err(e) = self.prep().await {
2214 if e.is::<crate::repo_lock::ResourceBlocked>() {
2215 return Err(e);
2216 }
2217 self.state.status = RunStatus::Blocked;
2218 self.state
2219 .prep_failure
2220 .get_or_insert_with(|| format!("{e:#}"));
2221 self.state
2222 .event("prep", format!("preparation stopped: {e:#}"));
2223 self.state.save()?;
2224 return Err(e);
2225 }
2226 if self.park_here()? {
2227 return Ok(());
2228 }
2229 self.advise().await?;
2230 if self.park_here()? {
2231 return Ok(());
2232 }
2233 self.implement().await?;
2234 if self.park_here()? {
2235 return Ok(());
2236 }
2237 if self.state.status == RunStatus::VerifiedNoop {
2241 return Ok(());
2242 }
2243 self.judge().await?;
2244 if self.park_here()? {
2245 return Ok(());
2246 }
2247 self.deliberate().await?;
2248 if self.park_here()? {
2249 return Ok(());
2250 }
2251 self.vote().await?;
2252 if self.park_here()? {
2253 return Ok(());
2254 }
2255 self.tally()?;
2256 if self.state.status == RunStatus::Stalled {
2261 self.state.save()?;
2265 return Ok(());
2266 }
2267 self.finish_after_tally().await?;
2268 Ok(())
2269 }
2270
2271 fn stop_for_resource(&mut self) -> Result<()> {
2276 if self.pause.parked() || self.interrupt.parked() {
2279 self.park_here()?;
2280 return Ok(());
2281 }
2282 if self.state.resource_block.is_some() {
2283 self.state.status = RunStatus::Blocked;
2284 }
2285 self.state.save()
2286 }
2287
2288 fn resource_halt(&mut self) -> Result<bool> {
2290 if self.state.resource_block.is_none() {
2291 return Ok(false);
2292 }
2293 self.stop_for_resource()?;
2294 Ok(true)
2295 }
2296
2297 fn park_here(&mut self) -> Result<bool> {
2304 let parking = self.pause.parked() || self.interrupt.parked();
2305 if !parking && self.state.resource_block.is_some() {
2306 self.stop_for_resource()?;
2307 return Ok(true);
2308 }
2309 if !self.pause.parked() && !self.interrupt.parked() {
2314 return Ok(false);
2315 }
2316 if self.state.parked {
2317 return Ok(true);
2318 }
2319 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320 if let Some(block) = &mut self.state.resource_block
2321 && matches!(
2322 block.cause,
2323 crate::run::ResourceBlockCause::Cancelled { .. }
2324 )
2325 {
2326 block.cause = crate::run::ResourceBlockCause::Cancelled {
2327 park: cause.clone(),
2328 };
2329 }
2330 let why = match cause.as_ref().map(ParkCause::label) {
2331 Some(reason) => format!(
2332 "parked after `{}` ({reason}) — resume to carry on from here",
2333 self.state.status.as_str()
2334 ),
2335 None => format!(
2336 "parked after `{}` — resume to carry on from here",
2337 self.state.status.as_str()
2338 ),
2339 };
2340 self.state.event("park", why);
2341 self.state
2342 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343 self.state.parked = true;
2344 self.state.save()?;
2345 Ok(true)
2346 }
2347
2348 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350 if self.state.prep_failure.is_none() {
2351 return Ok(());
2352 }
2353 let failed_at = self
2354 .state
2355 .events
2356 .iter()
2357 .rev()
2358 .find(|event| event.node == "prep")
2359 .map_or(self.state.updated_at, |event| event.at);
2360 anyhow::ensure!(
2361 approval_at > failed_at,
2362 "preparation retry requires a new explicit release after the failure"
2363 );
2364 self.state.prep_failure = None;
2365 self.state.prep_complete = Some(false);
2366 self.state
2367 .event("prep-retry", "operator approved retry of preparation");
2368 Ok(())
2369 }
2370
2371 pub fn on_pause(&mut self, pause: Pause) {
2373 self.pause = pause;
2374 }
2375
2376 pub fn watch_interrupt(&mut self, pause: Pause) {
2382 self.interrupt = pause;
2383 }
2384
2385 fn settle_questions(&mut self) {
2405 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407 }
2408 }
2409
2410 fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413 Box::pin(self.finish_after_tally_inner())
2414 }
2415
2416 async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417 self.graft().await?;
2420 if self.resource_halt()? {
2421 return Ok(());
2422 }
2423 self.fold_losers().await?;
2424 self.sync_to_base().await?;
2429 if self.hold_base_sync().await? || self.resource_halt()? {
2430 return Ok(());
2431 }
2432 if self.state.status == RunStatus::AlreadyInBase {
2433 return Ok(());
2434 }
2435 self.review_loop().await?;
2436 if self.resource_halt()? {
2437 return Ok(());
2438 }
2439 self.sync_to_base().await?;
2440 if self.hold_base_sync().await? || self.resource_halt()? {
2441 return Ok(());
2442 }
2443 if self.state.status == RunStatus::AlreadyInBase {
2444 return Ok(());
2445 }
2446 self.gate().await?;
2447 self.merge().await?;
2448 if self.resource_halt()? {
2449 return Ok(());
2450 }
2451 self.state.save()?;
2452 Ok(())
2453 }
2454
2455 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462 let result = match ensure_setup_config(&mut self.state, repo).await {
2463 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464 Err(e) => Err(e),
2465 };
2466 if let Err(e) = result {
2467 git::worktree_remove_clean(repo, wt).await.ok();
2468 if in_prep {
2469 self.rollback_prep(repo).await;
2470 }
2471 self.state.status = RunStatus::Blocked;
2472 self.state.event(
2473 "setup",
2474 format!("worktree setup failed in {}: {e:#}", wt.display()),
2475 );
2476 self.state.save()?;
2477 return Err(e);
2478 }
2479 Ok(())
2480 }
2481
2482 async fn rollback_prep(&mut self, repo: &Path) {
2484 let root = self.state.worktree_root();
2485 for c in std::mem::take(&mut self.state.candidates) {
2486 if matches!(
2487 git::worktree_remove_clean(repo, &c.worktree).await,
2488 Ok(true)
2489 ) {
2490 git::branch_delete(repo, &c.branch).await.ok();
2491 } else {
2492 self.state.candidates.push(c);
2493 }
2494 }
2495 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496 for j in 1..=seats {
2497 remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2498 .await
2499 .ok();
2500 }
2501 for k in 1..=self.state.config.graph.advisors {
2502 remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2503 .await
2504 .ok();
2505 }
2506 }
2507
2508 async fn prep(&mut self) -> Result<()> {
2509 if let Some(reason) = &self.state.prep_failure {
2510 bail!(
2511 "candidate preparation previously failed: {reason}; correct the source before retrying"
2512 );
2513 }
2514 if self
2515 .state
2516 .prep_complete
2517 .unwrap_or(!self.state.candidates.is_empty())
2518 {
2519 return Ok(());
2520 }
2521 let repo = self.state.repo.clone();
2522 let _coordination = self.coordinate(&repo).await?;
2525 let base = self.state.base_commit.clone();
2526 let plan = self.state.sources.plan(&repo, &base).await?;
2527 self.rollback_prep(&repo).await;
2528 if !self.state.candidates.is_empty() {
2529 bail!(
2530 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531 );
2532 }
2533 self.state.prep_complete = Some(false);
2534 self.state.status = RunStatus::Prep;
2535 self.state.save()?;
2536 let repo = self.state.repo.clone();
2537 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538 let root = self.state.worktree_root();
2539 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541 let hooks_dir = self.state.dir().join("hooks");
2544 if self.state.config.blind.commit_msg_hook {
2545 std::fs::create_dir_all(&hooks_dir)
2546 .with_context(|| format!("create {}", hooks_dir.display()))?;
2547 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548 let path = hooks_dir.join("commit-msg");
2549 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550 make_executable(&path)?;
2551 git::acquire_worktree_config(&repo).await?;
2559 self.state.enabled_worktree_config = true;
2560 }
2561
2562 ensure_setup_config(&mut self.state, &repo).await?;
2565
2566 for (index, (spec, label)) in self
2567 .roles
2568 .implementers
2569 .clone()
2570 .into_iter()
2571 .zip(labels)
2572 .enumerate()
2573 {
2574 let branch = self.state.branch_for(label);
2575 let worktree = root.join(format!("cand-{label}"));
2576 self.state.event(
2580 "prep-intent",
2581 serde_json::json!({
2582 "branch": branch,
2583 "worktree": worktree,
2584 "start": start,
2585 "picks": plan.picks,
2586 })
2587 .to_string(),
2588 );
2589 self.state.save()?;
2590 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591 if self.state.config.blind.commit_msg_hook {
2592 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593 }
2594 git::local_exclude(&worktree, "/.magi/").await?;
2595 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596 git::worktree_remove_clean(&repo, &worktree).await.ok();
2599 git::branch_delete(&repo, &branch).await.ok();
2600 self.rollback_prep(&repo).await;
2601 self.state.status = RunStatus::Blocked;
2602 self.state
2603 .event("prep", format!("worktree setup failed: {e:#}"));
2604 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605 self.state.save()?;
2606 return Err(e);
2607 }
2608 self.state.candidates.push(Candidate {
2609 index,
2610 label,
2611 agent: spec.id.clone(),
2612 branch,
2613 worktree,
2614 summary: String::new(),
2615 stat: String::new(),
2616 files: 0,
2617 commits: 0,
2618 empty: false,
2619 failed: None,
2620 verified_noop: None,
2621 duration_ms: 0,
2622 folded: false,
2623 });
2624 self.state.save()?;
2625 for pick in &plan.picks {
2626 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627 {
2628 self.state.status = RunStatus::Blocked;
2629 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630 self.state
2631 .event("prep", format!("cannot apply referenced commit: {e}"));
2632 self.state.save()?;
2633 return Err(e);
2634 }
2635 }
2636 }
2637
2638 for j in 1..=self.roles.judges.len() {
2639 let wt = root.join(format!("judge-{j}"));
2640 if !wt.exists() {
2641 git::worktree_add_detached(&repo, &wt, &base).await?;
2642 self.setup_seat_worktree(&repo, &wt, true).await?;
2643 }
2644 }
2645
2646 if self.state.config.graph.advise {
2654 for k in 1..=self.state.config.graph.advisors {
2655 let wt = root.join(format!("advisor-{k}"));
2656 if !wt.exists() {
2657 git::worktree_add_detached(&repo, &wt, &base).await?;
2658 self.setup_seat_worktree(&repo, &wt, true).await?;
2659 }
2660 }
2661 }
2662
2663 let authors: Vec<&str> = self
2668 .roles
2669 .implementers
2670 .iter()
2671 .map(|a| a.id.as_str())
2672 .collect();
2673 let overlap: Vec<String> = self
2674 .roles
2675 .judges
2676 .iter()
2677 .enumerate()
2678 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680 .collect();
2681 if !overlap.is_empty() {
2682 let note = format!(
2683 "{} also authored a candidate; blind, but the panel is less \
2684 independent than {} distinct agents would be",
2685 overlap.join(", "),
2686 self.roles.judges.len()
2687 );
2688 self.state.event("prep", note);
2689 }
2690
2691 self.state.event(
2692 "prep",
2693 format!(
2694 "{} candidates, {} judges, base {} ({})",
2695 self.state.candidates.len(),
2696 self.roles.judges.len(),
2697 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698 self.state.base_branch
2699 ),
2700 );
2701 self.state.prep_complete = Some(true);
2702 self.state.status = RunStatus::Implementing;
2703 self.state.save()?;
2704 Ok(())
2705 }
2706
2707 async fn advise(&mut self) -> Result<()> {
2740 let implement_untouched = self
2741 .state
2742 .candidates
2743 .iter()
2744 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745 if !self.state.config.graph.advise || self.state.advise_attempted {
2746 return Ok(());
2747 }
2748 if !implement_untouched {
2749 self.state.event(
2750 "advise",
2751 "skipping the design-deliberation stage: at least one \
2752 candidate already shows implementation progress, so this \
2753 run is past the point the stage exists to run before"
2754 .to_owned(),
2755 );
2756 self.state.advise_attempted = true;
2757 self.state.save()?;
2758 return Ok(());
2759 }
2760 let run_id = self.state.id.clone();
2761 let prompts = self.state.config.prompts.clone();
2762 let instruction = self.state.instruction.clone();
2763 let language = self.state.config.graph.language.clone();
2764 let root = self.state.worktree_root();
2765 let n = self.state.config.graph.advisors;
2766 let where_recorded = self.state.dir().join("run.json");
2767
2768 let seats = match self.state.config.advisors() {
2769 Ok(seats) if !seats.is_empty() => seats,
2770 Ok(_) => {
2771 self.state.event(
2772 "advise",
2773 format!(
2774 "[graph] advisors is 0; skipping the design-deliberation \
2775 stage and continuing without a synthesis brief (see {})",
2776 where_recorded.display()
2777 ),
2778 );
2779 self.state.advise_attempted = true;
2780 self.state.save()?;
2781 return Ok(());
2782 }
2783 Err(e) => {
2784 self.state.event(
2785 "advise",
2786 format!(
2787 "could not resolve advisor seats ({e:#}); continuing \
2788 without a design-deliberation brief (see {})",
2789 where_recorded.display()
2790 ),
2791 );
2792 self.state.advise_attempted = true;
2793 self.state.save()?;
2794 return Ok(());
2795 }
2796 };
2797
2798 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799 let artifacts = agent::artifacts_dir(&self.state.dir());
2800 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802 let mut jobs = Vec::new();
2803 for (i, spec) in seats.iter().cloned().enumerate() {
2804 let seat_key = format!("advisor-{}", i + 1);
2805 let seat = self.seat(&seat_key, &spec.id);
2806 jobs.push(SeatJob {
2807 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808 spec,
2809 seat,
2810 cwd: worktrees[i % worktrees.len()].clone(),
2811 timeout,
2812 allow_write: false,
2813 sessions: false,
2814 artifacts: artifacts.clone(),
2815 stem: seat_key,
2816 handover: None,
2817 });
2818 }
2819
2820 self.state.event(
2821 "advise",
2822 format!(
2823 "{} advisor seat(s) sketching a design in parallel",
2824 jobs.len()
2825 ),
2826 );
2827 let mut quota_losses = Vec::new();
2828 let cache = self.state.config.cache_dir();
2829 let ctx = WaveCtx {
2830 carry_seats: false,
2831 pauses: Some((&self.pause, &self.interrupt)),
2832 run: &run_id,
2833 node: "advise",
2834 prompts: &prompts,
2835 cache: cache.as_deref(),
2836 round: None,
2837 };
2838 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839 let results = ask_json_wave::<Proposal>(
2840 jobs,
2841 Arc::clone(&self.sem),
2842 self.state.config.graph.retries,
2843 &advisor_roster,
2844 &ctx,
2845 &mut quota_losses,
2846 &mut self.state,
2847 &|p: &Proposal| p.validate(),
2848 )
2849 .await;
2850 if self.state.resource_block.is_some() {
2851 return self.stop_for_resource();
2852 }
2853 self.state.quota.extend(quota_losses);
2854
2855 let mut records = Vec::with_capacity(results.len());
2856 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857 let agent_id = seat.agent.clone();
2858 self.state.seats.insert(seat.key.clone(), seat);
2859 match res {
2860 Ok((proposal, out)) => {
2861 self.state
2862 .event("advise", format!("advisor-{} proposed a design", i + 1));
2863 records.push(advise::AdvisorRecord::proposed(
2864 i + 1,
2865 agent_id,
2866 proposal,
2867 out.duration_ms,
2868 ));
2869 }
2870 Err(e) => {
2871 self.state.event(
2872 "advise",
2873 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874 );
2875 records.push(advise::AdvisorRecord::failed(
2876 i + 1,
2877 agent_id,
2878 e.to_string(),
2879 ));
2880 }
2881 }
2882 }
2883
2884 let mut advice = advise::Advice {
2885 records,
2886 synthesis: None,
2887 };
2888 if advice.proposals().is_empty() {
2889 self.state.event(
2890 "advise",
2891 "no advisor produced a usable proposal; continuing without a \
2892 synthesis brief"
2893 .to_owned(),
2894 );
2895 } else {
2896 match self
2897 .synthesize_brief(
2898 &advice,
2899 &instruction,
2900 &language,
2901 &worktrees[0],
2902 &artifacts,
2903 &run_id,
2904 &prompts,
2905 cache.as_deref(),
2906 )
2907 .await
2908 {
2909 Ok(Some(text)) => {
2910 self.state.event(
2911 "advise",
2912 "synthesized a design brief for the implementer".to_owned(),
2913 );
2914 advice.synthesis = Some(text);
2915 }
2916 Ok(None) => {
2917 self.state.event(
2918 "advise",
2919 "the synthesis seat produced nothing usable; continuing \
2920 without a design brief"
2921 .to_owned(),
2922 );
2923 }
2924 Err(e) => {
2925 if e.is::<crate::repo_lock::ResourceBlocked>() {
2926 return Err(e);
2927 }
2928 self.state.event(
2929 "advise",
2930 format!("could not synthesize a design brief: {e:#}"),
2931 );
2932 }
2933 }
2934 }
2935 advise::apply_reflection(&mut advice);
2936
2937 self.state.advice = Some(advice);
2938 self.state.advise_attempted = true;
2939 self.state.save()?;
2940 Ok(())
2941 }
2942
2943 #[allow(clippy::too_many_arguments)]
2955 async fn synthesize_brief(
2956 &mut self,
2957 advice: &advise::Advice,
2958 instruction: &str,
2959 language: &str,
2960 cwd: &Path,
2961 artifacts: &Path,
2962 run_id: &str,
2963 prompts: &Prompts,
2964 cache: Option<&Path>,
2965 ) -> Result<Option<String>> {
2966 let chain = agent::pick_chain(
2967 &self.state.config.agents,
2968 self.state.config.roles.synthesizer.as_ref(),
2969 &agent::installed,
2970 "synthesizer",
2971 )?;
2972 let proposals = advice.proposals();
2973 let mut prompt = prompt::with_overlay(
2974 prompt::synthesize_brief(instruction, &proposals, language),
2975 prompts.overlay("advise"),
2976 );
2977 if cache.is_some() {
2978 prompt.push('\n');
2983 prompt.push_str(&prompt::build_cache_note("advise", false));
2984 }
2985 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986 let mut last = None;
2991 for (n, spec) in chain.iter().enumerate() {
2992 if n > 0 {
2993 self.state
2994 .event("advise", format!("synthesis falling back to {}", spec.id));
2995 }
2996 let mut seat = self.seat("advise-synthesis", &spec.id);
2997 let wait = self.repository_wait();
2998 let outcome = wait
2999 .scope(agent::invoke(
3000 spec,
3001 &mut seat,
3002 &Invocation {
3003 coordination: self.state.config.coordination.as_ref(),
3004 fetched_base: None,
3005 cwd,
3006 prompt: &prompt,
3007 timeout,
3008 allow_write: false,
3009 sessions: false,
3010 artifacts,
3011 stem: &if n == 0 {
3012 "advise-synthesis".to_owned()
3013 } else {
3014 format!("advise-synthesis-{}", spec.id)
3015 },
3016 run: run_id,
3017 node: "advise",
3018 cache_dir: None,
3019 attachments: &[],
3020 writable: &[],
3021 },
3022 ))
3023 .await;
3024 if outcome.is_ok() {
3025 self.state.seats.insert(seat.key.clone(), seat);
3026 }
3027 let advance = agent::chain_advances(&outcome);
3028 last = Some(outcome);
3029 if !advance {
3030 break;
3031 }
3032 }
3033 let out = last.expect("a chain holds at least one agent")?;
3036 if !out.usable() {
3037 return Ok(None);
3038 }
3039 let text =
3040 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041 Ok((!text.trim().is_empty()).then_some(text))
3042 }
3043
3044 async fn implement(&mut self) -> Result<()> {
3047 self.persist_candidates().await?;
3048 let run_id = self.state.id.clone();
3053 let prompts = self.state.config.prompts.clone();
3054 let todo: Vec<usize> = self
3055 .state
3056 .candidates
3057 .iter()
3058 .enumerate()
3059 .filter(|(_, c)| {
3060 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061 && c.failed.is_none()
3062 && !c.empty
3063 })
3064 .map(|(i, _)| i)
3065 .collect();
3066 if todo.is_empty() {
3067 return self.after_implement();
3068 }
3069 self.state.status = RunStatus::Implementing;
3070
3071 let language = self.state.config.graph.language.clone();
3072 let github_language = self.state.config.graph.github_language.clone();
3073 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074 let sessions = self.state.config.graph.sessions;
3075 let artifacts = agent::artifacts_dir(&self.state.dir());
3076 let brief = self
3080 .state
3081 .advice
3082 .as_ref()
3083 .and_then(|a| a.synthesis.as_deref())
3084 .map(str::to_owned);
3085 let attachments = self.state.attachments.clone();
3086
3087 let mut jobs = Vec::new();
3088 for &i in &todo {
3089 let (index, label, worktree) = {
3090 let c = &self.state.candidates[i];
3091 (c.index, c.label, c.worktree.clone())
3092 };
3093 let spec = self.roles.implementers[index].clone();
3094 let seat_key = format!("impl-{label}");
3095 let seat = self.seat(&seat_key, &spec.id);
3096 let instruction = seeded_instruction(&self.state);
3097 jobs.push(SeatJob {
3098 spec,
3099 seat,
3100 prompt: prompt::implement_in(
3101 &instruction,
3102 &worktree.to_string_lossy(),
3103 &language,
3104 &github_language,
3105 brief.as_deref(),
3106 &attachments,
3107 ),
3108 cwd: worktree,
3109 timeout,
3110 allow_write: true,
3111 sessions,
3112 artifacts: artifacts.clone(),
3113 stem: format!("impl-{label}"),
3114 handover: None,
3115 });
3116 }
3117
3118 self.state.event(
3119 "implement",
3120 format!("{} candidates in parallel", jobs.len()),
3121 );
3122 let mut sent = jobs.clone();
3128 let cache = self.state.config.cache_dir();
3129 let ctx = WaveCtx {
3130 carry_seats: false,
3131 pauses: Some((&self.pause, &self.interrupt)),
3132 run: &run_id,
3133 node: "implement",
3134 prompts: &prompts,
3135 cache: cache.as_deref(),
3136 round: None,
3137 };
3138 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139 if !wave_blocked(&results) {
3140 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141 .await;
3142 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143 .await?;
3144 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145 .await;
3146 }
3147 if !results.is_empty()
3150 && results
3151 .iter()
3152 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153 {
3154 let losses: Vec<_> = results
3155 .iter()
3156 .map(|(_, seat, out)| {
3157 let AgentOutcome::Quota(o) = out else {
3158 unreachable!()
3159 };
3160 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161 })
3162 .collect();
3163 self.state.quota.extend(losses.clone());
3164 for (_, seat, _) in results {
3165 self.state.seats.insert(seat.key.clone(), seat);
3166 }
3167 for &i in &todo {
3168 let label = self.state.candidates[i].label;
3169 self.state.interrupted_implementations.insert(label);
3170 self.state
3171 .candidate_preservation
3172 .insert(label, "machine outage awaiting persistence".into());
3173 }
3174 block_machine_outage(&mut self.state, "implement", &losses);
3177 self.state.save()?;
3178 self.persist_candidates().await?;
3179 for &i in &todo {
3180 self.state
3181 .interrupted_implementations
3182 .insert(self.state.candidates[i].label);
3183 }
3184 return self.stop_for_resource();
3185 }
3186 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189 && results
3190 .iter()
3191 .any(|(_, _, out)| FailClass::of(out).is_some());
3192 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194 if self
3198 .state
3199 .candidate_preservation
3200 .contains_key(&self.state.candidates[i].label)
3201 {
3202 self.state.seats.insert(seat.key.clone(), seat);
3203 }
3204 continue;
3205 }
3206 let seat_key = seat.key.clone();
3207 let agent = seat.agent.clone();
3216 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217 self.state.seats.insert(seat.key.clone(), seat);
3218 let label = self.state.candidates[i].label;
3219 self.state.interrupted_implementations.remove(&label);
3220
3221 let (summary, duration, failed, verified_claim) = match out {
3222 AgentOutcome::Ok(o) => {
3223 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224 let failed = (!o.usable()).then(|| {
3225 if o.timed_out {
3226 "agent timed out".to_owned()
3227 } else {
3228 format!(
3229 "agent exited with {}",
3230 crate::notices::exit_status(o.exit_code)
3231 )
3232 }
3233 });
3234 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235 (text, o.duration_ms, failed, verified_claim)
3236 }
3237 AgentOutcome::Dropped(o) => {
3243 let why = o
3244 .dropped
3245 .as_ref()
3246 .map(|d| d.why.as_str())
3247 .unwrap_or("the CLI ended the stream without delivering its answer");
3248 (
3249 String::new(),
3250 o.duration_ms,
3251 Some(format!("the CLI dropped the stream ({why})")),
3252 None,
3253 )
3254 }
3255 AgentOutcome::Quota(o) => {
3256 self.state.quota.push(QuotaLoss::from_report(
3257 seat_key,
3258 "implement",
3259 o.quota.as_ref(),
3260 ));
3261 (
3262 String::new(),
3263 o.duration_ms,
3264 Some("rate limited (quota); produced no change".to_owned()),
3265 None,
3266 )
3267 }
3268 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269 (String::new(), 0, Some(e), None)
3270 }
3271 };
3272
3273 let c = &mut self.state.candidates[i];
3276 if !exhausted_the_fallback_chain {
3277 c.agent = agent;
3278 }
3279 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280 c.duration_ms = duration;
3281 c.failed = failed;
3282 c.verified_noop = verified_claim;
3283 self.state
3284 .candidate_preservation
3285 .insert(label, "awaiting persistence".to_owned());
3286 }
3287 self.state.save()?;
3288 self.persist_candidates().await?;
3289
3290 if resource_blocked {
3291 return self.stop_for_resource();
3292 }
3293 if selected_failed {
3294 self.state.status = RunStatus::Blocked;
3295 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296 self.state.save()?;
3297 bail!(
3298 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299 );
3300 }
3301 self.after_implement()
3302 }
3303
3304 async fn persist_candidates(&mut self) -> Result<()> {
3306 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307 for label in labels {
3308 let i = self
3309 .state
3310 .candidates
3311 .iter()
3312 .position(|c| c.label == label)
3313 .context("pending candidate is missing")?;
3314 let c = self.state.candidates[i].clone();
3315 let interrupted = self.state.interrupted_implementations.contains(&label);
3316 let result = async {
3317 let wait = crate::repo_lock::WaitPolicy::new(
3318 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319 || false,
3320 );
3321 let _coordination =
3322 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323 if let Err(e) = git::snapshot_uncommitted(
3326 &c.worktree,
3327 &self
3328 .state
3329 .dir()
3330 .join("artifacts")
3331 .join(format!("candidate-{label}-recovery")),
3332 )
3333 .await
3334 {
3335 self.state.event(
3336 "preserve",
3337 format!("candidate {label} snapshot incomplete: {e:#}"),
3338 );
3339 }
3340 let rescued = git::rescue_commit(
3341 &c.worktree,
3342 &format!("magi: candidate {label} (uncommitted work)"),
3343 )
3344 .await?;
3345 self.state.note_withheld("implement", &rescued.withheld);
3346 let base = &self.state.base_commit;
3347 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352 if c.empty && commits > 0 && !patch.trim().is_empty()
3353 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355 || self.state.tally.is_some())
3356 {
3357 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361 }
3362 let c = &mut self.state.candidates[i];
3363 c.commits = commits;
3364 c.stat = stat;
3365 c.files = files;
3366 let no_changes = commits == 0 || patch.trim().is_empty();
3367 c.empty = no_changes && !interrupted;
3368 if interrupted && no_changes {
3369 c.failed = None;
3370 c.verified_noop = None;
3371 }
3372 if !c.empty {
3373 c.failed = None;
3374 c.verified_noop = None;
3375 }
3376 self.state.event(
3377 "preserve",
3378 format!("candidate {label}: {files} files, {commits} commits"),
3379 );
3380 Ok::<_, anyhow::Error>(())
3381 }
3382 .await;
3383 match result {
3384 Ok(()) => {
3385 self.state.candidate_preservation.remove(&label);
3386 if !interrupted || self.state.candidates[i].files > 0 {
3387 self.state.interrupted_implementations.remove(&label);
3388 }
3389 }
3390 Err(e) => {
3391 let reason = format!("candidate {label} preservation failed: {e:#}");
3392 self.state.event("preserve", &reason);
3393 self.state.candidate_preservation.insert(label, reason);
3394 }
3395 }
3396 if self.state.candidate_preservation.is_empty()
3397 && self.state.status == RunStatus::Blocked
3398 && self.state.candidates.iter().all(|c| c.failed.is_none())
3399 {
3400 self.state.status = RunStatus::Implementing;
3404 self.state.event(
3405 "preserve",
3406 "candidate persistence recovered; implementation can continue",
3407 );
3408 }
3409 self.state.save()?;
3410 }
3411 if !self.state.candidate_preservation.is_empty() {
3412 self.state.status = RunStatus::Blocked;
3413 self.state.save()?;
3414 bail!(
3415 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416 );
3417 }
3418 Ok(())
3419 }
3420
3421 async fn resume_undelivered(
3449 &mut self,
3450 results: &mut [(usize, SeatState, AgentOutcome)],
3451 sent: &[SeatJob],
3452 prompts: &Prompts,
3453 run_id: &str,
3454 ) {
3455 for (wi, seat, out) in results.iter_mut() {
3456 let Some(dropped) = (match &*out {
3457 AgentOutcome::Dropped(o) => o.dropped.clone(),
3458 _ => None,
3459 }) else {
3460 continue;
3461 };
3462 let Some(job) = sent.get(*wi) else { continue };
3463 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465 self.state.event(
3466 "implement",
3467 format!(
3468 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469 work is in the tree",
3470 seat.key, dropped.output_tokens, dropped.why
3471 ),
3472 );
3473 continue;
3474 }
3475 if !has_context(&job.spec, seat, job.sessions) {
3483 self.state.event(
3484 "implement",
3485 format!(
3486 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487 is no session left to resume",
3488 seat.key, dropped.output_tokens, dropped.why
3489 ),
3490 );
3491 continue;
3492 }
3493 self.state.event(
3494 "implement",
3495 format!(
3496 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497 conversation",
3498 seat.key, dropped.output_tokens, dropped.why
3499 ),
3500 );
3501 let mut retry = job.clone();
3502 retry.seat = seat.clone();
3503 retry.prompt = prompt::resume_after_drop(&dropped.why);
3504 retry.timeout = retry_budget(job.timeout, true);
3505 retry.stem = format!("{}-resume", job.stem);
3506 let cache = self.state.config.cache_dir();
3507 let ctx = WaveCtx {
3508 carry_seats: false,
3509 pauses: Some((&self.pause, &self.interrupt)),
3510 run: run_id,
3511 node: "implement",
3512 prompts,
3513 cache: cache.as_deref(),
3514 round: None,
3515 };
3516 let (resumed_seat, resumed) =
3517 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518 *seat = resumed_seat;
3519 *out = resumed;
3520 }
3521 }
3522
3523 async fn resume_seat_handovers(
3588 &mut self,
3589 results: &mut [(usize, SeatState, AgentOutcome)],
3590 sent: &mut [SeatJob],
3591 prompts: &Prompts,
3592 run_id: &str,
3593 ) -> Result<()> {
3594 let instruction = seeded_instruction(&self.state);
3595 let language = self.state.config.graph.language.clone();
3596 let github_language = self.state.config.graph.github_language.clone();
3597 let brief = self
3598 .state
3599 .advice
3600 .as_ref()
3601 .and_then(|a| a.synthesis.as_deref())
3602 .map(str::to_owned);
3603 let attachments = self.state.attachments.clone();
3604 for (wi, seat, out) in results.iter_mut() {
3605 let others: BTreeSet<String> = sent
3608 .iter()
3609 .enumerate()
3610 .filter(|(j, _)| j != wi)
3611 .map(|(_, j)| j.spec.id.clone())
3612 .collect();
3613 let Some(job) = sent.get_mut(*wi) else {
3614 continue;
3615 };
3616 let start = self
3621 .roles
3622 .implementer_roster
3623 .iter()
3624 .position(|s| s.id == job.spec.id)
3625 .unwrap_or(0);
3626 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627 let mut fallback_attempt = 0usize;
3628 let mut prev: Option<FailClass> = None;
3629 while let Some(cur) = FailClass::of(&*out) {
3630 if !should_hand_over(prev.as_ref(), &cur) {
3631 break;
3632 }
3633 let Some(next) =
3634 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635 .cloned()
3636 else {
3637 break;
3638 };
3639 tried.insert(next.id.clone());
3640 fallback_attempt += 1;
3641
3642 let coordination = self.coordinate(&job.cwd).await?;
3643 if let Ok(r) = git::rescue_commit(
3644 &job.cwd,
3645 &format!(
3646 "magi: candidate {} (uncommitted work before {} fallback)",
3647 seat.key,
3648 if cur == FailClass::Quota {
3649 "quota"
3650 } else {
3651 "handover"
3652 }
3653 ),
3654 )
3655 .await
3656 {
3657 self.state.note_withheld("implement", &r.withheld);
3658 }
3659 drop(coordination);
3660
3661 record_handover(
3662 &mut self.state,
3663 "implement",
3664 &seat.key,
3665 &seat.agent,
3666 &next.id,
3667 &cur,
3668 &fail_reason(&*out),
3669 );
3670 prev = Some(cur.clone());
3671
3672 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674 job.spec = next.clone();
3682 let mut retry = job.clone();
3683 retry.seat = new_seat;
3684 retry.prompt = prompt::implement_in(
3685 &instruction,
3686 &job.cwd.to_string_lossy(),
3687 &language,
3688 &github_language,
3689 brief.as_deref(),
3690 &attachments,
3691 );
3692 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693 let cache = self.state.config.cache_dir();
3694 let ctx = WaveCtx {
3695 carry_seats: false,
3696 pauses: Some((&self.pause, &self.interrupt)),
3697 run: run_id,
3698 node: "implement",
3699 prompts,
3700 cache: cache.as_deref(),
3701 round: None,
3702 };
3703 let (fallback_seat, fallback_out) = run_one(
3704 retry,
3705 Arc::clone(&self.sem),
3706 &ctx,
3707 &mut self.state,
3708 fallback_attempt,
3709 )
3710 .await;
3711 *seat = fallback_seat;
3712 *out = fallback_out;
3713 }
3714 }
3715 Ok(())
3716 }
3717
3718 async fn resume_unconfirmed_commands(
3742 &mut self,
3743 results: &mut [(usize, SeatState, AgentOutcome)],
3744 sent: &[SeatJob],
3745 prompts: &Prompts,
3746 run_id: &str,
3747 ) {
3748 for (wi, seat, out) in results.iter_mut() {
3749 let AgentOutcome::Ok(o) = &*out else {
3750 continue;
3751 };
3752 if !has_unconfirmed_command(&o.commands) {
3753 continue;
3754 }
3755 let Some(job) = sent.get(*wi) else { continue };
3756 if !has_context(&job.spec, seat, job.sessions) {
3757 self.state.event(
3758 "implement",
3759 format!(
3760 "{}: the reply named a command whose own CLI never confirmed the exit \
3761 status of, but there is no session left to resume",
3762 seat.key
3763 ),
3764 );
3765 continue;
3766 }
3767 self.state.event(
3768 "implement",
3769 format!(
3770 "{}: the reply named a command whose own CLI never confirmed the exit \
3771 status of; resuming the conversation",
3772 seat.key
3773 ),
3774 );
3775 let mut retry = job.clone();
3776 retry.seat = seat.clone();
3777 retry.prompt = prompt::resume_incomplete(
3778 "a command in your last reply had no confirmed exit status",
3779 );
3780 retry.timeout = retry_budget(job.timeout, true);
3781 retry.stem = format!("{}-confirm", job.stem);
3782 let cache = self.state.config.cache_dir();
3783 let ctx = WaveCtx {
3784 carry_seats: false,
3785 pauses: Some((&self.pause, &self.interrupt)),
3786 run: run_id,
3787 node: "implement",
3788 prompts,
3789 cache: cache.as_deref(),
3790 round: None,
3791 };
3792 let (resumed_seat, resumed) =
3793 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794 *seat = resumed_seat;
3795 *out = resumed;
3796 }
3797 }
3798
3799 async fn continue_fix_report(
3820 &mut self,
3821 mut seat: SeatState,
3822 parse_err: String,
3823 job: &SeatJob,
3824 prompts: &Prompts,
3825 run_id: &str,
3826 round: usize,
3827 ) -> (
3828 SeatState,
3829 Option<FixReport>,
3830 Option<String>,
3831 ContinuationRecord,
3832 ) {
3833 let mut last_err = parse_err;
3834 let mut cumulative_wait_ms = 0u64;
3835 let mut attempts = 0usize;
3836 loop {
3837 if !has_context(&job.spec, &seat, job.sessions) {
3838 self.state.event(
3839 "fix",
3840 format!(
3841 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842 session left to resume into"
3843 ),
3844 );
3845 let outcome = if attempts == 0 {
3846 ContinuationOutcome::NoSession
3847 } else {
3848 ContinuationOutcome::Exhausted
3849 };
3850 return (
3851 seat,
3852 None,
3853 Some(format!("unparsable fix report: {last_err}")),
3854 ContinuationRecord {
3855 attempts,
3856 cumulative_wait_ms,
3857 outcome,
3858 },
3859 );
3860 }
3861 if attempts >= MAX_FIX_CONTINUATIONS {
3862 self.state.event(
3863 "fix",
3864 format!(
3865 "round {round}: fixer's reply still had no adoption report after \
3866 {attempts} continuation(s) ({last_err}); giving up"
3867 ),
3868 );
3869 return (
3870 seat,
3871 None,
3872 Some(format!(
3873 "unparsable fix report after {attempts} continuation(s): {last_err}"
3874 )),
3875 ContinuationRecord {
3876 attempts,
3877 cumulative_wait_ms,
3878 outcome: ContinuationOutcome::Exhausted,
3879 },
3880 );
3881 }
3882 attempts += 1;
3883 self.state.event(
3884 "fix",
3885 format!(
3886 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888 ),
3889 );
3890 let mut retry = job.clone();
3891 retry.seat = seat.clone();
3892 retry.prompt = prompt::resume_incomplete(&last_err);
3893 retry.timeout = retry_budget(job.timeout, true);
3894 retry.stem = format!("{}-continue{attempts}", job.stem);
3895 let cache = self.state.config.cache_dir();
3896 let ctx = WaveCtx {
3897 carry_seats: false,
3898 pauses: Some((&self.pause, &self.interrupt)),
3899 run: run_id,
3900 node: "fix",
3901 prompts,
3902 cache: cache.as_deref(),
3903 round: Some(round),
3904 };
3905 let (resumed_seat, resumed_out) = run_one(
3906 retry,
3907 Arc::clone(&self.sem),
3908 &ctx,
3909 &mut self.state,
3910 attempts,
3911 )
3912 .await;
3913 seat = resumed_seat;
3914 match resumed_out {
3915 AgentOutcome::Ok(o) => {
3916 cumulative_wait_ms += o.duration_ms;
3917 match verdict::extract_json::<FixReport>(&o.text) {
3918 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919 self.state.event(
3920 "fix",
3921 format!(
3922 "round {round}: fixer's adoption report recovered after \
3923 {attempts} continuation(s)"
3924 ),
3925 );
3926 return (
3927 seat,
3928 Some(report),
3929 None,
3930 ContinuationRecord {
3931 attempts,
3932 cumulative_wait_ms,
3933 outcome: ContinuationOutcome::Resumed,
3934 },
3935 );
3936 }
3937 Ok(_) => {
3945 last_err = "the reply parsed, but it reported a command whose own CLI \
3946 never confirmed an exit status"
3947 .to_owned();
3948 }
3949 Err(e) => last_err = e.to_string(),
3950 }
3951 }
3952 AgentOutcome::Quota(o) => {
3953 cumulative_wait_ms += o.duration_ms;
3954 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956 self.state.quota.push(loss);
3957 self.state.event(
3958 "fix",
3959 format!(
3960 "round {round}: continuation rate limited (quota); not retrying now"
3961 ),
3962 );
3963 return (
3964 seat,
3965 None,
3966 Some("rate limited (quota) while recovering the fix report".to_owned()),
3967 ContinuationRecord {
3968 attempts,
3969 cumulative_wait_ms,
3970 outcome: ContinuationOutcome::QuotaLost,
3971 },
3972 );
3973 }
3974 AgentOutcome::Dropped(o) => {
3975 cumulative_wait_ms += o.duration_ms;
3976 let why = o
3977 .dropped
3978 .as_ref()
3979 .map(|d| d.why.as_str())
3980 .unwrap_or("the CLI ended the stream without delivering its answer");
3981 last_err = format!("the CLI dropped the stream ({why})");
3982 }
3983 AgentOutcome::Failed(e) => last_err = e,
3984 AgentOutcome::ResourceBlocked(e) => {
3987 return (
3988 seat,
3989 None,
3990 Some(format!("the continuation was not started: {e}")),
3991 ContinuationRecord {
3992 attempts: attempts.saturating_sub(1),
3993 cumulative_wait_ms,
3994 outcome: ContinuationOutcome::NoSession,
3995 },
3996 );
3997 }
3998 }
3999 }
4000 }
4001
4002 fn after_implement(&mut self) -> Result<()> {
4003 if !self.state.candidate_preservation.is_empty() {
4004 bail!("candidate preservation incomplete; judging is unsafe");
4005 }
4006 if self.state.leaks.is_empty() {
4008 let cfg = self.state.config.blind.clone();
4009 let mut leaks = Vec::new();
4010 for c in &self.state.candidates {
4011 let Some(patch) =
4012 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013 else {
4014 continue;
4015 };
4016 leaks.extend(blind::scan(
4017 &format!("candidate {} patch", c.label),
4018 &patch,
4019 &cfg.vendor_tokens,
4020 ));
4021 }
4022 if !leaks.is_empty() {
4023 let summary = leaks
4024 .iter()
4025 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026 .collect::<Vec<_>>()
4027 .join(", ");
4028 match cfg.on_leak {
4029 LeakPolicy::Fail => {
4030 self.state.status = RunStatus::Failed;
4031 self.state
4032 .event("blind", format!("vendor text in a patch: {summary}"));
4033 self.state.leaks = leaks;
4034 self.state.save()?;
4035 self.settle_questions();
4036 bail!(
4037 "blind.on_leak = \"fail\" and vendor text reached a \
4038 judged patch: {summary}"
4039 );
4040 }
4041 LeakPolicy::Redact => self.state.event(
4042 "blind",
4043 format!("redacting vendor text for judging: {summary}"),
4044 ),
4045 LeakPolicy::Warn => self.state.event(
4046 "blind",
4047 format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048 ),
4049 }
4050 self.state.leaks = leaks;
4051 }
4052 }
4053
4054 if self.state.viable().is_empty() {
4055 if self.state.all_candidates_verified_noop() {
4056 self.state.status = RunStatus::VerifiedNoop;
4067 self.state.save()?;
4068 self.settle_questions();
4069 return Ok(());
4070 }
4071 self.state.status = RunStatus::Failed;
4072 self.state.save()?;
4073 self.settle_questions();
4074 bail!("no candidate produced a change; nothing to judge");
4075 }
4076 self.state.status = RunStatus::Judging;
4077 self.state.save()?;
4078 Ok(())
4079 }
4080
4081 async fn judge(&mut self) -> Result<()> {
4084 let run_id = self.state.id.clone();
4089 let prompts = self.state.config.prompts.clone();
4090 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091 return Ok(());
4092 }
4093 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094 if viable.len() == 1 {
4095 self.state.judge_skipped = true;
4102 self.state.event(
4103 "judge",
4104 format!(
4105 "only candidate {} produced a change; judging skipped",
4106 viable[0].label
4107 ),
4108 );
4109 self.state.save()?;
4110 return Ok(());
4111 }
4112 self.state.status = RunStatus::Judging;
4113
4114 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115 let language = self.state.config.graph.language.clone();
4116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117 let sessions = self.state.config.graph.sessions;
4118 let artifacts = agent::artifacts_dir(&self.state.dir());
4119 let root = self.state.worktree_root();
4120 let base_short = short(&self.state.base_commit);
4121
4122 let mut jobs = Vec::new();
4123 let mut orders = Vec::new();
4124 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128 let seat_key = format!("judge-{}", j + 1);
4129 let seat = self.seat(&seat_key, &spec.id);
4130 jobs.push(SeatJob {
4131 prompt: prompt::judge(
4132 &self.state.instruction,
4133 &views,
4134 self.roles.judges.len(),
4135 &base_short,
4136 &language,
4137 ),
4138 spec,
4139 seat,
4140 cwd: root.join(format!("judge-{}", j + 1)),
4141 timeout,
4142 allow_write: false,
4143 sessions,
4144 artifacts: artifacts.clone(),
4145 stem: format!("judge-{}", j + 1),
4146 handover: None,
4147 });
4148 }
4149
4150 self.state.event(
4151 "judge",
4152 format!(
4153 "{} judges ranking {} candidates blind",
4154 jobs.len(),
4155 viable.len()
4156 ),
4157 );
4158 let labels_for_check = labels.clone();
4159 let mut quota_losses = Vec::new();
4160 let cache = self.state.config.cache_dir();
4161 let ctx = WaveCtx {
4162 carry_seats: false,
4163 pauses: Some((&self.pause, &self.interrupt)),
4164 run: &run_id,
4165 node: "judge",
4166 prompts: &prompts,
4167 cache: cache.as_deref(),
4168 round: None,
4169 };
4170 let results = ask_json_wave::<Ranking>(
4171 jobs,
4172 Arc::clone(&self.sem),
4173 self.state.config.graph.retries,
4174 &self.roles.judge_roster,
4175 &ctx,
4176 &mut quota_losses,
4177 &mut self.state,
4178 &move |r: &Ranking| r.validate(&labels_for_check),
4179 )
4180 .await;
4181 if self.state.resource_block.is_some() {
4182 return self.stop_for_resource();
4183 }
4184 self.state.quota.extend(quota_losses);
4185
4186 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187 let agent_id = seat.agent.clone();
4188 self.state.seats.insert(seat.key.clone(), seat);
4189 let mut record = Judgement {
4190 judge: j + 1,
4191 seat: format!("judge-{}", j + 1),
4192 agent: agent_id,
4193 ranking: Vec::new(),
4194 reasons: BTreeMap::new(),
4195 confidence: None,
4196 strengths: BTreeMap::new(),
4197 order: orders[j].clone(),
4198 failed: None,
4199 duration_ms: 0,
4200 };
4201 match res {
4202 Ok((ranking, out)) => {
4203 record.ranking = ranking.normalized();
4204 record.reasons = ranking.reasons;
4205 record.strengths = ranking.strengths;
4206 record.confidence = ranking.confidence;
4207 record.duration_ms = out.duration_ms;
4208 self.state.event(
4209 "judge",
4210 format!(
4211 "judge {} ranked {}",
4212 j + 1,
4213 record.ranking.iter().collect::<String>()
4214 ),
4215 );
4216 }
4217 Err(e) => {
4218 record.failed = Some(e.to_string());
4219 self.state
4220 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221 }
4222 }
4223 self.state.judgements.push(record);
4224 self.state.save()?;
4225 }
4226 Ok(())
4227 }
4228
4229 async fn deliberate(&mut self) -> Result<()> {
4232 if self.hold_base_sync().await? {
4233 return Ok(());
4234 }
4235 let run_id = self.state.id.clone();
4240 let prompts = self.state.config.prompts.clone();
4241 let rounds = self.state.config.graph.deliberate_rounds;
4242 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244 return Ok(());
4245 }
4246 let tops: Vec<char> = self
4247 .state
4248 .judgements
4249 .iter()
4250 .filter_map(|j| j.ranking.first().copied())
4251 .collect();
4252 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254 self.state.event(
4255 "deliberate",
4256 format!("judges agreed on {} outright; no deliberation", tops[0]),
4257 );
4258 }
4259 self.state.status = RunStatus::Voting;
4260 self.state.save()?;
4261 return Ok(());
4262 }
4263
4264 self.state.status = RunStatus::Deliberating;
4265 self.state.event(
4266 "deliberate",
4267 format!(
4268 "split: first choices were {} — opening {rounds} round(s)",
4269 tops.iter().collect::<String>()
4270 ),
4271 );
4272
4273 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274 let language = self.state.config.graph.language.clone();
4275 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276 let sessions = self.state.config.graph.sessions;
4277 let artifacts = agent::artifacts_dir(&self.state.dir());
4278 let root = self.state.worktree_root();
4279 let base_short = short(&self.state.base_commit);
4280
4281 for round in self.state.deliberation.len() + 1..=rounds {
4285 let mut turns: Vec<DeliberationTurn> = Vec::new();
4286 let mut round_losses = Vec::new();
4287 let mut asked = 0;
4288 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289 if self.state.judgements[j].failed.is_some() {
4290 continue;
4291 }
4292 let seat_key = format!("judge-{}", j + 1);
4293 let spec = self.occupant(&seat_key, spec);
4294 let mut seat = self.seat(&seat_key, &spec.id);
4295 let transcript = self.transcript(&turns, j);
4296 let build = |context: Option<&str>| {
4297 prompt::deliberate(
4298 &self.state.instruction,
4299 context,
4300 &transcript,
4301 round,
4302 rounds,
4303 &language,
4304 )
4305 };
4306 let block = self.candidate_block(&viable, &base_short);
4307 let full = build(Some(&block));
4308 let text = if has_context(&spec, &seat, sessions) {
4309 build(None)
4310 } else {
4311 full.clone()
4312 };
4313 let job = SeatJob {
4314 spec,
4315 seat: seat.clone(),
4316 prompt: text,
4317 cwd: root.join(format!("judge-{}", j + 1)),
4318 timeout,
4319 allow_write: false,
4320 sessions,
4321 artifacts: artifacts.clone(),
4322 stem: format!("delib-{round}-judge-{}", j + 1),
4323 handover: Some(full),
4324 };
4325 let cache = self.state.config.cache_dir();
4326 let ctx = WaveCtx {
4327 carry_seats: false,
4328 pauses: Some((&self.pause, &self.interrupt)),
4329 run: &run_id,
4330 node: "deliberate",
4331 prompts: &prompts,
4332 cache: cache.as_deref(),
4333 round: None,
4334 };
4335 let mut losses = Vec::new();
4339 asked += 1;
4340 let mut results = ask_wave_with::<String>(
4341 vec![job],
4342 Arc::clone(&self.sem),
4343 0,
4344 &self.roles.judge_roster,
4345 &ctx,
4346 &mut losses,
4347 &mut self.state,
4348 &|text: &str| {
4349 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350 },
4351 )
4352 .await;
4353 if self.state.resource_block.is_some() {
4354 return self.stop_for_resource();
4355 }
4356 round_losses.extend(losses.iter().cloned());
4357 self.state.quota.extend(losses);
4358 let (updated, res, _) = results.pop().expect("one job in, one result out");
4359 seat = updated;
4360 let agent_id = seat.agent.clone();
4361 self.state.seats.insert(seat.key.clone(), seat);
4362 let body = match res {
4363 Ok((body, _)) => body,
4364 Err(e) => {
4367 self.state
4368 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369 continue;
4370 }
4371 };
4372 let tentative = verdict::extract_json::<Position>(&body)
4373 .ok()
4374 .and_then(|p| p.tentative)
4375 .and_then(|s| s.trim().chars().next())
4376 .map(|c| c.to_ascii_uppercase());
4377 self.state.event(
4378 "deliberate",
4379 format!(
4380 "round {round}: judge {} now favours {}",
4381 j + 1,
4382 tentative.map_or("—".to_owned(), |c| c.to_string())
4383 ),
4384 );
4385 turns.push(DeliberationTurn {
4386 judge: j + 1,
4387 agent: agent_id,
4388 body: blind::sanitize_prose(&body, &self.state.config.blind),
4389 tentative,
4390 });
4391 }
4392 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394 return self.stop_for_resource();
4395 }
4396 self.state
4397 .deliberation
4398 .push(DeliberationRound { round, turns });
4399 self.state.save()?;
4400 }
4401
4402 self.state.status = RunStatus::Voting;
4403 self.state.save()?;
4404 Ok(())
4405 }
4406
4407 async fn vote(&mut self) -> Result<()> {
4410 if self.hold_base_sync().await? {
4411 return Ok(());
4412 }
4413 let run_id = self.state.id.clone();
4418 let prompts = self.state.config.prompts.clone();
4419 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420 return Ok(());
4421 }
4422 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423 if viable.len() == 1 {
4424 return Ok(());
4425 }
4426 self.state.status = RunStatus::Voting;
4427
4428 let language = self.state.config.graph.language.clone();
4429 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430 let sessions = self.state.config.graph.sessions;
4431 let artifacts = agent::artifacts_dir(&self.state.dir());
4432 let root = self.state.worktree_root();
4433 let base_short = short(&self.state.base_commit);
4434 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436 let mut jobs = Vec::new();
4437 let mut seats_at = Vec::new();
4438 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439 if self
4440 .state
4441 .judgements
4442 .get(j)
4443 .is_some_and(|r| r.failed.is_some())
4444 {
4445 continue;
4446 }
4447 let seat_key = format!("judge-{}", j + 1);
4448 let spec = self.occupant(&seat_key, spec);
4449 let seat = self.seat(&seat_key, &spec.id);
4450 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451 let text = if has_context(&spec, &seat, sessions) {
4452 prompt::final_vote(&viable, &language)
4453 } else {
4454 full.clone()
4455 };
4456 jobs.push(SeatJob {
4457 spec,
4458 seat,
4459 prompt: text,
4460 cwd: root.join(format!("judge-{}", j + 1)),
4461 timeout,
4462 allow_write: false,
4463 sessions,
4464 artifacts: artifacts.clone(),
4465 stem: format!("vote-judge-{}", j + 1),
4466 handover: Some(full),
4467 });
4468 seats_at.push(j);
4469 }
4470
4471 self.state.event(
4472 "vote",
4473 format!(
4474 "collecting {} final votes one by one, privately",
4475 jobs.len()
4476 ),
4477 );
4478 let allowed = viable.clone();
4479 let mut quota_losses = Vec::new();
4480 let cache = self.state.config.cache_dir();
4481 let ctx = WaveCtx {
4482 carry_seats: false,
4483 pauses: Some((&self.pause, &self.interrupt)),
4484 run: &run_id,
4485 node: "vote",
4486 prompts: &prompts,
4487 cache: cache.as_deref(),
4488 round: None,
4489 };
4490 let results = ask_json_wave::<FinalVote>(
4491 jobs,
4492 Arc::clone(&self.sem),
4493 self.state.config.graph.retries,
4494 &self.roles.judge_roster,
4495 &ctx,
4496 &mut quota_losses,
4497 &mut self.state,
4498 &move |v: &FinalVote| match v.label() {
4499 Some(c) if allowed.contains(&c) => Ok(()),
4500 other => bail!("vote {other:?} is not one of {allowed:?}"),
4501 },
4502 )
4503 .await;
4504 if self.state.resource_block.is_some() {
4505 return self.stop_for_resource();
4506 }
4507 self.state.quota.extend(quota_losses);
4508
4509 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510 let agent_id = seat.agent.clone();
4511 self.state.seats.insert(seat.key.clone(), seat);
4512 let initial = self
4513 .state
4514 .judgements
4515 .get(j)
4516 .and_then(|r| r.ranking.first().copied());
4517 let mut record = VoteRecord {
4518 judge: j + 1,
4519 agent: agent_id,
4520 vote: None,
4521 reason: String::new(),
4522 changed: false,
4523 };
4524 match res {
4525 Ok((v, _)) => {
4526 record.vote = v.label();
4527 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529 self.state.event(
4530 "vote",
4531 format!(
4532 "judge {} voted {}{}",
4533 j + 1,
4534 record.vote.unwrap_or('?'),
4535 if record.changed { " (changed)" } else { "" }
4536 ),
4537 );
4538 }
4539 Err(e) => {
4540 self.state
4541 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542 }
4543 }
4544 self.state.votes.push(record);
4545 self.state.save()?;
4546 }
4547 Ok(())
4548 }
4549
4550 fn tally(&mut self) -> Result<()> {
4553 if self.state.tally.is_some() {
4554 return Ok(());
4555 }
4556 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557 let tops: Vec<char> = self
4558 .state
4559 .judgements
4560 .iter()
4561 .filter_map(|j| j.ranking.first().copied())
4562 .collect();
4563 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568 let mut cast: Vec<char> = Vec::new();
4569 for (i, j) in self.state.judgements.iter().enumerate() {
4570 let vote = self
4571 .state
4572 .votes
4573 .iter()
4574 .find(|v| v.judge == i + 1)
4575 .and_then(|v| v.vote)
4576 .or_else(|| j.ranking.first().copied());
4577 if let Some(v) = vote {
4578 *first_choice.entry(v).or_insert(0) += 1;
4579 cast.push(v);
4580 }
4581 }
4582
4583 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584 for j in &self.state.judgements {
4585 let n = j.ranking.len();
4586 for (pos, label) in j.ranking.iter().enumerate() {
4587 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588 }
4589 }
4590
4591 let best = first_choice.values().copied().max().unwrap_or(0);
4592 let mut leaders: Vec<char> = first_choice
4593 .iter()
4594 .filter(|(_, v)| **v == best)
4595 .map(|(k, _)| *k)
4596 .collect();
4597 let mut tie_break = None;
4598 if leaders.len() > 1 {
4599 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600 let borda_leaders: Vec<char> = leaders
4601 .iter()
4602 .copied()
4603 .filter(|l| borda[l] == top_borda)
4604 .collect();
4605 tie_break = Some(if borda_leaders.len() == 1 {
4606 format!(
4607 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608 leaders.len()
4609 )
4610 } else {
4611 format!(
4612 "{} way tie on both first-choice votes and Borda points, broken by label order",
4613 leaders.len()
4614 )
4615 });
4616 leaders = borda_leaders;
4617 leaders.sort_unstable();
4618 }
4619 let winner = *leaders
4620 .first()
4621 .or(viable.first())
4622 .context("no candidate to declare a winner from")?;
4623
4624 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626 let deliberated = !self.state.deliberation.is_empty();
4627
4628 let quota_seats: std::collections::BTreeSet<&str> =
4632 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633 let mut present = 0usize;
4634 for (i, j) in self.state.judgements.iter().enumerate() {
4635 if quota_seats.contains(j.seat.as_str()) {
4636 continue;
4637 }
4638 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639 let voted = self
4640 .state
4641 .votes
4642 .iter()
4643 .any(|v| v.judge == i + 1 && v.vote.is_some());
4644 if ranked || voted {
4645 present += 1;
4646 }
4647 }
4648 let needs_quorum = viable.len() > 1;
4654 let judges_total = if needs_quorum {
4655 self.roles.judges.len()
4656 } else {
4657 0
4658 };
4659 let quorum = if needs_quorum {
4660 judges_total / 2 + 1
4661 } else {
4662 0
4663 };
4664 let met_quorum = !needs_quorum || present >= quorum;
4665 let uncontested = (!needs_quorum).then(|| {
4666 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667 });
4668
4669 self.state.event(
4670 "tally",
4671 match &uncontested {
4672 Some(reason) => format!("winner {winner} — {reason}"),
4673 None => format!(
4674 "winner {winner} — votes {} | initial {} | {} changed | \
4675 {present}/{judges_total} judges{}",
4676 first_choice
4677 .iter()
4678 .map(|(k, v)| format!("{k}:{v}"))
4679 .collect::<Vec<_>>()
4680 .join(" "),
4681 if unanimous_initial {
4682 "unanimous"
4683 } else {
4684 "split"
4685 },
4686 changed_votes,
4687 if met_quorum {
4688 String::new()
4689 } else {
4690 format!(" — below quorum ({quorum} required)")
4691 },
4692 ),
4693 },
4694 );
4695 if !met_quorum {
4696 self.state.event(
4697 "stall",
4698 format!(
4699 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700 the run stops here, resumable"
4701 ),
4702 );
4703 }
4704 self.state.tally = Some(Tally {
4705 first_choice,
4706 borda,
4707 winner,
4708 rankings: tops.len(),
4709 unanimous_initial,
4710 deliberated,
4711 changed_votes,
4712 unanimous_final,
4713 tie_break,
4714 judges: judges_total,
4715 present,
4716 quorum,
4717 met_quorum,
4718 uncontested,
4719 });
4720 self.state.status = if met_quorum {
4721 RunStatus::Reviewing
4722 } else {
4723 RunStatus::Stalled
4724 };
4725 self.state.save()?;
4726 Ok(())
4727 }
4728
4729 #[allow(clippy::too_many_lines)]
4750 async fn recover_stall(&mut self) -> Result<bool> {
4751 let run_id = self.state.id.clone();
4756 let prompts = self.state.config.prompts.clone();
4757 let quota_seats: BTreeSet<&str> =
4762 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763 let absent: Vec<String> = self
4764 .state
4765 .judgements
4766 .iter()
4767 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768 .map(|j| j.seat.clone())
4769 .collect();
4770 if absent.is_empty() {
4771 return Ok(false);
4772 }
4773 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774 if viable.len() <= 1 {
4775 return Ok(false);
4776 }
4777 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778 let language = self.state.config.graph.language.clone();
4779 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780 let sessions = self.state.config.graph.sessions;
4781 let artifacts = agent::artifacts_dir(&self.state.dir());
4782 let root = self.state.worktree_root();
4783 let base_short = short(&self.state.base_commit);
4784 let candidates: Vec<Candidate> = viable.clone();
4785
4786 let mut positions: Vec<usize> = absent
4788 .iter()
4789 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790 .collect();
4791 if positions.is_empty() {
4792 return Ok(false);
4793 }
4794 positions.sort_unstable();
4795 positions.dedup();
4796
4797 let mut judge_jobs = Vec::new();
4799 for &j in &positions {
4800 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802 let seat_key = format!("judge-{}", j + 1);
4803 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804 let seat = self.seat(&seat_key, &spec.id);
4805 judge_jobs.push(SeatJob {
4806 spec,
4807 seat,
4808 prompt: prompt::judge(
4809 &self.state.instruction,
4810 &views,
4811 self.roles.judges.len(),
4812 &base_short,
4813 &language,
4814 ),
4815 cwd: root.join(seat_key),
4816 timeout,
4817 allow_write: false,
4818 sessions,
4819 artifacts: artifacts.clone(),
4820 stem: format!("judge-{}-recover", j + 1),
4821 handover: None,
4822 });
4823 }
4824
4825 let labels_for_check = labels.clone();
4826 let mut judge_losses = Vec::new();
4827 let retries = self.state.config.graph.retries;
4828 let cache = self.state.config.cache_dir();
4829 let ctx = WaveCtx {
4830 carry_seats: false,
4831 pauses: Some((&self.pause, &self.interrupt)),
4832 run: &run_id,
4833 node: "judge",
4834 prompts: &prompts,
4835 cache: cache.as_deref(),
4836 round: None,
4837 };
4838 let results = ask_json_wave::<Ranking>(
4839 judge_jobs,
4840 Arc::clone(&self.sem),
4841 retries,
4842 &self.roles.judge_roster,
4843 &ctx,
4844 &mut judge_losses,
4845 &mut self.state,
4846 &move |r: &Ranking| r.validate(&labels_for_check),
4847 )
4848 .await;
4849 if self.state.resource_block.is_some() {
4850 self.stop_for_resource()?;
4851 return Ok(false);
4852 }
4853
4854 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857 let agent_id = seat.agent.clone();
4858 self.state.seats.insert(seat.key.clone(), seat);
4859 let record = &mut self.state.judgements[j];
4860 match res {
4861 Ok((ranking, out)) => {
4862 record.agent = agent_id;
4863 record.ranking = ranking.normalized();
4864 record.reasons = ranking.reasons;
4865 record.strengths = ranking.strengths;
4866 record.confidence = ranking.confidence;
4867 record.failed = None;
4868 record.duration_ms = out.duration_ms;
4869 recovered.insert(j);
4870 self.state.event(
4871 "recover",
4872 format!("judge {} ranked again after the limit", j + 1),
4873 );
4874 }
4875 Err(e) => {
4876 self.state
4877 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878 }
4879 }
4880 }
4881
4882 let mut vote_jobs = Vec::new();
4884 let mut vote_pos: Vec<usize> = Vec::new();
4885 for &j in &recovered {
4886 let seat_key = format!("judge-{}", j + 1);
4887 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888 let seat = self.seat(&seat_key, &spec.id);
4889 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890 let text = if has_context(&spec, &seat, sessions) {
4891 prompt::final_vote(&labels, &language)
4892 } else {
4893 full.clone()
4894 };
4895 vote_jobs.push(SeatJob {
4896 spec,
4897 seat,
4898 prompt: text,
4899 cwd: root.join(seat_key),
4900 timeout,
4901 allow_write: false,
4902 sessions,
4903 artifacts: artifacts.clone(),
4904 stem: format!("vote-judge-{}-recover", j + 1),
4905 handover: Some(full),
4906 });
4907 vote_pos.push(j);
4908 }
4909 let allowed = labels.clone();
4910 let mut vote_losses = Vec::new();
4911 let vote_retries = self.state.config.graph.retries;
4912 let vote_cache = self.state.config.cache_dir();
4913 let ctx = WaveCtx {
4914 carry_seats: false,
4915 pauses: Some((&self.pause, &self.interrupt)),
4916 run: &run_id,
4917 node: "vote",
4918 prompts: &prompts,
4919 cache: vote_cache.as_deref(),
4920 round: None,
4921 };
4922 let votes = ask_json_wave::<FinalVote>(
4923 vote_jobs,
4924 Arc::clone(&self.sem),
4925 vote_retries,
4926 &self.roles.judge_roster,
4927 &ctx,
4928 &mut vote_losses,
4929 &mut self.state,
4930 &move |v: &FinalVote| match v.label() {
4931 Some(c) if allowed.contains(&c) => Ok(()),
4932 other => bail!("vote {other:?} is not one of {allowed:?}"),
4933 },
4934 )
4935 .await;
4936 if self.state.resource_block.is_some() {
4937 self.stop_for_resource()?;
4938 return Ok(false);
4939 }
4940 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941 let agent_id = seat.agent.clone();
4942 self.state.seats.insert(seat.key.clone(), seat);
4943 match res {
4944 Ok((v, _)) => {
4945 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946 rec.vote = v.label();
4947 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948 } else {
4949 self.state.votes.push(VoteRecord {
4950 judge: j + 1,
4951 agent: agent_id,
4952 vote: v.label(),
4953 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954 changed: false,
4955 });
4956 }
4957 self.state.event(
4958 "recover",
4959 format!("judge {} voted again after the limit", j + 1),
4960 );
4961 }
4962 Err(e) => {
4963 self.state
4964 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965 }
4966 }
4967 }
4968
4969 let recovered_keys: BTreeSet<String> = recovered
4973 .iter()
4974 .map(|&j| format!("judge-{}", j + 1))
4975 .collect();
4976 self.state
4977 .quota
4978 .retain(|q| !recovered_keys.contains(&q.seat));
4979 for loss in judge_losses.into_iter().chain(vote_losses) {
4983 if recovered_keys.contains(&loss.seat) {
4984 continue;
4985 }
4986 self.state.quota.retain(|q| q.seat != loss.seat);
4987 self.state.quota.push(loss);
4988 }
4989
4990 self.state.tally = None;
4992 self.tally()?;
4993 Ok(self
4994 .state
4995 .tally
4996 .as_ref()
4997 .map(|t| t.met_quorum)
4998 .unwrap_or(false))
4999 }
5000
5001 async fn graft(&mut self) -> Result<()> {
5017 let Err(e) = self.graft_inner().await else {
5018 return Ok(());
5019 };
5020 if e.is::<crate::repo_lock::ResourceBlocked>() {
5021 return Err(e);
5022 }
5023 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024 self.state.status = RunStatus::Blocked;
5025 self.state.event("graft", format!("blocked: {e:#}"));
5026 self.state.save()?;
5027 return Err(e);
5028 }
5029 if let Some(rec) = self.state.graft.clone()
5034 && !rec.outcome.settled()
5035 {
5036 if let Some(winner) = self.state.winner().cloned() {
5037 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038 if re.is::<crate::repo_lock::ResourceBlocked>() {
5039 return Err(re);
5040 }
5041 self.state.status = RunStatus::Blocked;
5042 self.state.event("graft", format!("blocked: {re:#}"));
5043 self.state.save()?;
5044 return Err(re);
5045 }
5046 }
5047 if let Some(g) = self.state.graft.as_mut() {
5048 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049 }
5050 }
5051 self.state.event("graft", format!("skipped: {e:#}"));
5052 if let Err(e) = self.state.save() {
5053 tracing::warn!("save after graft: {e:#}");
5054 }
5055 Ok(())
5056 }
5057
5058 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061 let _coordination = self.coordinate(wt).await?;
5062 crate::recovery::restore_repair(wt, pre)
5063 .await
5064 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065 }
5066
5067 fn graft_skipped(&mut self, why: &str) {
5069 let message = format!("skipped: {why}");
5070 if !self
5071 .state
5072 .events
5073 .iter()
5074 .any(|e| e.node == "graft" && e.message == message)
5075 {
5076 self.state.event("graft", message);
5077 }
5078 }
5079
5080 async fn graft_inner(&mut self) -> Result<()> {
5081 if let Some(rec) = self.state.graft.clone()
5084 && !rec.outcome.settled()
5085 {
5086 if self.state.reviews.is_empty()
5090 && let Some(winner) = self.state.winner().cloned()
5091 {
5092 self.graft_restore(&winner.worktree, &rec.pre).await?;
5093 }
5094 if rec.outcome == GraftOutcome::WaitingForRestore {
5095 self.state.graft = None;
5096 self.state.save()?;
5097 } else {
5098 if let Some(g) = self.state.graft.as_mut() {
5099 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100 }
5101 self.state.event(
5102 "graft",
5103 "rolled back: the previous round was interrupted; the winner is untouched",
5104 );
5105 self.state.save()?;
5106 return Ok(());
5107 }
5108 }
5109 let plan = match graft::plan(&self.state) {
5110 Ok(plan) => plan,
5111 Err(skip) => {
5112 if skip != graft::Skip::AlreadyDone {
5113 self.graft_skipped(skip.reason());
5114 self.state.save()?;
5115 }
5116 return Ok(());
5117 }
5118 };
5119 let Some(winner) = self.state.winner().cloned() else {
5120 return Ok(());
5121 };
5122 let wt = winner.worktree.clone();
5123 if !git::is_clean(&wt).await? {
5124 self.graft_skipped("the winner's worktree has uncommitted changes");
5125 self.state.save()?;
5126 return Ok(());
5127 }
5128 let pre = git::rev_parse(&wt, "HEAD").await?;
5129 self.state.graft = Some(GraftRecord {
5132 pre: pre.clone(),
5133 post: None,
5134 lines: None,
5135 limit: None,
5136 agent: None,
5137 outcome: GraftOutcome::Running,
5138 requested: graft::requested(&plan),
5139 adoption: None,
5140 adoption_error: None,
5141 });
5142 self.state.event(
5143 "graft",
5144 format!(
5145 "porting strengths of candidate(s) {} onto {}",
5146 plan.strengths
5147 .iter()
5148 .map(|(l, _, _)| l.to_string())
5149 .collect::<Vec<_>>()
5150 .join(", "),
5151 plan.winner
5152 ),
5153 );
5154 self.state.save()?;
5155
5156 let fix_prompt = prompt::graft(
5157 &self.state.instruction,
5158 plan.winner,
5159 &plan.strengths,
5160 &self.state.config.graph.language,
5161 );
5162 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163 let sessions = self.state.config.graph.sessions;
5164 let artifacts = agent::artifacts_dir(&self.state.dir());
5165 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166 let (_, seat, out) = self
5167 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168 SeatJob {
5169 prompt: fix_prompt.clone(),
5170 spec,
5171 seat,
5172 cwd: wt.clone(),
5173 timeout,
5174 allow_write: true,
5175 sessions,
5176 artifacts: artifacts.clone(),
5177 stem: "graft".to_owned(),
5178 handover: None,
5179 }
5180 })
5181 .await;
5182 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183 self.state.graft.as_mut().expect("graft record").outcome =
5186 GraftOutcome::WaitingForRestore;
5187 return self.stop_for_resource();
5188 }
5189 let mut why: Option<String> = None;
5190 match &out {
5191 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192 why = Some("the fixer produced nothing usable".to_owned());
5193 }
5194 AgentOutcome::Ok(_) => {}
5195 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196 AgentOutcome::Quota(o) => {
5197 self.state.quota.push(QuotaLoss::from_report(
5198 seat.key.clone(),
5199 "graft",
5200 o.quota.as_ref(),
5201 ));
5202 why = Some("rate limited (quota)".to_owned());
5203 }
5204 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205 why = Some(format!(
5206 "the fixer failed: {}",
5207 e.lines()
5208 .next()
5209 .unwrap_or("")
5210 .chars()
5211 .take(160)
5212 .collect::<String>()
5213 ));
5214 }
5215 }
5216 let agent_id = seat.agent.clone();
5217 self.state.seats.insert(seat.key.clone(), seat);
5218
5219 let mut post = None;
5220 let mut lines = None;
5221 let _coordination = self.coordinate(&wt).await?;
5222 if why.is_none() {
5223 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224 if let Ok(r) = &rescued {
5225 self.state.note_withheld("graft", &r.withheld);
5226 }
5227 let head = git::rev_parse(&wt, "HEAD").await?;
5228 let dirty = match &rescued {
5232 Ok(r) => git::status_porcelain(&wt)
5233 .await
5234 .map(|st| {
5235 st.lines().any(|l| {
5236 !r.withheld
5237 .iter()
5238 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239 })
5240 })
5241 .unwrap_or(true),
5242 Err(_) => true,
5243 };
5244 if dirty {
5245 why = Some("the fixer left changes that could not be committed".to_owned());
5246 } else if head == pre {
5247 why = Some("the fixer changed nothing".to_owned());
5248 } else {
5249 lines = git::changed_lines(&wt, &pre, &head).await;
5250 post = Some(head);
5251 }
5252 }
5253 let (adoption, adoption_error) = match &out {
5254 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255 _ => (None, None),
5256 };
5257 let requested = graft::requested(&plan);
5258 let outcome = match why {
5259 None => {
5260 self.state.event(
5261 "graft",
5262 format!(
5263 "applied: {}",
5264 lines.map_or_else(
5265 || "changed lines unavailable".to_owned(),
5266 |n| format!("{n} changed lines")
5267 )
5268 ),
5269 );
5270 GraftOutcome::Applied
5271 }
5272 Some(why) => {
5273 crate::recovery::restore_repair(&wt, &pre)
5276 .await
5277 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278 self.state.event(
5279 "graft",
5280 format!("rolled back: {why}; the winner is untouched"),
5281 );
5282 GraftOutcome::RolledBack(why)
5283 }
5284 };
5285 self.state.graft = Some(GraftRecord {
5286 pre,
5287 post,
5288 lines,
5289 limit: None,
5290 agent: Some(agent_id),
5291 outcome,
5292 requested,
5293 adoption,
5294 adoption_error,
5295 });
5296 self.state.save()?;
5297 Ok(())
5298 }
5299
5300 async fn fold_losers(&mut self) -> Result<()> {
5303 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304 return Ok(());
5305 };
5306 if !self.state.candidate_preservation.is_empty() {
5307 bail!("candidate preservation incomplete; refusing cleanup");
5308 }
5309 let repo = self.state.repo.clone();
5310 let _coordination = self.coordinate(&repo).await?;
5311 let mut folded = Vec::new();
5312 for i in 0..self.state.candidates.len() {
5313 let c = &self.state.candidates[i];
5314 if c.label == winner || c.folded {
5315 continue;
5316 }
5317 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318 remove_preserved_worktree(&repo, &wt).await?;
5319 if git::branch_exists(&repo, &branch).await?
5320 && !git::branch_delete(&repo, &branch).await?
5321 {
5322 bail!("candidate branch removal refused");
5323 }
5324 self.state.candidates[i].folded = true;
5325 self.state.save()?;
5326 folded.push(label.to_string());
5327 }
5328 let root = self.state.worktree_root();
5330 for j in 1..=self.roles.judges.len() {
5331 let wt = root.join(format!("judge-{j}"));
5332 if wt.exists() {
5333 remove_read_seat_worktree(&mut self.state, &wt).await?;
5334 self.state.save()?;
5335 }
5336 }
5337 if self.state.config.graph.advise {
5340 for k in 1..=self.state.config.graph.advisors {
5341 let wt = root.join(format!("advisor-{k}"));
5342 if wt.exists() {
5343 remove_read_seat_worktree(&mut self.state, &wt).await?;
5344 self.state.save()?;
5345 }
5346 }
5347 }
5348 if !folded.is_empty() {
5349 self.state
5350 .event("fold", format!("folded candidates {}", folded.join(", ")));
5351 self.state.save()?;
5352 }
5353 Ok(())
5354 }
5355
5356 async fn hold_base_sync(&mut self) -> Result<bool> {
5359 self.hold_base_sync_as(None).await
5360 }
5361
5362 async fn hold_base_sync_kind(
5363 &mut self,
5364 kind: crate::base_sync_recovery::StopKind,
5365 ) -> Result<()> {
5366 self.hold_base_sync_as(Some(kind)).await?;
5367 Ok(())
5368 }
5369
5370 async fn hold_base_sync_as(
5371 &mut self,
5372 kind: Option<crate::base_sync_recovery::StopKind>,
5373 ) -> Result<bool> {
5374 if !self.state.base_sync_blocked() {
5375 self.state.base_sync_recovery.stop = None;
5376 return Ok(false);
5377 }
5378 let head = match self.state.winner() {
5379 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5380 .await
5381 .unwrap_or_default(),
5382 None => String::new(),
5383 };
5384 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5385 && !authorization.consumed
5386 && authorization.stop.head != head
5387 {
5388 authorization.refuse(format!(
5389 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5390 authorization.stop.head
5391 ));
5392 }
5393 self.state.record_base_sync_stop(&head);
5394 if let Some(kind) = kind {
5395 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5396 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5397 stop.kind = kind;
5398 }
5399 }
5400 self.state.save()?;
5401 Ok(true)
5402 }
5403
5404 async fn sync_to_base(&mut self) -> Result<()> {
5446 if self.state.status == RunStatus::AlreadyInBase {
5447 return Ok(());
5448 }
5449 self.hold_base_sync().await?;
5450 let conflicted = self
5451 .state
5452 .base_sync
5453 .as_ref()
5454 .is_some_and(|s| s.conflict.is_some());
5455 let Some(winner) = self.state.winner().cloned() else {
5456 return Ok(());
5457 };
5458
5459 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5460
5461 let repo = self.state.repo.clone();
5462 let remote = self.state.config.merge.remote.clone();
5463 let base_branch = self.state.base_branch.clone();
5464 let tracking = format!("{remote}/{base_branch}");
5465
5466 let fetched_base =
5467 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5468 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5473 return Ok(());
5474 };
5475
5476 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5477 self.state.ensure_base_episode();
5478 let mut pending = self
5479 .state
5480 .base_sync_recovery
5481 .operations
5482 .last()
5483 .filter(|o| !o.finished)
5484 .cloned();
5485 let scratch = self.state.dir().join("base-sync");
5486 if git::rebase_in_progress(&scratch).await {
5487 if pending.is_none()
5490 && self.state.base_sync_recovery.operations.is_empty()
5491 && !conflicted
5492 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5493 .await
5494 .ok()
5495 .as_deref()
5496 == Some(head.as_str())
5497 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5498 && git::is_clean(&winner.worktree).await?
5499 {
5500 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5501 if self.state.base_sync.is_none() {
5502 self.state.base_sync = Some(BaseSync {
5503 tip: saved_tip.clone(),
5504 behind: 0,
5505 attempts: 0,
5506 conflict: None,
5507 already_in: None,
5508 });
5509 }
5510 self.state.reserve_base_operation(&head, &saved_tip);
5511 self.state.event(
5512 "base-recovery",
5513 "imported standing legacy rebase as a spent synchronization reservation",
5514 );
5515 self.state.save()?;
5516 pending = self.state.base_sync_recovery.operations.last().cloned();
5517 }
5518 let expected = pending.as_ref().map(|op| op.tip.as_str());
5519 if expected.is_none()
5520 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5521 {
5522 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5523 if self.state.base_sync.is_none() {
5524 self.state.base_sync = Some(BaseSync {
5525 tip: tip.clone(),
5526 behind: 0,
5527 attempts: 0,
5528 conflict: None,
5529 already_in: None,
5530 });
5531 }
5532 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5533 self.state.event("land", why);
5534 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535 .await?;
5536 return Ok(());
5537 }
5538 }
5539 if let Some(op) = &pending {
5540 let mut replay_proven =
5542 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5543 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5544 let wait = repository_wait(&self.state, None);
5545 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5546 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5547 Ok(true) => {
5548 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5549 self.state.save()?;
5550 git::worktree_remove(&repo, &scratch).await.ok();
5551 replay_proven = true;
5552 }
5553 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5554 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5555 }
5556 }
5557 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5558 if self.state.foreign_checkout.is_some() {
5559 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5560 .await
5561 .ok()
5562 .as_deref()
5563 == Some(op.head.as_str())
5564 && git::rev_parse(&repo, &format!("{head}^2"))
5565 .await
5566 .ok()
5567 .as_deref()
5568 == Some(op.tip.as_str());
5569 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5570 .await
5571 .is_ok_and(|m| m.is_empty())
5572 && let Ok(added) =
5573 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5574 {
5575 replay_proven = true;
5576 for commit in added {
5577 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5578 replay_proven = false;
5579 break;
5580 }
5581 }
5582 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5583 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5584 }
5585 }
5586 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590 || (head != op.head
5591 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5592 .await?
5593 .is_empty())
5594 || (!git::is_clean(&winner.worktree).await?
5595 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5596 .await
5597 .is_ok_and(|o| o.ok()))
5598 || !replay_proven
5599 {
5600 let why = format!(
5601 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5602 op.id
5603 );
5604 let sync = self
5605 .state
5606 .base_sync
5607 .as_mut()
5608 .context("reserved synchronization has no lifetime accounting")?;
5609 sync.conflict = Some(why.clone());
5610 self.state.event("land", why);
5611 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5612 .await?;
5613 return Ok(());
5614 }
5615 if head != op.head {
5616 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5617 }
5618 tip = op.tip.clone();
5619 }
5620 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5621 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5622 if pending.is_none()
5623 && !conflicted
5624 && fetched_base
5625 && self
5626 .state
5627 .base_sync_recovery
5628 .episodes
5629 .last()
5630 .is_some_and(|e| e.boundary.is_some())
5631 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5632 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5633 && owner.execution_runs().last() == Some(&self.state.id)
5634 && owner.status != crate::queue::TaskStatus::Done
5635 && !owner.fresh_start
5636 && owner.review_branch.is_none()
5637 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5638 && let Some(approval) = owner.resumption.as_ref()
5639 && approval.instruction == owner.instruction
5640 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5641 && git::is_clean(&winner.worktree).await?
5642 && self.state.renew_base_episode(&task, approval, &head, &tip)
5643 {
5644 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5645 self.state.save()?;
5646 }
5647
5648 if (behind > 0 || conflicted || head == tip)
5657 && self
5658 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5659 .await?
5660 {
5661 return Ok(());
5662 }
5663 if conflicted && self.state.base_recovery_pending() {
5664 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5665 let owner =
5666 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5667 let owner_valid = owner.as_ref().is_ok_and(|t| {
5668 t.execution_runs().last() == Some(&self.state.id)
5669 && t.status != crate::queue::TaskStatus::Done
5670 && !t.fresh_start
5671 && t.review_branch.is_none()
5672 });
5673 let invalidation = if !owner_valid {
5674 Some("latest task ownership changed")
5675 } else if !owner.as_ref().is_ok_and(|t| {
5676 authorization
5677 .generation
5678 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5679 }) {
5680 Some("owner approval generation changed")
5681 } else if authorization.question.as_ref().is_some_and(|question| {
5682 !crate::base_recovery_control::decision_current(
5683 &crate::queue::Queue::at(crate::run::home().join("queue")),
5684 &self.state,
5685 &authorization.task,
5686 question,
5687 )
5688 }) {
5689 Some("recovery decision ownership, routing or stop evidence changed")
5690 } else if authorization.request.is_some()
5691 && !owner
5692 .as_ref()
5693 .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5694 {
5695 Some("owner decision changed")
5696 } else if let Some(scope) = &authorization.scope {
5697 if scope.tip != tip {
5698 Some("fetched base changed")
5699 } else if !self
5700 .state
5701 .base_sync_recovery
5702 .episodes
5703 .last()
5704 .is_some_and(|e| e.id == scope.episode)
5705 {
5706 Some("active synchronization interval changed")
5707 } else if !owner
5708 .as_ref()
5709 .is_ok_and(|t| t.instruction == scope.instruction)
5710 {
5711 Some("task instruction changed")
5712 } else {
5713 None
5714 }
5715 } else {
5716 None
5717 };
5718 if let Some(reason) = invalidation {
5719 let authorization = self
5720 .state
5721 .base_sync_recovery
5722 .authorizations
5723 .last_mut()
5724 .unwrap();
5725 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5726 self.state.save()?;
5727 return Ok(());
5728 }
5729 if !fetched_base
5730 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5731 || !git::is_clean(&winner.worktree).await?
5732 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5733 {
5734 return Ok(());
5735 }
5736 }
5737 let recovery = conflicted && self.state.consume_base_recovery(&head);
5738 if conflicted {
5739 self.state.save()?;
5741 if !recovery {
5742 return Ok(());
5743 }
5744 }
5745
5746 if behind == 0 {
5747 if let Some(from) = self
5756 .state
5757 .rebase_fixes
5758 .iter()
5759 .rev()
5760 .find_map(|r| r.from.clone())
5761 && from != head
5762 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5763 .await
5764 .is_ok_and(|o| o.ok())
5765 {
5766 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5767 }
5768 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5774 if let Some(why) = &conflict {
5775 self.state.status = RunStatus::Blocked;
5776 self.state.event("land", why.clone());
5777 }
5778 self.state.base_sync = Some(BaseSync {
5779 tip,
5780 behind: 0,
5781 attempts,
5782 conflict,
5783 already_in: None,
5784 });
5785 if pending.is_some() {
5786 self.state.finish_base_operation(Some(&head));
5787 }
5788 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5789 .await?;
5790 self.state.save()?;
5791 return Ok(());
5792 }
5793
5794 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5795 let why = format!(
5796 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5797 synchronization(s); synchronizing again would only race it",
5798 winner.branch
5799 );
5800 self.state.status = RunStatus::Blocked;
5801 self.state.base_sync = Some(BaseSync {
5802 tip,
5803 behind,
5804 attempts,
5805 conflict: Some(why.clone()),
5806 already_in: None,
5807 });
5808 self.state.event("land", why);
5809 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5810 .await?;
5811 self.state.save()?;
5812 return Ok(());
5813 }
5814
5815 if self.state.foreign_checkout.is_some() {
5816 let wait = repository_wait(&self.state, None);
5819 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5820 if self.state.base_sync.is_none() {
5821 self.state.base_sync = Some(BaseSync {
5822 tip: tip.clone(),
5823 behind,
5824 attempts: 0,
5825 conflict: None,
5826 already_in: None,
5827 });
5828 }
5829 self.state.reserve_base_operation(&head, &tip);
5830 self.state.save()?;
5831 let merged = git::git_raw(
5832 &winner.worktree,
5833 &[
5834 "-c",
5835 "core.hooksPath=/dev/null",
5836 "merge",
5837 "--no-ff",
5838 "--no-autostash",
5839 "--no-edit",
5840 "-m",
5841 &format!("Merge {tracking} before verifying"),
5842 &tip,
5843 ],
5844 )
5845 .await?;
5846 let conflict = if merged.ok() {
5847 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5848 self.state.event(
5849 "land",
5850 format!("merged {tracking} into {} before verifying", winner.branch),
5851 );
5852 None
5853 } else {
5854 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5855 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5856 }
5857 let why = format!(
5858 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5859 winner.branch,
5860 merged.stderr.chars().take(600).collect::<String>()
5861 );
5862 self.state.status = RunStatus::Blocked;
5863 self.state.event("land", &why);
5864 Some(why)
5865 };
5866 let synced = conflict.is_none();
5867 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5868 self.state
5869 .finish_base_operation(synced.then_some(fresh.as_str()));
5870 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5871 self.state.base_sync = Some(BaseSync {
5872 tip,
5873 behind: if synced { 0 } else { behind },
5874 attempts,
5875 conflict,
5876 already_in: None,
5877 });
5878 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5879 .await?;
5880 self.state.save()?;
5881 return Ok(());
5882 }
5883
5884 self.state.event(
5885 "land",
5886 format!(
5887 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5888 winner.branch
5889 ),
5890 );
5891 self.state.save()?;
5892
5893 let branch_tracking = format!("{remote}/{}", winner.branch);
5899 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5900 let remote_tip = if self.state.task_pr_branch().is_none()
5901 && matches!(&fetched_branch, Ok(o) if o.ok())
5902 {
5903 git::rev_parse(&repo, &branch_tracking).await.ok()
5904 } else {
5905 None
5906 };
5907 if let Some(theirs) = &remote_tip
5911 && !git::is_ancestor(&repo, theirs, &head).await
5912 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5913 .await
5914 .is_ok_and(|missing| missing.is_empty())
5915 {
5916 let why = format!(
5917 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5918 them",
5919 short(theirs),
5920 winner.branch
5921 );
5922 self.state.status = RunStatus::Blocked;
5923 self.state.base_sync = Some(BaseSync {
5924 tip,
5925 behind,
5926 attempts,
5927 conflict: Some(why.clone()),
5928 already_in: None,
5929 });
5930 self.state.event("land", why);
5931 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5932 .await?;
5933 self.state.save()?;
5934 return Ok(());
5935 }
5936
5937 if self.state.base_sync.is_none() {
5938 self.state.base_sync = Some(BaseSync {
5939 tip: tip.clone(),
5940 behind,
5941 attempts: 0,
5942 conflict: None,
5943 already_in: None,
5944 });
5945 }
5946 let scratch = self.state.dir().join("base-sync");
5947 if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5948 {
5949 if error.is::<crate::repo_lock::ResourceBlocked>() {
5950 return Ok(());
5951 }
5952 return Err(error);
5953 }
5954 self.state.reserve_base_operation(&head, &tip);
5955 self.state.save()?;
5956 let rebased =
5957 match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
5958 .await
5959 {
5960 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5961 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5962 Err(e) => Err(e),
5963 };
5964 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5965 let stop_kind = match &rebased {
5966 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5967 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5968 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5969 };
5970 match rebased {
5971 Ok(None) => {
5972 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5976 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5977 self.state.finish_base_operation(Some(&fresh));
5978 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5979 let mut conflict = None;
5980 if let Some(pinned) = &remote_tip {
5981 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5982 match pushed {
5983 Ok(o) if o.ok() => self.state.event(
5984 "land",
5985 format!("pushed rebased {} to {remote}", winner.branch),
5986 ),
5987 Ok(o) => {
5988 conflict = Some(format!(
5989 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5990 winner.branch,
5991 short(pinned),
5992 o.stderr.chars().take(600).collect::<String>()
5993 ));
5994 }
5995 Err(e) => {
5996 conflict = Some(format!(
5997 "rebased {} locally but could not push it: {e:#}",
5998 winner.branch
5999 ));
6000 }
6001 }
6002 }
6003 if let Some(why) = &conflict {
6004 self.state.status = RunStatus::Blocked;
6005 self.state.event("land", why.clone());
6006 }
6007 self.state.base_sync = Some(BaseSync {
6008 tip: tip.clone(),
6009 behind: 0,
6010 attempts,
6011 conflict,
6012 already_in: None,
6013 });
6014 self.state
6015 .event("land", format!("rebased {} onto {tracking}", winner.branch));
6016 }
6017 Ok(Some(conflict)) => {
6018 self.state.finish_base_operation(None);
6019 let why = format!(
6020 "{} conflicts with {tracking} and did not rebase: {}",
6021 winner.branch,
6022 conflict.chars().take(600).collect::<String>()
6023 );
6024 self.state.status = RunStatus::Blocked;
6025 self.state.base_sync = Some(BaseSync {
6026 tip,
6027 behind,
6028 attempts,
6029 conflict: Some(why.clone()),
6030 already_in: None,
6031 });
6032 self.state.event("land", why);
6033 }
6034 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6038 self.state.save()?;
6039 return Ok(());
6040 }
6041 Err(e) => {
6042 self.state.finish_base_operation(None);
6043 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6044 self.state.status = RunStatus::Blocked;
6045 self.state.base_sync = Some(BaseSync {
6046 tip,
6047 behind,
6048 attempts,
6049 conflict: Some(why.clone()),
6050 already_in: None,
6051 });
6052 self.state.event("land", why);
6053 }
6054 }
6055 if !self.state.base_sync_blocked() {
6056 self.state.base_sync_recovery.stop = None;
6057 }
6058 self.hold_base_sync_kind(stop_kind).await?;
6059 self.state.save()?;
6060 Ok(())
6061 }
6062
6063 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6069 if self.state.task_pr_branch().is_some() {
6072 return None;
6073 }
6074 let repo = self.state.repo.clone();
6075 let remote = self.state.config.merge.remote.clone();
6076 let fetched = git::fetch(&repo, &remote, branch).await;
6077 if !matches!(&fetched, Ok(o) if o.ok()) {
6078 return None;
6079 }
6080 let branch_tracking = format!("{remote}/{branch}");
6081 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6082 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6083 return None;
6084 }
6085 if !crate::reconcile::origin_missing(&repo, head, &theirs)
6086 .await
6087 .is_ok_and(|missing| missing.is_empty())
6088 {
6089 return Some(format!(
6090 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6091 them",
6092 short(&theirs)
6093 ));
6094 }
6095 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6096 Ok(o) if o.ok() => {
6097 self.state
6098 .event("land", format!("pushed rebased {branch} to {remote}"));
6099 None
6100 }
6101 Ok(o) => Some(format!(
6102 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6103 someone may have pushed): {}",
6104 short(&theirs),
6105 o.stderr.chars().take(600).collect::<String>()
6106 )),
6107 Err(e) => Some(format!(
6108 "{branch} is rebased locally but could not be pushed: {e:#}"
6109 )),
6110 }
6111 }
6112
6113 async fn settle_already_in(
6130 &mut self,
6131 branch: &str,
6132 tip: &str,
6133 head: &str,
6134 attempts: usize,
6135 behind: usize,
6136 ) -> Result<bool> {
6137 let repo = self.state.repo.clone();
6138 let remote = self.state.config.merge.remote.clone();
6139 let start = self.state.base_commit.clone();
6140 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6141 Ok(Some(e)) => e,
6142 Ok(None) => return Ok(false),
6143 Err(e) => {
6144 tracing::warn!("already-in-base check for {branch}: {e:#}");
6145 return Ok(false);
6146 }
6147 };
6148 let mut verified = vec![head.to_owned()];
6149 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6150 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6151 if matches!(&fetched, Ok(o) if o.ok())
6152 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6153 && theirs != head
6154 {
6155 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6156 Ok(Some(_)) => verified.push(theirs),
6157 _ => return Ok(false),
6158 }
6159 }
6160 let base_branch = self.state.base_branch.clone();
6161 let message = format!(
6162 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6163 land",
6164 evidence.names(),
6165 evidence.proof.as_str()
6166 );
6167 let closed =
6168 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6169 .await;
6170 match closed {
6171 Ok(Ok(url)) => self
6172 .state
6173 .event("land", format!("closed {url}: superseded on {base_branch}")),
6174 Ok(Err(why)) => self
6175 .state
6176 .event("land", format!("did not close a pull request: {why}")),
6177 Err(e) => {
6178 let why = format!(
6179 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6180 not be closed ({e:#}); resume to retry"
6181 );
6182 self.state.status = RunStatus::Blocked;
6183 self.state.base_sync = Some(BaseSync {
6184 tip: tip.to_owned(),
6185 behind,
6186 attempts,
6187 conflict: Some(why.clone()),
6188 already_in: None,
6189 });
6190 self.state.event("land", why);
6191 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6192 .await?;
6193 self.state.save()?;
6194 return Ok(true);
6195 }
6196 }
6197 self.state.status = RunStatus::AlreadyInBase;
6198 self.state.base_sync_recovery.stop = None;
6199 self.state.base_sync = Some(BaseSync {
6200 tip: tip.to_owned(),
6201 behind,
6202 attempts,
6203 conflict: None,
6204 already_in: Some(evidence),
6205 });
6206 self.state.event("land", message);
6207 self.state.save()?;
6208 self.settle_questions();
6209 Ok(true)
6210 }
6211
6212 fn landing_base(&self) -> String {
6222 self.state
6223 .base_sync
6224 .as_ref()
6225 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6226 }
6227
6228 pub async fn fix_selected(
6261 &mut self,
6262 ids: &[String],
6263 reason: &str,
6264 allow_stale: bool,
6265 ) -> Result<()> {
6266 let reason = reason.trim();
6267 if reason.is_empty() {
6268 bail!("a fix request needs a reason — that is the operator's own record of why");
6269 }
6270 if ids.is_empty() {
6271 bail!("no finding id given");
6272 }
6273 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6274 bail!(
6275 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6276 has already concluded — can be given a targeted fix. A run still \
6277 in progress should simply be resumed; a `merged` run's branch has \
6278 already landed, so its answer is a fresh `magi review <branch>`, \
6279 not reopening this run's own record",
6280 self.state.id,
6281 self.state.status.as_str()
6282 );
6283 }
6284 let _claim = FixClaim::acquire(&self.state.dir())?;
6287 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6288 let Some(winner) = self.state.winner().cloned() else {
6289 bail!("run {} has no winning candidate to fix", self.state.id);
6290 };
6291 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6292 bail!(
6293 "branch `{}` no longer exists; this run cannot be extended",
6294 winner.branch
6295 );
6296 }
6297 let home = crate::run::home();
6298 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6299 bail!(
6300 "run {} is currently being worked on by another magi process",
6301 self.state.id
6302 );
6303 }
6304 let mut seen = BTreeSet::new();
6308 let mut findings = Vec::new();
6309 let mut missing = Vec::new();
6310 for id in ids {
6311 if !seen.insert(id.clone()) {
6312 continue;
6313 }
6314 match self.state.finding(id) {
6315 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6316 id: f.id.clone(),
6317 severity: f.severity,
6318 reviewer_vote: rec.vote,
6319 round: round.round,
6320 round_head: round.head.clone(),
6321 reviewer: rec.reviewer,
6322 agent: rec.agent.clone(),
6323 file: f.file.clone(),
6324 line: f.line,
6325 title: f.title.clone(),
6326 detail: f.detail.clone(),
6327 outcome: OperatorFixOutcome::Pending,
6328 }),
6329 None => missing.push(id.clone()),
6330 }
6331 }
6332 if !missing.is_empty() {
6333 bail!(
6334 "unknown finding id(s): {}; nothing was changed",
6335 missing.join(", ")
6336 );
6337 }
6338
6339 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6340 let stale_details: Vec<(String, String)> = findings
6341 .iter()
6342 .filter(|f| f.round_head != head_at_request)
6343 .map(|f| (f.id.clone(), f.round_head.clone()))
6344 .collect();
6345 let stale = !stale_details.is_empty();
6346 if stale && !allow_stale {
6347 bail!(
6348 "the branch has moved since some finding(s) were raised — {} — now \
6349 at {}; pass --allow-stale to fix anyway, or re-run review first",
6350 stale_details
6351 .iter()
6352 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6353 .collect::<Vec<_>>()
6354 .join(", "),
6355 short(&head_at_request)
6356 );
6357 }
6358
6359 let request = OperatorFixRequest {
6360 requested_at: Timestamp::now(),
6361 reason: reason.to_owned(),
6362 findings,
6363 head_at_request: head_at_request.clone(),
6364 allow_stale,
6365 stale,
6366 fix: None,
6367 result_head: None,
6368 follow_up_review_run: None,
6369 };
6370 self.state.event(
6371 "fix",
6372 format!(
6373 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6374 request.findings.len(),
6375 request
6376 .findings
6377 .iter()
6378 .map(|f| f.id.as_str())
6379 .collect::<Vec<_>>()
6380 .join(", "),
6381 ),
6382 );
6383 self.state.operator_fixes.push(request);
6390 self.state.save()?;
6391 let request_index = self.state.operator_fixes.len() - 1;
6392
6393 let repo = self.state.repo.clone();
6402 let coordination = self.coordinate(&repo).await?;
6403 if winner.worktree.exists() {
6404 let dirty = git::git(
6407 &winner.worktree,
6408 &["status", "--porcelain", "--untracked-files=all"],
6409 )
6410 .await?;
6411 let only_withheld = dirty.lines().all(|l| {
6412 l.strip_prefix("?? ")
6413 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6414 });
6415 if !only_withheld {
6416 bail!(
6417 "`{}` has uncommitted changes; refusing to touch it — commit or \
6418 discard them first",
6419 winner.worktree.display()
6420 );
6421 }
6422 git::worktree_remove(&self.state.repo, &winner.worktree)
6423 .await
6424 .ok();
6425 }
6426 let fix_worktree = self.state.worktree_root().join("operator-fix");
6427 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6428 git::git(
6429 &self.state.repo,
6430 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6431 )
6432 .await
6433 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6434 if !git::is_clean(&fix_worktree).await? {
6435 git::worktree_remove(&self.state.repo, &fix_worktree)
6436 .await
6437 .ok();
6438 bail!(
6439 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6440 winner.branch
6441 );
6442 }
6443 let repo_for_setup = self.state.repo.clone();
6444 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6445 Ok(()) => {
6446 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6447 }
6448 Err(e) => Err(e),
6449 };
6450 if let Err(e) = setup {
6451 git::worktree_remove(&self.state.repo, &fix_worktree)
6452 .await
6453 .ok();
6454 return Err(e);
6455 }
6456
6457 self.state.resource_block = None;
6460
6461 let run_id = self.state.id.clone();
6462 let prompts = self.state.config.prompts.clone();
6463 let language = self.state.config.graph.language.clone();
6464 let sessions = self.state.config.graph.sessions;
6465 let artifacts = agent::artifacts_dir(&self.state.dir());
6466 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6467 .findings
6468 .iter()
6469 .map(|f| Finding {
6470 id: f.id.clone(),
6471 severity: f.severity,
6472 file: f.file.clone(),
6473 line: f.line,
6474 title: f.title.clone(),
6475 detail: f.detail.clone(),
6476 })
6477 .collect();
6478 let fix_prompt = prompt::operator_fix_in(
6479 &self.state.instruction,
6480 &finding_list,
6481 reason,
6482 &stale_details,
6483 &head_at_request,
6484 &language,
6485 &self.state.config.graph.github_language,
6486 );
6487 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6488 drop(coordination);
6489 let (job, seat, out) = self
6490 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6491 prompt: fix_prompt.clone(),
6492 spec,
6493 seat,
6494 cwd: fix_worktree.clone(),
6495 timeout,
6496 allow_write: true,
6497 sessions,
6498 artifacts: artifacts.clone(),
6499 stem: "operator-fix".to_owned(),
6500 handover: None,
6501 })
6502 .await;
6503 let agent_id = seat.agent.clone();
6504
6505 let mut fix = FixRecord {
6506 agent: agent_id,
6507 addressed: Vec::new(),
6508 rejected: Vec::new(),
6509 notes: String::new(),
6510 committed: false,
6511 failed: None,
6512 duration_ms: 0,
6513 continuation: None,
6514 };
6515 let mut final_seat = seat.clone();
6516 match out {
6517 AgentOutcome::Ok(o) => {
6518 fix.duration_ms = o.duration_ms;
6519 let parsed = verdict::extract_json::<FixReport>(&o.text);
6520 let incomplete_reason = match &parsed {
6521 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6522 "the reply parsed, but it reported a command whose own CLI \
6523 never confirmed an exit status"
6524 .to_owned(),
6525 ),
6526 Ok(_) => None,
6527 Err(e) => Some(e.to_string()),
6528 };
6529 match incomplete_reason {
6530 None => {
6531 let report = parsed.expect("checked Ok above");
6532 fix.addressed = report.addressed;
6533 fix.rejected = report.rejected;
6534 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6535 }
6536 Some(reason) => {
6537 let (resumed_seat, resolved, failure, cont) = self
6538 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6539 .await;
6540 fix.duration_ms += cont.cumulative_wait_ms;
6541 fix.continuation = Some(cont);
6542 final_seat = resumed_seat;
6543 match resolved {
6544 Some(report) => {
6545 fix.addressed = report.addressed;
6546 fix.rejected = report.rejected;
6547 fix.notes =
6548 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6549 }
6550 None => fix.failed = failure,
6551 }
6552 }
6553 }
6554 }
6555 AgentOutcome::Dropped(o) => {
6556 fix.duration_ms = o.duration_ms;
6557 let why = o
6558 .dropped
6559 .as_ref()
6560 .map(|d| d.why.as_str())
6561 .unwrap_or("the CLI ended the stream without delivering its answer");
6562 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6563 }
6564 AgentOutcome::Quota(o) => {
6565 self.state.quota.push(QuotaLoss::from_report(
6566 final_seat.key.clone(),
6567 "fix",
6568 o.quota.as_ref(),
6569 ));
6570 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6571 }
6572 AgentOutcome::Failed(e) => fix.failed = Some(e),
6573 AgentOutcome::ResourceBlocked(e) => {
6574 fix.failed = Some(format!("the fixer was not started: {e}"));
6575 }
6576 }
6577 if fix.continuation.is_none() {
6580 fix.continuation = Some(ContinuationRecord::not_needed());
6581 }
6582 self.state.seats.insert(final_seat.key.clone(), final_seat);
6583
6584 let rescue_message = format!(
6585 "magi: operator-selected fix ({}) (uncommitted work)",
6586 self.state.operator_fixes[request_index]
6587 .findings
6588 .iter()
6589 .map(|f| f.id.as_str())
6590 .collect::<Vec<_>>()
6591 .join(", ")
6592 );
6593 let repo = self.state.repo.clone();
6594 let coordination = self.coordinate(&repo).await?;
6595 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6596 self.state.note_withheld("fix", &r.withheld);
6597 }
6598 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6599 fix.committed = after != head_at_request;
6600 git::worktree_remove(&self.state.repo, &fix_worktree)
6601 .await
6602 .ok();
6603 drop(coordination);
6604
6605 self.state.event(
6606 "fix",
6607 match &fix.failed {
6608 Some(reason) => format!(
6609 "operator fix: adoption report was lost ({reason}); {}",
6610 if fix.committed {
6611 "committed"
6612 } else {
6613 "NO new commit"
6614 }
6615 ),
6616 None => format!(
6617 "operator fix: {} addressed, {} rejected, {}",
6618 fix.addressed.len(),
6619 fix.rejected.len(),
6620 if fix.committed {
6621 "committed"
6622 } else {
6623 "NO new commit"
6624 }
6625 ),
6626 },
6627 );
6628
6629 for f in &mut self.state.operator_fixes[request_index].findings {
6636 f.outcome = if fix.failed.is_some() {
6637 OperatorFixOutcome::Unreported
6638 } else if fix.addressed.contains(&f.id) {
6639 OperatorFixOutcome::Addressed
6640 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6641 OperatorFixOutcome::Rejected { why: r.why.clone() }
6642 } else {
6643 OperatorFixOutcome::Unreported
6644 };
6645 }
6646
6647 let committed = fix.committed;
6648 if committed {
6649 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6650 }
6651 self.state.operator_fixes[request_index].fix = Some(fix);
6652 self.state.save()?;
6655 if self.state.resource_block.is_some() {
6656 return self.stop_for_resource();
6657 }
6658
6659 if committed {
6660 self.state.event(
6661 "fix",
6662 format!(
6663 "operator fix committed {}; opening a follow-up review-only run",
6664 short(&after)
6665 ),
6666 );
6667 let origin =
6670 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6671 match Self::review(
6672 &self.state.repo,
6673 &winner.branch,
6674 self.state.config.clone(),
6675 origin,
6676 )
6677 .await
6678 {
6679 Ok(mut follow_up) => {
6680 follow_up.state.event(
6681 "start",
6682 format!(
6683 "requested by an operator fix on run {} for finding(s) {}",
6684 self.state.id,
6685 self.state.operator_fixes[request_index]
6686 .findings
6687 .iter()
6688 .map(|f| f.id.as_str())
6689 .collect::<Vec<_>>()
6690 .join(", "),
6691 ),
6692 );
6693 follow_up.state.save()?;
6694 let follow_up_id = follow_up.state.id.clone();
6695 let adopted = match crate::direct::adopt(
6698 &follow_up.state,
6699 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6700 ) {
6701 Ok(a) => a,
6702 Err(e) => {
6703 self.state.event(
6706 "fix",
6707 format!(
6708 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6709 ),
6710 );
6711 self.state.save()?;
6712 return Ok(());
6713 }
6714 };
6715 let executed = follow_up.execute().await;
6716 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6717 if let Some(a) = adopted {
6718 a.finish(&follow_up.state, executed);
6719 }
6720 if let Some(e) = failure {
6721 self.state.event(
6722 "fix",
6723 format!(
6724 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6725 ),
6726 );
6727 }
6728 self.state.operator_fixes[request_index].follow_up_review_run =
6729 Some(follow_up_id);
6730 }
6731 Err(e) => {
6732 self.state.event(
6733 "fix",
6734 format!("committed the fix but could not open a follow-up review: {e:#}"),
6735 );
6736 }
6737 }
6738 self.state.save()?;
6739 }
6740
6741 Ok(())
6742 }
6743
6744 async fn ask_fixer(
6764 &mut self,
6765 winner: &Candidate,
6766 node: &'static str,
6767 round: Option<usize>,
6768 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6769 ) -> (SeatJob, SeatState, AgentOutcome) {
6770 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6771 self.ask_fixer_among(attempts, node, round, None, build)
6772 .await
6773 }
6774
6775 async fn ask_fixer_among(
6780 &mut self,
6781 attempts: Vec<(AgentSpec, String)>,
6782 node: &'static str,
6783 round: Option<usize>,
6784 restore: Option<(&Path, &str)>,
6785 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6786 ) -> (SeatJob, SeatState, AgentOutcome) {
6787 let run_id = self.state.id.clone();
6788 let prompts = self.state.config.prompts.clone();
6789 let cache = self.state.config.cache_dir();
6790 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6791 let mut last = None;
6792 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6793 let seat = self.seat(&key, &spec.id);
6794 let mut job = build(spec.clone(), seat);
6795 if i > 0 {
6796 job.stem = format!("{}-{}", job.stem, spec.id);
6797 }
6798 let ctx = WaveCtx {
6799 carry_seats: false,
6800 pauses: Some((&self.pause, &self.interrupt)),
6801 run: &run_id,
6802 node,
6803 prompts: &prompts,
6804 cache: cache.as_deref(),
6805 round,
6806 };
6807 let (seat, out) =
6808 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6809 let advances = match &out {
6812 AgentOutcome::Ok(o) => agent::output_advances(o),
6813 AgentOutcome::ResourceBlocked(_) => false,
6814 _ => true,
6815 };
6816 if let Some(next) = ids.get(i + 1)
6817 && advances
6818 {
6819 self.state.seats.insert(seat.key.clone(), seat.clone());
6820 let class =
6821 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6822 let reason = match &out {
6823 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6824 other => fail_reason(other),
6825 };
6826 if let Some((wt, rev)) = restore
6830 && let Err(e) = self.graft_restore(wt, rev).await
6831 {
6832 let why =
6833 format!("could not restore the worktree before the next fixer: {e:#}");
6834 tracing::warn!("{why}");
6835 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6836 AgentOutcome::ResourceBlocked(why)
6837 } else {
6838 AgentOutcome::Failed(why)
6839 };
6840 last = Some((job, seat, out));
6841 break;
6842 }
6843 record_handover(
6844 &mut self.state,
6845 node,
6846 &seat.key,
6847 &spec.id,
6848 next,
6849 &class,
6850 &reason,
6851 );
6852 continue;
6853 }
6854 last = Some((job, seat, out));
6855 break;
6856 }
6857 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6858 if let AgentOutcome::Quota(output) = &out
6859 && node != "graft"
6860 {
6861 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6866 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6867 self.state.quota.push(loss);
6868 self.state.seats.insert(seat.key.clone(), seat.clone());
6869 self.state.status = RunStatus::Blocked;
6870 return (
6871 job,
6872 seat,
6873 AgentOutcome::ResourceBlocked(
6874 "fixer unavailable; restore agent availability and resume".into(),
6875 ),
6876 );
6877 }
6878 (job, seat, out)
6879 }
6880
6881 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6892 use crate::recovery::{Mode, Repair};
6893 if !self.state.reviews.is_empty() {
6894 return Ok(());
6895 }
6896 let Some(mut rec) = self.state.recovery.clone() else {
6897 return Ok(());
6898 };
6899 if rec.mode != Mode::VerifyRepair {
6900 return Ok(());
6901 }
6902 let wt = winner.worktree.clone();
6903 let mut head = git::rev_parse(&wt, "HEAD").await?;
6904 match rec.repair.clone() {
6905 Repair::Done { .. } => return Ok(()),
6906 Repair::WaitingForRestore { before } => {
6907 self.graft_restore(&wt, &before).await?;
6908 head = before;
6909 rec.repair = Repair::NotStarted;
6910 self.state.recovery = Some(rec.clone());
6911 self.state.save()?;
6912 }
6913 Repair::Started { before } => {
6914 if let Err(e) = self.graft_restore(&wt, &before).await {
6918 if e.is::<crate::repo_lock::ResourceBlocked>() {
6919 return Err(e);
6920 }
6921 return self
6922 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6923 .await;
6924 }
6925 rec.repair = Repair::Done {
6926 before: before.clone(),
6927 after: before,
6928 committed: false,
6929 failed: Some("interrupted; rolled back".to_owned()),
6930 };
6931 self.state
6932 .event("recovery", "the repair was interrupted and rolled back");
6933 self.state.recovery = Some(rec);
6934 self.state.save()?;
6935 return Ok(());
6936 }
6937 Repair::NotStarted => {}
6938 }
6939 if crate::recovery::recheck(&mut rec, &head) {
6940 self.state.event("recovery", rec.summary());
6941 self.state.recovery = Some(rec);
6942 self.state.save()?;
6943 return Ok(());
6944 }
6945 rec.repair = Repair::Started {
6946 before: head.clone(),
6947 };
6948 self.state.recovery = Some(rec.clone());
6949 self.state.save()?;
6950
6951 let language = self.state.config.graph.language.clone();
6952 let sessions = self.state.config.graph.sessions;
6953 let artifacts = agent::artifacts_dir(&self.state.dir());
6954 let fix_prompt = prompt::recovery_fix(
6955 &self.state.instruction,
6956 &rec,
6957 self.state.config.graph.review_rounds,
6958 &language,
6959 );
6960 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6961 self.state.event(
6962 "recovery",
6963 format!(
6964 "repair: the fixer starts from the failed verification on {}",
6965 short(&head)
6966 ),
6967 );
6968 let (_job, seat, out) = self
6969 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6970 prompt: fix_prompt.clone(),
6971 spec,
6972 seat,
6973 cwd: wt.clone(),
6974 timeout,
6975 allow_write: true,
6976 sessions,
6977 artifacts: artifacts.clone(),
6978 stem: "fix-recovery".to_owned(),
6979 handover: None,
6980 })
6981 .await;
6982 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6983 rec.repair = Repair::WaitingForRestore { before: head };
6984 self.state.recovery = Some(rec);
6985 return self.stop_for_resource();
6986 }
6987 let failed = match &out {
6988 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6989 AgentOutcome::Quota(o) => {
6990 self.state.quota.push(QuotaLoss::from_report(
6991 seat.key.clone(),
6992 "fix",
6993 o.quota.as_ref(),
6994 ));
6995 Some("rate limited (quota); the fixer could not run".to_owned())
6996 }
6997 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6998 AgentOutcome::Failed(e) => Some(e.clone()),
6999 };
7000 self.state.seats.insert(seat.key.clone(), seat);
7001 let coordination = self.coordinate(&wt).await?;
7002 let failed = match failed {
7003 Some(why) => {
7005 if let Err(e) = git::restore_to(&wt, &head).await {
7006 return self
7007 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7008 .await;
7009 }
7010 Some(why)
7011 }
7012 None => {
7013 if let Ok(r) = git::rescue_commit(
7014 &wt,
7015 "magi: recovery repair of the failed verification (uncommitted work)",
7016 )
7017 .await
7018 {
7019 self.state.note_withheld("fix", &r.withheld);
7020 }
7021 None
7022 }
7023 };
7024 let after = git::rev_parse(&wt, "HEAD").await?;
7025 drop(coordination);
7026 self.state.event(
7027 "recovery",
7028 match &failed {
7029 Some(why) => format!("repair: nothing kept ({why})"),
7030 None if after != head => {
7031 format!("repair: committed {}", short(&after))
7032 }
7033 None => "repair: the fixer made no commit".to_owned(),
7034 },
7035 );
7036 rec.repair = Repair::Done {
7037 before: head.clone(),
7038 committed: after != head,
7039 after,
7040 failed,
7041 };
7042 self.state.recovery = Some(rec);
7043 self.state.save()?;
7044 Ok(())
7045 }
7046
7047 async fn stop_recovery(
7050 &mut self,
7051 mut rec: crate::recovery::RecoveryRecord,
7052 head: &str,
7053 why: String,
7054 ) -> Result<()> {
7055 rec.repair = crate::recovery::Repair::Started {
7057 before: head.to_owned(),
7058 };
7059 self.state.recovery = Some(rec);
7060 self.state
7061 .event("recovery", format!("repair could not be undone: {why}"));
7062 self.state.status = RunStatus::Blocked;
7063 self.state.save()?;
7064 bail!("recovery repair could not be rolled back: {why}")
7065 }
7066
7067 async fn review_loop(&mut self) -> Result<()> {
7068 if self.hold_base_sync().await? {
7069 return Ok(());
7070 }
7071 loop {
7072 if self
7073 .state
7074 .base_sync
7075 .as_ref()
7076 .is_some_and(|s| s.conflict.is_some())
7077 {
7078 return Ok(());
7079 }
7080 if self.catchup_repairable() {
7081 let record = self.state.reviews.last().expect("repairable round").clone();
7082 let winner = self
7083 .state
7084 .winner()
7085 .cloned()
7086 .context("catch-up repair has no winner")?;
7087 if !self.fix_review_round(&winner, record).await? {
7088 return Ok(());
7089 }
7090 }
7091 self.review_loop_pass().await?;
7092 if !self.catchup_repairable() {
7093 return Ok(());
7094 }
7095 }
7096 }
7097
7098 fn catchup_repairable(&self) -> bool {
7101 self.state.reviews.len() < self.state.config.graph.review_rounds
7102 && self.state.reviews.last().is_some_and(|r| {
7103 !r.clean
7104 && r.blocking == 0
7105 && r.fix.is_none()
7106 && r.e2e_status() == E2eStatus::Failed
7107 })
7108 }
7109
7110 async fn review_loop_pass(&mut self) -> Result<()> {
7111 if self
7116 .state
7117 .base_sync
7118 .as_ref()
7119 .is_some_and(|s| s.conflict.is_some())
7120 {
7121 return Ok(());
7122 }
7123 let run_id = self.state.id.clone();
7128 let prompts = self.state.config.prompts.clone();
7129 let Some(winner) = self.state.winner().cloned() else {
7130 return Ok(());
7131 };
7132 let granted_review = self
7133 .state
7134 .repair_grants
7135 .last()
7136 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7137 .cloned();
7138 let fresh_review = granted_review
7139 .as_ref()
7140 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7141 let max_rounds = granted_review
7142 .as_ref()
7143 .map_or(self.state.config.graph.review_rounds, |g| {
7144 g.review_start.saturating_add(1)
7145 });
7146 if let Some(status) =
7156 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7157 {
7158 if status == RunStatus::Gating {
7161 self.record_contested_handoff();
7162 }
7163 self.state.status = status;
7164 self.state.save()?;
7165 return Ok(());
7166 }
7167 self.state.status = RunStatus::Reviewing;
7168 if !fresh_review
7178 && self.state.reviews.last().is_some_and(|r| {
7179 r.e2e_status() == E2eStatus::ResourceBlocked
7180 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7181 })
7182 {
7183 let shell = self.state.config.shell();
7184 return self
7185 .stop_unresolved_review(
7186 "the last round's own verification never resolved",
7187 &shell,
7188 &winner.worktree,
7189 )
7190 .await;
7191 }
7192
7193 if granted_review.is_none() {
7194 self.recovery_repair(&winner).await?;
7195 }
7196 if self.resource_halt()? {
7197 return Ok(());
7198 }
7199
7200 let repo = self.state.repo.clone();
7201 let root = self.state.worktree_root();
7202 let language = self.state.config.graph.language.clone();
7203 let sessions = self.state.config.graph.sessions;
7204 let artifacts = self.merge_repair_artifacts();
7205 let base = self.landing_base();
7206 let base_short = short(&base);
7207 let reviewers = self.roles.reviewers.clone();
7208 let shell = self.state.config.shell();
7209
7210 for round in (self.state.reviews.len() + 1)..=max_rounds {
7211 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7212 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7213 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7214 let prev_verification = self
7223 .state
7224 .reviews
7225 .last()
7226 .and_then(|r| r.verification_summary(&head));
7227 let prior_section = match (&self.state.recovery, round) {
7230 (Some(rec), 1) if self.state.reviews.is_empty() => {
7231 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7232 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7233 Some(crate::recovery::head_of(
7234 &git::diff(&winner.worktree, reviewed, "HEAD")
7235 .await
7236 .unwrap_or_default(),
7237 crate::recovery::SINCE_DIFF_MAX,
7238 ))
7239 }
7240 _ => None,
7241 };
7242 Some(prompt::recovery_review(rec, since.as_deref()))
7243 }
7244 _ => None,
7245 };
7246
7247 let mut jobs = Vec::new();
7251 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7252 self.coordinate(&repo).await?
7253 } else {
7254 None
7255 };
7256 for (r, spec) in reviewers.iter().cloned().enumerate() {
7257 let wt = root.join(format!("review-{}", r + 1));
7258 if wt.exists() {
7261 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7262 } else {
7263 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7264 None
7265 } else {
7266 self.coordinate(&repo).await?
7267 };
7268 git::worktree_add_detached(&repo, &wt, &head).await?;
7269 }
7270 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7271 None
7272 } else {
7273 self.coordinate(&repo).await?
7274 };
7275 self.setup_seat_worktree(&repo, &wt, false).await?;
7276 let seat_key = format!("review-{}", r + 1);
7277 let spec = pick_start_spec(
7280 &self.roles.reviewer_roster,
7281 spec,
7282 self.state.seat_history.get(&seat_key),
7283 );
7284 let seat = self.seat(&seat_key, &spec.id);
7285 jobs.push(SeatJob {
7286 prompt: prompt::review(&prompt::ReviewCtx {
7287 instruction: &self.state.instruction,
7288 branch: &winner.branch,
7289 base_short: &base_short,
7290 stat: &stat,
7291 patch: &patch,
7292 verification: prev_verification.as_ref(),
7293 prior: prior_section.as_deref(),
7294 reviewers: reviewers.len(),
7295 round,
7296 rounds: max_rounds,
7297 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7300 lens: Lens::for_seat(r),
7301 language: &language,
7302 github_language: &self.state.config.graph.github_language,
7303 }),
7304 spec,
7305 seat,
7306 cwd: wt,
7307 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7308 allow_write: false,
7309 sessions,
7310 artifacts: artifacts.clone(),
7311 stem: format!("review-{round}-{}", r + 1),
7312 handover: None,
7313 });
7314 }
7315
7316 drop(coordination);
7317 self.state.event(
7318 "review",
7319 format!(
7320 "round {round}: {} reviewers on {}",
7321 jobs.len(),
7322 short(&head)
7323 ),
7324 );
7325 let mut quota_losses = Vec::new();
7326 let review_retries = self.state.config.graph.retries;
7327 let review_cache = self.state.config.cache_dir();
7328 let ctx = WaveCtx {
7329 carry_seats: true,
7330 pauses: Some((&self.pause, &self.interrupt)),
7331 run: &run_id,
7332 node: "review",
7333 prompts: &prompts,
7334 cache: review_cache.as_deref(),
7335 round: Some(round),
7336 };
7337 let results = ask_json_wave::<Review>(
7338 jobs,
7339 Arc::clone(&self.sem),
7340 review_retries,
7341 &self.roles.reviewer_roster,
7342 &ctx,
7343 &mut quota_losses,
7344 &mut self.state,
7345 &|_: &Review| Ok(()),
7346 )
7347 .await;
7348 if self.state.resource_block.is_some() {
7349 return self.stop_for_resource();
7350 }
7351 let round_quota_missing = quota_losses.len();
7355 self.state.quota.extend(quota_losses);
7356
7357 let mut records = Vec::new();
7358 let mut all_findings = Vec::new();
7359 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7360 let agent_id = seat.agent.clone();
7361 self.state.seats.insert(seat.key.clone(), seat);
7362 let mut record = ReviewRecord {
7363 reviewer: r + 1,
7364 agent: agent_id,
7365 summary: String::new(),
7366 findings: Vec::new(),
7367 vote: None,
7368 failed: None,
7369 duration_ms: 0,
7370 attempts,
7376 };
7377 match res {
7378 Ok((review, out)) => {
7379 record.summary =
7387 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7388 record.vote = Some(review.vote);
7389 record.duration_ms = out.duration_ms;
7390 for (n, mut f) in review.findings.into_iter().enumerate() {
7391 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7394 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7395 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7396 f.file = f
7402 .file
7403 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7404 all_findings.push(f.clone());
7405 record.findings.push(f);
7406 }
7407 self.state.event(
7408 "review",
7409 format!(
7410 "round {round}: reviewer {} voted {} with {} finding(s)",
7411 r + 1,
7412 review.vote.label(),
7413 record.findings.len()
7414 ),
7415 );
7416 }
7417 Err(e) => {
7418 record.failed = Some(e.to_string());
7419 self.state.event(
7420 "review",
7421 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7422 );
7423 }
7424 }
7425 records.push(record);
7426 }
7427
7428 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7435 let vote_split =
7436 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7437 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7438 if vote_split {
7439 self.state.event(
7440 "review",
7441 format!(
7442 "round {round}: votes split ({}) — one round of reconsideration",
7443 initial_votes
7444 .iter()
7445 .map(|v| v.label())
7446 .collect::<Vec<_>>()
7447 .join(", ")
7448 ),
7449 );
7450 let panel: Vec<ReviewSeatReport<'_>> = records
7453 .iter()
7454 .filter_map(|r| {
7455 r.vote.map(|vote| ReviewSeatReport {
7456 reviewer: r.reviewer,
7457 vote,
7458 summary: &r.summary,
7459 findings: &r.findings,
7460 })
7461 })
7462 .collect();
7463
7464 let mut jobs = Vec::new();
7465 let mut seats_at = Vec::new();
7466 for (r, spec) in reviewers.iter().cloned().enumerate() {
7467 if records[r].vote.is_none() {
7471 continue;
7472 }
7473 let wt = root.join(format!("review-{}", r + 1));
7474 let seat_key = format!("review-{}", r + 1);
7475 let spec = self.occupant(&seat_key, spec);
7476 let seat = self.seat(&seat_key, &spec.id);
7477 let build = |with_patch: bool| {
7485 prompt::review_reconsider(&ReviewReconsiderCtx {
7486 instruction: &self.state.instruction,
7487 reviewer: r + 1,
7488 lens: Lens::for_seat(r),
7489 panel: &panel,
7490 patch: with_patch.then_some(ReviewPatch {
7491 branch: &winner.branch,
7492 base_short: &base_short,
7493 stat: &stat,
7494 patch: &patch,
7495 }),
7496 round,
7497 rounds: max_rounds,
7498 language: &language,
7499 })
7500 };
7501 let full = build(true);
7502 let prompt = if has_context(&spec, &seat, sessions) {
7503 build(false)
7504 } else {
7505 full.clone()
7506 };
7507 jobs.push(SeatJob {
7508 prompt,
7509 spec,
7510 seat,
7511 cwd: wt,
7512 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7513 allow_write: false,
7514 sessions,
7515 artifacts: artifacts.clone(),
7516 stem: format!("review-{round}-reconsider-{}", r + 1),
7517 handover: Some(full),
7518 });
7519 seats_at.push(r);
7520 }
7521
7522 let mut recon_quota_losses = Vec::new();
7523 let recon_cache = self.state.config.cache_dir();
7524 let recon_ctx = WaveCtx {
7525 carry_seats: true,
7526 pauses: Some((&self.pause, &self.interrupt)),
7527 run: &run_id,
7528 node: "review",
7529 prompts: &prompts,
7530 cache: recon_cache.as_deref(),
7531 round: Some(round),
7532 };
7533 let recon_results = ask_json_wave::<ReviewRevote>(
7534 jobs,
7535 Arc::clone(&self.sem),
7536 review_retries,
7537 &self.roles.reviewer_roster,
7538 &recon_ctx,
7539 &mut recon_quota_losses,
7540 &mut self.state,
7541 &|_: &ReviewRevote| Ok(()),
7542 )
7543 .await;
7544 if self.state.resource_block.is_some() {
7545 return self.stop_for_resource();
7546 }
7547 self.state.quota.extend(recon_quota_losses);
7548
7549 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7550 let agent_id = seat.agent.clone();
7551 self.state.seats.insert(seat.key.clone(), seat);
7552 let mut rec = ReviewRevoteRecord {
7553 reviewer: r + 1,
7554 agent: agent_id,
7555 vote: None,
7556 reason: String::new(),
7557 failed: None,
7558 };
7559 match res {
7560 Ok((rv, _)) => {
7561 rec.vote = Some(rv.vote);
7562 rec.reason =
7563 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7564 self.state.event(
7565 "review",
7566 format!(
7567 "round {round}: reviewer {} revoted {}",
7568 r + 1,
7569 rv.vote.label()
7570 ),
7571 );
7572 }
7573 Err(e) => {
7574 rec.failed = Some(e.to_string());
7575 self.state.event(
7576 "review",
7577 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7578 );
7579 }
7580 }
7581 reconsideration.push(rec);
7582 }
7583 } else if initial_votes.len() > 1 {
7584 self.state.event(
7585 "review",
7586 format!(
7587 "round {round}: votes agreed ({}) — no reconsideration",
7588 initial_votes[0].label()
7589 ),
7590 );
7591 }
7592
7593 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7594 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7595 let defer_e2e =
7606 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7607 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7608 let reason =
7609 format!("{blocking} blocking finding(s) already required a fix this round");
7610 self.state.event(
7611 "verify",
7612 format!(
7613 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7614 {}); it will run once a round has none left",
7615 short(&head)
7616 ),
7617 );
7618 (Vec::new(), false, true, Some(reason))
7619 } else {
7620 let e2e_commands = self.state.config.verify.e2e.clone();
7621 let cache_dir = self.state.config.cache_dir();
7622 let context = format!("round {round}");
7623 let wait = self.repository_wait();
7624 let (e2e, verify_retried) = with_cache_lease(
7625 &mut self.state,
7626 cache_dir.as_deref(),
7627 "e2e",
7628 "e2e",
7629 &winner.worktree,
7630 &head,
7631 verify_timeout,
7632 &context,
7633 &wait,
7634 |state, budget| {
7635 let shell = shell.clone();
7636 let e2e_commands = e2e_commands.clone();
7637 let worktree = winner.worktree.clone();
7638 let context = context.clone();
7639 async move {
7640 run_e2e_with_retry(
7641 state,
7642 &shell,
7643 &e2e_commands,
7644 &worktree,
7645 budget,
7646 &context,
7647 )
7648 .await
7649 }
7650 },
7651 )
7652 .await?;
7653 (e2e, verify_retried, false, None)
7654 };
7655
7656 let expected = records.len();
7657 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7658 let incomplete = answered < expected;
7659 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7660 let policy = self.state.config.graph.incomplete_review;
7661 let clean = round_is_clean(
7662 blocking,
7663 e2e_ok,
7664 answered,
7665 expected,
7666 round_quota_missing,
7667 policy,
7668 );
7669
7670 let mut round_record = ReviewRound {
7671 round,
7672 head: head.clone(),
7673 verified_head: None,
7674 verified_at: None,
7675 reviews: records,
7676 e2e,
7677 verify_retried,
7678 e2e_deferred,
7679 e2e_defer_reason,
7680 fix: None,
7681 blocking,
7682 answered,
7683 expected,
7684 clean,
7685 progressed: false,
7686 vote_split,
7687 reconsideration,
7688 verdict: None,
7689 };
7690 round_record.verdict = ReviewVote::worst(
7694 round_record
7695 .final_votes()
7696 .into_iter()
7697 .map(|(_, _, vote)| vote),
7698 );
7699 if !matches!(
7710 round_record.e2e_status(),
7711 E2eStatus::Deferred | E2eStatus::NotConfigured
7712 ) {
7713 round_record.verified_head = Some(head.clone());
7714 round_record.verified_at = Some(Timestamp::now());
7715 }
7716
7717 if incomplete {
7718 let missing: Vec<String> = round_record
7719 .reviews
7720 .iter()
7721 .filter(|r| r.failed.is_some())
7722 .map(|r| format!("review-{}", r.reviewer))
7723 .collect();
7724 self.state.event(
7725 "review",
7726 format!(
7727 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7728 missing.join(", ")
7729 ),
7730 );
7731 }
7732
7733 if clean {
7734 self.state.event(
7735 "review",
7736 if incomplete && policy == IncompleteReviewPolicy::Warn {
7737 format!(
7738 "round {round}: clean (warn policy, incomplete panel) — no \
7739 blocking findings from the seats that answered, verification green"
7740 )
7741 } else if incomplete {
7742 format!(
7743 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7744 quorum) — no blocking findings from the seats that answered, \
7745 verification green",
7746 expected - answered
7747 )
7748 } else {
7749 format!("round {round}: clean — no blocking findings, verification green")
7750 },
7751 );
7752 self.state.reviews.push(round_record);
7753 self.state.status = RunStatus::Gating;
7754 self.state.save()?;
7755 return Ok(());
7756 }
7757
7758 if incomplete && blocking == 0 && e2e_ok {
7766 self.state.reviews.push(round_record);
7767 self.state.save()?;
7768 if round == max_rounds {
7769 self.state.status = RunStatus::Blocked;
7770 self.state.event(
7771 "review",
7772 format!(
7773 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7774 refusing to call it clean",
7775 expected - answered
7776 ),
7777 );
7778 return Ok(());
7779 }
7780 continue;
7781 }
7782
7783 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7795 self.state.reviews.push(round_record);
7796 return self
7797 .stop_unresolved_review(
7798 "the round's own verification could not run",
7799 &shell,
7800 &winner.worktree,
7801 )
7802 .await;
7803 }
7804
7805 if round == max_rounds {
7806 self.state.reviews.push(round_record);
7807 return self
7808 .stop_reviewing(
7809 &format!(
7810 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7811 ),
7812 &shell,
7813 &winner.worktree,
7814 )
7815 .await;
7816 }
7817
7818 if !self.fix_review_round(&winner, round_record).await? {
7819 return Ok(());
7820 }
7821 }
7822 Ok(())
7823 }
7824
7825 async fn fix_review_round(
7828 &mut self,
7829 winner: &Candidate,
7830 mut round_record: ReviewRound,
7831 ) -> Result<bool> {
7832 let round = round_record.round;
7833 let shell = self.state.config.shell();
7834 let step = self.run_fix_round(winner, &mut round_record).await?;
7835 if matches!(step, FixStep::Resource) {
7836 self.stop_for_resource()?;
7837 return Ok(false);
7838 }
7839 if self.state.reviews.len() < round {
7840 self.state.reviews.push(round_record);
7841 } else {
7842 self.state.reviews[round - 1] = round_record;
7843 }
7844 self.state.save()?;
7845
7846 if matches!(step, FixStep::Recorded { report_lost: true }) {
7859 self.stop_reviewing(
7860 "the fixer's adoption report never came back, even after resuming its \
7861 own seat; refusing to start another round against the same worktree \
7862 while that is unresolved",
7863 &shell,
7864 &winner.worktree,
7865 )
7866 .await?;
7867 return Ok(false);
7868 }
7869
7870 let streak = self
7871 .state
7872 .reviews
7873 .iter()
7874 .rev()
7875 .take_while(|r| !r.progressed)
7876 .count();
7877 if streak >= STAGNANT_LIMIT {
7878 self.stop_reviewing(
7879 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7880 &shell,
7881 &winner.worktree,
7882 )
7883 .await?;
7884 return Ok(false);
7885 }
7886 Ok(true)
7887 }
7888
7889 async fn run_fix_round(
7891 &mut self,
7892 winner: &Candidate,
7893 round_record: &mut ReviewRound,
7894 ) -> Result<FixStep> {
7895 let round = round_record.round;
7896 let max_rounds = self.state.config.graph.review_rounds;
7897 let run_id = self.state.id.clone();
7898 let prompts = self.state.config.prompts.clone();
7899 let language = self.state.config.graph.language.clone();
7900 let sessions = self.state.config.graph.sessions;
7901 let artifacts = self.merge_repair_artifacts();
7902 let base = self.landing_base();
7903 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7904 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7905 let this_round_verification = round_record.verification_summary(&head);
7906 let all_findings: Vec<_> = round_record
7907 .reviews
7908 .iter()
7909 .flat_map(|r| r.findings.iter().cloned())
7910 .collect();
7911 let blocking_findings: Vec<_> = all_findings
7914 .iter()
7915 .filter(|f| f.severity.blocks())
7916 .cloned()
7917 .collect();
7918 let fix_prompt = prompt::fix_in(
7919 &self.state.instruction,
7920 &blocking_findings,
7921 this_round_verification.as_ref(),
7922 round,
7923 max_rounds,
7924 &language,
7925 &self.state.config.graph.github_language,
7926 );
7927 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7928 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7929 let (job, seat, out) = self
7930 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7931 prompt: fix_prompt.clone(),
7932 spec,
7933 seat,
7934 cwd: winner.worktree.clone(),
7935 timeout,
7936 allow_write: true,
7937 sessions,
7938 artifacts: artifacts.clone(),
7939 stem: format!("fix-{round}"),
7940 handover: None,
7941 })
7942 .await;
7943 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7946 return Ok(FixStep::Resource);
7947 }
7948 let agent_id = seat.agent.clone();
7949
7950 let mut fix = FixRecord {
7951 agent: agent_id,
7952 addressed: Vec::new(),
7953 rejected: Vec::new(),
7954 notes: String::new(),
7955 committed: false,
7956 failed: None,
7957 duration_ms: 0,
7958 continuation: None,
7959 };
7960 let mut continuation = ContinuationRecord::not_needed();
7961 let mut final_seat = seat.clone();
7962 match out {
7963 AgentOutcome::Ok(o) => {
7964 fix.duration_ms = o.duration_ms;
7965 let parsed = verdict::extract_json::<FixReport>(&o.text);
7966 let incomplete_reason = match &parsed {
7973 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7974 "the reply parsed, but it reported a command whose own CLI never \
7975 confirmed an exit status"
7976 .to_owned(),
7977 ),
7978 Ok(_) => None,
7979 Err(e) => Some(e.to_string()),
7980 };
7981 match incomplete_reason {
7982 None => {
7983 let report = parsed.expect("checked Ok above");
7984 fix.addressed = report.addressed;
7985 fix.rejected = report.rejected;
7986 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7987 }
7988 Some(reason) => {
7989 let (resumed_seat, resolved, failure, cont) = self
7990 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7991 .await;
7992 fix.duration_ms += cont.cumulative_wait_ms;
7993 continuation = cont;
7994 final_seat = resumed_seat;
7995 match resolved {
7996 Some(report) => {
7997 fix.addressed = report.addressed;
7998 fix.rejected = report.rejected;
7999 fix.notes =
8000 blind::sanitize_prose(&report.notes, &self.state.config.blind);
8001 }
8002 None => fix.failed = failure,
8003 }
8004 }
8005 }
8006 }
8007 AgentOutcome::Dropped(o) => {
8009 fix.duration_ms = o.duration_ms;
8010 let why = o
8011 .dropped
8012 .as_ref()
8013 .map(|d| d.why.as_str())
8014 .unwrap_or("the CLI ended the stream without delivering its answer");
8015 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8016 }
8017 AgentOutcome::Quota(o) => {
8018 self.state.quota.push(QuotaLoss::from_report(
8019 final_seat.key.clone(),
8020 "fix",
8021 o.quota.as_ref(),
8022 ));
8023 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8024 }
8025 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8026 }
8027 if self.state.resource_block.is_some() {
8028 self.state.seats.insert(final_seat.key.clone(), final_seat);
8029 return Ok(FixStep::Resource);
8030 }
8031 fix.continuation = Some(continuation);
8032 self.state.seats.insert(final_seat.key.clone(), final_seat);
8033 let coordination = self.coordinate(&winner.worktree).await?;
8034 if let Ok(r) = git::rescue_commit(
8035 &winner.worktree,
8036 &format!("magi: review round {round} fixes (uncommitted work)"),
8037 )
8038 .await
8039 {
8040 self.state.note_withheld("fix", &r.withheld);
8041 }
8042 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8043 fix.committed = after != before;
8044 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8052 drop(coordination);
8053 let progressed = diff_after != patch;
8054 let commit_note = if fix.committed {
8055 "committed"
8056 } else {
8057 "NO new commit"
8058 };
8059 let tree_note = if progressed {
8060 "changed vs base"
8061 } else {
8062 "unchanged vs base"
8063 };
8064 self.state.event(
8065 "fix",
8066 match &fix.failed {
8067 Some(reason) => {
8073 format!(
8074 "round {round}: fixer's adoption report was lost ({reason}); \
8075 {commit_note}, tree {tree_note}"
8076 )
8077 }
8078 None => format!(
8079 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8080 {tree_note}{}",
8081 fix.addressed.len(),
8082 fix.rejected.len(),
8083 if continuation.outcome == ContinuationOutcome::Resumed {
8084 format!(
8085 " (adoption report recovered after {} continuation(s))",
8086 continuation.attempts
8087 )
8088 } else {
8089 String::new()
8090 },
8091 ),
8092 },
8093 );
8094 round_record.fix = Some(fix);
8095 round_record.progressed = progressed;
8096 Ok(FixStep::Recorded {
8097 report_lost: matches!(
8098 continuation.outcome,
8099 ContinuationOutcome::Exhausted
8100 | ContinuationOutcome::QuotaLost
8101 | ContinuationOutcome::NoSession
8102 ),
8103 })
8104 }
8105
8106 fn record_contested_handoff(&mut self) {
8140 if self.state.contested_handoff.is_some() {
8141 return;
8142 }
8143 let Some(contested) = self
8144 .state
8145 .reviews
8146 .last()
8147 .and_then(ReviewRound::contested_handoff)
8148 else {
8149 return;
8150 };
8151 self.state.event(
8152 "review",
8153 format!(
8154 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8155 wait for the owner's approval",
8156 contested.findings.len(),
8157 contested.rejecters.len()
8158 ),
8159 );
8160 self.state.contested_handoff = Some(contested);
8161 }
8162
8163 async fn stop_unresolved_review(
8164 &mut self,
8165 why: &str,
8166 shell: &[String],
8167 worktree: &Path,
8168 ) -> Result<()> {
8169 self.stop_reviewing_inner(why, shell, worktree, true).await
8170 }
8171
8172 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8173 self.stop_reviewing_inner(why, shell, worktree, false).await
8174 }
8175
8176 async fn stop_reviewing_inner(
8177 &mut self,
8178 why: &str,
8179 shell: &[String],
8180 worktree: &Path,
8181 repairable: bool,
8182 ) -> Result<()> {
8183 let round_idx = self.state.reviews.len() - 1;
8184 let needs_catchup_run = matches!(
8192 self.state.reviews[round_idx].e2e_status(),
8193 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8194 );
8195 if needs_catchup_run {
8196 let round = self.state.reviews[round_idx].round;
8197 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8198 let commands = self.state.config.verify.e2e.clone();
8199 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8200 let cache_dir = self.state.config.cache_dir();
8201 let context = format!(
8202 "round {round}: verification unresolved, catching up before the final decision"
8203 );
8204 let wait = self.repository_wait();
8205 let (outcomes, verify_retried) = with_cache_lease(
8206 &mut self.state,
8207 cache_dir.as_deref(),
8208 "e2e",
8209 "e2e",
8210 worktree,
8211 &attempted_head,
8212 timeout,
8213 &context,
8214 &wait,
8215 |state, budget| {
8216 let shell = shell.to_vec();
8217 let commands = commands.clone();
8218 let context = context.clone();
8219 async move {
8220 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8221 .await
8222 }
8223 },
8224 )
8225 .await?;
8226 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8227 self.state.record_base_progress(&attempted_head);
8228 }
8229 let last = &mut self.state.reviews[round_idx];
8230 last.e2e = outcomes;
8231 last.verify_retried = verify_retried;
8232 last.verified_head = Some(attempted_head);
8239 last.verified_at = Some(Timestamp::now());
8240 if verify_inconclusive(&last.e2e) {
8241 self.state.save()?;
8248 return Ok(());
8249 }
8250 last.e2e_deferred = false;
8251 if last.answered == last.expected {
8256 last.clean = round_is_clean(
8257 last.blocking,
8258 last.e2e.iter().all(CommandOutcome::ok),
8259 last.answered,
8260 last.expected,
8261 0,
8262 self.state.config.graph.incomplete_review,
8263 );
8264 }
8265 }
8266 let last = &self.state.reviews[round_idx];
8267 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8268
8269 match last.e2e_status() {
8270 E2eStatus::Failed => {
8271 let red: Vec<String> = last
8272 .e2e
8273 .iter()
8274 .filter(|o| !o.ok())
8275 .map(|o| {
8276 format!(
8277 "`{}` -> {:?}\n{}",
8278 o.command,
8279 o.code,
8280 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8281 )
8282 })
8283 .collect();
8284 let can_repair = last.blocking == 0 && last.fix.is_none();
8285 self.state.reviews[round_idx].clean = false;
8286 self.state
8287 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8288 self.state.status = RunStatus::Blocked;
8289 if repairable
8290 && can_repair
8291 && self.state.reviews.len() < self.state.config.graph.review_rounds
8292 {
8293 self.state.status = RunStatus::Reviewing;
8294 }
8295 }
8296 E2eStatus::ResourceBlocked => {
8301 self.state.event(
8302 "review",
8303 format!(
8304 "{why}; e2e could not run (resource unavailable); not \
8305 deciding yet"
8306 ),
8307 );
8308 }
8309 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8310 self.state.event(
8311 "review",
8312 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8313 );
8314 self.record_contested_handoff();
8315 self.state.status = RunStatus::Gating;
8316 }
8317 }
8318 self.state.save()?;
8319 Ok(())
8320 }
8321
8322 async fn gate(&mut self) -> Result<()> {
8325 if self.hold_base_sync().await? {
8326 return Ok(());
8327 }
8328 if self.state.status == RunStatus::Failed
8340 || self
8341 .state
8342 .base_sync
8343 .as_ref()
8344 .is_some_and(|s| s.conflict.is_some())
8345 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8346 != Some(RunStatus::Gating)
8347 {
8348 return Ok(());
8349 }
8350 let Some(winner) = self.state.winner().cloned() else {
8351 return Ok(());
8352 };
8353 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8354 self.state.reconcile_verification_head(&head);
8355 self.state.save()?;
8356 let wait = self.repository_wait();
8357 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8358 return Ok(());
8359 }
8360 if self.state.gate_ran {
8361 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8372 self.state.status = RunStatus::Blocked;
8373 self.state.save()?;
8374 }
8375 return Ok(());
8376 }
8377 self.state.status = RunStatus::Gating;
8378 let mut outcomes = self.run_gate(&winner).await?;
8379 let mut attempted_head = self.state.gate_head.clone();
8380 loop {
8381 if verify_inconclusive(&outcomes) {
8392 self.state.save()?;
8393 return Ok(());
8394 }
8395 if outcomes.iter().all(CommandOutcome::ok) {
8396 break;
8397 }
8398 let fixes_before = self.state.gate_fixes.len();
8399 match self.gate_fix_round(&winner, &outcomes).await? {
8400 GateFix::Retry => {
8401 outcomes = self.run_gate(&winner).await?;
8402 attempted_head = self.state.gate_head.clone();
8403 }
8404 GateFix::Stop => {
8405 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8406 if self.state.gate_fixes.len() > fixes_before
8407 && attempted_head.as_deref() != Some(current.as_str())
8408 && verify_current_head(&mut self.state, &winner, &wait).await?
8409 {
8410 outcomes = self.run_gate(&winner).await?;
8413 attempted_head = self.state.gate_head.clone();
8414 } else {
8415 break;
8416 }
8417 }
8418 GateFix::Defer => {
8419 self.state.save()?;
8420 return Ok(());
8421 }
8422 }
8423 }
8424 let passed = outcomes.iter().all(CommandOutcome::ok);
8425 self.state.gate = outcomes;
8426 self.state.gate_ran = true;
8427 self.state.gate_head = attempted_head;
8428 if !self.state.gate.is_empty()
8429 && !verify_inconclusive(&self.state.gate)
8430 && let Some(head) = self.state.gate_head.clone()
8431 {
8432 self.state.record_base_progress(&head);
8433 }
8434 self.state.save()?;
8435 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8436 return Ok(());
8437 }
8438 if !passed {
8439 self.state.status = RunStatus::Blocked;
8440 let spent = self.state.gate_fixes.len();
8441 self.state.event(
8442 "gate",
8443 if spent == 0 {
8444 "gate failed; not merging".to_owned()
8445 } else {
8446 format!("gate failed after {spent} gate-fix round(s); not merging")
8447 },
8448 );
8449 }
8450 self.state.save()?;
8451 Ok(())
8452 }
8453
8454 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8464 let commands = state.config.verify.pre_gate.clone();
8465 if commands.is_empty() {
8466 return;
8467 }
8468 let shell = state.config.shell();
8469 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8470 let (outcomes, _) = run_commands(
8471 state,
8472 "pre_gate",
8473 "pre_gate",
8474 0,
8475 &shell,
8476 &commands,
8477 &winner.worktree,
8478 timeout,
8479 )
8480 .await;
8481 for o in &outcomes {
8482 if !o.ok() {
8483 tracing::warn!(
8484 "pre_gate `{}` failed ({:?}); the gate decides",
8485 o.command,
8486 o.code
8487 );
8488 }
8489 state.event(
8490 "pre_gate",
8491 format!(
8492 "`{}` -> {}",
8493 o.command,
8494 if o.ok() {
8495 "pass".to_owned()
8496 } else {
8497 format!(
8498 "FAIL ({:?})\n{}",
8499 o.code,
8500 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8501 )
8502 }
8503 ),
8504 );
8505 }
8506 state.pre_gate = outcomes;
8507 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8508 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8509 Ok(head) => {
8510 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8511 state.pre_gate_commit = Some(head);
8512 }
8513 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8514 },
8515 Ok(false) => {}
8516 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8517 }
8518 if let Err(e) = state.save() {
8519 tracing::warn!("could not persist the pre_gate record: {e:#}");
8520 }
8521 }
8522
8523 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8526 let wait = self.repository_wait();
8527 run_landing_gate(&mut self.state, winner, &wait).await
8528 }
8529
8530 async fn gate_fix_round(
8543 &mut self,
8544 winner: &Candidate,
8545 outcomes: &[CommandOutcome],
8546 ) -> Result<GateFix> {
8547 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8548 g.phase == crate::repair_budget::Phase::Fixing
8549 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8550 }) {
8551 self.state.gate_fixes.len().saturating_add(1)
8552 } else {
8553 self.state.config.graph.gate_fix_rounds
8554 };
8555 let spent = self.state.gate_fixes.len();
8556 if spent >= cap {
8557 if cap > 0 {
8558 self.state.event(
8559 "gate",
8560 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8561 );
8562 }
8563 return Ok(GateFix::Stop);
8564 }
8565 if !gate_fixable(outcomes) {
8566 self.state.event(
8567 "gate",
8568 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8569 command or similar); not spending a fix round on it",
8570 );
8571 return Ok(GateFix::Stop);
8572 }
8573 let min_free = self.state.config.disk.min_free_bytes;
8574 if min_free > 0 {
8575 match crate::disk::free_bytes(&winner.worktree) {
8576 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8577 Ok(free) => {
8578 self.state.event(
8579 "gate",
8580 format!(
8581 "only {free} bytes free ({min_free} required by `[disk] \
8582 min_free_bytes`); not spending a fix round on a failure the disk \
8583 may explain"
8584 ),
8585 );
8586 return Ok(GateFix::Stop);
8587 }
8588 Err(e) => {
8589 self.state.event(
8590 "gate",
8591 format!("free disk space could not be measured ({e:#}); no fix round"),
8592 );
8593 return Ok(GateFix::Stop);
8594 }
8595 }
8596 }
8597
8598 let attempt = spent + 1;
8599 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8600 let base = self.landing_base();
8601 let fix_prompt = prompt::gate_fix_in(
8602 &self.state.instruction,
8603 &failed,
8604 attempt,
8605 cap,
8606 &self.state.config.graph.language,
8607 &self.state.config.graph.github_language,
8608 );
8609 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8610 let sessions = self.state.config.graph.sessions;
8611 let artifacts = self.merge_repair_artifacts();
8612 self.state.event(
8613 "gate",
8614 format!("gate failed; gate-fix round {attempt} of {cap}"),
8615 );
8616 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8617 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8618 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8621 head: before.clone(),
8622 passed: false,
8623 outcomes: Vec::new(),
8624 });
8625 self.state.save()?;
8626 let (_, seat, out) = self
8627 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8628 prompt: fix_prompt.clone(),
8629 spec,
8630 seat,
8631 cwd: winner.worktree.clone(),
8632 timeout,
8633 allow_write: true,
8634 sessions,
8635 artifacts: artifacts.clone(),
8636 stem: format!("gate-fix-{attempt}"),
8637 handover: None,
8638 })
8639 .await;
8640 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8643 return Ok(GateFix::Defer);
8644 }
8645 let mut record = GateFixRecord {
8646 agent: seat.agent.clone(),
8647 failed,
8648 notes: String::new(),
8649 committed: false,
8650 error: None,
8651 };
8652 match out {
8653 AgentOutcome::Ok(o) => {
8654 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8657 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8658 }
8659 }
8660 AgentOutcome::Dropped(_) => {
8661 record.error = Some("the CLI dropped the stream".to_owned());
8662 }
8663 AgentOutcome::Quota(o) => {
8664 self.state.quota.push(QuotaLoss::from_report(
8665 seat.key.clone(),
8666 "gate-fix",
8667 o.quota.as_ref(),
8668 ));
8669 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8670 }
8671 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8672 }
8673 self.state.seats.insert(seat.key.clone(), seat);
8674 let coordination = self.coordinate(&winner.worktree).await?;
8675 if let Ok(r) = git::rescue_commit(
8676 &winner.worktree,
8677 &format!("magi: gate fix {attempt} (uncommitted work)"),
8678 )
8679 .await
8680 {
8681 self.state.note_withheld("gate-fix", &r.withheld);
8682 }
8683 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8684 record.committed = after != before;
8685 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8686 let note = record.error.clone();
8687 self.state.gate_fixes.push(record);
8688 self.state.save()?;
8689 if !changed {
8690 self.state.event(
8691 "gate",
8692 match note {
8693 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8694 None => format!("gate-fix round {attempt}: the tree did not change"),
8695 },
8696 );
8697 return Ok(GateFix::Stop);
8698 }
8699 self.state.event(
8700 "gate",
8701 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8702 );
8703
8704 drop(coordination);
8705 let wait = self.repository_wait();
8706 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8707 }
8708
8709 async fn guarded_pr_message(
8716 &mut self,
8717 winner: &Candidate,
8718 facts: Option<&BranchFacts>,
8719 posting: bool,
8720 ) -> Result<Option<PrMessage>> {
8721 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8722 let head = match final_head.as_deref() {
8723 Some(h) => crate::pr_outcome::FinalHead::At(h),
8724 None => crate::pr_outcome::FinalHead::Unknown,
8725 };
8726 let summarized = if posting {
8728 Some(self.summarize_change(winner, final_head.as_deref()).await)
8729 } else {
8730 None
8731 };
8732 if self.state.resource_block.is_some() {
8737 self.state.event(
8738 "pr-summary",
8739 "a shared resource was busy; nothing was published, resume to summarize again",
8740 );
8741 return Ok(None);
8742 }
8743 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8744 anyhow::bail!(
8745 "the final commit moved during public description generation; retry before publishing"
8746 );
8747 }
8748 let note = match (&summarized, &self.state.change_summary) {
8749 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8750 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8751 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8752 (None, _) => ChangeNote::NotAsked,
8753 };
8754 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8755 if !posting {
8756 let identity = crate::scrub::Identity::current();
8757 return Ok(Some(PrMessage {
8758 title: crate::scrub::scrub(&pr.title, &identity),
8759 body: crate::scrub::scrub(&pr.body, &identity),
8760 }));
8761 }
8762 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8765 if let Some(known) = self
8766 .state
8767 .github_text
8768 .clone()
8769 .filter(|g| g.fingerprint == raw_fp)
8770 {
8771 return self.settle_withheld(winner, &pr, known).await;
8772 }
8773 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8774 let mut violations =
8775 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8776 let identity = crate::scrub::Identity::current();
8777 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8778 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8779 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8780 {
8781 violations.push(crate::github_text::Violation::SensitiveData);
8782 }
8783 let has_language = violations
8787 .iter()
8788 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8789 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8790 self.state.event(
8791 "github-text",
8792 if has_language {
8793 format!("description rejected: {violations:?}; requesting one rewrite")
8794 } else {
8795 format!("description rejected: {violations:?}; asking the owner")
8796 },
8797 );
8798 let mut rewritten = false;
8799 let mut rejected_rewrite = None;
8800 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8801 {
8802 let output = text;
8803 if let Some(title) = summary_title(&output) {
8804 let raw_body = summary_without_title(&output);
8807 let clean_title = crate::scrub::scrub(&title, &identity);
8808 let mut body = crate::scrub::scrub(&raw_body, &identity);
8809 let redacted = clean_title != title || body != raw_body;
8810 let title = clean_title;
8811 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8812 if !complete {
8813 self.state.event(
8814 "github-text",
8815 "description rewrite omitted required findings",
8816 );
8817 }
8818 if !body.trim().is_empty() {
8819 body.push_str("\n\n");
8820 body.push_str(&public_pr_evidence(&self.state, head));
8821 }
8822 let marker = format!("magi:run/{}", self.state.id);
8823 if !body.trim().is_empty() && !body.contains(&marker) {
8824 body.push_str(&format!("\n\n{marker}"));
8825 }
8826 let rewrite = PrMessage {
8827 title: title.clone(),
8828 body: body.clone(),
8829 };
8830 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8831 let mut failed = crate::github_text::check_for(
8832 &self.state.config.graph,
8833 &title,
8834 &body,
8835 redecision,
8836 );
8837 if (!crate::github_text::shareable(&title)
8838 || !crate::github_text::shareable(&body))
8839 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8840 {
8841 failed.push(crate::github_text::Violation::SensitiveData);
8842 }
8843 if complete
8844 && !title.trim().is_empty()
8845 && !body.trim().is_empty()
8846 && !failed.is_empty()
8847 {
8848 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8849 title: title.clone(),
8850 body: body.clone(),
8851 failed: failed
8852 .iter()
8853 .map(|v| crate::github_text::category(*v).to_owned())
8854 .collect(),
8855 });
8856 }
8857 if complete
8858 && !title.trim().is_empty()
8859 && !raw_body.trim().is_empty()
8860 && failed.is_empty()
8861 {
8862 decision = redecision;
8863 pr = PrMessage { title, body };
8864 rewritten = true;
8865 self.state
8866 .event("github-text", "description rewrite passed");
8867 if redacted {
8868 self.state.event(
8869 "github-text",
8870 "sensitive data removed from description rewrite",
8871 );
8872 }
8873 }
8874 }
8875 }
8876 if !rewritten {
8877 self.state.event(
8878 "github-text",
8879 "description rewrite unavailable or rejected; asking the owner",
8880 );
8881 if let Some(w) =
8882 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8883 {
8884 self.retire_github_text_question("the withheld text changed");
8887 let record = crate::run::GithubTextAsk {
8888 rewrite: rejected_rewrite,
8889 fingerprint: raw_fp,
8890 title: w.title,
8891 body: w.body,
8892 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8893 question: None,
8894 asks: 0,
8895 resolved: false,
8896 chosen_title: None,
8897 choice: None,
8898 publication: None,
8899 };
8900 return self.settle_withheld(winner, &pr, record).await;
8901 }
8902 }
8903 }
8904 let (title, body) =
8905 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8906 Ok(Some(PrMessage { title, body }))
8907 }
8908
8909 async fn summarize_change(
8920 &mut self,
8921 winner: &Candidate,
8922 head: Option<&str>,
8923 ) -> std::result::Result<(), String> {
8924 let Some(head) = head else {
8925 return Err("the final commit could not be read".to_owned());
8926 };
8927 let remote = self.state.config.merge.remote.clone();
8928 let tracking = format!("{remote}/{}", self.state.base_branch);
8929 if !git::rev_exists(&winner.worktree, &tracking).await {
8930 return Err(format!("the remote base `{tracking}` could not be read"));
8931 }
8932 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8933 Ok(b) if !b.is_empty() => b,
8934 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8935 };
8936 let evidence = crate::pr_outcome::summary_evidence(
8937 &self.state,
8938 crate::pr_outcome::FinalHead::At(head),
8939 );
8940 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8941 if self
8942 .state
8943 .change_summary
8944 .as_ref()
8945 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8946 {
8947 return Ok(());
8948 }
8949 if self.state.change_summary.take().is_some() {
8950 self.state.event(
8951 "pr-summary",
8952 "the final diff or evidence changed; the earlier change summary was discarded",
8953 );
8954 }
8955 if let Err(e) = write_artifact(
8957 &self.state,
8958 &format!("public-description-{fingerprint}.md"),
8959 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8960 ) {
8961 tracing::warn!("could not save public description evidence: {e:#}");
8962 }
8963 let chain = match agent::pick_chain(
8964 &self.state.config.agents,
8965 self.state.config.roles.synthesizer.as_ref(),
8966 &agent::installed,
8967 "synthesizer",
8968 ) {
8969 Ok(chain) => chain,
8970 Err(e) => {
8971 tracing::warn!("no agent for the change summary: {e:#}");
8972 self.state.event("pr-summary", "no summary agent available");
8973 return Err("no agent was available to summarize it".to_owned());
8974 }
8975 };
8976 let subjects = git::subjects(&winner.worktree, &base, head)
8977 .await
8978 .unwrap_or_default();
8979 let stat = git::diff_stat(&winner.worktree, &base, head)
8980 .await
8981 .unwrap_or_default();
8982 let stat: String = stat.chars().take(6_000).collect();
8983 let mut prompt = crate::prompt::change_summary_in(
8984 &base,
8985 head,
8986 &subjects,
8987 stat.trim(),
8988 &self.state.config.graph.github_language,
8989 );
8990 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8991 let sessions = self.state.config.graph.sessions;
8992 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8993 let prompts = self.state.config.prompts.clone();
8994 let cache = self.state.config.cache_dir();
8995 let run = self.state.id.clone();
8996 for (i, spec) in chain.into_iter().enumerate() {
8997 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8998 let github_spec = match agent::for_github_text(
8999 &spec,
9000 &self.state.config.graph.github_language,
9001 &winner.worktree,
9002 ) {
9003 Ok(spec) => spec,
9004 Err(e) => {
9005 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9006 continue;
9007 }
9008 };
9009 let job = SeatJob {
9010 spec: github_spec,
9011 seat,
9012 cwd: winner.worktree.clone(),
9013 prompt: prompt.clone(),
9014 timeout,
9015 allow_write: false,
9016 sessions,
9017 artifacts: agent::artifacts_dir(&self.state.dir()),
9018 stem: if i == 0 {
9019 "change-summary".to_owned()
9020 } else {
9021 format!("change-summary-{}", spec.id)
9022 },
9023 handover: None,
9024 };
9025 let ctx = WaveCtx {
9026 run: &run,
9027 node: "pr-summary",
9028 prompts: &prompts,
9029 cache: cache.as_deref(),
9030 round: None,
9031 carry_seats: false,
9032 pauses: Some((&self.pause, &self.interrupt)),
9033 };
9034 let (seat, outcome) =
9035 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9036 self.state.seats.insert(seat.key.clone(), seat);
9037 let text = match outcome {
9038 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9039 AgentOutcome::ResourceBlocked(why) => {
9040 if self.state.resource_block.is_none() {
9043 self.state.resource_block = Some(crate::run::ResourceBlock {
9044 cause: crate::run::ResourceBlockCause::Other,
9045 node: "pr-summary".to_owned(),
9046 kind: "shared resource".to_owned(),
9047 detail: why,
9048 at: Timestamp::now(),
9049 });
9050 }
9051 return Err("a shared resource was busy".to_owned());
9052 }
9053 _ => continue,
9054 };
9055 let blind = &self.state.config.blind;
9056 let Some(title) = summary_title(&text) else {
9057 continue;
9058 };
9059 let body: String = summary_without_title(&text)
9060 .lines()
9061 .filter(|l| !l.contains("magi:run/"))
9062 .collect::<Vec<_>>()
9063 .join("\n");
9064 let body = blind::sanitize_prose(body.trim(), blind);
9065 let title = blind::sanitize_prose(&title, blind);
9066 if body.trim().is_empty() || title.trim().is_empty() {
9067 continue;
9068 }
9069 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9070 continue;
9071 }
9072 if git::rev_parse(&winner.worktree, "HEAD")
9074 .await
9075 .ok()
9076 .as_deref()
9077 != Some(head)
9078 {
9079 return Err("the branch moved while the summary was written".to_owned());
9080 }
9081 if git::merge_base(&winner.worktree, &tracking, head)
9082 .await
9083 .ok()
9084 .as_deref()
9085 != Some(base.as_str())
9086 || crate::pr_outcome::summary_evidence(
9087 &self.state,
9088 crate::pr_outcome::FinalHead::At(head),
9089 ) != evidence
9090 {
9091 return Err("the summary evidence changed while it was written".to_owned());
9092 }
9093 self.state.change_summary = Some(crate::run::ChangeSummary {
9094 head: head.to_owned(),
9095 base,
9096 evidence: fingerprint,
9097 title,
9098 body,
9099 });
9100 self.state.event(
9101 "pr-summary",
9102 format!("change summary written by {} for the final diff", spec.id),
9103 );
9104 if let Err(e) = self.state.save() {
9105 tracing::warn!("could not save the change summary: {e:#}");
9106 }
9107 return Ok(());
9108 }
9109 self.state
9110 .event("pr-summary", "no agent produced a change summary");
9111 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9112 }
9113
9114 async fn rewrite_pr_text(
9124 &mut self,
9125 winner: &Candidate,
9126 pr: &PrMessage,
9127 violations: &[crate::github_text::Violation],
9128 ) -> Option<String> {
9129 let prompt = format!(
9130 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9131 pr.title, pr.body
9132 );
9133 let sessions = self.state.config.graph.sessions;
9134 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9135 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9138 let key = format!("impl-{}", winner.label);
9139 if let Some(spec) = self
9140 .state
9141 .config
9142 .agents
9143 .iter()
9144 .find(|a| a.id == winner.agent)
9145 .cloned()
9146 && let Some(existing) = self.state.seats.get(&key)
9147 && existing.agent == spec.id
9148 && agent::has_session(spec.kind, existing, sessions)
9149 {
9150 attempts.push((spec, Some(existing.clone())));
9151 }
9152 match agent::pick_chain(
9153 &self.state.config.agents,
9154 self.state.config.roles.language_judge.as_ref(),
9155 &agent::installed,
9156 "language_judge",
9157 ) {
9158 Ok(chain) => {
9159 for spec in chain {
9160 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9161 continue;
9162 }
9163 attempts.push((spec, None));
9164 }
9165 }
9166 Err(e) if attempts.is_empty() => {
9167 tracing::warn!("no agent for the description rewrite: {e:#}");
9168 self.state
9169 .event("github-text", "no rewrite agent available");
9170 return None;
9171 }
9172 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9173 }
9174 let prompts = self.state.config.prompts.clone();
9175 let cache = self.state.config.cache_dir();
9176 let run = self.state.id.clone();
9177 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9178 let resumed = resumed_seat.is_some();
9179 let seat = resumed_seat.unwrap_or_else(|| {
9180 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9181 });
9182 let github_spec = match agent::for_github_text(
9183 &spec,
9184 &self.state.config.graph.github_language,
9185 &winner.worktree,
9186 ) {
9187 Ok(spec) => spec,
9188 Err(e) => {
9189 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9190 continue;
9191 }
9192 };
9193 let job = SeatJob {
9194 spec: github_spec,
9195 seat,
9196 cwd: winner.worktree.clone(),
9197 prompt: prompt.clone(),
9198 timeout: retry_budget(timeout_full, resumed),
9199 allow_write: false,
9200 sessions,
9201 artifacts: agent::artifacts_dir(&self.state.dir()),
9202 stem: if i == 0 {
9203 "github-text-rewrite".to_owned()
9204 } else {
9205 format!("github-text-rewrite-{}", spec.id)
9206 },
9207 handover: None,
9208 };
9209 let ctx = WaveCtx {
9210 run: &run,
9211 node: "github-text",
9212 prompts: &prompts,
9213 cache: cache.as_deref(),
9214 round: None,
9215 carry_seats: false,
9216 pauses: Some((&self.pause, &self.interrupt)),
9217 };
9218 let (seat, outcome) =
9219 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9220 self.state.seats.insert(seat.key.clone(), seat);
9221 match outcome {
9222 AgentOutcome::Ok(o)
9223 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9224 {
9225 return Some(o.text);
9226 }
9227 AgentOutcome::ResourceBlocked(_) => return None,
9228 _ => {}
9229 }
9230 }
9231 None
9232 }
9233
9234 fn retire_github_text_question(&mut self, why: &str) {
9236 let Some(id) = self
9237 .state
9238 .github_text
9239 .as_ref()
9240 .and_then(|g| g.question.clone())
9241 else {
9242 return;
9243 };
9244 let _ = ask::Questions::open().update(&id, |q| {
9245 q.abandon(why);
9246 Ok(())
9247 });
9248 }
9249
9250 async fn settle_withheld(
9254 &mut self,
9255 winner: &Candidate,
9256 pr: &PrMessage,
9257 mut g: crate::run::GithubTextAsk,
9258 ) -> Result<Option<PrMessage>> {
9259 use crate::github_text as gt;
9260 let store = ask::Questions::open();
9261 let timeout = self.state.config.graph.answer_timeout;
9262 if !g.resolved {
9263 let mut standing = g
9264 .question
9265 .as_ref()
9266 .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9267 if let Some(q) = standing.as_ref()
9268 && q.status.open()
9269 && !store.held_reply(&q.id)
9270 && gt::expired(q, timeout)
9271 {
9272 let why = format!("no answer within {}s of asking", timeout.max(1));
9273 standing = store
9274 .update(&q.id, |q| {
9275 q.abandon(&why);
9276 Ok(())
9277 })
9278 .ok()
9279 .map(|(q, ())| q);
9280 }
9281 let mut ask_again: Option<Option<Vec<String>>> = None;
9282 match standing {
9283 None if g.asks <= 1 => {
9286 g.asks = 0;
9287 ask_again = Some(None);
9288 }
9289 None => {
9290 g.resolved = true;
9291 g.chosen_title = None;
9292 g.choice = Some(GithubTextChoice::Fallback);
9293 }
9294 Some(q) if q.status.open() => {
9295 self.park_on_github_text(&g)?;
9296 return Ok(None);
9297 }
9298 Some(q) => match gt::read_reply(&q) {
9299 gt::Reply::Fallback => {
9300 g.resolved = true;
9301 g.chosen_title = None;
9302 g.choice = Some(GithubTextChoice::Fallback);
9303 }
9304 gt::Reply::Original => {
9305 g.resolved = true;
9306 g.chosen_title = None;
9307 g.choice = Some(GithubTextChoice::Original);
9308 }
9309 gt::Reply::Rewrite => {
9310 g.resolved = true;
9311 g.chosen_title = None;
9312 g.choice = Some(if g.rewrite.is_some() {
9313 GithubTextChoice::Rewrite
9314 } else {
9315 GithubTextChoice::Fallback
9316 });
9317 }
9318 gt::Reply::Title(text) => {
9319 let first = text
9320 .lines()
9321 .map(str::trim)
9322 .find(|l| !l.is_empty())
9323 .unwrap_or("");
9324 let decision = if first.is_empty() {
9325 None
9326 } else {
9327 let candidate = PrMessage {
9328 title: first.to_owned(),
9329 body: String::new(),
9330 };
9331 self.judge_pr_language(&winner.worktree, &candidate).await
9332 };
9333 match gt::vet_title(
9334 &text,
9335 decision,
9336 self.state.config.graph.github_english(),
9337 ) {
9338 Ok(title) => {
9339 g.resolved = true;
9340 g.chosen_title = Some(title);
9341 g.choice = Some(GithubTextChoice::MyTitle);
9342 }
9343 Err(cats) if g.asks < 2 => {
9344 ask_again = Some(Some(
9345 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9346 ))
9347 }
9348 Err(_) => {
9349 g.resolved = true;
9350 g.chosen_title = None;
9351 g.choice = Some(GithubTextChoice::Fallback);
9352 }
9353 }
9354 }
9355 },
9356 }
9357 if let Some(rejected) = ask_again {
9358 let w = gt::Withheld {
9361 title: g.title,
9362 body: g.body,
9363 categories: g.categories.clone(),
9364 };
9365 let artifact = match crate::run::write_artifact(
9368 &self.state,
9369 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9370 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9371 ) {
9372 Ok(p) => Some(p),
9373 Err(e) => {
9374 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9375 None
9376 }
9377 };
9378 let (detail, comparison) = gt::question_content(
9379 &w,
9380 &pr.title,
9381 &pr.body,
9382 &self.state.id,
9383 rejected.as_deref(),
9384 artifact.as_deref(),
9385 g.rewrite.as_ref(),
9386 );
9387 let mut q = ask::Question::new(
9388 self.state.id.clone(),
9389 gt::ASK_NODE.to_owned(),
9390 gt::ASK_SEAT.to_owned(),
9391 gt::question_summary(&self.state.config.graph.language, &w),
9392 detail,
9393 gt::question_choices(&w, g.rewrite.as_ref()),
9394 );
9395 q.posting_comparison = Some(comparison);
9396 q.answer_timeout = timeout;
9397 g.question = Some(q.id.clone());
9400 g.asks += 1;
9401 self.state.github_text = Some(g.clone());
9402 self.state.event(
9403 "github-text",
9404 format!("asking the owner about the withheld text ({})", q.short()),
9405 );
9406 self.state.save()?;
9407 store
9408 .put_executable(&mut q)
9409 .context("file the posting-gate question")?;
9410 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9411 tracing::warn!(
9412 "could not notify about posting-gate question {}: {e:#}",
9413 q.short()
9414 );
9415 }
9416 self.park_on_github_text(&g)?;
9417 return Ok(None);
9418 }
9419 }
9420 let resolved = g.chosen_title.clone();
9421 let original = g.choice == Some(GithubTextChoice::Original);
9422 self.state.event(
9424 "github-text",
9425 if original {
9426 format!(
9427 "posting the original text for: {}",
9428 match (g.title, g.body) {
9429 (true, true) => "title, description",
9430 (true, false) => "title",
9431 _ => "description",
9432 }
9433 )
9434 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9435 "posting the saved rewrite title and description".to_owned()
9436 } else if resolved.is_some() {
9437 "posting the owner's replacement title".to_owned()
9438 } else {
9439 "posting the neutral text for the withheld fields".to_owned()
9440 },
9441 );
9442 self.state.github_text = Some(g.clone());
9443 self.state.save()?;
9444 if g.choice == Some(GithubTextChoice::Rewrite)
9445 && let Some(r) = g.rewrite.as_ref()
9446 {
9447 return Ok(Some(PrMessage {
9448 title: r.title.clone(),
9449 body: r.body.clone(),
9450 }));
9451 }
9452 let w = gt::Withheld {
9453 title: g.title,
9454 body: g.body,
9455 categories: g.categories.clone(),
9456 };
9457 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9458 if let Some(t) = resolved {
9459 title = t;
9460 }
9461 let english = gt::LanguageDecision {
9463 title_english: true,
9464 body_english: true,
9465 body_complete: true,
9466 };
9467 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9468 if original {
9469 if g.title {
9472 title = pr.title.clone();
9473 }
9474 if g.body {
9475 body = pr.body.clone();
9476 }
9477 }
9478 Ok(Some(PrMessage { title, body }))
9479 }
9480
9481 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9482 self.state.github_text = Some(g.clone());
9483 self.state.end_pass_parked(ParkCause::Question);
9484 self.state.parked = true;
9485 self.state.event(
9486 "github-text",
9487 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9488 );
9489 self.state.save()
9490 }
9491
9492 async fn judge_pr_language(
9495 &mut self,
9496 cwd: &Path,
9497 pr: &PrMessage,
9498 ) -> Option<crate::github_text::LanguageDecision> {
9499 self.state.config.roles.language_judge.as_ref()?;
9500 if !self.state.config.graph.language_gate() {
9501 return None;
9502 }
9503 let decision =
9504 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9505 self.state.event(
9506 "github-text",
9507 if decision.is_some() {
9508 "language decided by the language judge"
9509 } else {
9510 "language judge unavailable; using built-in heuristics"
9511 },
9512 );
9513 decision
9514 }
9515
9516 fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9517 Box::pin(self.merge_inner())
9518 }
9519
9520 async fn merge_inner(&mut self) -> Result<()> {
9521 if self.resource_halt()? || self.hold_base_sync().await? {
9522 return Ok(());
9523 }
9524 if self
9539 .state
9540 .base_sync
9541 .as_ref()
9542 .is_some_and(|s| s.conflict.is_some())
9543 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9544 != Some(RunStatus::Gating)
9545 || !self.state.gate_allows_progress()
9553 {
9554 return Ok(());
9555 }
9556 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9565 return Ok(());
9566 }
9567 let Some(winner) = self.state.winner().cloned() else {
9568 return Ok(());
9569 };
9570 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9571 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9572 self.state.status = RunStatus::Blocked;
9573 self.state.event(
9574 "gate",
9575 "winner changed after gate; fresh verification required",
9576 );
9577 self.state.save()?;
9578 return Ok(());
9579 }
9580 let wait = self.repository_wait();
9581 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9582 return Ok(());
9583 }
9584 let repo = self.state.repo.clone();
9585 let base = self.state.base_branch.clone();
9586 let mode = self.state.config.merge.mode;
9587 let style = self.state.config.merge.style;
9588 let facts = if is_review_run(&self.state) {
9589 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9590 let start = review_base(
9591 &repo,
9592 &self.state.config.merge.remote,
9593 &base,
9594 &self.state.base_commit,
9595 &winner.branch,
9596 )
9597 .await;
9598 branch_facts(&repo, &start, &winner.branch).await
9599 } else {
9600 None
9601 };
9602 let leaked = if is_review_run(&self.state) {
9605 leaked_subjects(&self.state, &winner.branch).await
9606 } else {
9607 Some(Vec::new())
9608 };
9609 let Some(pr) = self
9613 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9614 .await?
9615 else {
9616 return Ok(());
9617 };
9618 let message = pr.commit_message();
9619
9620 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9621 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9622 return Ok(());
9623 }
9624 let outcome = match mode {
9625 MergeMode::None => MergeOutcome {
9626 mode,
9627 ok: true,
9628 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9629 empty: false,
9630 },
9631 MergeMode::Pr | MergeMode::Local
9632 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9633 {
9634 MergeOutcome {
9635 mode,
9636 ok: false,
9637 detail: empty_candidate_detail(&self.state, &base),
9638 empty: true,
9639 }
9640 }
9641 MergeMode::Local => {
9642 let on = git::current_branch(&repo).await?;
9643 if on.as_deref() != Some(base.as_str()) {
9644 MergeOutcome {
9645 mode,
9646 ok: false,
9647 detail: format!(
9648 "{} has {} checked out, not the base branch {base}",
9649 repo.display(),
9650 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9651 ),
9652 empty: false,
9653 }
9654 } else if !git::is_clean(&repo).await? {
9655 MergeOutcome {
9656 mode,
9657 ok: false,
9658 detail: format!("{} is dirty; refusing to merge", repo.display()),
9659 empty: false,
9660 }
9661 } else {
9662 let out = match style {
9663 MergeStyle::Merge => {
9664 git::merge_no_ff(&repo, &winner.branch, &message).await?
9665 }
9666 MergeStyle::Squash => {
9667 git::merge_squash(&repo, &winner.branch, &message).await?
9668 }
9669 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9670 };
9671 MergeOutcome {
9672 mode,
9673 ok: out.ok(),
9674 detail: if out.ok() { out.stdout } else { out.stderr },
9675 empty: false,
9676 }
9677 }
9678 }
9679 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9680 self.state.pr_publication_pending = true;
9681 self.state.save()?;
9682 let out =
9683 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9684 self.state.pr_publication_pending = out.is_err();
9685 MergeOutcome {
9686 mode,
9687 ok: out.is_ok(),
9688 detail: match out {
9689 Ok(url) => url,
9690 Err(e) => format!("{e:#}"),
9691 },
9692 empty: false,
9693 }
9694 }
9695 MergeMode::Pr => {
9696 let remote = self.state.config.merge.remote.clone();
9697 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9698 let pushed = if self.state.foreign_checkout.is_some() {
9699 let refspec = format!("{delivery_head}:refs/heads/{public}");
9700 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9701 } else {
9702 git::push(&winner.worktree, &remote, &winner.branch).await?
9703 };
9704 if !pushed.ok() {
9705 MergeOutcome {
9706 mode,
9707 ok: false,
9708 detail: pushed.stderr,
9709 empty: false,
9710 }
9711 } else {
9712 if self.state.foreign_checkout.is_some() {
9713 self.state.pr_pushed_head = Some(delivery_head.clone());
9714 }
9715 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9721 let out = match pr_merge_plan(found) {
9722 PrPlan::Create => {
9723 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9724 .await
9725 }
9726 PrPlan::Adopt { url, title } => {
9727 self.state
9728 .event("merge", format!("Pr: adopted open pull request {url}"));
9729 if self.state.explicit_github_text_selection() {
9730 land::set_pr_text_verbatim(
9733 &winner.worktree,
9734 &url,
9735 &pr.title,
9736 &pr.body,
9737 )
9738 .await
9739 .map(|()| url)
9740 } else {
9741 if title != pr.title
9742 && (!is_review_run(&self.state)
9743 || leaked
9744 .as_deref()
9745 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9746 && let Err(e) =
9747 if self.state.github_text.as_ref().is_some_and(|g| {
9748 g.resolved
9749 && g.title
9750 && g.choice == Some(GithubTextChoice::Original)
9751 }) {
9752 land::set_pr_title_verbatim(
9754 &winner.worktree,
9755 &url,
9756 &pr.title,
9757 )
9758 .await
9759 } else {
9760 land::set_pr_title(
9761 &mut self.state,
9762 &winner.worktree,
9763 &url,
9764 &pr.title,
9765 )
9766 .await
9767 }
9768 {
9769 tracing::warn!("could not refresh title of {url}: {e:#}");
9770 self.state
9771 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9772 }
9773 Ok(url)
9774 }
9775 }
9776 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9777 };
9778 match out {
9779 Ok(url) => MergeOutcome {
9780 mode,
9781 ok: true,
9782 detail: url,
9783 empty: false,
9784 },
9785 Err(e) => MergeOutcome {
9786 mode,
9787 ok: false,
9788 detail: e.to_string(),
9789 empty: false,
9790 },
9791 }
9792 }
9793 }
9794 };
9795
9796 self.state.status = match (mode, outcome.ok) {
9797 (MergeMode::None, _) => RunStatus::Ready,
9798 (_, true) if outcome.empty => RunStatus::Ready,
9799 (MergeMode::Pr, true) => RunStatus::Ready,
9800 (_, true) => RunStatus::Merged,
9801 (_, false) => RunStatus::Blocked,
9802 };
9803 self.state.event(
9804 "merge",
9805 format!(
9806 "{:?}: {}",
9807 mode,
9808 outcome.detail.lines().next().unwrap_or("")
9809 ),
9810 );
9811 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9812 self.state.record_github_text_publication(&outcome.detail);
9813 }
9814 self.state.merge = Some(outcome);
9815 self.state.save()?;
9816
9817 if self.state.config.graph.land
9823 && mode == MergeMode::Pr
9824 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9825 {
9826 self.run_land().await?;
9827 }
9828 self.settle_questions();
9833 Ok(())
9834 }
9835
9836 fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9847 Box::pin(self.run_land_inner())
9848 }
9849
9850 async fn run_land_inner(&mut self) -> Result<()> {
9851 crate::pr_head::ensure_owner(&self.state)?;
9852 let url = self
9853 .state
9854 .pr
9855 .as_ref()
9856 .map(|pr| pr.url.clone())
9857 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9858 .unwrap_or_default();
9859 let url = url.lines().next().unwrap_or("").trim().to_owned();
9860 if !url.starts_with("http") {
9861 return Ok(());
9862 }
9863 loop {
9866 match land::land(&mut self.state, &url).await {
9867 Ok(_)
9868 if self
9869 .state
9870 .merge_repairs
9871 .last()
9872 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9873 {
9874 if self.repair_merge_review().await? {
9875 continue;
9876 }
9877 }
9878 Ok(_) if self.state.resource_block.is_some() => {
9879 return self.stop_for_resource();
9880 }
9881 Ok(pr) if self.state.parked => {
9882 let _ = pr;
9886 }
9887 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9888 self.state.status = RunStatus::Superseded;
9889 self.state.save()?;
9890 }
9891 Ok(pr) => {
9892 self.state.status = match pr.state {
9893 land::PrLifecycle::Merged => RunStatus::Merged,
9894 _ => RunStatus::Blocked,
9895 };
9896 if bump::should_release_bump(self.state.status)
9903 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9904 {
9905 self.state
9911 .event("bump", format!("release bump skipped: {e:#}"));
9912 }
9913 if self.state.status == RunStatus::Merged {
9916 crate::followup::after_merge(&mut self.state, &pr.url).await;
9917 }
9918 self.state.save()?;
9919 }
9920 Err(e) => {
9921 if e.is::<crate::repo_lock::ResourceBlocked>() {
9922 return Err(e);
9923 }
9924 self.state.status = RunStatus::Blocked;
9925 self.state.event("land", format!("gave up: {e}"));
9926 self.state.save()?;
9927 }
9928 }
9929 break;
9930 }
9931 Ok(())
9932 }
9933
9934 async fn repair_merge_review(&mut self) -> Result<bool> {
9937 use crate::run::MergeRepairPhase as Phase;
9938 crate::pr_head::ensure_owner(&self.state)?;
9939 let i = self.state.merge_repairs.len() - 1;
9940 let request = self.state.merge_repairs[i].clone();
9941 if request.phase == Phase::Fixing {
9942 let restored = match self.state.winner().cloned() {
9943 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9944 None => Err(anyhow::anyhow!("repair has no winning branch")),
9945 };
9946 let why = match restored {
9947 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9948 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9949 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9950 };
9951 self.hold_merge_repair(i, why)?;
9952 return Ok(false);
9953 }
9954 if request.phase == Phase::Reviewing && self.state.gate_ran {
9959 let winner = self
9960 .state
9961 .winner()
9962 .context("repair has no winning branch")?;
9963 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9964 if self.state.gate_head.as_deref() != Some(head.as_str())
9965 || !git::is_clean(&winner.worktree).await?
9966 {
9967 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9968 return Ok(false);
9969 }
9970 }
9971 let failure = if i >= self.state.config.graph.land_rounds
9972 || self.state.config.graph.review_rounds == 0
9973 {
9974 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9975 } else {
9976 None
9977 };
9978 if let Some(why) = failure {
9979 self.hold_merge_repair(i, why)?;
9980 return Ok(false);
9981 }
9982 if request.phase == Phase::Requested {
9983 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9984 Ok(pr) => pr,
9985 Err(e) => {
9986 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9987 return Ok(false);
9988 }
9989 };
9990 self.state.merge_repairs[i].phase = Phase::Fixing;
9991 self.state.land_approval = None;
9992 self.state.land_armed_head = None;
9993 self.state.save()?;
9994 let evidence = serde_json::to_string_pretty(&request)?;
9997 let reason = format!(
9998 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9999 );
10000 let budget = self.state.config.graph.land_rounds;
10001 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10002 match result {
10003 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10004 Ok(land::Fixed::Deferred) => {
10005 let worktree = self
10009 .state
10010 .winner()
10011 .context("repair has no winning branch")?
10012 .worktree
10013 .clone();
10014 self.graft_restore(&worktree, &request.head).await?;
10015 self.state.merge_repairs[i].phase = Phase::Requested;
10016 self.stop_for_resource()?;
10017 return Ok(false);
10018 }
10019 Ok(land::Fixed::Failed(why)) => {
10020 self.hold_merge_repair(
10021 i,
10022 format!(
10023 "repair failed: {why}; inspect the branch and run an explicit review"
10024 ),
10025 )?;
10026 return Ok(false);
10027 }
10028 Err(e) => {
10029 self.hold_merge_repair(
10030 i,
10031 format!(
10032 "repair failed: {e:#}; inspect the branch and run an explicit review"
10033 ),
10034 )?;
10035 return Ok(false);
10036 }
10037 }
10038 self.begin_merge_repair_review(i)?;
10042 }
10043 self.review_loop().await?;
10044 self.gate().await?;
10045 if self.resource_halt()? || self.state.parked {
10046 return Ok(false);
10047 }
10048 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10049 != Some(RunStatus::Gating)
10050 || !self.state.gate_allows_progress()
10051 {
10052 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10053 return Ok(false);
10054 }
10055 let winner = self
10058 .state
10059 .winner()
10060 .cloned()
10061 .context("repair has no winning branch")?;
10062 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10063 if self.state.gate_head.as_deref() != Some(after.as_str())
10064 || !git::is_clean(&winner.worktree).await?
10065 {
10066 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10067 return Ok(false);
10068 }
10069 let expected = self
10070 .state
10071 .pr_pushed_head
10072 .as_deref()
10073 .unwrap_or(&request.head)
10074 .to_owned();
10075 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10076 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10079 return Ok(false);
10080 }
10081 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10082 let refspec = format!("{after}:refs/heads/{public}");
10083 let push = git::git_raw(
10084 &winner.worktree,
10085 &["push", &lease, &self.state.config.merge.remote, &refspec],
10086 )
10087 .await;
10088 match push {
10089 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10090 Ok(out) => {
10091 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10092 return Ok(false);
10093 }
10094 Err(e) => {
10095 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10096 return Ok(false);
10097 }
10098 }
10099 self.state.merge_repairs[i].phase = Phase::Complete;
10100 self.state.status = RunStatus::Landing;
10101 self.state.event(
10102 "repair",
10103 "new review and gate completed; fresh owner merge decision required",
10104 );
10105 self.state.save()?;
10106 Ok(true)
10107 }
10108
10109 fn merge_repair_artifacts(&self) -> PathBuf {
10110 let artifacts = agent::artifacts_dir(&self.state.dir());
10111 match self
10112 .state
10113 .merge_repairs
10114 .last()
10115 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10116 {
10117 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10118 None => artifacts,
10119 }
10120 }
10121
10122 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10123 if let Some(e2e) = self.state.head_e2e.take() {
10124 self.state
10125 .verification_history
10126 .push(crate::run::VerificationEvidence {
10127 superseded_by: None,
10128 archived_at: Some(Timestamp::now()),
10129 fixes_spent: Some(self.state.gate_fixes.len()),
10130 gate_head: None,
10131 gate_ran: false,
10132 gate: Vec::new(),
10133 e2e: Some(e2e),
10134 });
10135 }
10136 self.state.reviews.clear();
10137 self.state.gate.clear();
10138 self.state.gate_ran = false;
10139 self.state.gate_head = None;
10140 self.state.gate_fixes.clear();
10141 self.state.gate_repair_e2e = None;
10142 self.state.pre_gate.clear();
10143 self.state.pre_gate_commit = None;
10144 self.state.contested_handoff = None;
10145 self.state.land_reruns = None;
10146 self.state.land_approval = None;
10147 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10148 self.state.status = RunStatus::Reviewing;
10149 self.state.save()
10150 }
10151
10152 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10153 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10154 self.state.merge_repairs[index].failure = Some(why.clone());
10155 self.state.status = RunStatus::Blocked;
10156 self.state.event("repair", why);
10157 self.state.save()
10158 }
10159
10160 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10164 if let Some(existing) = self.state.seats.get(key)
10165 && existing.agent == agent
10166 {
10167 return existing.clone();
10168 }
10169 let fresh = if self.state.seats.contains_key(key) {
10172 handover_seat(key, agent, self.state.next_seat_seed())
10173 } else {
10174 SeatState::new(key, agent, self.state.seed)
10175 };
10176 self.state.seats.insert(key.to_owned(), fresh.clone());
10177 fresh
10178 }
10179
10180 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10186 match self.state.seats.get(key) {
10187 Some(s) if s.agent != spec.id => {
10188 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10189 }
10190 _ => spec,
10191 }
10192 }
10193
10194 fn view(&self, c: &Candidate) -> CandidateView {
10196 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10197 .unwrap_or_default();
10198 let (patch, _) = blind::sanitize_patch(
10199 &format!("candidate {} patch", c.label),
10200 &raw,
10201 &self.state.config.blind,
10202 );
10203 CandidateView {
10204 label: c.label,
10205 branch: c.branch.clone(),
10206 summary: c.summary.clone(),
10207 stat: c.stat.clone(),
10208 patch,
10209 }
10210 }
10211
10212 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10214 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10215 prompt::judge(
10216 "(see above)",
10217 &views,
10218 self.roles.judges.len(),
10219 base_short,
10220 "en",
10221 )
10222 }
10223
10224 fn vote_prompt_full(
10230 &self,
10231 j: usize,
10232 labels: &[char],
10233 language: &str,
10234 candidates: &[Candidate],
10235 base_short: &str,
10236 ) -> String {
10237 let mut text = format!(
10238 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10239 prompt::final_vote(labels, language),
10240 self.state.instruction,
10241 self.candidate_block(candidates, base_short)
10242 );
10243 if let Some(own) = self
10244 .state
10245 .judgements
10246 .get(j)
10247 .filter(|r| !r.ranking.is_empty())
10248 {
10249 let reasons = own
10250 .reasons
10251 .iter()
10252 .map(|(k, v)| format!("- {k}: {v}"))
10253 .collect::<Vec<_>>()
10254 .join("\n");
10255 text.push_str(&format!(
10256 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10257 own.ranking.iter().collect::<String>(),
10258 if reasons.is_empty() {
10259 ""
10260 } else {
10261 ", because:\n"
10262 }
10263 ));
10264 }
10265 if !self.state.deliberation.is_empty() {
10266 text.push_str("\n# What was argued before this vote\n");
10267 for t in self.transcript(&[], j) {
10268 text.push_str(&format!(
10269 "\n## {}{}\n\n{}\n",
10270 t.who,
10271 if t.is_self { " (you)" } else { "" },
10272 t.body.trim()
10273 ));
10274 }
10275 }
10276 text
10277 }
10278
10279 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10286 let mut turns = Vec::new();
10287 for j in &self.state.judgements {
10288 if j.ranking.is_empty() {
10289 continue;
10290 }
10291 let reasons = j
10292 .reasons
10293 .iter()
10294 .map(|(k, v)| format!("- {k}: {v}"))
10295 .collect::<Vec<_>>()
10296 .join("\n");
10297 turns.push(Turn {
10298 who: format!("Judge {} (opening ranking)", j.judge),
10299 is_self: j.judge == self_idx + 1,
10300 body: format!(
10301 "Ranked {}{}{reasons}",
10302 j.ranking.iter().collect::<String>(),
10303 if reasons.is_empty() {
10304 ""
10305 } else {
10306 ", because:\n"
10307 }
10308 ),
10309 });
10310 }
10311 for t in self
10312 .state
10313 .deliberation
10314 .iter()
10315 .flat_map(|r| r.turns.iter())
10316 .chain(current)
10317 {
10318 turns.push(Turn {
10319 who: format!("Judge {}", t.judge),
10320 is_self: t.judge == self_idx + 1,
10321 body: t.body.clone(),
10322 });
10323 }
10324 turns
10325 }
10326}
10327
10328fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10330 agent::has_session(spec.kind, seat, sessions)
10331}
10332
10333fn next_untried_in_roster<'a>(
10354 roster: &'a [AgentSpec],
10355 start: usize,
10356 tried: &BTreeSet<String>,
10357) -> Option<&'a AgentSpec> {
10358 roster
10359 .get(start + 1..)?
10360 .iter()
10361 .find(|s| !tried.contains(&s.id))
10362}
10363
10364fn pick_successor<'a>(
10385 roster: &'a [AgentSpec],
10386 start: usize,
10387 tried: &BTreeSet<String>,
10388 carried: Option<&BTreeSet<String>>,
10389 others: &BTreeSet<String>,
10390) -> Option<&'a AgentSpec> {
10391 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10392 roster
10393 .get(start + 1..)
10394 .and_then(|tail| {
10395 tail.iter()
10396 .filter(free)
10397 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10398 })
10399 .or_else(|| {
10400 carried
10401 .is_some()
10402 .then(|| roster.iter().find(free))
10403 .flatten()
10404 })
10405 .or_else(|| match carried {
10406 Some(c) => next_for_seat(roster, start, tried, c),
10407 None => next_untried_in_roster(roster, start, tried),
10408 })
10409}
10410
10411fn next_for_seat<'a>(
10421 roster: &'a [AgentSpec],
10422 start: usize,
10423 round_tried: &BTreeSet<String>,
10424 carried_failed: &BTreeSet<String>,
10425) -> Option<&'a AgentSpec> {
10426 roster
10427 .get(start + 1..)?
10428 .iter()
10429 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10430 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10431}
10432
10433fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10439 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10440 return spec;
10441 };
10442 let ok = |id: &str| !h.failed.contains(id);
10443 if let Some(last) = h.last_ok.as_deref()
10444 && ok(last)
10445 && let Some(s) = roster.iter().find(|s| s.id == last)
10446 {
10447 return s.clone();
10448 }
10449 if ok(&spec.id) {
10450 return spec;
10451 }
10452 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10453 roster
10454 .get(from + 1..)
10455 .into_iter()
10456 .flatten()
10457 .chain(roster.iter())
10458 .find(|s| ok(&s.id))
10459 .cloned()
10460 .unwrap_or(spec)
10461}
10462
10463pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10467 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10468}
10469
10470const TIMED_OUT: &str = "timed out";
10473
10474impl FailClass {
10475 fn of(out: &AgentOutcome) -> Option<Self> {
10477 match out {
10478 AgentOutcome::Ok(_) => None,
10479 AgentOutcome::Quota(_) => Some(Self::Quota),
10480 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10481 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10482 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10483 AgentOutcome::ResourceBlocked(_) => None,
10486 }
10487 }
10488
10489 fn stem_word(&self) -> &'static str {
10491 match self {
10492 Self::Quota => "quota",
10493 _ => "handover",
10494 }
10495 }
10496}
10497
10498fn failure_signature(msg: &str) -> String {
10502 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10503 if let Some(code) = line
10506 .strip_prefix("process exited with no usable output (exit code ")
10507 .and_then(|s| s.strip_suffix(')'))
10508 .and_then(|s| s.parse::<i32>().ok())
10509 {
10510 line = format!("exited with some({code}) and no usable output");
10511 } else if matches!(
10512 line.as_str(),
10513 "process exited with no usable output (no exit code reported)"
10514 | "process exited with no usable output (exit code unavailable)"
10515 ) {
10516 line = "exited with none and no usable output".to_owned();
10517 }
10518 let mut out = Vec::new();
10519 for word in line.split_whitespace() {
10520 if word.contains('/') || word.contains('\\') {
10521 out.push("<path>".to_owned());
10522 continue;
10523 }
10524 let mut w = String::new();
10525 let mut in_digits = false;
10526 for c in word.chars() {
10527 if c.is_ascii_digit() {
10528 if !in_digits {
10529 w.push('#');
10530 }
10531 in_digits = true;
10532 } else {
10533 in_digits = false;
10534 w.push(c);
10535 }
10536 }
10537 out.push(w);
10538 }
10539 out.join(" ").chars().take(120).collect()
10540}
10541
10542fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10549 match cur {
10550 FailClass::Quota | FailClass::Timeout => true,
10551 FailClass::Other(_) => prev != Some(cur),
10552 }
10553}
10554
10555fn fail_reason(out: &AgentOutcome) -> String {
10557 match out {
10558 AgentOutcome::Ok(_) => String::new(),
10559 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10560 AgentOutcome::Dropped(o) => format!(
10561 "the CLI dropped the stream ({})",
10562 o.dropped
10563 .as_ref()
10564 .map(|d| d.why.as_str())
10565 .unwrap_or("it ended without delivering its answer")
10566 ),
10567 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10568 e.lines().next().unwrap_or("").chars().take(160).collect()
10569 }
10570 }
10571}
10572
10573pub(crate) fn record_handover(
10576 state: &mut RunState,
10577 node: &str,
10578 seat: &str,
10579 from: &str,
10580 to: &str,
10581 class: &FailClass,
10582 reason: &str,
10583) {
10584 let message = if *class == FailClass::Quota {
10585 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10586 } else {
10587 format!("{seat}: handed over {from} -> {to} ({reason})")
10588 };
10589 state.event(node, message);
10590 state.handovers.push(Handover {
10591 at: Timestamp::now(),
10592 node: node.to_owned(),
10593 seat: seat.to_owned(),
10594 from: from.to_owned(),
10595 to: to.to_owned(),
10596 reason: reason.to_owned(),
10597 });
10598}
10599
10600fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10614 commands.iter().any(|c| c.exit_code.is_none())
10615}
10616
10617fn verified_noop_claim(
10630 usable: bool,
10631 commands: &[agent::CommandEvidence],
10632 text: &str,
10633) -> Option<String> {
10634 (usable && !has_unconfirmed_command(commands))
10635 .then(|| verdict::verified_noop(text))
10636 .flatten()
10637}
10638
10639fn short(commit: &str) -> String {
10640 commit.chars().take(7).collect()
10641}
10642
10643fn make_executable(path: &Path) -> Result<()> {
10644 #[cfg(unix)]
10645 {
10646 use std::os::unix::fs::PermissionsExt as _;
10647 let mut perms = std::fs::metadata(path)?.permissions();
10648 perms.set_mode(0o755);
10649 std::fs::set_permissions(path, perms)?;
10650 }
10651 #[cfg(not(unix))]
10652 {
10653 let _ = path;
10654 }
10655 Ok(())
10656}
10657
10658struct WaveCtx<'a> {
10665 run: &'a str,
10668 node: &'a str,
10670 prompts: &'a Prompts,
10671 cache: Option<&'a Path>,
10673 round: Option<usize>,
10676 carry_seats: bool,
10680 pauses: Option<(&'a Pause, &'a Pause)>,
10683}
10684
10685fn repository_wait(
10687 state: &RunState,
10688 pauses: Option<(&Pause, &Pause)>,
10689) -> crate::repo_lock::WaitPolicy {
10690 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10691 let inherited = crate::repo_lock::WaitPolicy::current();
10692 crate::repo_lock::WaitPolicy::new(
10693 Duration::from_secs(state.config.graph.timeout_resource_wait),
10694 move || {
10695 inherited.cancelled()
10696 || pauses
10697 .as_ref()
10698 .is_some_and(|(a, b)| a.parked() || b.parked())
10699 },
10700 )
10701}
10702
10703fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10704 if error.is::<crate::repo_lock::ResourceBlocked>() {
10705 state.resource_block = Some(crate::run::ResourceBlock {
10706 cause: crate::run::ResourceBlockCause::Other,
10707 node: "repository".into(),
10708 kind: if error.to_string().contains("disk pressure") {
10709 "disk pressure"
10710 } else if error.to_string().contains("disk admission") {
10711 "disk admission"
10712 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10713 "incremental cache admission"
10714 } else {
10715 "repository coordination"
10716 }
10717 .into(),
10718 detail: error.to_string(),
10719 at: Timestamp::now(),
10720 });
10721 state.event("resource", error.to_string());
10722 }
10723}
10724
10725async fn coordinate_operation(
10726 state: &mut RunState,
10727 cwd: &Path,
10728 wait: &crate::repo_lock::WaitPolicy,
10729) -> Result<Option<crate::repo_lock::SeatLease>> {
10730 let result =
10731 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10732 .await;
10733 if let Err(e) = &result {
10734 record_repository_block(state, e);
10735 }
10736 result
10737}
10738
10739async fn run_one(
10741 job: SeatJob,
10742 sem: Arc<Semaphore>,
10743 ctx: &WaveCtx<'_>,
10744 state: &mut RunState,
10745 attempt: usize,
10746) -> (SeatState, AgentOutcome) {
10747 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10748 .await
10749 .pop()
10750 .expect("one job in, one result out");
10751 (seat, out)
10752}
10753
10754async fn wave(
10760 jobs: Vec<SeatJob>,
10761 sem: Arc<Semaphore>,
10762 ctx: &WaveCtx<'_>,
10763 state: &mut RunState,
10764 attempt: usize,
10765) -> Vec<(usize, SeatState, AgentOutcome)> {
10766 let WaveCtx {
10767 run,
10768 node,
10769 prompts,
10770 cache,
10771 round,
10772 carry_seats: _,
10773 pauses,
10774 } = *ctx;
10775 for job in &jobs {
10776 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10777 }
10778 if let Err(e) = state.save() {
10779 tracing::warn!("could not persist in-progress seats: {e:#}");
10784 }
10785 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10801 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10805 let cache_guard = if let Some(cache_dir) = cache {
10806 if jobs_had_a_writer {
10807 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10808 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10809 match acquire_cache_lease(
10810 state,
10811 cache_dir,
10812 &owner,
10813 resource_limit,
10814 node,
10815 node,
10816 &cancel,
10817 )
10818 .await
10819 {
10820 Ok(Some(g)) => Some(g),
10821 Ok(None) => {
10824 if let Some(block) = &mut state.resource_block {
10825 block.cause = crate::run::ResourceBlockCause::Cancelled {
10826 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10827 };
10828 }
10829 return block_wave(
10830 jobs,
10831 state,
10832 node,
10833 attempt,
10834 "parked while waiting for the shared build cache",
10835 );
10836 }
10837 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10840 Err(e) => {
10841 let why = format!("{e:#}");
10845 state.event(
10846 "resource",
10847 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10848 );
10849 return block_wave(jobs, state, node, attempt, &why);
10850 }
10851 }
10852 } else {
10853 None
10854 }
10855 } else {
10856 None
10857 };
10858 state.seats_spawning(node, attempt);
10861 if let Err(e) = state.save() {
10862 tracing::warn!("could not persist the start of a wave: {e:#}");
10863 }
10864 let slot_limit = resource_limit
10868 + jobs
10869 .iter()
10870 .map(|j| j.timeout)
10871 .max()
10872 .unwrap_or_default()
10873 .saturating_mul(jobs.len() as u32);
10874 let mut set = tokio::task::JoinSet::new();
10875 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10878 let overlay = prompts.overlay(node);
10879 let repository_wait = repository_wait(state, pauses);
10880 for (i, mut job) in jobs.into_iter().enumerate() {
10881 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10882 if cache.is_some() {
10883 job.prompt.push('\n');
10884 job.prompt
10885 .push_str(&prompt::build_cache_note(node, job.allow_write));
10886 }
10887 let sem = Arc::clone(&sem);
10888 let run = run.to_owned();
10889 let node = node.to_owned();
10890 let attachments = if node == "implement" {
10893 state.attachments.clone()
10894 } else {
10895 Vec::new()
10896 };
10897 let cache = cache
10908 .filter(|_| job.allow_write && cache_guard.is_some())
10909 .map(Path::to_path_buf);
10910 let spawned_tx = spawned_tx.clone();
10911 let repository_wait = repository_wait.clone();
10912 let coordination = state.config.coordination.clone();
10913 let fetched_base = state.fetched_base.clone();
10914 set.spawn(async move {
10915 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10919 Ok(p) => p,
10920 Err(_) => {
10921 return (
10922 i,
10923 job.seat,
10924 AgentOutcome::ResourceBlocked(format!(
10925 "no parallel slot became free within {}s",
10926 slot_limit.as_secs()
10927 )),
10928 false,
10929 );
10930 }
10931 };
10932 spawned_tx.send(job.seat.key.clone()).ok();
10933 let mut seat = job.seat;
10934 let out = repository_wait
10935 .scope(agent::invoke(
10936 &job.spec,
10937 &mut seat,
10938 &Invocation {
10939 coordination: coordination.as_ref(),
10940 fetched_base: fetched_base.as_ref(),
10941 cwd: &job.cwd,
10942 prompt: &job.prompt,
10943 timeout: job.timeout,
10944 allow_write: job.allow_write,
10945 sessions: job.sessions,
10946 artifacts: &job.artifacts,
10947 stem: &job.stem,
10948 run: &run,
10949 node: &node,
10950 cache_dir: cache.as_deref(),
10951 attachments: &attachments,
10952 writable: &[],
10953 },
10954 ))
10955 .await;
10956 let interrupted = out
10957 .as_ref()
10958 .err()
10959 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10960 let out = match out {
10961 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10962 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10963 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10971 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10972 Ok(o) => AgentOutcome::Failed(format!(
10973 "process exited with no usable output ({})",
10974 crate::notices::exit_status(o.exit_code)
10975 )),
10976 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10977 AgentOutcome::ResourceBlocked(e.to_string())
10978 }
10979 Err(e) => AgentOutcome::Failed(e.to_string()),
10980 };
10981 (i, seat, out, interrupted)
10982 });
10983 }
10984 drop(spawned_tx);
10985 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10986 loop {
10987 let joined = tokio::select! {
10988 Some(key) = spawned_rx.recv() => {
10989 state.seat_spawned(&key);
10991 if let Err(e) = state.save() {
10992 tracing::warn!("could not persist a seat's start: {e:#}");
10993 }
10994 continue;
10995 }
10996 joined = set.join_next() => joined,
10997 };
10998 let Some(joined) = joined else {
10999 break;
11000 };
11001 let (i, seat, out, interrupted) = match joined {
11002 Ok(v) => v,
11003 Err(e) => {
11007 tracing::error!("agent task panicked: {e}");
11008 continue;
11009 }
11010 };
11011 state.seat_finished(&seat.key);
11012 if interrupted
11013 && node == "implement"
11014 && let Some(label) = state
11015 .candidates
11016 .iter()
11017 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11018 .map(|candidate| candidate.label)
11019 {
11020 state.candidate_preservation.insert(
11021 label,
11022 "interrupted implementation awaiting persistence".into(),
11023 );
11024 state.interrupted_implementations.insert(label);
11025 }
11026 if let AgentOutcome::ResourceBlocked(why) = &out
11027 && state.resource_block.is_none()
11028 {
11029 state.resource_block = Some(crate::run::ResourceBlock {
11030 cause: crate::run::ResourceBlockCause::Other,
11031 node: node.to_owned(),
11032 kind: if why.contains("disk pressure") {
11033 "disk pressure"
11034 } else if why.contains("disk admission") {
11035 "disk admission"
11036 } else if why.contains(crate::incremental::BLOCKED) {
11037 "incremental cache admission"
11038 } else if why.contains("repository coordination") {
11039 "repository coordination"
11040 } else {
11041 "parallel slot"
11042 }
11043 .to_owned(),
11044 detail: why.clone(),
11045 at: Timestamp::now(),
11046 });
11047 let activity = if interrupted {
11048 "agent execution interrupted"
11049 } else {
11050 "no agent was started"
11051 };
11052 state.event("resource", format!("{node}: {activity} - {why}"));
11053 }
11054 record_jobs(state, node, round, &seat.key, &out);
11055 if let Err(e) = state.save() {
11056 tracing::warn!("could not persist a seat's completion: {e:#}");
11057 }
11058 if collected.len() <= i {
11059 collected.resize_with(i + 1, || None);
11060 }
11061 collected[i] = Some((i, seat, out));
11062 }
11063 if state
11069 .active
11070 .values()
11071 .any(|a| a.node == node && a.attempt == attempt)
11072 {
11073 state
11074 .active
11075 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11076 if let Err(e) = state.save() {
11077 tracing::warn!("could not persist the end of a wave: {e:#}");
11078 }
11079 }
11080 if let Some(cache_dir) = cache
11087 && jobs_had_a_writer
11088 {
11089 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11090 }
11091 if let Some(guard) = cache_guard {
11092 guard.release();
11093 }
11094 collected.into_iter().flatten().collect()
11095}
11096
11097fn record_jobs(
11108 state: &mut RunState,
11109 node: &str,
11110 round: Option<usize>,
11111 seat: &str,
11112 out: &AgentOutcome,
11113) {
11114 let commands: &[agent::CommandEvidence] = match out {
11115 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11116 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11117 };
11118 let checked_at = Timestamp::now();
11119 for c in commands {
11120 state.jobs.push(JobRecord {
11121 node: node.to_owned(),
11122 round,
11123 seat: seat.to_owned(),
11124 id: c.id.clone(),
11125 description: c.description.clone(),
11126 checked_at,
11127 status: match c.exit_code {
11128 Some(0) => JobStatus::Completed,
11129 Some(_) => JobStatus::Failed,
11130 None => JobStatus::Unknown,
11131 },
11132 exit_code: c.exit_code,
11133 result_summary: c.result_summary.clone(),
11134 source: c.source.clone(),
11135 });
11136 }
11137}
11138
11139fn round_is_clean(
11160 blocking: usize,
11161 e2e_ok: bool,
11162 answered: usize,
11163 expected: usize,
11164 quota_missing: usize,
11165 policy: IncompleteReviewPolicy,
11166) -> bool {
11167 if blocking != 0 || !e2e_ok {
11168 return false;
11169 }
11170 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11171 return true;
11172 }
11173 answered > 0 && expected - answered <= quota_missing
11174}
11175
11176pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11200 if max_rounds == 0
11201 || reviews.last().is_some_and(|r| {
11202 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11203 })
11204 {
11205 return Some(RunStatus::Gating);
11206 }
11207 let last = reviews.last()?;
11208 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11209 if reviews.len() < max_rounds && !stagnant {
11210 return None;
11211 }
11212 if last.incomplete() && last.blocking == 0 {
11213 return Some(RunStatus::Blocked);
11214 }
11215 if last.e2e_status() == E2eStatus::ResourceBlocked
11216 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11217 {
11218 return None;
11219 }
11220 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11221 RunStatus::Gating
11222 } else {
11223 RunStatus::Blocked
11224 })
11225}
11226
11227pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11242 if nudged {
11243 (full / 4).max(Duration::from_secs(120)).min(full)
11244 } else {
11245 full
11246 }
11247}
11248
11249#[allow(clippy::too_many_arguments)]
11278async fn ask_json_wave<T>(
11279 jobs: Vec<SeatJob>,
11280 sem: Arc<Semaphore>,
11281 retries: usize,
11282 roster: &[AgentSpec],
11283 ctx: &WaveCtx<'_>,
11284 losses: &mut Vec<QuotaLoss>,
11285 state: &mut RunState,
11286 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11287) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11288where
11289 T: serde::de::DeserializeOwned + Send + 'static,
11290{
11291 ask_wave_with(
11292 jobs,
11293 sem,
11294 retries,
11295 roster,
11296 ctx,
11297 losses,
11298 state,
11299 &|text: &str| {
11300 let v = verdict::extract_json::<T>(text)?;
11301 validate(&v)?;
11302 Ok(v)
11303 },
11304 )
11305 .await
11306}
11307
11308#[allow(clippy::too_many_arguments)]
11317async fn ask_wave_with<T>(
11318 jobs: Vec<SeatJob>,
11319 sem: Arc<Semaphore>,
11320 retries: usize,
11321 roster: &[AgentSpec],
11322 ctx: &WaveCtx<'_>,
11323 losses: &mut Vec<QuotaLoss>,
11324 state: &mut RunState,
11325 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11326) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11327where
11328 T: Send + 'static,
11329{
11330 let n = jobs.len();
11331 let originals: Vec<SeatJob> = jobs;
11332 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11333 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11334 let mut nudges: Vec<usize> = vec![0; n];
11341 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11345 let mut tried: Vec<BTreeSet<String>> = specs
11346 .iter()
11347 .map(|s| BTreeSet::from([s.id.clone()]))
11348 .collect();
11349 let starts: Vec<usize> = specs
11350 .iter()
11351 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11352 .collect();
11353 let carry = ctx.carry_seats && !roster.is_empty();
11354 let carried: Vec<BTreeSet<String>> = originals
11357 .iter()
11358 .map(|j| {
11359 state
11360 .seat_history
11361 .get(&j.seat.key)
11362 .filter(|_| carry)
11363 .map(|h| h.failed.clone())
11364 .unwrap_or_default()
11365 })
11366 .collect();
11367 let mut prev: Vec<Option<FailClass>> = originals
11368 .iter()
11369 .map(|j| {
11370 state
11371 .seat_history
11372 .get(&j.seat.key)
11373 .filter(|_| carry)
11374 .and_then(|h| h.last_fail.clone())
11375 })
11376 .collect();
11377 let next_agent =
11378 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11379 let others: BTreeSet<String> = specs
11380 .iter()
11381 .enumerate()
11382 .filter(|(j, _)| *j != i)
11383 .map(|(_, s)| s.id.clone())
11384 .collect();
11385 pick_successor(
11386 roster,
11387 starts[i],
11388 tried,
11389 carry.then(|| &carried[i]),
11390 &others,
11391 )
11392 .cloned()
11393 };
11394 let mut fresh: Vec<Option<String>> = vec![None; n];
11395 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11396 let mut pending: Vec<usize> = (0..n).collect();
11397
11398 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11403 for round in 0..max_rounds {
11404 if pending.is_empty() {
11405 break;
11406 }
11407 let mut batch = Vec::with_capacity(pending.len());
11408 let mut renudged: Vec<&str> = Vec::new();
11409 for &i in &pending {
11410 let src = &originals[i];
11411 let full: &str = match &src.handover {
11416 Some(h) if specs[i].id != src.spec.id => h,
11417 _ => &src.prompt,
11418 };
11419 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11422 (
11423 full.to_owned(),
11424 src.timeout,
11425 format!("{}-{word}-{}", src.stem, specs[i].id),
11426 )
11427 } else if nudges[i] == 0 {
11428 (full.to_owned(), src.timeout, src.stem.clone())
11429 } else {
11430 renudged.push(src.seat.key.as_str());
11431 let why = done[i]
11432 .as_ref()
11433 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11434 .unwrap_or_else(|| "no parsable answer".to_owned());
11435 let nudge = prompt::nudge(&why);
11436 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11437 let prompt = if nudged {
11438 nudge
11439 } else {
11440 format!("{full}\n\n---\n\n{nudge}")
11441 };
11442 (
11443 prompt,
11444 retry_budget(src.timeout, nudged),
11445 format!("{}-retry{}", src.stem, nudges[i]),
11446 )
11447 };
11448 batch.push(SeatJob {
11449 spec: specs[i].clone(),
11450 seat: seats[i].clone(),
11451 cwd: src.cwd.clone(),
11452 prompt,
11453 timeout,
11454 allow_write: src.allow_write,
11455 sessions: src.sessions,
11456 artifacts: src.artifacts.clone(),
11457 stem,
11458 handover: None,
11459 });
11460 }
11461
11462 if !renudged.is_empty() {
11463 state.event(
11464 ctx.node,
11465 format!("retry {round}: re-asking {}", renudged.join(", ")),
11466 );
11467 }
11468 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11469 let mut still = Vec::new();
11470 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11471 seats[i] = seat;
11472 if let AgentOutcome::ResourceBlocked(why) = &out {
11475 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11476 continue;
11477 }
11478 let class = FailClass::of(&out);
11479 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11482 && nudges[i] < retries
11483 && !roster.is_empty();
11484 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11485 let next = should_hand_over(prev[i].as_ref(), &cur)
11486 .then(|| next_agent(i, &tried[i], &specs))
11487 .flatten();
11488 if carry {
11489 let h = state
11490 .seat_history
11491 .entry(originals[i].seat.key.clone())
11492 .or_default();
11493 h.failed.insert(specs[i].id.clone());
11494 h.last_fail = Some(cur.clone());
11495 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11496 h.last_ok = None;
11497 }
11498 if let Err(e) = state.save() {
11501 tracing::warn!("could not persist a seat's failure history: {e:#}");
11502 }
11503 }
11504 if let Some(next) = next {
11505 record_handover(
11506 state,
11507 ctx.node,
11508 &originals[i].seat.key,
11509 &specs[i].id,
11510 &next.id,
11511 &cur,
11512 &fail_reason(&out),
11513 );
11514 tried[i].insert(next.id.clone());
11515 prev[i] = Some(cur.clone());
11516 seats[i] =
11517 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11518 specs[i] = next;
11519 fresh[i] = Some(cur.stem_word().to_owned());
11520 nudges[i] = 0;
11521 done[i] = Some(Err(anyhow::anyhow!(
11522 "handed over after: {}",
11523 fail_reason(&out)
11524 )));
11525 still.push(i);
11526 continue;
11527 }
11528 }
11529 if carry
11530 && !nudge_first
11531 && let Some(cur) = class.clone()
11532 {
11533 let h = state
11535 .seat_history
11536 .entry(originals[i].seat.key.clone())
11537 .or_default();
11538 h.failed.insert(specs[i].id.clone());
11539 h.last_fail = Some(cur);
11540 }
11541 let parsed = match out {
11542 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11543 AgentOutcome::Quota(o) => {
11544 last_quota[i] = o.quota.clone();
11545 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11546 }
11547 AgentOutcome::Dropped(o) => {
11551 let why = o
11552 .dropped
11553 .as_ref()
11554 .map(|d| d.why.as_str())
11555 .unwrap_or("the CLI ended the stream without delivering its answer");
11556 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11557 }
11558 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11559 Err(anyhow::anyhow!(e))
11560 }
11561 };
11562 let quota = class == Some(FailClass::Quota);
11563 let failed = parsed.is_err();
11564 if !failed && matches!(ctx.node, "judge" | "vote") {
11565 state
11568 .quota
11569 .retain(|loss| loss.seat != originals[i].seat.key);
11570 }
11571 done[i] = Some(parsed);
11572 if carry && !failed {
11573 let h = state
11574 .seat_history
11575 .entry(originals[i].seat.key.clone())
11576 .or_default();
11577 h.failed.remove(&specs[i].id);
11578 h.last_ok = Some(specs[i].id.clone());
11579 h.last_fail = None;
11580 }
11581 let agent_failure = class.is_some()
11589 && !nudge_first
11590 && !roster.is_empty()
11591 && next_agent(i, &tried[i], &specs).is_some();
11592 if failed && !quota && !agent_failure && nudges[i] < retries {
11593 nudges[i] += 1;
11594 still.push(i);
11595 }
11596 }
11597 pending = still;
11598 }
11599
11600 for (i, q) in last_quota.into_iter().enumerate() {
11603 if let Some(quota) = q {
11604 losses.push(QuotaLoss::from_report(
11605 originals[i].seat.key.clone(),
11606 ctx.node,
11607 Some("a),
11608 ));
11609 }
11610 }
11611
11612 if state.status != RunStatus::Stalled
11615 && ctx.node != "deliberate"
11616 && n > 0
11617 && losses.len() == n
11618 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11619 {
11620 block_machine_outage(state, ctx.node, losses);
11621 state.quota.append(losses);
11622 }
11623
11624 seats
11625 .into_iter()
11626 .zip(done)
11627 .zip(nudges)
11628 .map(|((seat, res), attempts)| {
11629 (
11630 seat,
11631 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11632 attempts,
11633 )
11634 })
11635 .collect()
11636}
11637
11638pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11641 let mut auth: Vec<_> = losses
11642 .iter()
11643 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11644 .map(|loss| loss.agent.clone())
11645 .collect();
11646 auth.sort();
11647 auth.dedup();
11648 let (kind, detail) = if auth.is_empty() {
11649 (
11650 "agent usage limit",
11651 "All seats reached their agent usage limits; resume after usage is available"
11652 .to_owned(),
11653 )
11654 } else {
11655 (
11656 "agent authentication",
11657 format!(
11658 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11659 auth.join(", ")
11660 ),
11661 )
11662 };
11663 state.resource_block = Some(crate::run::ResourceBlock {
11664 node: node.to_owned(),
11665 kind: kind.to_owned(),
11666 cause: crate::run::ResourceBlockCause::Other,
11667 detail: detail.clone(),
11668 at: Timestamp::now(),
11669 });
11670 state.event(node, detail);
11671}
11672
11673fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11675 results
11676 .iter()
11677 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11678}
11679
11680fn block_wave(
11684 jobs: Vec<SeatJob>,
11685 state: &mut RunState,
11686 node: &str,
11687 attempt: usize,
11688 why: &str,
11689) -> Vec<(usize, SeatState, AgentOutcome)> {
11690 state.resource_wait = None;
11691 state
11692 .active
11693 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11694 if let Err(e) = state.save() {
11695 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11696 }
11697 jobs.into_iter()
11698 .enumerate()
11699 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11700 .collect()
11701}
11702
11703async fn acquire_cache_lease(
11716 state: &mut RunState,
11717 cache_dir: &Path,
11718 owner: &crate::cache::Owner,
11719 budget: Duration,
11720 node: &str,
11721 context: &str,
11722 cancel: &(dyn Fn() -> bool + Sync),
11723) -> Result<Option<crate::cache::Guard>> {
11724 let home = crate::run::home();
11725 let started = Instant::now();
11726 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11727 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11728 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11729 Err(e) => {
11730 state.event(
11731 "verify",
11732 format!("{context}: could not check the shared build cache: {e:#}"),
11733 );
11734 if let Err(e2) = state.save() {
11735 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11736 }
11737 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11738 }
11739 };
11740 state.event(
11741 "verify",
11742 format!(
11743 "{context}: waiting for the shared build cache at {} ({})",
11744 cache_dir.display(),
11745 busy.describe()
11746 ),
11747 );
11748 state.resource_wait = Some(crate::run::ResourceWait {
11751 node: context.to_owned(),
11752 kind: "shared build cache".to_owned(),
11753 holder: busy.describe(),
11754 since: Timestamp::now(),
11755 limit_secs: budget.as_secs(),
11756 });
11757 if let Err(e) = state.save() {
11758 tracing::warn!("could not persist a cache wait: {e:#}");
11759 }
11760 let remaining = budget.saturating_sub(started.elapsed());
11761 let waited = crate::cache::wait_for_unless(
11762 &home,
11763 cache_dir,
11764 owner,
11765 remaining,
11766 Duration::from_secs(5),
11767 cancel,
11768 )
11769 .await;
11770 state.resource_wait = None;
11771 let cause = match &waited {
11772 Ok(Some(_)) => None,
11773 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11774 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11775 Some(crate::run::ResourceBlockCause::Timeout {
11776 waited_secs: started.elapsed().as_secs(),
11777 limit_secs: budget.as_secs(),
11778 })
11779 }
11780 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11781 };
11782 if let Some(cause) = cause {
11783 let detail = match &waited {
11784 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11785 Err(error) => format!("{error:#}"),
11786 };
11787 state.resource_block = Some(crate::run::ResourceBlock {
11788 cause,
11789 node: node.to_owned(),
11790 kind: "shared build cache".into(),
11791 detail,
11792 at: Timestamp::now(),
11793 });
11794 state.event(
11795 "resource",
11796 state.resource_block.as_ref().unwrap().description(),
11797 );
11798 }
11799 if let Err(error) = state.save() {
11800 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11801 }
11802 waited
11803}
11804
11805#[derive(Debug)]
11810struct CacheCheckError(String);
11811
11812impl std::fmt::Display for CacheCheckError {
11813 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11814 f.write_str(&self.0)
11815 }
11816}
11817
11818impl std::error::Error for CacheCheckError {}
11819
11820fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11822 state
11823 .resource_block
11824 .get_or_insert_with(|| crate::run::ResourceBlock {
11825 cause: crate::run::ResourceBlockCause::Other,
11826 node: node.to_owned(),
11827 kind: "shared build cache".into(),
11828 detail: format!("{error:#}"),
11829 at: Timestamp::now(),
11830 });
11831 if let Err(error) = state.save() {
11832 tracing::warn!("could not persist cache resource block: {error:#}");
11833 }
11834}
11835
11836#[allow(clippy::too_many_arguments)]
11857async fn with_cache_lease<'s, F, Fut>(
11858 state: &'s mut RunState,
11859 cache_dir: Option<&Path>,
11860 node: &str,
11861 seat: &str,
11862 worktree: &Path,
11863 head: &str,
11864 budget: Duration,
11865 context: &str,
11866 wait: &crate::repo_lock::WaitPolicy,
11867 body: F,
11868) -> Result<(Vec<CommandOutcome>, bool)>
11869where
11870 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11871 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11872{
11873 let started = Instant::now();
11876 let home = crate::run::home();
11877 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11878 let guard = if let Some(cache_dir) = cache_dir {
11879 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11880 match acquire_cache_lease(
11881 state,
11882 cache_dir,
11883 &owner,
11884 resource_limit,
11885 node,
11886 context,
11887 &|| wait.cancelled(),
11888 )
11889 .await
11890 .and_then(|g| {
11891 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11892 }) {
11893 Ok(g) => Some(g),
11894 Err(e) => {
11895 record_cache_error(state, node, &e);
11896 return Ok((
11897 vec![CommandOutcome {
11898 command: "(waiting for the shared build cache)".into(),
11899 code: None,
11900 output_tail: e.to_string(),
11901 duration_ms: started.elapsed().as_millis() as u64,
11902 resource_blocked: true,
11903 }],
11904 false,
11905 ));
11906 }
11907 }
11908 } else {
11909 None
11910 };
11911 let _coordination = coordinate_operation(state, worktree, wait).await?;
11912 if let Some(cache_dir) = cache_dir {
11913 let identity = crate::cache::Identity::new(worktree, head);
11914 if let Err(e) = crate::cache::ensure_fresh(
11915 &home,
11916 cache_dir,
11917 &identity,
11918 &state.config.verify.cargo_manifests(),
11919 ) {
11920 record_cache_error(state, node, &e);
11921 state.event(
11922 "verify",
11923 format!(
11924 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11925 worktree.display(),
11926 short(head)
11927 ),
11928 );
11929 if let Some(guard) = guard {
11930 guard.release();
11931 }
11932 return Ok((
11933 vec![CommandOutcome {
11934 command: "(confirming the shared build cache is fresh)".into(),
11935 code: None,
11936 output_tail: e.to_string(),
11937 duration_ms: started.elapsed().as_millis() as u64,
11938 resource_blocked: true,
11939 }],
11940 false,
11941 ));
11942 }
11943 }
11944 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11945 if !timed_out_pids.is_empty() {
11946 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11947 }
11948 drop(_coordination);
11951 if let Some(guard) = guard {
11952 guard.release();
11953 }
11954 Ok((outcomes, retried))
11955}
11956
11957async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11969 wait_for_pids_with(
11970 pids,
11971 crate::proc::pid_alive,
11972 LEASE_RELEASE_POLL,
11973 LEASE_RELEASE_MAX_WAIT,
11974 )
11975 .await;
11976}
11977
11978async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11984 pids: &[u32],
11985 alive: F,
11986 poll: Duration,
11987 max_wait: Duration,
11988) {
11989 let deadline = Instant::now() + max_wait;
11990 loop {
11991 if pids.iter().all(|&pid| !alive(pid)) {
11992 return;
11993 }
11994 if Instant::now() >= deadline {
11995 return;
11996 }
11997 tokio::time::sleep(poll).await;
11998 }
11999}
12000
12001fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12008 outcomes.iter().any(|o| o.resource_blocked)
12009}
12010
12011pub(crate) enum GateFix {
12013 Retry,
12015 Stop,
12018 Defer,
12021}
12022
12023pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12031 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12032 red.peek().is_some()
12033 && red.all(|o| {
12034 !o.resource_blocked
12035 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12036 && !o.output_tail.trim().is_empty()
12037 })
12038}
12039
12040fn e2e_outcome_label(o: &CommandOutcome) -> String {
12044 if o.ok() {
12045 return "pass".to_owned();
12046 }
12047 let reason = if o.build_failed() {
12048 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12049 } else {
12050 format!("FAIL ({:?})", o.code)
12051 };
12052 format!(
12053 "{reason}\n{}",
12054 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12055 )
12056}
12057
12058async fn run_e2e_with_retry(
12066 state: &mut RunState,
12067 shell: &[String],
12068 commands: &[String],
12069 worktree: &Path,
12070 timeout: Duration,
12071 context: &str,
12072) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12073 let (mut e2e, mut timed_out_pids) = run_commands(
12074 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12075 )
12076 .await;
12077 for o in &e2e {
12078 state.event(
12079 "verify",
12080 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12081 );
12082 }
12083 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12087 if verify_retried {
12088 state.event(
12089 "verify",
12090 format!(
12091 "{context}: verify could not build/link, not a test result — retrying once \
12092 before concluding"
12093 ),
12094 );
12095 let retried = run_commands(
12096 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12097 )
12098 .await;
12099 e2e = retried.0;
12100 timed_out_pids.extend(retried.1);
12103 for o in &e2e {
12104 state.event(
12105 "verify",
12106 format!(
12107 "{context}: retry `{}` -> {}",
12108 o.command,
12109 e2e_outcome_label(o)
12110 ),
12111 );
12112 }
12113 }
12114 (e2e, verify_retried, timed_out_pids)
12115}
12116
12117#[allow(clippy::too_many_arguments)]
12133async fn run_commands(
12134 state: &mut RunState,
12135 node: &str,
12136 task: &str,
12137 attempt: usize,
12138 shell: &[String],
12139 commands: &[String],
12140 cwd: &Path,
12141 timeout: Duration,
12142) -> (Vec<CommandOutcome>, Vec<u32>) {
12143 if commands.is_empty() {
12144 return (Vec::new(), Vec::new());
12149 }
12150 let mut out = Vec::new();
12151 let mut timed_out_pids = Vec::new();
12152 let total = commands.len();
12153 for (idx, command) in commands.iter().enumerate() {
12154 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12155 if let Err(e) = state.save() {
12156 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12157 }
12158 let started = Instant::now();
12159 let mut cmd = tokio::process::Command::new(&shell[0]);
12160 cmd.quiet();
12161 for key in crate::worker::INVOCATION_ENV {
12165 cmd.env_remove(key);
12166 }
12167 cmd.args(&shell[1..])
12168 .arg(command)
12169 .current_dir(cwd)
12170 .stdin(std::process::Stdio::null())
12171 .stdout(std::process::Stdio::piped())
12172 .stderr(std::process::Stdio::piped())
12173 .kill_on_drop(true);
12174 let mut outputs = Vec::new();
12175 let protected = (|| -> Result<()> {
12176 crate::incremental::configure(&mut cmd, cwd)?;
12177 let mut paths = vec![cwd.to_owned()];
12178 if let Some(cache) = state.config.cache_dir() {
12179 paths.push(cache);
12180 }
12181 outputs.extend(crate::retention::OutputLease::admit(
12182 &crate::run::home(),
12183 &paths,
12184 state.config.disk.min_free_bytes,
12185 )?);
12186 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12187 outputs.push(guard);
12188 }
12189 if let Some(cache) = state.config.cache_dir() {
12190 outputs.push(crate::retention::OutputLease::cache(
12191 &crate::run::home(),
12192 &cache,
12193 false,
12194 )?);
12195 }
12196 Ok(())
12197 })();
12198 if let Err(e) = protected {
12199 let detail = format!("execution admission unavailable: {e:#}");
12200 state.resource_block = Some(crate::run::ResourceBlock {
12201 cause: crate::run::ResourceBlockCause::Other,
12202 node: node.to_owned(),
12203 kind: if detail.contains("disk admission") {
12204 "disk admission"
12205 } else {
12206 "output ownership"
12207 }
12208 .to_owned(),
12209 detail: detail.clone(),
12210 at: Timestamp::now(),
12211 });
12212 out.push(CommandOutcome {
12213 command: command.clone(),
12214 code: None,
12215 output_tail: detail,
12216 duration_ms: 0,
12217 resource_blocked: true,
12218 });
12219 break;
12220 }
12221 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12222 let (code, body) = match spawned {
12223 Ok(mut running) => {
12224 let pid = running.child().id();
12225 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12226 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12227 let mut paths = vec![cwd.to_owned()];
12228 if let Some(cache) = state.config.cache_dir() {
12229 paths.push(cache);
12230 }
12231 let mut pressure = None;
12232 let status = tokio::select! {
12233 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12234 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12235 pressure = Some(reason); None
12236 }
12237 };
12238 let (code, failure) = match status {
12239 Some(Ok(Ok(status))) => (
12240 status.code(),
12241 status
12242 .code()
12243 .is_none()
12244 .then(|| format!("command terminated: {status}")),
12245 ),
12246 Some(Ok(Err(error))) => {
12247 (None, Some(format!("failed waiting for command: {error}")))
12248 }
12249 Some(Err(_)) => {
12250 if !running.coordinated()
12254 && let Some(pid) = pid
12255 {
12256 timed_out_pids.push(pid);
12257 }
12258 (
12259 None,
12260 Some(format!("timed out after {}s", timeout.as_secs())),
12261 )
12262 }
12263 None => (None, None),
12264 };
12265 running.finish().await;
12266 if let Some(reason) = pressure {
12267 state.resource_block = Some(crate::run::ResourceBlock {
12268 cause: crate::run::ResourceBlockCause::Other,
12269 node: node.to_owned(),
12270 kind: "disk pressure".to_owned(),
12271 detail: reason.clone(),
12272 at: Timestamp::now(),
12273 });
12274 out.push(CommandOutcome {
12275 command: command.clone(),
12276 code: None,
12277 output_tail: reason,
12278 duration_ms: started.elapsed().as_millis() as u64,
12279 resource_blocked: true,
12280 });
12281 break;
12282 }
12283 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12284 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12285 if let Some(failure) = failure {
12286 body.push('\n');
12287 body.push_str(&failure);
12288 }
12289 (code, body)
12290 }
12291 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12292 };
12293 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12294 if !body.is_empty() {
12295 match crate::run::write_verification_artifact(state, task, &body) {
12299 Ok(name) if summary != body => {
12300 summary = format!("[full output: artifacts/{name}]\n{summary}");
12301 }
12302 Ok(_) => {}
12303 Err(e) => {
12304 let warning = format!("could not save the full {task} output: {e:#}");
12305 tracing::warn!("{warning}");
12306 state.event(node, warning);
12307 if summary != body {
12308 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12309 }
12310 }
12311 }
12312 }
12313 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12314 out.push(CommandOutcome {
12315 command: command.clone(),
12316 code,
12317 output_tail: summary,
12318 duration_ms: started.elapsed().as_millis() as u64,
12319 resource_blocked,
12320 });
12321 if resource_blocked {
12322 state.resource_block = Some(crate::run::ResourceBlock {
12323 cause: crate::run::ResourceBlockCause::Other,
12324 node: node.to_owned(),
12325 kind: "incremental cache admission".into(),
12326 detail: body,
12327 at: Timestamp::now(),
12328 });
12329 break;
12330 }
12331 }
12332 state.task_finished(task);
12333 if let Err(e) = state.save() {
12334 tracing::warn!("could not persist the end of {task}: {e:#}");
12335 }
12336 (out, timed_out_pids)
12337}
12338
12339fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12351 let repo = repo.display();
12352 match style {
12353 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12354 MergeStyle::Squash => {
12355 let subject = message
12358 .lines()
12359 .next()
12360 .unwrap_or(branch)
12361 .replace(['\\', '"', '$', '`'], "");
12362 format!(
12363 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12364 )
12365 }
12366 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12367 }
12368}
12369
12370const PR_TITLE_MAX: usize = 240;
12381
12382fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12393 let line = queue::first_line(text)?;
12394 let chars: Vec<char> = line.chars().collect();
12395 let (kept, mark) = if chars.len() <= max {
12396 (line.to_owned(), "")
12397 } else if let Some(end) = sentence_end(&chars, max) {
12398 (chars[..end].iter().collect::<String>(), "")
12399 } else {
12400 let room = max.saturating_sub(3);
12401 let cut = if chars[room].is_whitespace() {
12404 room
12405 } else {
12406 chars[..room]
12407 .iter()
12408 .rposition(|c| c.is_whitespace())
12409 .unwrap_or(room)
12410 };
12411 let head: String = chars[..cut].iter().collect();
12412 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12413 (head.to_owned(), "...")
12414 };
12415 if kept.is_empty() {
12416 return None;
12417 }
12418 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12419 return None;
12420 }
12421 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12422 return None;
12423 }
12424 Some(format!("{kept}{mark}"))
12425}
12426
12427fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12431 const MIN: usize = 20;
12432 for i in MIN..max.min(chars.len()) {
12433 if !matches!(chars[i], '.' | '!' | '?') {
12434 continue;
12435 }
12436 let Some(&next) = chars.get(i + 1) else {
12437 continue;
12438 };
12439 if !next.is_whitespace() {
12440 continue;
12441 }
12442 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12443 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12444 continue;
12445 }
12446 let word: String = chars[..i]
12447 .iter()
12448 .rev()
12449 .take_while(|c| !c.is_whitespace())
12450 .collect::<Vec<_>>()
12451 .into_iter()
12452 .rev()
12453 .collect();
12454 let word = word.to_ascii_lowercase();
12455 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12456 continue;
12457 }
12458 let end = chars[..i]
12459 .iter()
12460 .rposition(|c| !c.is_whitespace())
12461 .map_or(i, |p| p + 1);
12462 return Some(end);
12463 }
12464 None
12465}
12466
12467struct PrMessage {
12472 title: String,
12473 body: String,
12474}
12475
12476impl PrMessage {
12477 fn commit_message(&self) -> String {
12481 format!("{}\n\n{}", self.title, self.body)
12482 }
12483}
12484
12485fn title_marker(line: &str) -> Option<&str> {
12487 let line = line.trim();
12488 let head = line.get(..6)?;
12489 head.eq_ignore_ascii_case("title:")
12490 .then(|| line[6..].trim())
12491}
12492
12493fn summary_title(summary: &str) -> Option<String> {
12498 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12499 let raw = title_marker(first)?;
12500 if raw.is_empty() {
12501 return None;
12502 }
12503 let title = queue::title_from(raw, PR_TITLE_MAX);
12504 let lower = title.to_ascii_lowercase();
12505 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12506 return None;
12507 }
12508 Some(title)
12509}
12510
12511const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12513
12514pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12516
12517pub(crate) fn is_review_run(state: &RunState) -> bool {
12521 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12522}
12523
12524fn review_title(state: &RunState) -> Option<String> {
12529 english_subject(
12530 state.reviewed_commits.as_ref()?.first()?,
12531 state.config.graph.github_english(),
12532 )
12533}
12534
12535fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12538 let raw = raw.trim();
12539 if raw.is_empty() {
12540 return None;
12541 }
12542 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12543 return None;
12544 }
12545 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12546 return None;
12547 }
12548 let title = queue::title_from(raw, PR_TITLE_MAX);
12549 let lower = title.to_ascii_lowercase();
12550 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12551 return None;
12552 }
12553 Some(title)
12554}
12555
12556#[derive(Debug, Clone, PartialEq, Eq)]
12559struct BranchFacts {
12560 commits: Vec<(String, String)>,
12562 stat: String,
12564}
12565
12566const STAT_FILE_LINES: usize = 25;
12568const COMMITS_MAX_CHARS: usize = 20_000;
12570
12571async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12574 let commits = git::commit_log(repo, base, branch).await.ok()?;
12575 if commits.is_empty() {
12576 return None;
12577 }
12578 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12579 let lines: Vec<&str> = stat.lines().collect();
12580 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12581 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12582 let more = format!(" ... {omitted} more file(s)");
12583 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12584 kept.push(&more);
12585 kept.push(lines[lines.len() - 1]);
12586 kept.join("\n")
12587 } else {
12588 lines.join("\n")
12589 };
12590 Some(BranchFacts { commits, stat })
12591}
12592
12593fn markdown_safe(text: &str) -> String {
12596 text.replace("</details>", "</details>")
12597 .replace("\x60\x60\x60", "~~~")
12598}
12599
12600fn neutral_title(state: &RunState, winner: char) -> String {
12601 format!(
12602 "chore: land candidate {} of run {}",
12603 winner.to_ascii_uppercase(),
12604 state.id
12605 )
12606}
12607
12608pub(crate) async fn repair_landing_gate(
12611 state: &mut RunState,
12612 winner: &Candidate,
12613 outcomes: &[CommandOutcome],
12614) -> Result<GateFix> {
12615 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12616 return Ok(GateFix::Stop);
12617 }
12618 let mut runner = Runner {
12619 roles: state.config.resolve_roles()?,
12620 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12621 state: state.clone(),
12622 pause: Pause::new(),
12623 interrupt: Pause::new(),
12624 };
12625 let result = runner.gate_fix_round(winner, outcomes).await;
12626 *state = runner.state;
12627 result
12628}
12629
12630pub(crate) async fn verify_current_head(
12633 state: &mut RunState,
12634 winner: &Candidate,
12635 wait: &crate::repo_lock::WaitPolicy,
12636) -> Result<bool> {
12637 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12638 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12640 state
12641 .verification_history
12642 .push(crate::run::VerificationEvidence {
12643 superseded_by: Some(head.clone()),
12644 archived_at: Some(Timestamp::now()),
12645 fixes_spent: Some(state.gate_fixes.len()),
12646 gate_head: None,
12647 gate_ran: false,
12648 gate: Vec::new(),
12649 e2e: state.head_e2e.take(),
12650 });
12651 }
12652 if let Some(e) = &state.head_e2e
12653 && e.head == head
12654 {
12655 if e.passed {
12656 return Ok(true);
12657 }
12658 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12659 state.status = RunStatus::Blocked;
12660 state.save()?;
12661 return Ok(false);
12662 }
12663 } else if state.reviews.last().is_some_and(|r| {
12664 r.verified_head.as_deref() == Some(head.as_str())
12665 && (r.e2e_status() == E2eStatus::Passed
12666 || (r.e2e_status() == E2eStatus::NotConfigured
12667 && state.config.verify.e2e.is_empty()))
12668 }) || state
12669 .gate_repair_e2e
12670 .as_ref()
12671 .is_some_and(|e| e.head == head && e.passed)
12672 {
12673 return Ok(true);
12674 }
12675 if let Some(e) = &state.gate_repair_e2e
12676 && e.head == head
12677 && !e.passed
12678 && !e.outcomes.is_empty()
12679 && !verify_inconclusive(&e.outcomes)
12680 {
12681 state.status = RunStatus::Blocked;
12682 state.save()?;
12683 return Ok(false);
12684 }
12685 state.head_e2e = Some(crate::run::GateRepairE2e {
12686 head: head.clone(),
12687 passed: false,
12688 outcomes: Vec::new(),
12689 });
12690 state.save()?;
12691 let commands = state.config.verify.e2e.clone();
12692 let shell = state.config.shell();
12693 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12694 let cache_dir = state.config.cache_dir();
12695 let outcomes = if commands.is_empty() {
12696 Vec::new()
12697 } else {
12698 let (outcomes, _) = with_cache_lease(
12699 state,
12700 cache_dir.as_deref(),
12701 "e2e",
12702 "e2e",
12703 &winner.worktree,
12704 &head,
12705 timeout,
12706 "current head e2e",
12707 wait,
12708 |state, budget| async move {
12709 run_e2e_with_retry(
12710 state,
12711 &shell,
12712 &commands,
12713 &winner.worktree,
12714 budget,
12715 "current head e2e",
12716 )
12717 .await
12718 },
12719 )
12720 .await?;
12721 outcomes
12722 };
12723 let unresolved = verify_inconclusive(&outcomes);
12724 let passed = !unresolved
12725 && outcomes.iter().all(CommandOutcome::ok)
12726 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12727 if !outcomes.is_empty() && !unresolved {
12728 state.record_base_progress(&head);
12729 }
12730 state.head_e2e = Some(crate::run::GateRepairE2e {
12731 head: head.clone(),
12732 passed,
12733 outcomes,
12734 });
12735 if !passed && !unresolved {
12736 state.status = RunStatus::Blocked;
12737 state.event(
12738 "e2e",
12739 format!("verify.e2e failed on head {head}; nothing published"),
12740 );
12741 }
12742 state.save()?;
12743 Ok(passed)
12744}
12745
12746pub(crate) async fn check_gate_repair_e2e(
12749 state: &mut RunState,
12750 winner: &Candidate,
12751 wait: &crate::repo_lock::WaitPolicy,
12752) -> Result<GateFix> {
12753 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12754 return Ok(GateFix::Retry);
12755 }
12756 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12757 if state
12758 .gate_repair_e2e
12759 .as_ref()
12760 .is_some_and(|r| r.head == head && r.passed)
12761 {
12762 return Ok(GateFix::Retry);
12763 }
12764 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12765 head: head.clone(),
12766 passed: false,
12767 outcomes: Vec::new(),
12768 });
12769 state.save()?;
12770 let commands = state.config.verify.e2e.clone();
12771 let outcomes = if commands.is_empty() {
12772 Vec::new()
12773 } else {
12774 let shell = state.config.shell();
12775 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12776 let cache_dir = state.config.cache_dir();
12777 let (outcomes, _) = with_cache_lease(
12778 state,
12779 cache_dir.as_deref(),
12780 "e2e",
12781 "e2e",
12782 &winner.worktree,
12783 &head,
12784 timeout,
12785 "gate repair e2e",
12786 wait,
12787 |state, budget| async move {
12788 run_e2e_with_retry(
12789 state,
12790 &shell,
12791 &commands,
12792 &winner.worktree,
12793 budget,
12794 "gate repair e2e",
12795 )
12796 .await
12797 },
12798 )
12799 .await?;
12800 outcomes
12801 };
12802 let deferred = verify_inconclusive(&outcomes);
12803 let passed = outcomes.iter().all(CommandOutcome::ok)
12804 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12805 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12806 head,
12807 passed,
12808 outcomes,
12809 });
12810 state.save()?;
12811 if deferred {
12812 Ok(GateFix::Defer)
12813 } else if !passed {
12814 state.event(
12815 "gate",
12816 "verify.e2e failed after gate repair; nothing published",
12817 );
12818 Ok(GateFix::Stop)
12819 } else {
12820 Ok(GateFix::Retry)
12821 }
12822}
12823
12824pub(crate) async fn run_landing_gate(
12826 state: &mut RunState,
12827 winner: &Candidate,
12828 wait: &crate::repo_lock::WaitPolicy,
12829) -> Result<Vec<CommandOutcome>> {
12830 let shell = state.config.shell();
12831 let gate_commands = state.config.verify.gate.clone();
12832 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12833 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12834 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12835 Vec::new()
12836 } else {
12837 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12838 let cache_dir = state.config.cache_dir();
12839 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12840 let (outcomes, _) = with_cache_lease(
12841 state,
12842 cache_dir.as_deref(),
12843 "gate",
12844 "gate",
12845 &winner.worktree,
12846 &head,
12847 timeout,
12848 "final gate",
12849 wait,
12850 |state, budget| {
12851 let shell = shell.clone();
12852 let gate_commands = gate_commands.clone();
12853 let winner = winner.clone();
12854 async move {
12855 Runner::run_pre_gate(state, &winner).await;
12856 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12858 let (outcomes, timed_out_pids) = run_commands(
12859 state,
12860 "gate",
12861 "gate",
12862 0,
12863 &shell,
12864 &gate_commands,
12865 &winner.worktree,
12866 budget,
12867 )
12868 .await;
12869 (outcomes, false, timed_out_pids)
12870 }
12871 },
12872 )
12873 .await?;
12874 outcomes
12875 };
12876 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12877 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12878 let mut outcomes = outcomes;
12879 outcomes.push(CommandOutcome {
12880 command: "gate head stability".into(), code: None,
12881 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12882 duration_ms: 0, resource_blocked: false,
12883 });
12884 return Ok(outcomes);
12885 }
12886 if outcomes.is_empty() {
12887 state.event("gate", "no gate commands configured; not tested");
12892 }
12893 for o in &outcomes {
12894 state.event(
12895 "gate",
12896 format!(
12897 "`{}` -> {}",
12898 o.command,
12899 if o.ok() {
12900 "pass".to_owned()
12901 } else {
12902 format!(
12903 "FAIL ({:?})\n{}",
12904 o.code,
12905 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12906 )
12907 }
12908 ),
12909 );
12910 }
12911 Ok(outcomes)
12912}
12913
12914pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12920 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12921 ""
12922 } else {
12923 pr_title
12924 }
12925}
12926
12927pub fn landing_subject_source(state: &RunState) -> String {
12931 if is_review_run(state) {
12932 let winner = state.candidates.first().map_or('A', |c| c.label);
12933 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12934 }
12935 state.instruction.clone()
12936}
12937
12938fn summary_without_title(summary: &str) -> String {
12941 let mut lines = summary.trim().lines().peekable();
12942 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12943 lines.next();
12944 }
12945 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12946}
12947
12948#[cfg(test)]
12960fn pr_message(state: &RunState, winner: char) -> PrMessage {
12961 pr_message_with(state, winner, None)
12962}
12963
12964#[cfg(test)]
12967fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12968 let raw = pr_message_raw(
12969 state,
12970 winner,
12971 facts,
12972 crate::pr_outcome::FinalHead::Unreported,
12973 ChangeNote::NotAsked,
12974 );
12975 let id = crate::scrub::Identity::current();
12976 PrMessage {
12977 title: crate::scrub::scrub(&raw.title, &id),
12978 body: crate::scrub::scrub(&raw.body, &id),
12979 }
12980}
12981
12982#[derive(Debug, Clone, Copy)]
12984enum ChangeNote<'a> {
12985 NotAsked,
12987 Summary(&'a crate::run::ChangeSummary),
12989 Unavailable(&'a str),
12991}
12992
12993fn pr_message_raw(
12994 state: &RunState,
12995 winner: char,
12996 facts: Option<&BranchFacts>,
12997 head: crate::pr_outcome::FinalHead<'_>,
12998 change: ChangeNote<'_>,
12999) -> PrMessage {
13000 if !matches!(change, ChangeNote::NotAsked) {
13001 let (title, mut body) = match change {
13002 ChangeNote::Summary(c) => (
13003 c.title.clone(),
13004 format!("## Summary\n\n{}\n\n", c.body.trim()),
13005 ),
13006 ChangeNote::Unavailable(why) => (
13007 neutral_title(state, winner),
13008 format!(
13009 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13010 ),
13011 ),
13012 ChangeNote::NotAsked => unreachable!(),
13013 };
13014 body.push_str(&public_pr_evidence(state, head));
13015 body.push_str(&format!(
13016 "\n---\nmagi:run/{} magi:candidate-{}\n",
13017 state.id,
13018 winner.to_ascii_lowercase()
13019 ));
13020 return PrMessage { title, body };
13021 }
13022 pr_preview(state, winner, facts, head)
13023}
13024
13025fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13028 let mut body = crate::pr_outcome::review_rounds(state);
13029 body.push_str(&crate::pr_outcome::final_verification(state, head));
13030 let open = state.last_round_findings();
13031 if !open.is_empty() {
13032 body.push_str("## Open review findings\n\n");
13033 for f in open {
13034 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13035 }
13036 body.push('\n');
13037 }
13038 if let Some(c) = state.change_summary.as_ref()
13039 && head == crate::pr_outcome::FinalHead::At(&c.head)
13040 {
13041 body.push_str(&format!(
13042 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13043 c.base, c.head
13044 ));
13045 }
13046 body
13047}
13048
13049fn pr_preview(
13051 state: &RunState,
13052 winner: char,
13053 facts: Option<&BranchFacts>,
13054 head: crate::pr_outcome::FinalHead<'_>,
13055) -> PrMessage {
13056 let summary = state
13057 .candidates
13058 .iter()
13059 .find(|c| c.label == winner)
13060 .map(|c| c.summary.as_str())
13061 .unwrap_or_default();
13062 let review = is_review_run(state);
13065 let title = if review {
13066 facts
13067 .and_then(|f| {
13068 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13069 })
13070 .or_else(|| review_title(state))
13071 .or_else(|| {
13072 state
13073 .candidates
13074 .iter()
13075 .find(|c| c.label == winner)
13076 .filter(|c| !c.branch.starts_with("magi/"))
13077 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13078 })
13079 .unwrap_or_else(|| neutral_title(state, winner))
13080 } else {
13081 summary_title(summary).unwrap_or_else(|| {
13082 english_title(
13083 &state.instruction,
13084 PR_TITLE_MAX,
13085 state.config.graph.github_english(),
13086 )
13087 .unwrap_or_else(|| neutral_title(state, winner))
13088 })
13089 };
13090
13091 let mut body = String::new();
13092 let what = summary_without_title(summary);
13093 if !what.is_empty() {
13094 body.push_str("## Summary\n\n");
13095 body.push_str(&what);
13096 body.push_str("\n\n");
13097 }
13098
13099 body.push_str(&crate::pr_outcome::review_rounds(state));
13101 body.push_str(&crate::pr_outcome::final_verification(state, head));
13102 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13103
13104 let open = state.open_findings();
13105 if !open.is_empty() {
13106 body.push_str("## Open review findings\n\n");
13107 for f in &open {
13108 body.push_str(&format!(
13109 "- `{}` [{:?}] {}\n",
13110 f.id, f.severity, "Context retained in local review records."
13111 ));
13112 }
13113 body.push('\n');
13114 }
13115
13116 if let Some(fix) = fix
13117 && !fix.rejected.is_empty()
13118 {
13119 body.push_str("## Declined by the fixer\n\n");
13120 for r in &fix.rejected {
13121 body.push_str(&format!(
13122 "- `{}`: {}\n",
13123 r.id, "Reason retained in local review records."
13124 ));
13125 }
13126 body.push('\n');
13127 }
13128
13129 if review {
13130 body.push_str("## Commits under review\n\n");
13132 if let Some(facts) = facts {
13133 let mut left = COMMITS_MAX_CHARS;
13134 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13135 let mut entry = format!("- {}\n", markdown_safe(subject));
13136 for l in markdown_safe(text).lines() {
13137 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13138 }
13139 if left == 0 {
13140 body.push_str(&format!(
13141 "- ... {} more commit(s)\n",
13142 facts.commits.len() - i
13143 ));
13144 break;
13145 }
13146 if entry.len() > left {
13147 let mut end = left;
13150 while !entry.is_char_boundary(end) {
13151 end -= 1;
13152 }
13153 entry.truncate(end);
13154 entry.push_str("\n ... (truncated)\n");
13155 left = 0;
13156 } else {
13157 left -= entry.len();
13158 }
13159 body.push_str(&entry);
13160 }
13161 if !facts.stat.trim().is_empty() {
13162 body.push_str(&format!(
13163 "\n## Diff stat\n\n```\n{}\n```\n",
13164 markdown_safe(facts.stat.trim())
13165 ));
13166 }
13167 } else {
13168 match &state.reviewed_commits {
13169 Some(subjects) => {
13170 for s in subjects {
13171 body.push_str(&format!("- {}\n", s.trim()));
13172 }
13173 }
13174 None => {
13175 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13178 body.push_str(rest.trim());
13179 body.push('\n');
13180 }
13181 }
13182 }
13183 } else {
13184 let task = state.instruction.trim();
13185 let task = if task.is_empty() {
13186 "(empty task)"
13187 } else {
13188 task
13189 };
13190 body.push_str(&format!(
13191 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13192 task.replace("</details>", "</details>")
13193 ));
13194 }
13195
13196 body.push_str(&format!(
13197 "\n---\nmagi:run/{} magi:candidate-{}\n",
13198 state.id,
13199 winner.to_ascii_lowercase()
13200 ));
13201
13202 PrMessage { title, body }
13203}
13204
13205fn seeded_instruction(state: &RunState) -> String {
13208 let mut inputs = Vec::new();
13209 if let Some(start) = &state.sources.start_from {
13210 inputs.push(format!(
13211 "Start from `{}` at `{}` (including ancestry).",
13212 start.token, start.sha
13213 ));
13214 }
13215 for pick in &state.sources.pick_commits {
13216 inputs.push(format!(
13217 "Pick `{}` at `{}` (single commit).",
13218 pick.token, pick.sha
13219 ));
13220 }
13221 if inputs.is_empty() {
13222 state.instruction.clone()
13223 } else {
13224 format!(
13225 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13226 state.instruction,
13227 inputs.join("\n")
13228 )
13229 }
13230}
13231
13232async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13236 let base = &state.base_branch;
13237 let mut against = base.clone();
13238 if mode == MergeMode::Pr {
13239 let remote = &state.config.merge.remote;
13243 let tracking = format!("{remote}/{base}");
13244 let fetched = git::fetch(repo, remote, base).await;
13245 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13246 against = tracking;
13247 } else {
13248 return false;
13249 }
13250 }
13251 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13252}
13253
13254fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13257 let mut detail = format!(
13258 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13259 nothing to open a pull request for"
13260 );
13261 detail.push_str(
13262 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13263 );
13264 detail
13265}
13266
13267#[derive(Debug, PartialEq, Eq)]
13270enum PrPlan {
13271 Create,
13272 Adopt { url: String, title: String },
13273 Stop(String),
13274}
13275
13276fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13279 match found {
13280 Ok(land::OpenPr::None) => PrPlan::Create,
13281 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13282 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13283 "several open pull requests exist for this branch, not picking one: {}",
13284 urls.join(" ")
13285 )),
13286 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13287 }
13288}
13289
13290pub(crate) async fn gh_pr_create(
13292 cwd: &Path,
13293 base: &str,
13294 head: &str,
13295 title: &str,
13296 body: &str,
13297) -> Result<String> {
13298 let out = tokio::process::Command::new("gh")
13299 .args([
13300 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13301 ])
13302 .current_dir(cwd)
13303 .quiet()
13304 .stdin(std::process::Stdio::null())
13305 .output()
13306 .await
13307 .context("spawn gh")?;
13308 if out.status.success() {
13309 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13310 } else {
13311 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13312 }
13313}
13314
13315async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13317 match std::fs::symlink_metadata(path) {
13318 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13319 git::worktree_remove_clean(repo, path).await?;
13322 return Ok(());
13323 }
13324 Err(e) => return Err(e.into()),
13325 Ok(_) => {}
13326 }
13327 if !git::is_clean(path).await? {
13328 bail!(
13329 "worktree has uncommitted work; refusing removal: {}",
13330 path.display()
13331 );
13332 }
13333 if !git::worktree_remove_clean(repo, path).await? {
13334 bail!("worktree removal refused: {}", path.display());
13335 }
13336 Ok(())
13337}
13338
13339async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13342 if !crate::retention::read_seat(path) || !path.exists() {
13343 return remove_preserved_worktree(&state.repo, path).await;
13344 }
13345 let owned = std::fs::symlink_metadata(path)?.is_dir()
13346 && path.parent().map(Path::canonicalize).transpose()?
13347 == Some(state.worktree_root().canonicalize()?)
13348 && crate::retention::worktrees(state)
13349 .await?
13350 .contains(&path.canonicalize()?);
13351 let common = crate::repo_lock::common_dir(&state.repo).await?;
13352 if !owned
13353 || common.is_none()
13354 || crate::repo_lock::common_dir(path).await? != common
13355 || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13356 .await?
13357 .ok()
13358 || state.candidates.iter().any(|c| c.worktree == path)
13359 {
13360 return remove_preserved_worktree(&state.repo, path).await;
13361 }
13362 let changes = git::status_porcelain(path).await?;
13363 if !git::worktree_remove(&state.repo, path).await? {
13364 bail!(
13365 "read-only seat worktree removal refused: {}",
13366 path.display()
13367 );
13368 }
13369 if !changes.is_empty() {
13370 let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13371 let paths: String = paths.chars().take(1024).collect();
13372 state.event(
13373 "fold",
13374 format!(
13375 "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13376 path.file_name().unwrap_or_default().to_string_lossy(),
13377 changes.lines().count()
13378 ),
13379 );
13380 }
13381 Ok(())
13382}
13383
13384pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13393 fold_run_with_policy(state, drop_winner, home, false).await
13394}
13395
13396pub async fn fold_run_overriding_recovery_retention(
13398 state: &mut RunState,
13399 drop_winner: bool,
13400 home: &Path,
13401) -> Result<Vec<String>> {
13402 fold_run_with_policy(state, drop_winner, home, true).await
13403}
13404
13405async fn fold_run_with_policy(
13406 state: &mut RunState,
13407 drop_winner: bool,
13408 home: &Path,
13409 override_recovery_retention: bool,
13410) -> Result<Vec<String>> {
13411 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state)
13412 && !override_recovery_retention
13413 {
13414 bail!("refusing fold: {reason}");
13415 }
13416 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13417 bail!("candidate preservation incomplete; refusing cleanup");
13418 }
13419 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13423 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13424 if override_recovery_retention {
13426 *state = RunState::load_under(&state.id, home)?;
13427 anyhow::ensure!(
13428 state.status.done() && !state.status.resumable(),
13429 "recovery retention override requires a finished, non-resumable run"
13430 );
13431 let claimed = crate::daemon::is_working_on_other(home, &state.id, Timestamp::now())?;
13432 anyhow::ensure!(
13433 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
13434 "run is live or its ownership is unknown"
13435 );
13436 anyhow::ensure!(
13437 state.candidate_preservation.is_empty() && state.interrupted_implementations.is_empty(),
13438 "candidate preservation incomplete; refusing cleanup"
13439 );
13440 }
13441 let retention_reason = crate::pr_recovery::source_retention_reason(home, state);
13442 if let Some(reason) = &retention_reason
13443 && !override_recovery_retention
13444 {
13445 bail!("refusing fold: {reason}");
13446 }
13447 let _driver = crate::driver::maintenance(home, &state.id)?;
13448 let mut output_guards = Vec::new();
13449 for entry in std::fs::read_dir(state.worktree_root())
13450 .into_iter()
13451 .flatten()
13452 .flatten()
13453 {
13454 if entry.path().join(".git").is_file() {
13455 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13456 }
13457 }
13458 let repo = state.repo.clone();
13459 let wait = repository_wait(state, None);
13462 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13463 Ok(lease) => lease,
13464 Err(e) => {
13465 state.save_under(home)?;
13466 return Err(e);
13467 }
13468 };
13469 let _source_outputs = if retention_reason.is_some()
13472 || (state.status == RunStatus::Merged && state.merge.as_ref().is_some_and(|merge| merge.ok))
13473 {
13474 verify_merged_fold_sources(state, drop_winner, home).await?
13475 } else {
13476 Vec::new()
13477 };
13478 if let Some(reason) = retention_reason {
13479 state.event(
13480 "fold",
13481 format!(
13482 "operator CLI --override-recovery-retention authorized deletion for run {}: bypassed {reason}",
13483 state.id
13484 ),
13485 );
13486 state.save_under(home)?;
13487 }
13488 let root = state.worktree_root();
13489 let winner = state.tally.as_ref().map(|t| t.winner);
13490 let mut removed = Vec::new();
13491
13492 for i in 0..state.candidates.len() {
13493 let c = state.candidates[i].clone();
13494 let is_winner = Some(c.label) == winner;
13495 if is_winner && !drop_winner {
13496 continue;
13497 }
13498 let existed = c.worktree.exists();
13499 remove_preserved_worktree(&repo, &c.worktree).await?;
13502 if existed {
13503 removed.push(c.worktree.to_string_lossy().into_owned());
13504 }
13505 let handed_over = state.released_branches.contains(&c.branch);
13508 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13509 if !git::branch_delete(&repo, &c.branch).await? {
13510 bail!("candidate branch removal refused");
13511 }
13512 removed.push(c.branch.clone());
13513 }
13514 state.candidates[i].folded = true;
13515 }
13516
13517 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13518 let path = name.path();
13519 let keep = !drop_winner
13520 && winner.is_some_and(|w| {
13521 path.file_name()
13522 .is_some_and(|n| n == format!("cand-{w}").as_str())
13523 });
13524 if keep {
13525 continue;
13526 }
13527 remove_read_seat_worktree(state, &path).await?;
13528 removed.push(path.to_string_lossy().into_owned());
13529 }
13530
13531 remove_if_empty(&root);
13540
13541 if state.enabled_worktree_config && drop_winner {
13542 git::release_worktree_config(&repo).await.ok();
13546 state.enabled_worktree_config = false;
13547 }
13548 state.save_under(home)?;
13549 Ok(removed)
13550}
13551
13552async fn verify_merged_fold_sources(
13553 state: &RunState,
13554 drop_winner: bool,
13555 home: &Path,
13556) -> Result<Vec<std::fs::File>> {
13557 let winner = state.tally.as_ref().map(|t| t.winner);
13558 let mut heads = BTreeSet::new();
13559 for candidate in &state.candidates {
13560 if Some(candidate.label) == winner && !drop_winner {
13561 continue;
13562 }
13563 anyhow::ensure!(
13564 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13565 "candidate recovery source is outside the inventoried run root: {}",
13566 candidate.worktree.display()
13567 );
13568 match std::fs::symlink_metadata(&candidate.worktree) {
13569 Ok(_) => {}
13570 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13571 Err(error) => return Err(error.into()),
13572 }
13573 if !state.released_branches.contains(&candidate.branch)
13574 && git::branch_exists(&state.repo, &candidate.branch).await?
13575 {
13576 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13577 }
13578 }
13579 let mut cargo_guards = Vec::new();
13580 let entries = match std::fs::read_dir(state.worktree_root()) {
13581 Ok(entries) => Some(entries),
13582 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13583 Err(error) => return Err(error.into()),
13584 };
13585 for entry in entries.into_iter().flatten() {
13586 let path = entry?.path();
13587 let metadata = std::fs::symlink_metadata(&path)?;
13588 anyhow::ensure!(
13589 metadata.is_dir() && !metadata.file_type().is_symlink(),
13590 "non-directory or symlink recovery source retained: {}",
13591 path.display()
13592 );
13593 if !drop_winner
13594 && winner.is_some_and(|w| {
13595 path.file_name()
13596 .is_some_and(|name| name == format!("cand-{w}").as_str())
13597 })
13598 {
13599 continue;
13600 }
13601 anyhow::ensure!(
13602 path.join(".git").is_file(),
13603 "unregistered recovery source retained: {}",
13604 path.display()
13605 );
13606 anyhow::ensure!(
13607 git::git(
13608 &path,
13609 &[
13610 "status",
13611 "--porcelain",
13612 "--untracked-files=all",
13613 "--ignored",
13614 "--",
13615 ".",
13616 ":(exclude)target"
13617 ]
13618 )
13619 .await?
13620 .is_empty(),
13621 "source changes or ignored recovery files retained: {}",
13622 path.display()
13623 );
13624 match std::fs::symlink_metadata(path.join("target")) {
13625 Ok(_) => {
13626 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13627 }
13628 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13629 Err(error) => return Err(error.into()),
13630 }
13631 for checkpoint in [
13632 "rebase-merge",
13633 "rebase-apply",
13634 "sequencer",
13635 "CHERRY_PICK_HEAD",
13636 "MERGE_HEAD",
13637 "REBASE_HEAD",
13638 "REVERT_HEAD",
13639 "BISECT_LOG",
13640 "AUTO_MERGE",
13641 "MERGE_MSG",
13642 ] {
13643 let location = git::git(
13644 &path,
13645 &[
13646 "rev-parse",
13647 "--path-format=absolute",
13648 "--git-path",
13649 checkpoint,
13650 ],
13651 )
13652 .await?;
13653 anyhow::ensure!(
13654 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13655 if e.kind() == std::io::ErrorKind::NotFound),
13656 "Git recovery checkpoint retained: {checkpoint}"
13657 );
13658 }
13659 heads.insert(git::rev_parse(&path, "HEAD").await?);
13660 }
13661 crate::fold_evidence::preserve(state, home, &heads).await?;
13662
13663 Ok(cargo_guards)
13664}
13665
13666fn remove_if_empty(dir: &Path) {
13677 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13678 std::fs::remove_dir(dir).ok();
13679 }
13680}
13681
13682pub fn worst_open(state: &RunState) -> Option<Severity> {
13684 state
13685 .reviews
13686 .last()?
13687 .reviews
13688 .iter()
13689 .flat_map(|r| r.findings.iter())
13690 .map(|f| f.severity)
13691 .max()
13692}
13693
13694async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13698 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13699 git::acquire_worktree_config(repo).await?;
13700 state.enabled_worktree_config = true;
13701 }
13702 Ok(())
13703}
13704
13705#[cfg(test)]
13706mod tests {
13707 #[tokio::test]
13708 async fn launcher_base_fetch_requires_remote_success() {
13709 let tmp = tempfile::tempdir().unwrap();
13710 let repo = tmp.path().join("repo");
13711 let remote = tmp.path().join("remote");
13712 std::fs::create_dir(&repo).unwrap();
13713 std::fs::create_dir(&remote).unwrap();
13714 git::git(&remote, &["init", "--bare", "-b", "main"])
13715 .await
13716 .unwrap();
13717 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13718 git::git(
13719 &repo,
13720 &[
13721 "-c",
13722 "user.name=Fixture",
13723 "-c",
13724 "user.email=fixture@example.invalid",
13725 "commit",
13726 "--allow-empty",
13727 "-m",
13728 "fixture",
13729 ],
13730 )
13731 .await
13732 .unwrap();
13733 git::git(
13734 &repo,
13735 &["remote", "add", "origin", remote.to_str().unwrap()],
13736 )
13737 .await
13738 .unwrap();
13739 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13740 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13741 assert_eq!(
13742 resolve_base(&repo, "main", "origin", None)
13743 .await
13744 .unwrap()
13745 .commit,
13746 head
13747 );
13748 #[cfg(unix)]
13749 {
13750 let policy: crate::config::Coordination = toml::from_str(
13751 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13752 ).unwrap();
13753 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13754 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13755 let held = crate::repo_lock::SeatLease::operation_with(
13756 &repo,
13757 Some(&policy),
13758 &crate::repo_lock::WaitPolicy::default(),
13759 )
13760 .await
13761 .unwrap();
13762 assert!(held.is_some());
13763 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13764 tokio::pin!(fetch);
13765 assert!(
13766 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13767 .await
13768 .is_err()
13769 );
13770 assert!(
13771 !repo.join(".git/FETCH_HEAD").exists(),
13772 "no fetch before admission"
13773 );
13774 drop(held);
13775 let fetched = fetch.await.unwrap();
13776 assert_eq!(fetched.commit, head);
13777 assert_eq!(fetched.remote, "origin");
13778 assert_eq!(fetched.branch, "main");
13779 }
13780 git::git(
13782 &repo,
13783 &[
13784 "remote",
13785 "set-url",
13786 "origin",
13787 tmp.path().join("missing").to_str().unwrap(),
13788 ],
13789 )
13790 .await
13791 .unwrap();
13792 let error = resolve_base(&repo, "main", "origin", None)
13793 .await
13794 .unwrap_err()
13795 .to_string();
13796 assert!(error.contains("cannot read origin/main"));
13797 assert!(error.contains("refusing to branch off"));
13798 }
13799
13800 #[test]
13801 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13802 let leaked = vec!["chore(deps): update a crate".to_owned()];
13803 let own = "fix(daemon): apply a chosen action";
13804 assert!(should_retitle("", own, &leaked));
13805 assert!(should_retitle(
13806 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13807 own,
13808 &leaked
13809 ));
13810 assert!(should_retitle(
13811 "chore: land candidate A of run 1",
13812 own,
13813 &leaked
13814 ));
13815 assert!(should_retitle(
13816 "magi: candidate A (uncommitted work)",
13817 own,
13818 &leaked
13819 ));
13820 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13821 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13822 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13823 }
13824
13825 #[test]
13826 fn pr_merge_plan_creates_adopts_or_stops() {
13827 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13828 assert_eq!(
13829 pr_merge_plan(Ok(land::OpenPr::One {
13830 url: "u".into(),
13831 title: "t".into()
13832 })),
13833 PrPlan::Adopt {
13834 url: "u".into(),
13835 title: "t".into()
13836 }
13837 );
13838 let PrPlan::Stop(many) =
13839 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13840 else {
13841 panic!("many must stop");
13842 };
13843 assert!(many.contains('a') && many.contains('b'));
13844 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13845 panic!("a failed lookup must stop");
13846 };
13847 assert!(err.contains("bad token"));
13848 }
13849
13850 use super::*;
13851 use crate::run::GateStatus;
13852 use std::collections::BTreeMap;
13853 use std::time::Duration;
13854
13855 fn conductor() -> AgentSpec {
13856 AgentSpec {
13857 id: "conductor".to_owned(),
13858 kind: crate::config::AgentKind::Command,
13859 model: None,
13860 command: vec!["true".to_owned()],
13861 extra_args: Vec::new(),
13862 env: BTreeMap::new(),
13863 prompt_delivery: None,
13864 max_turn_secs: None,
13865 }
13866 }
13867
13868 fn spec(id: &str) -> AgentSpec {
13869 AgentSpec {
13870 id: id.to_owned(),
13871 kind: crate::config::AgentKind::Command,
13872 model: None,
13873 command: vec!["true".to_owned()],
13874 extra_args: Vec::new(),
13875 env: BTreeMap::new(),
13876 prompt_delivery: None,
13877 max_turn_secs: None,
13878 }
13879 }
13880
13881 fn ids(xs: &[&str]) -> BTreeSet<String> {
13882 xs.iter().map(|s| (*s).to_owned()).collect()
13883 }
13884
13885 #[test]
13886 fn pick_successor_skips_an_agent_another_seat_holds() {
13887 let roster = [spec("a"), spec("b"), spec("c")];
13889 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13890 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13891 }
13892
13893 #[test]
13894 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13895 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13897 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13898 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13899 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13900 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13901 }
13902
13903 #[test]
13904 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13905 let roster = [spec("a"), spec("b")];
13906 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13907 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13908 let carried = ids(&["b"]);
13909 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13910 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13911 }
13912
13913 #[test]
13914 fn pick_successor_returns_none_without_any_untried_successor() {
13915 let roster = [spec("a"), spec("b")];
13916 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13917 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13918 let carried = ids(&["a"]);
13919 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13920 }
13921
13922 #[test]
13923 fn pick_successor_rescue_avoids_another_seats_occupant() {
13924 let roster = [spec("a"), spec("b"), spec("c")];
13926 let carried = ids(&["b", "c"]);
13927 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13928 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13929 }
13930
13931 #[test]
13932 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13933 let roster = [spec("a"), spec("b"), spec("c")];
13934 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13935 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13936 }
13937
13938 #[test]
13939 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13940 let roster = [spec("a"), spec("b"), spec("c")];
13941 let failed = ids(&["a", "b", "c"]);
13942 let mut tried = ids(&["b"]);
13944 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13945 assert_eq!(first.id, "a");
13946 tried.insert(first.id.clone());
13947 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13948 assert_eq!(second.id, "c");
13949 tried.insert(second.id.clone());
13950 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13951 }
13952
13953 #[test]
13954 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13955 let roster = [spec("a"), spec("b")];
13956 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13957 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13958 }
13959
13960 #[test]
13961 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13962 let roster = [spec("a"), spec("b"), spec("c")];
13963 let h = SeatHistory {
13964 failed: ids(&["a"]),
13965 last_ok: Some("b".to_owned()),
13966 last_fail: None,
13967 };
13968 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13969 let gone = SeatHistory {
13971 last_ok: Some("zzz".to_owned()),
13972 ..h.clone()
13973 };
13974 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13975 let failed = SeatHistory {
13976 failed: ids(&["a", "b"]),
13977 last_ok: Some("b".to_owned()),
13978 last_fail: None,
13979 };
13980 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13981 }
13982
13983 #[test]
13984 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13985 let first = SeatState::new("review-1", "alpha", 7);
13986 let next = handover_seat("review-1", "gamma", 7);
13987 assert_ne!(first.claude_session, next.claude_session);
13988 }
13989
13990 #[test]
13991 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13992 let mut state = state_with_summary("x", "y");
13993 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13994 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13995 assert_ne!(a.claude_session, b.claude_session);
13996 }
13997
13998 #[test]
13999 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
14000 let roster = [spec("a"), spec("b")];
14001 let h = SeatHistory {
14002 failed: ids(&["a", "b"]),
14003 ..SeatHistory::default()
14004 };
14005 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
14006 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
14007 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
14008 }
14009
14010 #[test]
14017 fn failure_signature_ignores_numbers_and_paths() {
14018 assert_eq!(
14019 failure_signature("exited with Some(2) and no usable output"),
14020 failure_signature("exited with Some(137) and no usable output")
14021 );
14022 assert_eq!(
14023 failure_signature("process exited with no usable output (exit code 2)"),
14024 failure_signature("process exited with no usable output (exit code 137)")
14025 );
14026 for (old, new) in [
14027 (
14028 "exited with Some(2) and no usable output",
14029 "process exited with no usable output (exit code 2)",
14030 ),
14031 (
14032 "exited with None and no usable output",
14033 "process exited with no usable output (exit code unavailable)",
14034 ),
14035 ] {
14036 let previous = FailClass::Other(failure_signature(old));
14037 let current = FailClass::Other(failure_signature(new));
14038 assert_eq!(previous, current, "resumption retains the failure class");
14039 assert!(!should_hand_over(Some(&previous), ¤t));
14040 }
14041 assert_eq!(
14042 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
14043 failure_signature("cannot open /var/x.txt: denied")
14044 );
14045 assert_ne!(failure_signature("boom"), failure_signature("bang"));
14046 }
14047
14048 #[test]
14049 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
14050 let other = FailClass::Other("x".into());
14051 assert!(should_hand_over(None, &FailClass::Quota));
14052 assert!(should_hand_over(Some(&other), &FailClass::Quota));
14053 assert!(should_hand_over(
14054 Some(&FailClass::Timeout),
14055 &FailClass::Timeout
14056 ));
14057 assert!(should_hand_over(None, &other));
14058 assert!(!should_hand_over(Some(&other), &other));
14059 assert!(should_hand_over(
14060 Some(&other),
14061 &FailClass::Other("y".into())
14062 ));
14063 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14065 assert!(should_hand_over(Some(&FailClass::Quota), &other));
14066 }
14067
14068 #[test]
14069 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14070 let a = SeatState::new("judge-1", "alpha", 7);
14071 let b = handover_seat("judge-1", "beta", 7);
14072 assert_ne!(a.claude_session, b.claude_session);
14073 assert_eq!(b.turns, 0);
14074 }
14075
14076 #[test]
14077 fn a_timeout_is_classified_apart_from_other_failures() {
14078 assert_eq!(
14079 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14080 Some(FailClass::Timeout)
14081 );
14082 assert!(matches!(
14083 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14084 Some(FailClass::Other(_))
14085 ));
14086 }
14087
14088 #[test]
14089 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14090 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14091 let tried = BTreeSet::from(["beta".to_owned()]);
14092 let next = next_untried_in_roster(&roster, 1, &tried);
14095 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14096 }
14097
14098 #[test]
14099 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14100 let roster = vec![spec("alpha"), spec("beta")];
14101 let tried = BTreeSet::from(["beta".to_owned()]);
14102 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14106 }
14107
14108 #[test]
14109 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14110 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14111 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14112 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14116 }
14117
14118 #[test]
14119 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14120 let roster = vec![spec("a"), spec("a"), spec("b")];
14121 let tried = BTreeSet::from(["a".to_owned()]);
14122 let next = next_untried_in_roster(&roster, 0, &tried);
14123 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14124 }
14125
14126 #[test]
14127 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14128 let roster = vec![spec("a"), spec("b")];
14129 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14130 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14131 }
14132
14133 #[test]
14134 fn remove_if_empty_only_ever_takes_a_bare_directory() {
14135 let dir = tempfile::tempdir().unwrap();
14136 let bay = dir.path().join("ffff");
14137
14138 remove_if_empty(&bay);
14140 assert!(!bay.exists());
14141
14142 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14145 remove_if_empty(&bay);
14146 assert!(bay.exists(), "non-empty directory must survive");
14147
14148 std::fs::remove_dir(bay.join("cand-A")).unwrap();
14150 remove_if_empty(&bay);
14151 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14152 }
14153
14154 #[test]
14163 fn a_full_panel_that_found_nothing_is_clean() {
14164 assert!(round_is_clean(
14165 0,
14166 true,
14167 2,
14168 2,
14169 0,
14170 IncompleteReviewPolicy::Block
14171 ));
14172 }
14173
14174 #[test]
14175 fn a_missing_seat_is_never_clean_under_the_default_policy() {
14176 assert!(!round_is_clean(
14177 0,
14178 true,
14179 1,
14180 2,
14181 0,
14182 IncompleteReviewPolicy::Block
14183 ));
14184 }
14185
14186 #[test]
14187 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14188 assert!(!round_is_clean(
14189 1,
14190 true,
14191 1,
14192 2,
14193 0,
14194 IncompleteReviewPolicy::Warn
14195 ));
14196 }
14197
14198 #[test]
14199 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14200 assert!(round_is_clean(
14201 0,
14202 true,
14203 1,
14204 2,
14205 0,
14206 IncompleteReviewPolicy::Warn
14207 ));
14208 }
14209
14210 #[test]
14211 fn a_full_panel_with_an_open_finding_is_not_clean() {
14212 assert!(!round_is_clean(
14213 1,
14214 true,
14215 2,
14216 2,
14217 0,
14218 IncompleteReviewPolicy::Block
14219 ));
14220 }
14221
14222 #[test]
14223 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14224 assert!(!round_is_clean(
14225 0,
14226 false,
14227 2,
14228 2,
14229 0,
14230 IncompleteReviewPolicy::Block
14231 ));
14232 }
14233
14234 #[test]
14241 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14242 assert!(round_is_clean(
14245 0,
14246 true,
14247 1,
14248 2,
14249 1,
14250 IncompleteReviewPolicy::Block
14251 ));
14252 }
14253
14254 #[test]
14255 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14256 assert!(!round_is_clean(
14259 0,
14260 true,
14261 1,
14262 2,
14263 0,
14264 IncompleteReviewPolicy::Block
14265 ));
14266 }
14267
14268 #[test]
14269 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14270 assert!(!round_is_clean(
14271 1,
14272 true,
14273 1,
14274 2,
14275 1,
14276 IncompleteReviewPolicy::Block
14277 ));
14278 assert!(!round_is_clean(
14279 0,
14280 false,
14281 1,
14282 2,
14283 1,
14284 IncompleteReviewPolicy::Block
14285 ));
14286 }
14287
14288 #[test]
14289 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14290 assert!(!round_is_clean(
14294 0,
14295 true,
14296 0,
14297 2,
14298 2,
14299 IncompleteReviewPolicy::Block
14300 ));
14301 }
14302
14303 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14304 CommandOutcome {
14305 command: "test".to_owned(),
14306 code,
14307 output_tail: String::new(),
14308 duration_ms: 0,
14309 resource_blocked,
14310 }
14311 }
14312
14313 #[cfg(unix)]
14314 #[tokio::test]
14315 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14316 ask_test_home();
14317 let temp = tempfile::tempdir().unwrap();
14318 let mut runner = runner_at(RunStatus::Gating);
14319 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14320 let commands = vec![
14321 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14322 "touch must-not-run".to_owned(),
14323 ];
14324 let (outcomes, retried, pids) = run_e2e_with_retry(
14325 &mut runner.state,
14326 &shell,
14327 &commands,
14328 temp.path(),
14329 Duration::from_secs(5),
14330 "fixture",
14331 )
14332 .await;
14333 assert_eq!(outcomes.len(), 1);
14334 assert!(outcomes[0].resource_blocked);
14335 assert!(!outcomes[0].build_failed());
14336 assert!(!retried);
14337 assert!(pids.is_empty());
14338 assert!(!temp.path().join("must-not-run").exists());
14339 assert_eq!(
14340 runner.state.resource_block.as_ref().unwrap().kind,
14341 "incremental cache admission"
14342 );
14343 }
14344
14345 #[test]
14346 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14347 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14348 assert!(
14349 !verify_inconclusive(&[outcome(Some(1), false)]),
14350 "an ordinary failure is still evidence about the patch"
14351 );
14352 assert!(verify_inconclusive(&[outcome(None, true)]));
14353 assert!(
14354 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14355 "one inconclusive outcome taints the whole batch"
14356 );
14357 assert!(!verify_inconclusive(&[]));
14358 }
14359
14360 #[tokio::test]
14361 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14362 let calls = std::sync::atomic::AtomicUsize::new(0);
14366 let started = Instant::now();
14367 wait_for_pids_with(
14368 &[123],
14369 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14370 Duration::from_millis(5),
14371 Duration::from_secs(5),
14372 )
14373 .await;
14374 assert!(
14375 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14376 "must keep checking rather than deciding on the first answer"
14377 );
14378 assert!(
14379 started.elapsed() < Duration::from_secs(1),
14380 "must return the moment it is confirmed dead, not wait out the ceiling"
14381 );
14382 }
14383
14384 #[tokio::test]
14385 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14386 let started = Instant::now();
14387 wait_for_pids_with(
14388 &[123],
14389 |_| true, Duration::from_millis(5),
14391 Duration::from_millis(30),
14392 )
14393 .await;
14394 let elapsed = started.elapsed();
14395 assert!(
14396 elapsed >= Duration::from_millis(30),
14397 "must not give up before its own ceiling: {elapsed:?}"
14398 );
14399 assert!(
14400 elapsed < Duration::from_secs(1),
14401 "must not wait past its own ceiling either: {elapsed:?}"
14402 );
14403 }
14404
14405 #[tokio::test]
14406 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14407 let started = Instant::now();
14408 wait_for_pids_with(
14409 &[],
14410 |_| true,
14411 Duration::from_secs(5),
14412 Duration::from_secs(5),
14413 )
14414 .await;
14415 assert!(
14416 started.elapsed() < Duration::from_millis(200),
14417 "an empty pid list has nothing to confirm"
14418 );
14419 }
14420
14421 fn review_round(
14427 clean: bool,
14428 blocking: usize,
14429 answered: usize,
14430 expected: usize,
14431 progressed: bool,
14432 e2e_ok: bool,
14433 ) -> ReviewRound {
14434 ReviewRound {
14435 round: 1,
14436 head: "h".to_owned(),
14437 verified_head: None,
14438 verified_at: None,
14439 reviews: Vec::new(),
14440 e2e: vec![CommandOutcome {
14441 command: "test".to_owned(),
14442 code: Some(if e2e_ok { 0 } else { 1 }),
14443 output_tail: String::new(),
14444 duration_ms: 0,
14445 resource_blocked: false,
14446 }],
14447 verify_retried: false,
14448 e2e_deferred: false,
14449 e2e_defer_reason: None,
14450 fix: None,
14451 blocking,
14452 answered,
14453 expected,
14454 clean,
14455 progressed,
14456 vote_split: false,
14457 reconsideration: Vec::new(),
14458 verdict: None,
14459 }
14460 }
14461
14462 #[test]
14463 fn review_conclusion_is_none_when_nothing_has_run() {
14464 assert_eq!(review_conclusion(&[], 3), None);
14465 }
14466
14467 #[test]
14468 fn review_conclusion_is_none_while_rounds_remain() {
14469 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14470 assert_eq!(review_conclusion(&rounds, 3), None);
14471 }
14472
14473 #[test]
14474 fn review_conclusion_is_gating_once_a_round_is_clean() {
14475 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14476 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14477 }
14478
14479 #[test]
14480 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14481 let rounds = vec![
14482 review_round(false, 1, 2, 2, true, true),
14483 review_round(false, 1, 2, 2, true, true),
14484 ];
14485 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14486 }
14487
14488 #[test]
14489 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14490 let rounds = vec![
14491 review_round(false, 1, 2, 2, true, true),
14492 review_round(false, 1, 2, 2, true, false),
14493 ];
14494 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14495 }
14496
14497 #[test]
14498 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14499 let mut blocked = review_round(false, 1, 2, 2, true, false);
14506 blocked.e2e[0].resource_blocked = true;
14507 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14508 assert_eq!(review_conclusion(&rounds, 2), None);
14509 }
14510
14511 #[test]
14512 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14513 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14515 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14516 }
14517
14518 #[test]
14519 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14520 let rounds = vec![
14521 review_round(false, 1, 2, 2, false, true),
14522 review_round(false, 1, 2, 2, false, true),
14523 ];
14524 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14525 }
14526
14527 fn secs(n: u64) -> Duration {
14528 Duration::from_secs(n)
14529 }
14530
14531 #[tokio::test]
14534 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14535 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14536 if std::env::var_os(CHILD).is_none() {
14537 let output = std::process::Command::new(std::env::current_exe().unwrap())
14541 .args([
14542 "--exact",
14543 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14544 "--nocapture",
14545 ])
14546 .env(CHILD, "1")
14547 .env_remove("MAGI_WORKER_HOME")
14548 .env_remove("MAGI_WORKER_ID")
14549 .output()
14550 .unwrap();
14551 assert!(
14552 output.status.success()
14553 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14554 "{}\n{}",
14555 String::from_utf8_lossy(&output.stdout),
14556 String::from_utf8_lossy(&output.stderr)
14557 );
14558 return;
14559 }
14560 ask_test_home();
14561 let tmp = tempfile::tempdir().unwrap();
14562 let repo = tmp.path().join("repo");
14563 std::fs::create_dir_all(&repo).unwrap();
14564 init_repo(&repo);
14565 let g = |args: &[&str]| {
14566 let out = std::process::Command::new("git")
14567 .args(args)
14568 .current_dir(&repo)
14569 .quiet()
14570 .output()
14571 .unwrap();
14572 assert!(
14573 out.status.success(),
14574 "{}",
14575 String::from_utf8_lossy(&out.stderr)
14576 );
14577 };
14578 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14579 g(&["checkout", "-b", "historical"]);
14580 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14581 g(&["add", "."]);
14582 g(&["commit", "-m", "old alternative"]);
14583 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14584 g(&["checkout", "-b", "magi/task-selected", &base]);
14585 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14586 g(&["add", "."]);
14587 g(&["commit", "-m", "measurement infrastructure"]);
14588 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14589 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14590 g(&["add", "."]);
14591 g(&["commit", "-m", "dependent implementation"]);
14592 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14593 g(&["checkout", "main"]);
14594 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14595 g(&["add", "."]);
14596 g(&["commit", "-m", "unrelated base change"]);
14597 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14598 let origin = tmp.path().join("origin.git");
14599 g(&[
14600 "clone",
14601 "--bare",
14602 repo.to_str().unwrap(),
14603 origin.to_str().unwrap(),
14604 ]);
14605 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14606 g(&["fetch", "origin"]);
14607 let selection = crate::pr_recovery::Selection {
14608 url: "https://example.test/project/pull/1".into(),
14609 branch: "magi/task-selected".into(),
14610 head: head.clone(),
14611 source_head: None,
14612 source_run: None,
14613 source_proof: None,
14614 disposition: Some(crate::pr_recovery::Disposition::Review),
14615 instruction: "preserve measured behavior".into(),
14616 approved_at: jiff::Timestamp::now(),
14617 };
14618 let mut template = runner_at(RunStatus::Reviewing);
14619 template.state.repo = repo.clone();
14620 template.state.base_commit = current.clone();
14621 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14622 template.state.origin = Some(crate::run::Origin::queue("selected"));
14623 template.state.pr_head = Some(selection.branch.clone());
14624 template.state.pr_recovery = Some(selection.clone());
14625 let mut runner =
14626 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14627 .await
14628 .unwrap();
14629 let wt = runner.state.winner().unwrap().worktree.clone();
14630 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14631 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14632 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14633 assert!(runner.state.seeds.is_empty());
14634 assert_eq!(runner.state.instruction, "preserve measured behavior");
14635 runner.sync_to_base().await.unwrap();
14636 assert_eq!(
14637 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14638 "prerequisite"
14639 );
14640 assert_eq!(
14641 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14642 "measured implementation"
14643 );
14644 assert_eq!(
14645 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14646 "current main change"
14647 );
14648 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14649 assert_eq!(
14650 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14651 head
14652 );
14653
14654 let mut partial = runner_at(RunStatus::Reviewing);
14656 partial.state.repo = repo.clone();
14657 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14658 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14659 run: Some("exit 1".into()),
14660 ..Default::default()
14661 }];
14662 let selection = runner.state.pr_recovery.clone().unwrap();
14663 partial.state.pr_recovery = Some(selection.clone());
14664 partial.state.pr_head = Some(selection.branch.clone());
14665 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14666 let id = partial.state.id.clone();
14667 let error = match Runner::open_selected(
14668 &repo,
14669 partial.state,
14670 partial.roles,
14671 1,
14672 runner.state.base_commit.clone(),
14673 selection,
14674 )
14675 .await
14676 {
14677 Ok(_) => panic!("setup must fail"),
14678 Err(e) => e,
14679 };
14680 assert_eq!(
14681 error
14682 .downcast_ref::<crate::run::StartupFailure>()
14683 .unwrap()
14684 .run,
14685 id
14686 );
14687 let failed = RunState::load(&id).unwrap();
14688 assert_eq!(failed.status, RunStatus::Failed);
14689 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14690 assert!(Runner::resume(&id).is_err());
14691 runner.state.status = RunStatus::Ready;
14692 runner.state.driver_exited = true;
14693 runner.state.save().unwrap();
14694 let mut retry = runner_at(RunStatus::Reviewing);
14695 retry.state.repo = repo.clone();
14696 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14697 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14698 reselected.approved_at = jiff::Timestamp::now();
14699 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14700 retry.state.pr_head = Some(reselected.branch.clone());
14701 retry.state.pr_recovery = Some(reselected.clone());
14702 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14703 .await
14704 .unwrap();
14705 let restarted = Runner::open_selected(
14706 &repo,
14707 retry.state,
14708 retry.roles,
14709 1,
14710 runner.state.base_commit.clone(),
14711 reselected,
14712 )
14713 .await
14714 .unwrap();
14715 assert_eq!(
14716 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14717 .await
14718 .unwrap(),
14719 head
14720 );
14721 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14722 assert_eq!(
14723 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14724 head
14725 );
14726 }
14727
14728 #[tokio::test]
14729 #[cfg(unix)]
14730 async fn fallback_and_interrupted_restoration_require_coordination() {
14731 use std::os::fd::AsRawFd as _;
14732 ask_test_home();
14733 let tmp = tempfile::tempdir().unwrap();
14734 let mut runner = registered_runner(tmp.path()).await;
14735 runner.state.config.graph.timeout_resource_wait = 1;
14736 let winner = runner.state.candidates[0].clone();
14737 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14738 let leftover = winner.worktree.join("leftover");
14739 std::fs::write(&leftover, "untrusted edits").unwrap();
14740 let next = tmp.path().join("next-fixer");
14741 let mut failed = spec("first");
14742 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14743 let mut successor = spec("second");
14744 successor.command = vec![
14745 "sh".into(),
14746 "-c".into(),
14747 format!("touch '{}'; echo done", next.display()),
14748 ];
14749 let attempts = vec![
14750 (failed.clone(), "first".into()),
14751 (successor.clone(), "second".into()),
14752 ];
14753 let lock_path = runner.state.repo.join(".git/updater.lock");
14754 let external = std::fs::File::create(&lock_path).unwrap();
14755 assert_eq!(
14756 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14757 0
14758 );
14759 let artifacts = tmp.path().join("artifacts");
14760 let build = |spec, seat| SeatJob {
14761 spec,
14762 seat,
14763 cwd: winner.worktree.clone(),
14764 prompt: "fixture".into(),
14765 timeout: Duration::from_secs(10),
14766 allow_write: true,
14767 sessions: false,
14768 artifacts: artifacts.clone(),
14769 stem: "fix".into(),
14770 handover: None,
14771 };
14772 let (_, _, out) = runner
14773 .ask_fixer_among(
14774 attempts.clone(),
14775 "graft",
14776 None,
14777 Some((&winner.worktree, &head)),
14778 build,
14779 )
14780 .await;
14781 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14782 assert!(leftover.exists(), "contention must prevent clean");
14783 assert!(!next.exists());
14784 assert!(runner.state.handovers.is_empty());
14785 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14787 from_run: "earlier".into(),
14788 generation: 1,
14789 mode: crate::recovery::Mode::VerifyRepair,
14790 reasons: Vec::new(),
14791 reviewed_head: None,
14792 rounds: Vec::new(),
14793 open_findings: Vec::new(),
14794 evidence: None,
14795 criteria_inherited: true,
14796 repair: crate::recovery::Repair::Started {
14797 before: head.clone(),
14798 },
14799 });
14800 assert!(
14801 runner
14802 .recovery_repair(&winner)
14803 .await
14804 .unwrap_err()
14805 .is::<crate::repo_lock::ResourceBlocked>()
14806 );
14807 assert!(matches!(
14808 runner.state.recovery.as_ref().unwrap().repair,
14809 crate::recovery::Repair::Started { .. }
14810 ));
14811 assert!(leftover.exists());
14812 drop(external);
14813 runner.recovery_repair(&winner).await.unwrap();
14814 assert!(!leftover.exists());
14815 assert!(matches!(
14816 runner.state.recovery.as_ref().unwrap().repair,
14817 crate::recovery::Repair::Done { .. }
14818 ));
14819 std::fs::remove_file(&lock_path).unwrap();
14821 std::fs::create_dir(&lock_path).unwrap();
14822 std::fs::write(&leftover, "untrusted edits").unwrap();
14823 let (_, _, out) = runner
14824 .ask_fixer_among(
14825 attempts,
14826 "graft",
14827 None,
14828 Some((&winner.worktree, &head)),
14829 build,
14830 )
14831 .await;
14832 assert!(matches!(out, AgentOutcome::Failed(_)));
14833 assert!(leftover.exists());
14834 assert!(!next.exists());
14835 assert_eq!(
14836 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14837 head
14838 );
14839 }
14840
14841 #[cfg(unix)]
14842 async fn registered_runner(root: &Path) -> Runner {
14843 repository_runner(root, crate::config::test_coordination()).await
14844 }
14845
14846 async fn repository_runner(
14847 root: &Path,
14848 coordination: Option<crate::config::Coordination>,
14849 ) -> Runner {
14850 ask_test_home();
14851 let repo = root.join("repo");
14852 std::fs::create_dir_all(&repo).unwrap();
14853 init_repo(&repo);
14854 if coordination.is_some() {
14855 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14856 }
14857 let mut runner = runner_at(RunStatus::Prep);
14858 runner.state.repo = repo.clone();
14859 runner.state.config.coordination = coordination;
14860 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14861 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14862 runner.state.config.graph.advise = false;
14863 runner.roles.implementers = vec![spec("implement")];
14864 runner.roles.judges = vec![spec("judge")];
14865 runner.prep().await.unwrap();
14866 runner
14867 }
14868
14869 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14870
14871 #[tokio::test]
14872 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14873 let tmp = tempfile::tempdir().unwrap();
14874 let mut runner = repository_runner(tmp.path(), None).await;
14875 let prompts = runner.state.config.prompts.clone();
14876 let run = runner.state.id.clone();
14877 let ctx = WaveCtx {
14878 carry_seats: false,
14879 pauses: None,
14880 run: &run,
14881 node: "advise",
14882 prompts: &prompts,
14883 cache: None,
14884 round: None,
14885 };
14886 let jobs: Vec<_> = ["alpha", "beta"]
14887 .into_iter()
14888 .map(|id| SeatJob {
14889 spec: scripted(id, AUTH_OUTAGE),
14890 seat: SeatState::new(id, id, 7),
14891 prompt: "propose".into(),
14892 cwd: runner.state.candidates[0].worktree.clone(),
14893 timeout: Duration::from_secs(10),
14894 allow_write: false,
14895 sessions: false,
14896 artifacts: agent::artifacts_dir(&runner.state.dir()),
14897 stem: id.into(),
14898 handover: None,
14899 })
14900 .collect();
14901 let mut losses = Vec::new();
14902 let results = ask_wave_with(
14903 jobs.clone(),
14904 runner.sem.clone(),
14905 2,
14906 &[],
14907 &ctx,
14908 &mut losses,
14909 &mut runner.state,
14910 &|_: &str| Ok(()),
14911 )
14912 .await;
14913 assert!(
14914 results
14915 .iter()
14916 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14917 );
14918 assert_eq!(runner.state.quota.len(), 2);
14919 assert!(
14920 runner
14921 .state
14922 .quota
14923 .iter()
14924 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14925 );
14926 assert!(!runner.state.advise_attempted);
14927 runner.stop_for_resource().unwrap();
14928 assert_eq!(runner.state.status, RunStatus::Blocked);
14929 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14930 assert_eq!(
14931 runner.state.resource_block.as_ref().unwrap().kind,
14932 "agent authentication"
14933 );
14934 runner.state.resource_block = None;
14937 runner.state.quota.clear();
14938 let mut mixed = jobs;
14939 mixed[1].spec.command = vec!["false".into()];
14940 let results = ask_wave_with(
14941 mixed,
14942 runner.sem.clone(),
14943 0,
14944 &[],
14945 &ctx,
14946 &mut losses,
14947 &mut runner.state,
14948 &|_: &str| Ok(()),
14949 )
14950 .await;
14951 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14952 assert!(runner.state.resource_block.is_none());
14953 assert_eq!(losses.len(), 1);
14954 }
14955
14956 #[tokio::test]
14957 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14958 let tmp = tempfile::tempdir().unwrap();
14959 let mut runner = repository_runner(tmp.path(), None).await;
14960 let implementer = scripted("implement", AUTH_OUTAGE);
14961 runner.roles.implementers = vec![implementer.clone()];
14962 runner.state.config.agents = vec![implementer];
14963 runner.implement().await.unwrap();
14964 assert_eq!(runner.state.status, RunStatus::Blocked);
14965 assert_eq!(runner.state.quota.len(), 1);
14966 assert_eq!(
14967 runner.state.quota[0].cause,
14968 agent::QuotaCause::Authentication
14969 );
14970 let candidate = &runner.state.candidates[0];
14971 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14972 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14973 }
14974
14975 #[tokio::test]
14976 #[cfg(unix)]
14977 async fn parked_implementer_persists_completed_work() {
14978 for registered in [false, true] {
14979 let tmp = tempfile::tempdir().unwrap();
14980 let mut runner = registered_runner(tmp.path()).await;
14981 if !registered {
14982 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14983 }
14984 let marker = tmp.path().join("started");
14985 let release = tmp.path().join("release");
14986 let mut implementer = spec("implement");
14987 implementer.command = vec![
14988 "sh".into(),
14989 "-c".into(),
14990 format!(
14991 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14992 marker.display(),
14993 release.display()
14994 ),
14995 ];
14996 runner.roles.implementers = vec![implementer.clone()];
14997 runner.state.config.agents = vec![implementer];
14998 let pause = runner.pause.clone();
14999 let (result, ()) = tokio::join!(runner.implement(), async {
15000 tokio::time::timeout(Duration::from_secs(10), async {
15001 while !marker.exists() {
15002 tokio::time::sleep(Duration::from_millis(10)).await;
15003 }
15004 })
15005 .await
15006 .unwrap();
15007 pause.park();
15008 std::fs::write(&release, "go").unwrap();
15009 });
15010 result.unwrap();
15011 let candidate = &runner.state.candidates[0];
15012 assert!(candidate.commits > 0, "{candidate:?}");
15013 assert!(candidate.summary.contains("saved-result"));
15014 assert_eq!(candidate.files, 1);
15015 let before = candidate.commits;
15016 runner.implement().await.unwrap();
15017 assert_eq!(runner.state.candidates[0].commits, before);
15018 }
15019 }
15020
15021 #[tokio::test]
15022 #[cfg(unix)]
15023 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
15024 let tmp = tempfile::tempdir().unwrap();
15025 let mut runner = registered_runner(tmp.path()).await;
15026 let winner = runner.state.candidates[0].clone();
15027 let cache = tmp.path().join("cache");
15028 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
15029 runner.state.config.graph.timeout_resource_wait = 0;
15030 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
15031 let crate::cache::AcquireOutcome::Acquired(held) =
15032 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
15033 else {
15034 panic!("cache must be free")
15035 };
15036 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15037 from_run: "earlier".into(),
15038 generation: 1,
15039 mode: crate::recovery::Mode::VerifyRepair,
15040 reasons: Vec::new(),
15041 reviewed_head: None,
15042 rounds: Vec::new(),
15043 open_findings: Vec::new(),
15044 evidence: None,
15045 criteria_inherited: true,
15046 repair: crate::recovery::Repair::NotStarted,
15047 });
15048 runner.recovery_repair(&winner).await.unwrap();
15049 assert!(matches!(
15050 runner.state.recovery.as_ref().unwrap().repair,
15051 crate::recovery::Repair::WaitingForRestore { .. }
15052 ));
15053 drop(held);
15054 runner.state.resource_block = None;
15055 runner.state.config.graph.timeout_resource_wait = 1;
15056 runner.recovery_repair(&winner).await.unwrap();
15057 assert!(matches!(
15058 runner.state.recovery.as_ref().unwrap().repair,
15059 crate::recovery::Repair::Done { .. }
15060 ));
15061 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15062 }
15063
15064 #[tokio::test]
15065 #[cfg(unix)]
15066 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15067 use std::os::fd::AsRawFd as _;
15068 for graft in [false, true] {
15069 let tmp = tempfile::tempdir().unwrap();
15070 let mut runner = registered_runner(tmp.path()).await;
15071 runner.state.config.graph.timeout_resource_wait = 1;
15072 let winner = runner.state.candidates[0].clone();
15073 let marker = tmp.path().join("fixer-started");
15074 let mut fixer = spec("fixer");
15075 fixer.command = vec![
15076 "sh".into(),
15077 "-c".into(),
15078 format!("echo started >> '{}'; echo done", marker.display()),
15079 ];
15080 runner.roles.fixer = Some(vec![fixer]);
15081 if graft {
15082 let mut loser = winner.clone();
15083 loser.label = 'B';
15084 loser.index = 1;
15085 runner.state.candidates.push(loser);
15086 runner.state.config.graph.graft_runner_up = true;
15087 runner.state.tally = Some(Tally {
15088 first_choice: BTreeMap::new(),
15089 borda: BTreeMap::new(),
15090 winner: winner.label,
15091 rankings: 1,
15092 unanimous_initial: true,
15093 deliberated: false,
15094 changed_votes: 0,
15095 unanimous_final: true,
15096 tie_break: None,
15097 judges: 1,
15098 present: 1,
15099 quorum: 1,
15100 met_quorum: true,
15101 uncontested: None,
15102 });
15103 runner.state.judgements.push(crate::run::Judgement {
15104 judge: 1,
15105 seat: "judge-1".into(),
15106 agent: "judge".into(),
15107 ranking: vec![winner.label, 'B'],
15108 reasons: BTreeMap::new(),
15109 confidence: None,
15110 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15111 order: vec![0, 1],
15112 failed: None,
15113 duration_ms: 0,
15114 });
15115 } else {
15116 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15117 from_run: "earlier".into(),
15118 generation: 1,
15119 mode: crate::recovery::Mode::VerifyRepair,
15120 reasons: Vec::new(),
15121 reviewed_head: None,
15122 rounds: Vec::new(),
15123 open_findings: Vec::new(),
15124 evidence: None,
15125 criteria_inherited: true,
15126 repair: crate::recovery::Repair::NotStarted,
15127 });
15128 }
15129 let reader = crate::repo_lock::SeatLease::for_checkout(
15130 &winner.worktree,
15131 false,
15132 crate::config::test_coordination().as_ref(),
15133 )
15134 .await
15135 .unwrap();
15136 let pause = runner.pause.clone();
15137 let (result, ()) = tokio::join!(
15138 async {
15139 if graft {
15140 runner.graft().await
15141 } else {
15142 runner.recovery_repair(&winner).await
15143 }
15144 },
15145 async {
15146 tokio::time::sleep(Duration::from_millis(100)).await;
15147 pause.park();
15148 }
15149 );
15150 result.unwrap();
15151 assert!(!marker.exists(), "a blocked seat must not spawn");
15152 runner.state = RunState::load(&runner.state.id).unwrap();
15154 if graft {
15155 assert_eq!(
15156 runner.state.graft.as_ref().unwrap().outcome,
15157 GraftOutcome::WaitingForRestore
15158 );
15159 } else {
15160 assert!(matches!(
15161 runner.state.recovery.as_ref().unwrap().repair,
15162 crate::recovery::Repair::WaitingForRestore { .. }
15163 ));
15164 }
15165 drop(reader);
15166 runner.pause = Pause::new();
15167 runner.state.resource_block = None;
15168 let external =
15170 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15171 assert_eq!(
15172 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15173 0
15174 );
15175 let leftover = winner.worktree.join("leftover");
15176 std::fs::write(&leftover, "earlier fallback edits").unwrap();
15177 let result = if graft {
15178 runner.graft().await
15179 } else {
15180 runner.recovery_repair(&winner).await
15181 };
15182 assert!(
15183 result
15184 .unwrap_err()
15185 .is::<crate::repo_lock::ResourceBlocked>()
15186 );
15187 assert!(leftover.exists());
15188 assert!(!marker.exists());
15189 drop(external);
15190 runner.state.resource_block = None;
15191 let result = if graft {
15192 runner.graft().await
15193 } else {
15194 runner.recovery_repair(&winner).await
15195 };
15196 result.unwrap();
15197 assert!(!leftover.exists(), "restore must precede retry");
15198 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15199 if graft {
15200 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15201 } else {
15202 assert!(matches!(
15203 runner.state.recovery.as_ref().unwrap().repair,
15204 crate::recovery::Repair::Done { .. }
15205 ));
15206 }
15207 }
15208 }
15209
15210 #[tokio::test]
15211 #[cfg(unix)]
15212 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15213 use std::os::fd::AsRawFd as _;
15214 let tmp = tempfile::tempdir().unwrap();
15215 let mut runner = registered_runner(tmp.path()).await;
15216 runner.state.config.graph.timeout_resource_wait = 1;
15217 let winner = runner.state.candidates[0].clone();
15218 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15219 runner.state.tally = Some(Tally {
15220 first_choice: BTreeMap::new(),
15221 borda: BTreeMap::new(),
15222 winner: winner.label,
15223 rankings: 0,
15224 unanimous_initial: false,
15225 deliberated: false,
15226 changed_votes: 0,
15227 unanimous_final: false,
15228 tie_break: None,
15229 judges: 0,
15230 present: 0,
15231 quorum: 0,
15232 met_quorum: true,
15233 uncontested: Some("fixture".into()),
15234 });
15235 runner.state.graft = Some(GraftRecord {
15236 pre: pre.clone(),
15237 post: None,
15238 lines: None,
15239 limit: None,
15240 agent: None,
15241 outcome: GraftOutcome::Running,
15242 requested: Vec::new(),
15243 adoption: None,
15244 adoption_error: None,
15245 });
15246 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15247 git::commit_all(&winner.worktree, "unfinished graft")
15248 .await
15249 .unwrap();
15250 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15251 let leftover = winner.worktree.join("leftover");
15252 std::fs::write(&leftover, "untrusted edits").unwrap();
15253 let path = runner.state.repo.join(".git/updater.lock");
15254 let external = std::fs::File::create(&path).unwrap();
15255 assert_eq!(
15256 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15257 0
15258 );
15259 assert!(
15260 runner
15261 .graft()
15262 .await
15263 .unwrap_err()
15264 .is::<crate::repo_lock::ResourceBlocked>()
15265 );
15266 assert!(matches!(
15267 runner.state.graft.as_ref().unwrap().outcome,
15268 GraftOutcome::Running
15269 ));
15270 assert_eq!(
15271 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15272 unfinished
15273 );
15274 assert!(leftover.exists());
15275 drop(external);
15276 std::fs::remove_file(&path).unwrap();
15277 std::fs::create_dir(&path).unwrap();
15278 assert!(runner.graft().await.is_err());
15279 assert!(matches!(
15280 runner.state.graft.as_ref().unwrap().outcome,
15281 GraftOutcome::Running
15282 ));
15283 assert_eq!(
15284 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15285 unfinished
15286 );
15287 assert!(leftover.exists());
15288 std::fs::remove_dir(&path).unwrap();
15289 runner.graft().await.unwrap();
15290 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15291 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15292 assert!(!leftover.exists());
15293 }
15294
15295 #[tokio::test]
15296 #[cfg(unix)]
15297 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15298 use std::os::fd::AsRawFd as _;
15299 let tmp = tempfile::tempdir().unwrap();
15300 let mut runner = registered_runner(tmp.path()).await;
15301 runner.state.prep_complete = Some(false);
15302 runner.state.config.graph.timeout_resource_wait = 1;
15303 runner.state.save().unwrap();
15304 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15305 assert_eq!(
15306 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15307 0
15308 );
15309 runner.execute().await.unwrap();
15310 let disk = RunState::load(&runner.state.id).unwrap();
15311 assert!(disk.driver_exited);
15312 assert_eq!(disk.driver_pid, None);
15313 assert_eq!(runner.state.status, RunStatus::Blocked);
15314 assert_eq!(
15315 runner.state.resource_block.as_ref().unwrap().kind,
15316 "repository coordination"
15317 );
15318 assert!(runner.state.prep_failure.is_none());
15319 assert!(runner.state.seats.is_empty());
15320 runner.pause.park();
15321 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15324 .await
15325 .unwrap()
15326 .unwrap();
15327 assert!(runner.state.parked);
15328 assert!(runner.state.prep_failure.is_none());
15329 let disk = RunState::load(&runner.state.id).unwrap();
15330 assert!(disk.driver_exited);
15331 assert_eq!(disk.driver_pid, None);
15332 }
15333
15334 #[tokio::test]
15335 #[cfg(unix)]
15336 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15337 use std::os::fd::AsRawFd as _;
15338 let tmp = tempfile::tempdir().unwrap();
15339 let mut runner = registered_runner(tmp.path()).await;
15340 runner.state.prep_complete = Some(false);
15341 runner.state.config.graph.timeout_resource_wait = 30;
15342 runner.state.save().unwrap();
15343 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15344 assert_eq!(
15345 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15346 0
15347 );
15348 assert!(
15349 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15350 .await
15351 .is_err()
15352 );
15353 let disk = RunState::load(&runner.state.id).unwrap();
15354 assert!(disk.driver_exited);
15355 assert_eq!(disk.driver_pid, None);
15356 assert!(disk.status.resumable());
15357 assert!(runner.state.driver_exited);
15358 assert_eq!(runner.state.driver_pid, None);
15359 runner.state.save().unwrap();
15360 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15361 }
15362
15363 #[tokio::test]
15364 #[cfg(unix)]
15365 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15366 let tmp = tempfile::tempdir().unwrap();
15367 let mut runner = registered_runner(tmp.path()).await;
15368 let winner = runner.state.candidates[0].clone();
15369 let cache = tmp.path().join("cache");
15370 let started = tmp.path().join("pre-gate-started");
15371 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15372 runner.state.config.verify.gate =
15373 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15374 let home = crate::run::home();
15375 let owner =
15376 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15377 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15378 crate::cache::AcquireOutcome::Acquired(g) => g,
15379 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15380 };
15381 let mut gate = Box::pin(runner.run_gate(&winner));
15382 assert!(
15383 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15384 .await
15385 .is_err()
15386 );
15387 assert!(!started.exists(), "pre-gate must wait for the cache too");
15388 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15392 let editor = crate::repo_lock::SeatLease::checkout_with(
15393 &winner.worktree,
15394 true,
15395 crate::config::test_coordination().as_ref(),
15396 &wait,
15397 )
15398 .await
15399 .unwrap();
15400 drop(editor);
15401 guard.release();
15402 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15403 .await
15404 .unwrap()
15405 .unwrap();
15406 drop(gate);
15407 assert!(outcomes.iter().all(CommandOutcome::ok));
15408 assert!(started.exists());
15409 }
15410
15411 #[tokio::test]
15412 #[cfg(unix)]
15413 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15414 use std::os::fd::AsRawFd as _;
15415
15416 ask_test_home();
15417 let tmp = tempfile::tempdir().unwrap();
15418 let repo = tmp.path().join("repo");
15419 std::fs::create_dir_all(&repo).unwrap();
15420 init_repo(&repo);
15421 let common = repo.join(".git");
15422 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15423 let mut runner = runner_at(RunStatus::Prep);
15424 runner.state.repo = repo.clone();
15425 runner.state.config.coordination = crate::config::test_coordination();
15426 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15427 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15428 runner.state.config.graph.advise = false;
15429 runner.roles.implementers = vec![spec("implement")];
15430 runner.roles.judges = vec![spec("judge")];
15431 runner.prep().await.unwrap();
15432 let winner = runner.state.candidates[0].clone();
15433 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15436 let completed = tmp.path().join("seat-completed");
15437 runner.roles.implementers[0].command = vec![
15438 "sh".into(),
15439 "-c".into(),
15440 format!("touch '{}'; echo completed", completed.display()),
15441 ];
15442 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15443 assert_eq!(
15444 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15445 0
15446 );
15447 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15448 let mut implement = Box::pin(runner.implement());
15449 tokio::select! {
15450 result = &mut implement => panic!("rescue must wait: {result:?}"),
15451 result = tokio::time::timeout(Duration::from_secs(10), async {
15452 while !completed.exists() {
15453 tokio::time::sleep(Duration::from_millis(20)).await;
15454 }
15455 }) => result.unwrap(),
15456 }
15457 assert!(
15458 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15459 .await
15460 .is_err()
15461 );
15462 assert_eq!(
15463 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15464 before
15465 );
15466 assert_eq!(
15467 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15468 0
15469 );
15470 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15471 .await
15472 .unwrap()
15473 .unwrap();
15474 drop(implement);
15475 assert_ne!(
15476 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15477 before
15478 );
15479 assert!(git::is_clean(&winner.worktree).await.unwrap());
15480
15481 let started = tmp.path().join("pre-gate-started");
15482 let proceed = tmp.path().join("pre-gate-proceed");
15483 let checked = tmp.path().join("gate-ran");
15484 runner.state.config.verify.pre_gate = vec![format!(
15485 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15486 started.display(),
15487 proceed.display()
15488 )];
15489 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15490 assert_eq!(
15491 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15492 0
15493 );
15494 let mut gate = Box::pin(runner.run_gate(&winner));
15495 assert!(
15496 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15497 .await
15498 .is_err()
15499 );
15500 assert!(!started.exists());
15501 assert!(!checked.exists());
15502 assert!(git::is_clean(&winner.worktree).await.unwrap());
15503 assert_eq!(
15504 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15505 0
15506 );
15507 tokio::select! {
15508 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15509 result = tokio::time::timeout(Duration::from_secs(10), async {
15510 while !started.exists() {
15511 tokio::time::sleep(Duration::from_millis(20)).await;
15512 }
15513 }) => result.unwrap(),
15514 }
15515 assert_eq!(
15516 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15517 -1,
15518 "mechanical edits must hold the real lock"
15519 );
15520 std::fs::write(proceed, "ready").unwrap();
15521 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15522 .await
15523 .unwrap()
15524 .unwrap();
15525 drop(gate);
15526 assert!(outcomes.iter().all(CommandOutcome::ok));
15527 assert!(checked.exists());
15528 assert!(runner.state.pre_gate_commit.is_some());
15529 assert!(git::is_clean(&winner.worktree).await.unwrap());
15530 assert_eq!(
15531 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15532 0,
15533 "gate completion must release coordination"
15534 );
15535 drop(outsider);
15536 std::fs::remove_file(common.join("updater.lock")).unwrap();
15537 std::fs::create_dir(common.join("updater.lock")).unwrap();
15538 std::fs::remove_file(&started).unwrap();
15539 std::fs::remove_file(&checked).unwrap();
15540 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15541 let error = runner.run_gate(&winner).await.unwrap_err();
15542 assert!(
15543 error
15544 .to_string()
15545 .contains("acquire mandatory repository lock")
15546 );
15547 assert!(
15548 !started.exists(),
15549 "failed acquisition must prevent pre-gate"
15550 );
15551 assert!(!checked.exists(), "failed acquisition must prevent gate");
15552 assert!(git::is_clean(&winner.worktree).await.unwrap());
15553 assert_eq!(
15554 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15555 head
15556 );
15557 }
15558
15559 #[tokio::test]
15560 #[cfg(unix)]
15561 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15562 use std::os::fd::AsRawFd as _;
15563 use std::time::Duration;
15564
15565 ask_test_home();
15566 let tmp = tempfile::tempdir().unwrap();
15567 let repo = tmp.path().join("repo");
15568 std::fs::create_dir_all(&repo).unwrap();
15569 init_repo(&repo);
15570 let common = repo.join(".git");
15571 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15572 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15573 assert_eq!(
15574 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15575 0
15576 );
15577 let mut runner = runner_at(RunStatus::Prep);
15578 runner.state.repo = repo.clone();
15579 runner.state.config.coordination = crate::config::test_coordination();
15580 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15581 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15582 runner.state.config.graph.advise = true;
15583 runner.state.config.graph.advisors = 1;
15584 runner.roles.implementers = vec![spec("implement")];
15585 runner.roles.judges = vec![spec("judge")];
15586 let root = runner.state.worktree_root();
15587 let started = tmp.path().join("setup-started");
15588 let proceed = tmp.path().join("setup-proceed");
15589 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15590 run: Some(format!(
15591 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15592 started.display(),
15593 proceed.display()
15594 )),
15595 ..Default::default()
15596 }];
15597 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15598 let config = git::git(&repo, &["config", "--local", "--list"])
15599 .await
15600 .unwrap();
15601 let mut prep = Box::pin(runner.prep());
15602 assert!(
15603 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15604 .await
15605 .is_err()
15606 );
15607 assert!(
15608 !root.exists(),
15609 "contention must prevent even checkout creation"
15610 );
15611 assert!(!started.exists(), "contention must prevent setup");
15612 assert_eq!(
15613 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15614 branches
15615 );
15616 assert_eq!(
15617 git::git(&repo, &["config", "--local", "--list"])
15618 .await
15619 .unwrap(),
15620 config
15621 );
15622 drop(lock);
15623 tokio::select! {
15624 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15625 result = tokio::time::timeout(Duration::from_secs(10), async {
15626 while !started.exists() {
15627 tokio::time::sleep(Duration::from_millis(20)).await;
15628 }
15629 }) => result.unwrap(),
15630 }
15631 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15632 assert_eq!(
15633 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15634 -1
15635 );
15636 assert!(
15637 tokio::time::timeout(
15638 Duration::from_millis(100),
15639 crate::repo_lock::SeatLease::for_checkout(
15640 &repo,
15641 true,
15642 crate::config::test_coordination().as_ref()
15643 )
15644 )
15645 .await
15646 .is_err(),
15647 "preparation must also exclude local editors"
15648 );
15649 std::fs::write(proceed, "ready").unwrap();
15650 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15651 .await
15652 .unwrap()
15653 .unwrap();
15654 drop(prep);
15655 assert_eq!(runner.state.prep_complete, Some(true));
15656 assert!(root.join("judge-1").exists());
15657 assert!(root.join("advisor-1").exists());
15658 assert_eq!(
15659 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15660 0
15661 );
15662 }
15663
15664 #[tokio::test]
15665 #[cfg(unix)]
15666 async fn preparation_lock_failure_prevents_repository_changes() {
15667 ask_test_home();
15668 let tmp = tempfile::tempdir().unwrap();
15669 let repo = tmp.path().join("repo");
15670 std::fs::create_dir_all(&repo).unwrap();
15671 init_repo(&repo);
15672 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15673 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15676 let mut runner = runner_at(RunStatus::Prep);
15677 runner.state.repo = repo.clone();
15678 runner.state.config.coordination = crate::config::test_coordination();
15679 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15680 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15681 runner.roles.implementers = vec![spec("implement")];
15682 let error = runner.prep().await.unwrap_err();
15683 assert!(
15684 error
15685 .to_string()
15686 .contains("acquire mandatory repository lock")
15687 );
15688 assert!(!runner.state.worktree_root().exists());
15689 assert!(runner.state.candidates.is_empty());
15690 assert!(!repo.join(".git/config.worktree").exists());
15691 assert_eq!(
15692 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15693 "* main"
15694 );
15695 }
15696
15697 #[tokio::test]
15698 async fn preparation_records_creation_intent_before_git_can_fail() {
15699 ask_test_home();
15700 let tmp = tempfile::tempdir().unwrap();
15701 let repo = tmp.path().join("repo");
15702 std::fs::create_dir_all(&repo).unwrap();
15703 init_repo(&repo);
15704 let mut runner = runner_at(RunStatus::Prep);
15705 runner.state.repo = repo.clone();
15706 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15707 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15708 runner.state.config.blind.commit_msg_hook = false;
15709 runner.roles.implementers = vec![spec("one")];
15710 let label = blind::assign_labels(1, runner.state.seed)[0];
15711 let branch = runner.state.branch_for(label);
15712 git::git(&repo, &["branch", &branch]).await.unwrap();
15715 assert!(runner.prep().await.is_err());
15716 let saved = RunState::load(&runner.state.id).unwrap();
15717 assert!(saved.candidates.is_empty());
15718 let event = saved
15719 .events
15720 .iter()
15721 .find(|e| e.node == "prep-intent")
15722 .unwrap();
15723 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15724 assert_eq!(intent["branch"], branch);
15725 assert_eq!(intent["start"], runner.state.base_commit);
15726 assert_eq!(
15727 intent["worktree"],
15728 runner
15729 .state
15730 .worktree_root()
15731 .join(format!("cand-{label}"))
15732 .to_string_lossy()
15733 .as_ref()
15734 );
15735 assert_eq!(intent["picks"], serde_json::json!([]));
15736 }
15737
15738 #[tokio::test]
15739 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15740 ask_test_home();
15741 let tmp = tempfile::tempdir().unwrap();
15742 let repo = tmp.path().join("repo");
15743 std::fs::create_dir_all(&repo).unwrap();
15744 init_repo(&repo);
15745 let mut runner = runner_at(RunStatus::Prep);
15746 runner.state.repo = repo.clone();
15747 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15748 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15749 runner.state.sources.pick_commits.push(refs::Input {
15750 token: "missing".into(),
15751 sha: "0".repeat(40),
15752 provenance: refs::Provenance::ExplicitCli,
15753 });
15754 assert!(runner.prep().await.is_err());
15755 assert!(runner.state.candidates.is_empty());
15756 assert!(!runner.state.worktree_root().exists());
15757 }
15758
15759 #[tokio::test]
15760 async fn prose_and_legacy_inference_never_import_into_candidates() {
15761 ask_test_home();
15762 let tmp = tempfile::tempdir().unwrap();
15763 let repo = tmp.path().join("repo");
15764 std::fs::create_dir_all(&repo).unwrap();
15765 init_repo(&repo);
15766 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15767 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15768 .await
15769 .unwrap();
15770 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15771 git::git(&repo, &["add", "."]).await.unwrap();
15772 git::git(&repo, &["commit", "-m", "historical implementation"])
15773 .await
15774 .unwrap();
15775 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15776 git::git(&repo, &["checkout", "main"]).await.unwrap();
15777 for instruction in [
15778 format!("Example: magi/history/A at {source}"),
15779 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15780 format!("Never import magi/history/A or {source}"),
15781 format!("Conversation answer: I mentioned {source} only as background"),
15782 ] {
15783 let mut runner = runner_at(RunStatus::Prep);
15784 runner.state.repo = repo.clone();
15785 runner.state.base_commit = base.clone();
15786 runner.state.instruction = instruction.clone();
15787 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15788 runner.state.config.blind.commit_msg_hook = false;
15789 runner.state.seeds = vec![refs::Seed {
15790 token: "magi/history/A".into(),
15791 sha: source.clone(),
15792 kind: refs::SeedKind::Unmerged,
15793 branch: true,
15794 detail: String::new(),
15795 }];
15796 runner.roles.implementers = vec![spec("one")];
15797 assert_eq!(seeded_instruction(&runner.state), instruction);
15798 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15800 legacy.as_object_mut().unwrap().remove("sources");
15801 runner.state = serde_json::from_value(legacy).unwrap();
15802 assert_eq!(runner.state.sources, refs::Sources::default());
15803 runner.state.save().unwrap();
15804 runner.state = RunState::load(&runner.state.id).unwrap();
15805 runner.prep().await.unwrap();
15806 for candidate in &runner.state.candidates {
15807 assert_eq!(
15808 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15809 base
15810 );
15811 assert!(!candidate.worktree.join("unwanted").exists());
15812 }
15813 }
15814 }
15815
15816 #[tokio::test]
15817 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15818 ask_test_home();
15819 let tmp = tempfile::tempdir().unwrap();
15820 let repo = tmp.path().join("repo");
15821 std::fs::create_dir_all(&repo).unwrap();
15822 init_repo(&repo);
15823 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15824 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15825 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15826 git::git(&repo, &["add", "."]).await.unwrap();
15827 git::git(&repo, &["commit", "-m", "required seed"])
15828 .await
15829 .unwrap();
15830 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15831 git::git(&repo, &["checkout", "main"]).await.unwrap();
15832 let mut runner = runner_at(RunStatus::Prep);
15833 runner.state.repo = repo.clone();
15834 runner.state.base_commit = base.clone();
15835 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15836 runner.state.config.graph.advise = false;
15837 runner.state.config.blind.commit_msg_hook = false;
15838 runner.state.sources =
15839 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15840 .await
15841 .unwrap();
15842 runner.roles.implementers = vec![spec("one"), spec("two")];
15843 runner.roles.judges = vec![spec("judge")];
15844 let label = blind::assign_labels(2, runner.state.seed)[0];
15845 let branch = runner.state.branch_for(label);
15846 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15847 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15848 .await
15849 .unwrap();
15850 runner.state.candidates.push(Candidate {
15851 index: 0,
15852 label,
15853 agent: "one".into(),
15854 branch,
15855 worktree,
15856 summary: String::new(),
15857 stat: String::new(),
15858 files: 0,
15859 commits: 0,
15860 empty: false,
15861 failed: None,
15862 verified_noop: None,
15863 duration_ms: 0,
15864 folded: false,
15865 });
15866 runner.state.save().unwrap();
15867 runner.state = RunState::load(&runner.state.id).unwrap();
15869 assert_eq!(runner.state.prep_complete, Some(false));
15870 runner.prep().await.unwrap();
15871 assert_eq!(runner.state.prep_complete, Some(true));
15872 assert_eq!(runner.state.candidates.len(), 2);
15873 for candidate in &runner.state.candidates {
15874 assert_eq!(
15875 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15876 git::tree_of(&repo, &seed).await.unwrap()
15877 );
15878 assert_eq!(
15879 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15880 "required commit"
15881 );
15882 }
15883 assert!(runner.state.worktree_root().join("judge-1").exists());
15884 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15885 runner.prep().await.unwrap();
15886 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15887
15888 runner.state.prep_complete = Some(false);
15889 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15890 run: Some(format!(
15891 "test -f '{}'",
15892 tmp.path().join("dependency-ready").display()
15893 )),
15894 ..Default::default()
15895 }];
15896 assert!(runner.prep().await.is_err());
15897 assert!(runner.state.prep_failure.is_some());
15898 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15899 runner.state = RunState::load(&runner.state.id).unwrap();
15900 assert!(
15901 runner.prep().await.is_err(),
15902 "correcting the cause alone must not auto-retry"
15903 );
15904 let failed_at = runner.state.events.last().unwrap().at;
15905 assert!(
15906 runner.retry_preparation(failed_at).is_err(),
15907 "old approvals cannot unlock the hold"
15908 );
15909 let mut task = crate::queue::Task::new(
15910 "retry".into(),
15911 "retry setup".into(),
15912 repo.clone(),
15913 crate::queue::Source::Human,
15914 );
15915 task.start(runner.state.id.clone());
15916 task.hold_machine(Some("setup failed".into()));
15917 task.release_operator("cli", None);
15918 runner
15919 .retry_preparation(task.resumption.as_ref().unwrap().at)
15920 .unwrap();
15921 runner.prep().await.unwrap();
15922 assert!(runner.state.prep_failure.is_none());
15923 assert_eq!(runner.state.prep_complete, Some(true));
15924 assert_eq!(runner.state.candidates.len(), 2);
15925 assert!(runner.state.worktree_root().join("judge-1").exists());
15926 for candidate in &runner.state.candidates {
15927 assert_eq!(
15928 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15929 git::tree_of(&repo, &seed).await.unwrap()
15930 );
15931 }
15932 }
15933
15934 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15935 let mut heads = Vec::new();
15936 for candidate in candidates {
15937 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15938 }
15939 heads
15940 }
15941
15942 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15943 ask_test_home();
15944 let tmp = tempfile::tempdir().unwrap();
15945 let repo = tmp.path().join("repo");
15946 std::fs::create_dir(&repo).unwrap();
15947 init_repo(&repo);
15948 let mut runner = runner_at(RunStatus::Reviewing);
15949 runner.state.repo = repo.clone();
15950 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15951 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15952 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15953 let branch = format!("candidate-{label}");
15954 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15955 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15956 .await
15957 .unwrap();
15958 if label == 'B' {
15959 std::fs::write(worktree.join("change"), "winner").unwrap();
15960 git::rescue_commit(&worktree, "candidate change")
15961 .await
15962 .unwrap();
15963 }
15964 runner.state.candidates.push(Candidate {
15965 index,
15966 label,
15967 agent: "alpha".into(),
15968 branch,
15969 worktree,
15970 summary: "completed reply".into(),
15971 stat: String::new(),
15972 files: usize::from(label == 'B'),
15973 commits: usize::from(label == 'B'),
15974 empty: label == 'A',
15975 failed: None,
15976 verified_noop: None,
15977 duration_ms: 1,
15978 folded: false,
15979 });
15980 }
15981 runner.state.judge_skipped = true;
15982 runner.state.tally = Some(Tally {
15983 first_choice: BTreeMap::new(),
15984 borda: BTreeMap::new(),
15985 winner: 'B',
15986 rankings: 0,
15987 unanimous_initial: false,
15988 deliberated: false,
15989 changed_votes: 0,
15990 unanimous_final: false,
15991 tie_break: None,
15992 judges: 0,
15993 present: 0,
15994 quorum: 0,
15995 met_quorum: true,
15996 uncontested: Some("only B produced a change".into()),
15997 });
15998 runner.state.save().unwrap();
15999 (tmp, runner)
16000 }
16001
16002 #[tokio::test]
16003 async fn fold_losers_persists_discard_before_later_teardown_failure() {
16004 let (_tmp, mut runner) = preservation_resume_fixture().await;
16005 runner.state.candidates[0].folded = true;
16006 runner.state.config.graph.advise = true;
16007 runner.state.config.graph.advisors = 2;
16008 let repo = runner.state.repo.clone();
16009 let root = runner.state.worktree_root();
16010 let discarded = root.join("advisor-1");
16011 git::worktree_add_detached(&repo, &discarded, "main")
16012 .await
16013 .unwrap();
16014 std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
16015 let protected = root.join("advisor-2");
16016 git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
16017 .await
16018 .unwrap();
16019 std::fs::write(protected.join("keep"), "uncommitted").unwrap();
16020 runner.state.save().unwrap();
16021
16022 assert!(runner.fold_losers().await.is_err());
16023 assert!(!discarded.exists());
16024 assert!(protected.join("keep").exists());
16025 let saved = RunState::load(&runner.state.id).unwrap();
16026 assert!(saved.events.iter().any(|event| {
16027 event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
16028 }));
16029 }
16030
16031 #[tokio::test]
16032 async fn read_seat_teardown_discards_only_owned_detached_seats() {
16033 let (tmp, mut runner) = preservation_resume_fixture().await;
16034 let repo = runner.state.repo.clone();
16035 let root = runner.state.worktree_root();
16036 for name in ["advisor-3", "judge-1", "review-2"] {
16037 let path = root.join(name);
16038 git::worktree_add_detached(&repo, &path, "main")
16039 .await
16040 .unwrap();
16041 std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
16042 std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
16043 remove_read_seat_worktree(&mut runner.state, &path)
16044 .await
16045 .unwrap();
16046 assert!(!path.exists());
16047 }
16048 let outside = tmp.path().join("advisor-1");
16049 let unknown = root.join("unknown");
16050 let attached = root.join("advisor-4");
16051 git::worktree_add_detached(&repo, &outside, "main")
16052 .await
16053 .unwrap();
16054 git::worktree_add_detached(&repo, &unknown, "main")
16055 .await
16056 .unwrap();
16057 git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16058 .await
16059 .unwrap();
16060 let foreign = tmp.path().join("foreign");
16061 std::fs::create_dir(&foreign).unwrap();
16062 init_repo(&foreign);
16063 let wrong_repo = root.join("judge-2");
16064 git::worktree_add_detached(&foreign, &wrong_repo, "main")
16065 .await
16066 .unwrap();
16067 let candidate = runner.state.candidates[0].worktree.clone();
16068 for path in [outside, unknown, attached, wrong_repo, candidate] {
16069 std::fs::write(path.join("uncommitted"), "keep").unwrap();
16070 assert!(
16071 remove_read_seat_worktree(&mut runner.state, &path)
16072 .await
16073 .is_err()
16074 );
16075 assert_eq!(
16076 std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16077 "keep"
16078 );
16079 }
16080 }
16081
16082 #[tokio::test]
16083 #[cfg(unix)]
16084 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16085 for (solo, failed_checkpoint) in
16086 [(true, false), (false, false), (true, true), (false, true)]
16087 {
16088 let (_tmp, mut runner) = preservation_resume_fixture().await;
16089 if solo {
16090 runner.state.candidates.retain(|c| c.label == 'A');
16091 }
16092 runner.state.candidates[0].empty = false;
16093 runner.state.candidates[0].summary.clear();
16094 runner.state.tally = None;
16095 runner.state.judge_skipped = false;
16096 runner.state.status = RunStatus::Implementing;
16097 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16098 let mut implementer = spec("alpha");
16099 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16100 runner.roles.implementers = vec![implementer];
16101 let patch = runner.state.dir().join("artifacts/cand-A.patch");
16102 if failed_checkpoint {
16103 std::fs::create_dir_all(&patch).unwrap();
16104 }
16105 runner.state.save().unwrap();
16106 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16107 drop(pressure);
16108 runner.state = RunState::load(&runner.state.id).unwrap();
16109 assert_eq!(runner.state.status, RunStatus::Blocked);
16110 assert!(runner.state.interrupted_implementations.contains(&'A'));
16111 assert_eq!(runner.state.candidates[0].commits, 0);
16112 assert!(!runner.state.candidates[0].empty);
16113 assert!(runner.state.candidates[0].failed.is_none());
16114 if failed_checkpoint {
16115 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16116 std::fs::remove_dir(patch).unwrap();
16117 }
16118 runner.state.resource_block = None;
16119 runner.state.config.disk.min_free_bytes = 0;
16120 runner.roles.implementers[0].command = vec![
16121 "sh".into(),
16122 "-c".into(),
16123 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16124 ];
16125 runner.implement().await.unwrap();
16126 assert!(runner.state.interrupted_implementations.is_empty());
16127 assert!(runner.state.candidate_preservation.is_empty());
16128 assert_eq!(runner.state.candidates[0].commits, 1);
16129 assert!(!runner.state.candidates[0].empty);
16130 assert!(
16131 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16132 .unwrap()
16133 .contains("retried empty interrupted seat")
16134 );
16135 if !solo {
16136 assert_eq!(runner.state.candidates[1].commits, 1);
16137 }
16138 }
16139 }
16140
16141 #[tokio::test]
16142 #[cfg(unix)]
16143 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16144 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16145 let (_tmp, mut runner) = preservation_resume_fixture().await;
16146 runner.state.candidates.retain(|c| c.label == 'A');
16147 runner.state.candidates[0].empty = false;
16148 runner.state.candidates[0].summary.clear();
16149 runner.state.tally = None;
16150 runner.state.judge_skipped = false;
16151 runner.state.status = RunStatus::Implementing;
16152 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16155 let wt = runner.state.candidates[0].worktree.clone();
16156 std::fs::write(
16157 wt.join("interrupted-source"),
16158 "retain implementation work\n",
16159 )
16160 .unwrap();
16161 if committed {
16162 git::rescue_commit(&wt, "implementation checkpoint")
16163 .await
16164 .unwrap();
16165 }
16166 let lock = git::git(
16167 &wt,
16168 &[
16169 "rev-parse",
16170 "--path-format=absolute",
16171 "--git-path",
16172 "index.lock",
16173 ],
16174 )
16175 .await
16176 .unwrap();
16177 if block_rescue {
16178 std::fs::write(&lock, "fixture-owned lock").unwrap();
16179 }
16180 let mut implementer = spec("alpha");
16181 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16182 runner.roles.implementers = vec![implementer];
16183 runner.state.save().unwrap();
16184 let result = runner.implement().await;
16185 drop(pressure);
16186 assert_eq!(result.is_err(), block_rescue);
16187 assert_eq!(
16188 runner.state.resource_block.as_ref().unwrap().kind,
16189 "disk pressure"
16190 );
16191 assert!(
16192 runner
16193 .state
16194 .events
16195 .iter()
16196 .any(|e| e.message.contains("agent execution interrupted"))
16197 );
16198 assert!(
16199 runner
16200 .state
16201 .dir()
16202 .join("artifacts/candidate-A-recovery/staged.patch")
16203 .exists()
16204 );
16205 if block_rescue {
16206 let saved = RunState::load(&runner.state.id).unwrap();
16207 assert!(saved.candidate_preservation.contains_key(&'A'));
16208 assert!(!saved.candidates[0].empty);
16209 assert_eq!(
16210 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16211 "retain implementation work\n"
16212 );
16213 std::fs::remove_file(lock).unwrap();
16214 runner.state = saved;
16215 } else {
16216 assert_eq!(runner.state.status, RunStatus::Blocked);
16217 assert_eq!(runner.state.candidates[0].commits, 1);
16218 }
16219 runner.roles.implementers.clear();
16222 runner.state.config.disk.min_free_bytes = 0;
16223 runner.state.resource_block = None;
16224 runner.implement().await.unwrap();
16225 assert!(runner.state.candidate_preservation.is_empty());
16226 assert_eq!(runner.state.candidates[0].commits, 1);
16227 assert!(!runner.state.candidates[0].empty);
16228 assert!(git::is_clean(&wt).await.unwrap());
16229 assert!(
16230 crate::run::read_artifact(&runner.state, "cand-A.patch")
16231 .unwrap()
16232 .contains("retain implementation work")
16233 );
16234 }
16235 }
16236
16237 #[tokio::test]
16238 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16239 let (_tmp, mut runner) = preservation_resume_fixture().await;
16240 runner.implement().await.unwrap();
16242 assert!(runner.state.candidate_preservation.is_empty());
16243 assert!(
16244 !runner
16245 .state
16246 .dir()
16247 .join("artifacts/candidate-A-recovery")
16248 .exists()
16249 );
16250 runner.fold_losers().await.unwrap();
16251 assert!(!runner.state.candidates[0].worktree.exists());
16252 for schema in [14, crate::run::SCHEMA] {
16253 runner.state.schema = schema;
16254 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16255 runner.state.save().unwrap();
16256 runner.state = RunState::load(&runner.state.id).unwrap();
16257 runner.implement().await.unwrap();
16258 assert!(runner.state.candidate_preservation.is_empty());
16259 assert!(runner.state.candidates[0].folded);
16260 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16261 }
16262 }
16263
16264 #[tokio::test]
16265 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16266 let (_tmp, mut runner) = preservation_resume_fixture().await;
16267 runner.state.candidates.retain(|c| c.label == 'A');
16268 runner.state.tally = None;
16269 runner.state.judge_skipped = false;
16270 let candidate = &mut runner.state.candidates[0];
16271 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16272 candidate.failed = Some("deployment validation unavailable".into());
16273 std::fs::write(
16274 candidate.worktree.join("repair"),
16275 "future launcher policy\n",
16276 )
16277 .unwrap();
16278 runner
16279 .state
16280 .candidate_preservation
16281 .insert('A', "awaiting persistence".into());
16282 runner.persist_candidates().await.unwrap();
16283 let candidate = &runner.state.candidates[0];
16284 assert!(!candidate.empty);
16285 assert_eq!(candidate.commits, 1);
16286 assert_eq!(candidate.files, 1);
16287 assert!(candidate.failed.is_none());
16288 assert!(candidate.summary.contains("EPERM"));
16289 assert!(
16290 crate::run::read_artifact(&runner.state, "cand-A.patch")
16291 .unwrap()
16292 .contains("future launcher policy")
16293 );
16294 assert!(runner.state.candidate_preservation.is_empty());
16295 }
16296
16297 #[tokio::test]
16298 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16299 let (_tmp, mut runner) = preservation_resume_fixture().await;
16300 let wt = runner.state.candidates[0].worktree.clone();
16301 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16302 runner.state.schema = 14;
16303 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16304 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16305 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16306 for _ in 0..2 {
16307 assert!(runner.implement().await.is_err());
16308 assert_eq!(runner.state.status, RunStatus::Blocked);
16309 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16310 assert!(
16311 runner.state.candidates[0].empty,
16312 "old verdict must remain unusable"
16313 );
16314 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16315 assert!(runner.state.judge_skipped);
16316 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16317 assert!(
16318 crate::run::read_artifact(&runner.state, "cand-A.patch")
16319 .unwrap()
16320 .contains("owned edits")
16321 );
16322 assert!(wt.exists());
16323 assert_eq!(
16324 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16325 .await
16326 .unwrap(),
16327 1
16328 );
16329 runner.state = RunState::load(&runner.state.id).unwrap();
16330 }
16331 }
16332
16333 #[tokio::test]
16334 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16335 let (_tmp, mut runner) = preservation_resume_fixture().await;
16336 let wt = runner.state.candidates[0].worktree.clone();
16337 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16338 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16339 let index_lock = PathBuf::from(
16340 git::git(
16341 &wt,
16342 &[
16343 "rev-parse",
16344 "--path-format=absolute",
16345 "--git-path",
16346 "index.lock",
16347 ],
16348 )
16349 .await
16350 .unwrap(),
16351 );
16352 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16353 runner.state.schema = 14;
16354 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16355 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16356 assert!(runner.implement().await.is_err());
16357 assert!(runner.after_implement().is_err());
16358 assert!(runner.fold_losers().await.is_err());
16359 assert_eq!(
16360 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16361 "failed seat edits"
16362 );
16363 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16364 runner.state = RunState::load(&runner.state.id).unwrap();
16365 std::fs::remove_file(index_lock).unwrap();
16366 assert!(runner.implement().await.is_err());
16368 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16369 assert!(runner.state.candidates[0].empty);
16370 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16371 assert!(
16372 crate::run::read_artifact(&runner.state, "cand-A.patch")
16373 .unwrap()
16374 .contains("failed seat edits")
16375 );
16376 assert!(wt.exists());
16377 }
16378
16379 #[tokio::test]
16380 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16381 let (_tmp, mut runner) = preservation_resume_fixture().await;
16382 let repo = runner.state.repo.clone();
16383 let common = git::git(
16384 &repo,
16385 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16386 )
16387 .await
16388 .unwrap();
16389 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16390 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16391 assert!(runner.fold_losers().await.is_err());
16392 assert!(!runner.state.candidates[0].worktree.exists());
16393 assert!(!runner.state.candidates[0].folded);
16394 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16395 runner.state = RunState::load(&runner.state.id).unwrap();
16396 std::fs::remove_file(branch_lock).unwrap();
16397 let judge = runner.state.worktree_root().join("judge-1");
16398 git::worktree_add_detached(&repo, &judge, "main")
16399 .await
16400 .unwrap();
16401 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16402 git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16403 .await
16404 .unwrap();
16405 runner.roles.judges = vec![spec("judge")];
16406 assert!(runner.fold_losers().await.is_err());
16407 runner.state = RunState::load(&runner.state.id).unwrap();
16408 assert!(
16409 runner.state.candidates[0].folded,
16410 "candidate progress saved before judge failure"
16411 );
16412 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16413 assert!(judge.join("uncommitted").exists());
16414 git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16415 .await
16416 .unwrap();
16417 runner.fold_losers().await.unwrap();
16418 assert!(!judge.exists());
16419 runner.fold_losers().await.unwrap();
16420 }
16421
16422 #[tokio::test]
16423 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16424 let (_tmp, mut runner) = preservation_resume_fixture().await;
16425 runner.state.candidates.retain(|c| c.label == 'B');
16426 runner.state.tally = None;
16427 runner.state.judge_skipped = false;
16428 runner.state.prep_complete = Some(true);
16429 runner.state.config.graph.advise = false;
16430 runner.state.config.graph.selected_implementer = Some("alpha".into());
16431 let wt = runner.state.candidates[0].worktree.clone();
16432 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16433 runner
16434 .state
16435 .candidate_preservation
16436 .insert('B', "awaiting persistence".into());
16437 let index_lock = git::git(
16438 &wt,
16439 &[
16440 "rev-parse",
16441 "--path-format=absolute",
16442 "--git-path",
16443 "index.lock",
16444 ],
16445 )
16446 .await
16447 .unwrap();
16448 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16449 runner.state.save().unwrap();
16450 assert!(runner.execute_graph().await.is_err());
16451 assert_eq!(runner.state.status, RunStatus::Blocked);
16452 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16453 std::fs::remove_file(index_lock).unwrap();
16454 for _ in 0..2 {
16457 runner.state = RunState::load(&runner.state.id).unwrap();
16458 runner.pause.park();
16459 runner.execute_graph().await.unwrap();
16460 assert_eq!(runner.state.status, RunStatus::Implementing);
16461 assert!(runner.state.candidate_preservation.is_empty());
16462 assert_eq!(runner.state.candidates[0].commits, 2);
16463 assert!(runner.state.tally.is_none());
16464 }
16465 runner.implement().await.unwrap();
16466 assert_eq!(runner.state.status, RunStatus::Judging);
16467 runner.state.status = RunStatus::Blocked;
16469 runner.state.candidates[0].failed = Some("implementation failed".into());
16470 runner.state.save().unwrap();
16471 let error = runner.execute_graph().await.unwrap_err();
16472 assert!(
16473 error
16474 .to_string()
16475 .contains("selected implementation is held")
16476 );
16477 }
16478
16479 #[tokio::test]
16480 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16481 let (_tmp, mut runner) = preservation_resume_fixture().await;
16482 let repo = runner.state.repo.clone();
16483 let candidate = runner.state.candidates[0].clone();
16484 let registered = git::worktree_holding(&repo, &candidate.branch)
16487 .await
16488 .unwrap()
16489 .unwrap();
16490 assert_eq!(
16491 registered.canonicalize().unwrap(),
16492 candidate.worktree.canonicalize().unwrap()
16493 );
16494 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16495 assert_eq!(
16496 git::worktree_holding(&repo, &candidate.branch)
16497 .await
16498 .unwrap(),
16499 Some(registered)
16500 );
16501 assert!(
16502 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16503 "the stale registration initially keeps the branch checked out"
16504 );
16505 runner.fold_losers().await.unwrap();
16506 assert!(runner.state.candidates[0].folded);
16507 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16508 assert_eq!(
16509 git::worktree_holding(&repo, &candidate.branch)
16510 .await
16511 .unwrap(),
16512 None
16513 );
16514 runner.state = RunState::load(&runner.state.id).unwrap();
16515 runner.fold_losers().await.unwrap();
16516 }
16517
16518 #[tokio::test]
16519 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16520 let (_tmp, mut runner) = preservation_resume_fixture().await;
16521 let repo = runner.state.repo.clone();
16522 let candidate = runner.state.candidates[0].clone();
16523 let winner = runner.state.candidates[1].clone();
16524 let registered = git::worktree_holding(&repo, &candidate.branch)
16525 .await
16526 .unwrap()
16527 .unwrap();
16528 assert_eq!(
16529 registered.canonicalize().unwrap(),
16530 candidate.worktree.canonicalize().unwrap()
16531 );
16532 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16533 assert_eq!(
16534 git::worktree_holding(&repo, &candidate.branch)
16535 .await
16536 .unwrap(),
16537 Some(registered)
16538 );
16539 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16540
16541 for _ in 0..2 {
16542 fold_run(&mut runner.state, false, &crate::run::home())
16543 .await
16544 .unwrap();
16545 runner.state = RunState::load(&runner.state.id).unwrap();
16546 assert!(runner.state.candidates[0].folded);
16547 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16548 assert_eq!(
16549 git::worktree_holding(&repo, &candidate.branch)
16550 .await
16551 .unwrap(),
16552 None
16553 );
16554 assert!(!runner.state.candidates[1].folded);
16555 assert!(winner.worktree.exists());
16556 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16557 }
16558 }
16559 #[tokio::test]
16560 async fn fold_recovery_override_archives_sources_and_keeps_safety_checks() {
16561 for sidecar in [false, true] {
16562 let tmp = tempfile::tempdir().unwrap();
16563 let home = tmp.path().join("home");
16564 let repo = tmp.path().join("repo");
16565 std::fs::create_dir(&repo).unwrap();
16566 init_repo(&repo);
16567 let mut state = runner_at(RunStatus::Superseded).state;
16568 state.repo = repo.clone();
16569 state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16570 state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16571 state.driver_exited = true;
16572 state.reserve_under(&home).unwrap();
16573 git::git(
16574 &repo,
16575 &["update-ref", "refs/remotes/origin/main", &state.base_commit],
16576 )
16577 .await
16578 .unwrap();
16579 let wt = state.worktree_root().join("cand-A");
16580 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16581 .await
16582 .unwrap();
16583 std::fs::write(wt.join("independent"), "retained implementation\n").unwrap();
16584 git::commit_all(&wt, "fixture: independent source")
16585 .await
16586 .unwrap();
16587 let head = git::rev_parse(&wt, "HEAD").await.unwrap();
16588 state.candidates.push(Candidate {
16589 index: 0,
16590 label: 'A',
16591 agent: "fixture".into(),
16592 branch: "candidate".into(),
16593 worktree: wt.clone(),
16594 summary: String::new(),
16595 stat: String::new(),
16596 files: 0,
16597 commits: 0,
16598 empty: false,
16599 failed: None,
16600 verified_noop: None,
16601 duration_ms: 0,
16602 folded: false,
16603 });
16604 let mut task = crate::queue::Task::new(
16605 "task".into(),
16606 "task".into(),
16607 repo.clone(),
16608 crate::queue::Source::Human,
16609 );
16610 task.runs.push(state.id.clone());
16611 crate::queue::Queue::at(home.join("queue"))
16612 .put(&mut task)
16613 .unwrap();
16614 if sidecar {
16615 crate::pr_recovery::retain_sources(&task, &home).unwrap();
16616 }
16617 state.save_under(&home).unwrap();
16618 let reason = crate::pr_recovery::source_retention_reason(&home, &state).unwrap();
16619 assert!(
16620 fold_run(&mut state, true, &home)
16621 .await
16622 .unwrap_err()
16623 .to_string()
16624 .contains(&reason)
16625 );
16626 let mut disk = state.config.disk.clone();
16627 disk.fold_grace_secs = 0;
16628 assert_eq!(
16629 crate::clean::fold_due(
16630 &home.join("runs"),
16631 &home,
16632 &state.worktree_root(),
16633 &disk,
16634 Timestamp::now() + std::time::Duration::from_secs(10),
16635 )
16636 .await
16637 .unwrap(),
16638 (0, 0)
16639 );
16640 assert!(wt.exists());
16641 state.status = RunStatus::Blocked;
16642 state.save_under(&home).unwrap();
16643 assert!(
16644 fold_run_overriding_recovery_retention(&mut state, true, &home)
16645 .await
16646 .is_err()
16647 );
16648 state.status = RunStatus::Superseded;
16649 state.driver_exited = false;
16650 state.driver_claimed = None;
16651 state.save_under(&home).unwrap();
16652 assert!(
16653 fold_run_overriding_recovery_retention(&mut state, true, &home)
16654 .await
16655 .is_err()
16656 );
16657 state.driver_pid = Some(std::process::id());
16658 state.driver_started_at = crate::proc::process_started_at(std::process::id());
16659 state.save_under(&home).unwrap();
16660 assert!(
16661 fold_run_overriding_recovery_retention(&mut state, true, &home)
16662 .await
16663 .is_err()
16664 );
16665 state.driver_exited = true;
16666 state.driver_pid = None;
16667 state.driver_started_at = None;
16668 state.save_under(&home).unwrap();
16669 let lease = crate::driver::maintenance(&home, &state.id).unwrap();
16670 assert!(
16671 fold_run_overriding_recovery_retention(&mut state, true, &home)
16672 .await
16673 .is_err()
16674 );
16675 drop(lease);
16676 #[cfg(unix)]
16677 {
16678 let output = crate::retention::OutputLease::take(&wt, false).unwrap();
16679 assert!(
16680 fold_run_overriding_recovery_retention(&mut state, true, &home)
16681 .await
16682 .is_err()
16683 );
16684 drop(output);
16685 }
16686 let owner = crate::pr_head::owner_lock(&home, &state.id).unwrap();
16687 assert!(
16688 fold_run_overriding_recovery_retention(&mut state, true, &home)
16689 .await
16690 .is_err()
16691 );
16692 drop(owner);
16693 let checkpoint = git::git(
16694 &wt,
16695 &[
16696 "rev-parse",
16697 "--path-format=absolute",
16698 "--git-path",
16699 "CHERRY_PICK_HEAD",
16700 ],
16701 )
16702 .await
16703 .unwrap();
16704 std::fs::write(&checkpoint, &head).unwrap();
16705 assert!(
16706 fold_run_overriding_recovery_retention(&mut state, true, &home)
16707 .await
16708 .is_err()
16709 );
16710 std::fs::remove_file(checkpoint).unwrap();
16711 std::fs::write(wt.join("dirty"), "keep").unwrap();
16712 assert!(
16713 fold_run_overriding_recovery_retention(&mut state, true, &home)
16714 .await
16715 .is_err()
16716 );
16717 assert!(wt.join("dirty").exists());
16718 std::fs::remove_file(wt.join("dirty")).unwrap();
16719 let evidence = home
16720 .join("runs")
16721 .join(&state.id)
16722 .join("artifacts/fold-evidence");
16723 std::fs::create_dir_all(evidence.parent().unwrap()).unwrap();
16724 std::fs::write(&evidence, "blocked archive").unwrap();
16725 assert!(
16726 fold_run_overriding_recovery_retention(&mut state, true, &home)
16727 .await
16728 .is_err()
16729 );
16730 assert!(wt.exists());
16731 assert!(
16732 !RunState::load_under(&state.id, &home)
16733 .unwrap()
16734 .events
16735 .iter()
16736 .any(|event| event.message.contains("--override-recovery-retention"))
16737 );
16738 std::fs::remove_file(&evidence).unwrap();
16739 fold_run_overriding_recovery_retention(&mut state, true, &home)
16740 .await
16741 .unwrap();
16742 assert!(!wt.exists());
16743 assert!(!git::branch_exists(&repo, "candidate").await.unwrap());
16744 assert!(
16745 git::worktree_holding(&repo, "candidate")
16746 .await
16747 .unwrap()
16748 .is_none()
16749 );
16750 assert!(
16751 std::fs::read_to_string(evidence.join(format!("{head}.patch")))
16752 .unwrap()
16753 .contains("retained implementation")
16754 );
16755 assert!(evidence.join(format!("{head}.index")).is_file());
16756 let saved = RunState::load_under(&state.id, &home).unwrap();
16757 assert_eq!(saved.status, RunStatus::Superseded);
16758 assert!(saved.candidates[0].folded);
16759 assert!(saved.events.iter().any(|event| {
16760 event.message.contains("--override-recovery-retention")
16761 && event.message.contains(&state.id)
16762 && event.message.contains(&reason)
16763 }));
16764 assert_eq!(
16765 crate::pr_recovery::explicit_sources_retained(&home, &state.id),
16766 sidecar
16767 );
16768 }
16769 }
16770
16771 #[tokio::test]
16772 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16773 ask_test_home();
16774 let tmp = tempfile::tempdir().unwrap();
16775 let repo = tmp.path().join("repo");
16776 std::fs::create_dir(&repo).unwrap();
16777 init_repo(&repo);
16778 let mut runner = runner_at(RunStatus::Implementing);
16779 runner.state.repo = repo.clone();
16780 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16781 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16782 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16783 let branch = format!("candidate-{label}");
16784 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16785 git::worktree_add_branch(&repo, &wt, &branch, "main")
16786 .await
16787 .unwrap();
16788 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16789 git::git(&wt, &["add", "README.md"]).await.unwrap();
16790 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16791 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16792 runner.state.candidates.push(Candidate {
16793 index,
16794 label,
16795 agent: "alpha".to_owned(),
16796 branch,
16797 worktree: wt,
16798 summary: "completed implementation".to_owned(),
16799 stat: String::new(),
16800 files: 0,
16801 commits: 0,
16802 empty: false,
16803 failed: None,
16804 verified_noop: None,
16805 duration_ms: 42,
16806 folded: false,
16807 });
16808 runner
16809 .state
16810 .candidate_preservation
16811 .insert(label, "awaiting persistence".to_owned());
16812 }
16813 let wt = runner.state.candidates[0].worktree.clone();
16814 let index_lock = git::git(
16815 &wt,
16816 &[
16817 "rev-parse",
16818 "--path-format=absolute",
16819 "--git-path",
16820 "index.lock",
16821 ],
16822 )
16823 .await
16824 .unwrap();
16825 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16826 runner.state.save().unwrap();
16827 assert!(runner.persist_candidates().await.is_err());
16828 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16829 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16830 assert_eq!(
16831 runner.state.candidates[1].commits, 1,
16832 "other results are still collected"
16833 );
16834 assert!(
16835 !runner.state.candidates[0].empty,
16836 "failed rescue is not no change"
16837 );
16838 assert_eq!(
16839 runner.state.candidates[0].summary,
16840 "completed implementation"
16841 );
16842 assert!(runner.after_implement().is_err());
16843 let home = tmp.path().join("fold-home");
16844 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16845 assert!(wt.join("new-file").exists());
16846 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16847 assert!(recovery.join("staged.patch").exists());
16848 assert!(recovery.join("unstaged.patch").exists());
16849 assert_eq!(
16850 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16851 "untracked edits\n"
16852 );
16853 runner.state.begin_pass();
16854 runner.state.end_pass_parked(ParkCause::Upgrade {
16855 from: "1.0".into(),
16856 to: Some("1.1".into()),
16857 });
16858 runner.state.save().unwrap();
16859 runner.state = RunState::load(&runner.state.id).unwrap();
16860 assert!(matches!(
16861 runner.state.passes.last().unwrap().end,
16862 Some(crate::run::PassEnd::Parked {
16863 cause: ParkCause::Upgrade { .. }
16864 })
16865 ));
16866 std::fs::remove_file(index_lock).unwrap();
16867 runner.implement().await.unwrap();
16869 assert!(runner.state.candidate_preservation.is_empty());
16870 assert!(
16871 runner
16872 .state
16873 .candidates
16874 .iter()
16875 .all(|c| c.commits == 1 && !c.empty)
16876 );
16877 assert!(git::is_clean(&wt).await.unwrap());
16878 assert!(
16879 crate::run::read_artifact(&runner.state, "cand-A.patch")
16880 .unwrap()
16881 .contains("later tracked edits")
16882 );
16883 }
16884
16885 #[tokio::test]
16886 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16887 ask_test_home();
16888 let tmp = tempfile::tempdir().unwrap();
16889 let repo = tmp.path().join("repo");
16890 std::fs::create_dir(&repo).unwrap();
16891 init_repo(&repo);
16892 let wt = tmp.path().join("candidate");
16893 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16894 .await
16895 .unwrap();
16896 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16897 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16898 assert!(wt.join("uncommitted").exists());
16899 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16900 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16901 let mut runner = runner_at(RunStatus::Implementing);
16902 runner.state.repo = repo.clone();
16903 runner.state.base_commit = "missing-base".to_owned();
16904 runner.state.candidates.push(Candidate {
16905 index: 0,
16906 label: 'A',
16907 agent: "alpha".into(),
16908 branch: "candidate".into(),
16909 worktree: wt.clone(),
16910 summary: "completed".into(),
16911 stat: String::new(),
16912 files: 0,
16913 commits: 0,
16914 empty: false,
16915 failed: None,
16916 verified_noop: None,
16917 duration_ms: 1,
16918 folded: false,
16919 });
16920 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16921 assert!(
16922 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16923 .await
16924 .is_err()
16925 );
16926 assert!(!runner.state.candidates[0].folded);
16927 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16928 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16929 runner
16930 .state
16931 .candidate_preservation
16932 .insert('A', "pending".into());
16933 assert!(runner.persist_candidates().await.is_err());
16934 assert!(!runner.state.candidates[0].empty);
16935 assert!(!runner.state.candidates[0].folded);
16936 assert!(wt.exists());
16937 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16938 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16939 std::fs::create_dir_all(&patch_path).unwrap();
16940 assert!(
16941 runner.persist_candidates().await.is_err(),
16942 "artifact failure stays pending"
16943 );
16944 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16945 assert!(!runner.state.candidates[0].empty);
16946 std::fs::remove_dir(patch_path).unwrap();
16947 runner.persist_candidates().await.unwrap();
16948 assert!(
16949 runner.state.candidates[0].empty,
16950 "only successful inspection confirms no change"
16951 );
16952 }
16953
16954 fn init_repo(dir: &Path) {
16955 let run = |args: &[&str]| {
16956 let out = std::process::Command::new("git")
16957 .args(args)
16958 .current_dir(dir)
16959 .quiet()
16960 .output()
16961 .expect("spawn git");
16962 assert!(
16963 out.status.success(),
16964 "git {args:?} failed: {}",
16965 String::from_utf8_lossy(&out.stderr)
16966 );
16967 };
16968 run(&["init", "-b", "main"]);
16969 run(&["config", "user.name", "magi test"]);
16970 run(&["config", "user.email", "magi@example.com"]);
16971 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16972 run(&["add", "-A"]);
16973 run(&["commit", "-m", "init"]);
16974 }
16975
16976 fn ask_test_home() {
16984 crate::run::pin_test_home();
16985 }
16986
16987 #[test]
16988 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16989 ask_test_home();
16990 let mut runner = runner_at(RunStatus::Prep);
16991 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16992 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16995 let error = guard
16996 .finish(
16997 &mut runner.state,
16998 Err(
16999 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
17000 .into(),
17001 ),
17002 )
17003 .unwrap_err();
17004 assert_eq!(
17005 error.downcast_ref::<std::io::Error>().unwrap().kind(),
17006 std::io::ErrorKind::PermissionDenied
17007 );
17008 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
17009 let message = format!("{error:#}");
17010 assert!(message.contains("original refusal"));
17011 assert!(!message.contains("could not save terminal"));
17012 assert_eq!(
17013 error.chain().count(),
17014 2,
17015 "only startup attribution wraps the original error"
17016 );
17017 let saved = RunState::load(&runner.state.id).unwrap();
17018 assert_eq!(saved.status, RunStatus::Blocked);
17019 assert!(!saved.startup.unwrap().complete);
17020 }
17021
17022 #[test]
17023 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
17024 ask_test_home();
17025 let mut runner = runner_at(RunStatus::Prep);
17026 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17027 let id = runner.state.id.clone();
17028 drop(guard);
17029 let saved = RunState::load(&id).unwrap();
17030 assert_eq!(saved.status, RunStatus::Blocked);
17031 assert!(!saved.startup.as_ref().unwrap().complete);
17032 assert!(
17033 saved
17034 .startup
17035 .unwrap()
17036 .failure
17037 .unwrap()
17038 .contains("interrupted")
17039 );
17040 assert!(Runner::resume(&id).is_err());
17041 }
17042
17043 #[test]
17044 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
17045 ask_test_home();
17046 let mut runner = runner_at(RunStatus::Prep);
17047 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17048 let error = guard
17049 .finish(
17050 &mut runner.state,
17051 Err(
17052 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
17053 .into(),
17054 ),
17055 )
17056 .unwrap_err();
17057 assert_eq!(
17058 error.downcast_ref::<std::io::Error>().unwrap().kind(),
17059 std::io::ErrorKind::PermissionDenied
17060 );
17061 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
17062 let mut task = crate::queue::Task::new(
17063 "review".into(),
17064 "criteria".into(),
17065 runner.state.repo.clone(),
17066 crate::queue::Source::Human,
17067 );
17068 task.record_startup_failure(failure.clone());
17069 task.record_startup_failure(failure.clone());
17070 assert_eq!(task.startup_failures.len(), 1);
17071 assert!(task.runs.is_empty());
17072 assert_eq!(task.attempts, 0);
17073 }
17074
17075 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
17076 ask_test_home();
17077 let tmp = tempfile::tempdir().unwrap();
17078 let repo = tmp.path().join("repo");
17079 std::fs::create_dir(&repo).unwrap();
17080 init_repo(&repo);
17081 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17082 let origin = tmp.path().join("origin.git");
17083 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
17084 .await
17085 .unwrap();
17086 git::git(
17087 &repo,
17088 &["remote", "add", "origin", origin.to_str().unwrap()],
17089 )
17090 .await
17091 .unwrap();
17092 git::git(&repo, &["checkout", "-b", "fix/retained"])
17093 .await
17094 .unwrap();
17095 std::fs::write(repo.join("retained"), "retained change").unwrap();
17096 git::git(&repo, &["add", "."]).await.unwrap();
17097 git::git(&repo, &["commit", "-m", "retained implementation"])
17098 .await
17099 .unwrap();
17100 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
17101 git::git(&repo, &["checkout", "main"]).await.unwrap();
17102 std::fs::write(repo.join("base-change"), "new base").unwrap();
17103 git::git(&repo, &["add", "."]).await.unwrap();
17104 git::git(&repo, &["commit", "-m", "base moved"])
17105 .await
17106 .unwrap();
17107 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17108 git::git(&repo, &["checkout", "fix/retained"])
17109 .await
17110 .unwrap();
17111 let mut runner = runner_at(RunStatus::Voting);
17112 runner.state.repo = repo.clone();
17113 runner.state.base_commit = base;
17114 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17115 runner.state.config.disk.min_free_bytes = 0;
17116 runner.state.config.graph.review_rounds = 1;
17117 runner.state.config.verify.e2e = vec!["true".into()];
17118 runner.state.config.verify.gate = vec!["true".into()];
17119 runner.state.candidates = vec![Candidate {
17120 index: 0,
17121 label: 'A',
17122 agent: "fixture".into(),
17123 branch: "fix/retained".into(),
17124 worktree: repo.clone(),
17125 summary: String::new(),
17126 stat: String::new(),
17127 files: 1,
17128 commits: 1,
17129 empty: false,
17130 failed: None,
17131 verified_noop: None,
17132 duration_ms: 0,
17133 folded: false,
17134 }];
17135 runner.state.tally = Some(Tally {
17136 first_choice: BTreeMap::new(),
17137 borda: BTreeMap::new(),
17138 winner: 'A',
17139 rankings: 0,
17140 unanimous_initial: true,
17141 deliberated: false,
17142 changed_votes: 0,
17143 unanimous_final: true,
17144 tie_break: None,
17145 judges: 0,
17146 present: 0,
17147 quorum: 0,
17148 met_quorum: true,
17149 uncontested: Some("review-only".into()),
17150 });
17151 let mut task = crate::queue::Task::new(
17152 "fixture".into(),
17153 "fixture".into(),
17154 repo,
17155 crate::queue::Source::Human,
17156 );
17157 task.link_run(&runner.state.id);
17158 crate::queue::Queue::open().put(&mut task).unwrap();
17159 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
17160 runner.state.base_sync = Some(BaseSync {
17161 tip: String::new(),
17162 behind: 1,
17163 attempts: BASE_SYNC_ROUNDS,
17164 conflict: None,
17165 already_in: None,
17166 });
17167 runner.sync_to_base().await.unwrap();
17168 assert_eq!(runner.state.status, RunStatus::Blocked);
17169 (tmp, runner, head, task.id)
17170 }
17171
17172 #[tokio::test]
17173 async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
17174 let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
17175 runner.state.status = RunStatus::Voting;
17176 runner.state.base_sync = None;
17177 runner.state.base_sync_recovery = Default::default();
17178 let scratch = runner.state.dir().join("base-sync");
17179 std::fs::create_dir_all(&scratch).unwrap();
17180 std::fs::write(scratch.join("obstruction"), "keep").unwrap();
17181 runner.sync_to_base().await.unwrap();
17182 assert!(runner.state.resource_block.is_some());
17183 assert!(!runner.state.base_sync_blocked());
17184 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
17185 assert_eq!(runner.state.base_episode_used(), 0);
17186 assert!(runner.state.base_sync_recovery.operations.is_empty());
17187 assert!(runner.state.rebase_fixes.is_empty());
17188 assert_eq!(
17189 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17190 head
17191 );
17192 std::fs::remove_dir_all(&scratch).unwrap();
17193 runner.state.resource_block = None;
17194 runner.sync_to_base().await.unwrap();
17195 assert!(!runner.state.base_sync_blocked());
17196 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17197 assert_eq!(runner.state.base_episode_used(), 1);
17198 assert!(runner.state.base_sync_recovery.operations[0].finished);
17199 assert!(runner.state.rebase_fixes.is_empty());
17200 assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
17201 assert!(git::is_clean(&runner.state.repo).await.unwrap());
17202 }
17203
17204 #[cfg(unix)]
17205 #[tokio::test]
17206 async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
17207 for routed in [false, true] {
17208 let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
17209 let home = crate::run::home();
17210 let queue = crate::queue::Queue::open();
17211 let task = queue.get(&owner).unwrap();
17212 runner.state.task_instruction = Some(task.instruction.clone());
17213 runner.state.driver_exited = true;
17214 runner.state.save().unwrap();
17215 let questions = crate::ask::Questions::at(home.join("questions"));
17216 let mut q = crate::ask::Question::new(
17217 runner.state.id.clone(),
17218 "base-recovery".into(),
17219 "owner".into(),
17220 "Recover bounded base movement?".into(),
17221 String::new(),
17222 vec!["recover".into()],
17223 );
17224 q.actions.insert(
17225 "recover".into(),
17226 crate::ask::ChoiceAction::RecoverBase {
17227 run: runner.state.id.clone(),
17228 stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
17229 episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
17230 },
17231 );
17232 q.routed_task = Some(owner.clone());
17233 q.answer(crate::ask::Answer::Choice("recover".into()))
17234 .unwrap();
17235 questions.put(&mut q).unwrap();
17236 crate::base_recovery_control::recover(
17237 &queue,
17238 &home,
17239 crate::base_recovery_control::Request {
17240 run: runner.state.id.clone(),
17241 head: head.clone(),
17242 actor: "decisions".into(),
17243 request: q.id.clone(),
17244 decision: Some((owner.clone(), q.answered_at.unwrap())),
17245 expected_stop: Some((
17246 runner.state.base_sync_recovery.stop.clone().unwrap(),
17247 runner.state.base_sync_recovery.episodes.last().unwrap().id,
17248 )),
17249 },
17250 )
17251 .await
17252 .unwrap();
17253 if routed {
17254 questions
17255 .update(&q.id, |q| {
17256 q.routed_task = Some("another-owner".into());
17257 Ok(())
17258 })
17259 .unwrap();
17260 } else {
17261 let mut task = queue.get(&owner).unwrap();
17262 task.hold_manual(Some("new owner decision".into()));
17263 task.status = crate::queue::TaskStatus::Queued;
17265 queue.put(&mut task).unwrap();
17266 }
17267 runner.state = RunState::load(&runner.state.id).unwrap();
17268 runner.sync_to_base().await.unwrap();
17269 let approval = runner
17270 .state
17271 .base_sync_recovery
17272 .authorizations
17273 .last()
17274 .unwrap();
17275 assert!(approval.consumed);
17276 assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17277 "routing"
17278 } else {
17279 "owner decision"
17280 }));
17281 assert_eq!(
17282 runner.state.base_sync.as_ref().unwrap().attempts,
17283 BASE_SYNC_ROUNDS
17284 );
17285 assert_eq!(
17286 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17287 head
17288 );
17289 }
17290 }
17291
17292 #[cfg(unix)]
17293 #[tokio::test]
17294 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17295 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17296 if std::env::var_os(CHILD).is_none() {
17297 let home = tempfile::tempdir().unwrap();
17298 let output = std::process::Command::new(std::env::current_exe().unwrap())
17299 .args([
17300 "--exact",
17301 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17302 "--nocapture",
17303 ])
17304 .env(CHILD, "1")
17305 .env("MAGI_HOME", home.path())
17306 .env("MAGI_WORKER_HOME", home.path())
17307 .env("MAGI_WORKER_ID", "abcdef")
17308 .output()
17309 .unwrap();
17310 assert!(
17311 output.status.success()
17312 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17313 "{}\n{}",
17314 String::from_utf8_lossy(&output.stdout),
17315 String::from_utf8_lossy(&output.stderr)
17316 );
17317 return;
17318 }
17319 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17320 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17321 let home = crate::run::home();
17322 let queue = crate::queue::Queue::open();
17323 let mut owner = queue.get(&task).unwrap();
17324 owner.release_operator("test-release", None);
17325 owner.start(runner.state.id.clone());
17326 queue.put(&mut owner).unwrap();
17327 runner.state.driver_exited = true;
17328 runner.state.task_instruction = Some(owner.instruction.clone());
17329 runner.state.save().unwrap();
17330 let _claim = queue.claim(&task).unwrap();
17331 let reserve = |id: &str| {
17332 let directory = home.join("workers/jobs").join(id);
17333 std::fs::create_dir_all(&directory).unwrap();
17334 let job = crate::worker::Job {
17335 id: id.into(),
17336 sequence: 1,
17337 kind: "task".into(),
17338 key: format!("task:{task}"),
17339 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17340 generation: "fixture-generation".into(),
17341 version: None,
17342 status: crate::worker::Status::Running,
17343 result: None,
17344 pid: Some(std::process::id()),
17345 created_at: 0,
17346 finished_at: None,
17347 };
17348 std::fs::write(
17349 directory.join("job.json"),
17350 serde_json::to_vec(&job).unwrap(),
17351 )
17352 .unwrap();
17353 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17354 };
17355 let _own = reserve("abcdef");
17356 let other = reserve("fedcba");
17357 assert!(
17358 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17359 .await
17360 .is_err()
17361 );
17362 assert!(
17363 RunState::load(&runner.state.id)
17364 .unwrap()
17365 .base_sync_recovery
17366 .authorizations
17367 .is_empty()
17368 );
17369 drop(other);
17370 assert!(crate::daemon::is_working_on(
17371 &home,
17372 &runner.state.id,
17373 Timestamp::now()
17374 ));
17375 assert!(
17376 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17377 );
17378 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17379 .await
17380 .unwrap();
17381 assert!(
17382 RunState::load(&runner.state.id)
17383 .unwrap()
17384 .base_recovery_pending()
17385 );
17386 }
17387
17388 #[cfg(unix)]
17389 #[tokio::test]
17390 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17391 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17392 let home = crate::run::home();
17393 runner.state.driver_pid = Some(std::process::id());
17394 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17395 runner.state.driver_exited = false;
17396 runner.state.begin_pass();
17397 runner.state.save().unwrap();
17398 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17399 driver.record(&runner.state).unwrap();
17400 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17401 drop(driver);
17403 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17404 assert_eq!(
17405 runner.state.liveness_under(&home, false),
17406 crate::run::Liveness::Dead
17407 );
17408 let queue = crate::queue::Queue::open();
17409 let _claim = queue.claim(&task).unwrap();
17410 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17411 .await
17412 .unwrap();
17413 let disk = RunState::load(&runner.state.id).unwrap();
17414 assert!(!disk.driver_exited);
17415 assert!(disk.base_recovery_pending());
17416 }
17417
17418 #[cfg(unix)]
17419 #[tokio::test]
17420 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17421 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17422 if std::env::var_os(CHILD).is_none() {
17423 use std::os::unix::fs::PermissionsExt;
17424 let tmp = tempfile::tempdir().unwrap();
17425 let gh = tmp.path().join("gh");
17426 std::fs::write(&gh, r#"#!/bin/sh
17427case "$1 $2" in
17428 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17429 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17430 *) exit 2 ;;
17431esac
17432"#).unwrap();
17433 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17434 let mut paths = vec![tmp.path().to_path_buf()];
17435 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17436 let output = std::process::Command::new(std::env::current_exe().unwrap())
17437 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17438 .env(CHILD, "1")
17439 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17440 .env("PATH", std::env::join_paths(paths).unwrap())
17441 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17442 .output().unwrap();
17443 assert!(
17444 output.status.success()
17445 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17446 "{}\n{}",
17447 String::from_utf8_lossy(&output.stdout),
17448 String::from_utf8_lossy(&output.stderr)
17449 );
17450 return;
17451 }
17452 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17453 let repo = runner.state.repo.clone();
17454 git::git(&repo, &["checkout", "main"]).await.unwrap();
17455 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17456 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17457 git::git(&repo, &["checkout", "fix/retained"])
17458 .await
17459 .unwrap();
17460 runner.state.pr = Some(crate::run::PrRecord {
17461 url: "https://example.test/project/pull/1".into(),
17462 number: 1,
17463 state: "open".into(),
17464 checks: "unknown".into(),
17465 round: 0,
17466 rounds: 1,
17467 red_at_merge: Vec::new(),
17468 });
17469 runner.sync_to_base().await.unwrap();
17470 assert_eq!(runner.state.status, RunStatus::Blocked);
17471 assert_eq!(
17472 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17473 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17474 );
17475 assert!(runner.state.base_sync_closure_pending());
17476 assert!(crate::handover::can_resume(&runner.state));
17477 let queue = crate::queue::Queue::open();
17478 assert!(!crate::base_sync_recovery::held_task(
17479 &queue,
17480 &queue.get(&task).unwrap()
17481 ));
17482 assert!(runner.state.authorize_base_sync(&task).is_err());
17483 runner.state.base_sync_recovery.stop = None;
17485 runner.state.status = RunStatus::Voting;
17486 runner.state.gate_ran = true;
17487 runner.state.config.graph.review_rounds = 0;
17488 runner.state.save().unwrap();
17489 assert!(crate::handover::can_resume(&runner.state));
17490 runner.state = RunState::load(&runner.state.id).unwrap();
17491 runner.execute_graph().await.unwrap();
17492 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17493 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17494 assert_eq!(
17495 runner.state.base_sync.as_ref().unwrap().attempts,
17496 BASE_SYNC_ROUNDS
17497 );
17498 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17499 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17500 assert!(runner.state.reviews.is_empty());
17501 assert!(runner.state.gate.is_empty());
17502 }
17503
17504 #[cfg(unix)]
17505 #[tokio::test]
17506 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17507 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17508 runner.state.driver_exited = true;
17509 runner.state.save().unwrap();
17510 let home = crate::run::home();
17511 let queue = crate::queue::Queue::open();
17512 let _claim = queue.claim(&task).unwrap();
17513 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17514 assert!(
17515 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17516 .await
17517 .is_err()
17518 );
17519 drop(lease);
17520 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17521 assert!(
17522 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17523 .await
17524 .is_err()
17525 );
17526 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17527 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17528 .await
17529 .unwrap();
17530 let disk = RunState::load(&runner.state.id).unwrap();
17531 assert!(disk.base_recovery_pending());
17532 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17533 assert!(
17534 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17535 .await
17536 .is_err()
17537 );
17538 assert_eq!(
17539 RunState::load(&runner.state.id)
17540 .unwrap()
17541 .base_sync_recovery
17542 .authorizations
17543 .len(),
17544 1
17545 );
17546 let mut owner = queue.get(&task).unwrap();
17547 owner.runs.push("20260101-000000-ffff".into());
17548 queue.put(&mut owner).unwrap();
17549 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17550 .await
17551 .unwrap_err();
17552 assert!(error.to_string().contains("run task ownership changed"));
17553 }
17554
17555 #[cfg(unix)]
17556 #[tokio::test]
17557 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17558 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17559 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17560 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17561 legacy["status"] = serde_json::json!("voting");
17562 runner.state = serde_json::from_value(legacy).unwrap();
17563 runner.state.save().unwrap();
17564 for _ in 0..3 {
17565 runner.execute_graph().await.unwrap();
17566 assert_eq!(runner.state.status, RunStatus::Blocked);
17567 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17568 assert!(!crate::handover::can_resume(&runner.state));
17569 assert_eq!(
17570 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17571 head
17572 );
17573 assert!(runner.state.reviews.is_empty());
17574 assert!(!runner.state.gate_ran);
17575 }
17576 let notice = crate::notices::run_ended(&runner.state).unwrap();
17577 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17578 runner.state.begin_pass();
17579 runner.state.status = RunStatus::Voting;
17580 runner.deliberate().await.unwrap();
17581 runner.vote().await.unwrap();
17582 runner.finish_after_tally().await.unwrap();
17583 assert_eq!(runner.state.status, RunStatus::Blocked);
17584 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17585 assert_eq!(
17586 crate::notices::run_ended(&runner.state).unwrap().since,
17587 notice.since
17588 );
17589 }
17590
17591 #[cfg(unix)]
17592 #[tokio::test]
17593 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17594 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17595 for detail in [
17596 "fix/retained conflicts with origin/main and did not rebase: conflict",
17597 "rebased fix/retained locally but origin refused the push",
17598 "unknown failure",
17599 ] {
17600 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17601 runner.state.record_base_sync_stop(&head);
17602 assert!(runner.state.authorize_base_sync(&task).is_err());
17603 runner.state.status = RunStatus::Voting;
17604 runner.execute_graph().await.unwrap();
17605 assert_eq!(runner.state.status, RunStatus::Blocked);
17606 assert!(!crate::handover::can_resume(&runner.state));
17607 assert_eq!(
17608 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17609 head
17610 );
17611 }
17612 }
17613
17614 #[cfg(unix)]
17615 #[tokio::test]
17616 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17617 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17618 runner.state.authorize_base_sync(&task).unwrap();
17619 assert!(runner.state.consume_base_recovery(&head));
17620 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17621 runner.state.reserve_base_operation(&head, &tip);
17622 runner.state.save().unwrap();
17623 runner.state = RunState::load(&runner.state.id).unwrap();
17624 assert!(!runner.state.consume_base_recovery(&head));
17625 runner.sync_to_base().await.unwrap();
17626 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17627 assert_eq!(runner.state.base_episode_used(), 1);
17628 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17629 assert!(
17630 runner
17631 .state
17632 .base_sync_recovery
17633 .operations
17634 .last()
17635 .unwrap()
17636 .finished
17637 );
17638 assert!(!runner.state.base_sync_blocked());
17639 runner.sync_to_base().await.unwrap();
17640 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17641 }
17642
17643 #[cfg(unix)]
17644 #[tokio::test]
17645 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17646 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17647 let original = runner
17649 .state
17650 .base_sync
17651 .as_ref()
17652 .unwrap()
17653 .conflict
17654 .clone()
17655 .unwrap();
17656 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17657 original
17658 .replace("synchronization(s)", "rebase(s)")
17659 .replace("synchronizing", "rebasing"),
17660 );
17661 runner.state.base_sync_recovery = Default::default();
17662 runner.state.record_base_sync_stop(&head);
17663 assert_eq!(
17664 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17665 crate::base_sync_recovery::StopKind::Exhausted
17666 );
17667 runner.state.authorize_base_sync(&task).unwrap();
17668 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17669 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17670 .as_object_mut()
17671 .unwrap();
17672 auth.remove("interval");
17673 auth.remove("scope");
17674 runner.state = serde_json::from_value(legacy).unwrap();
17675 assert!(runner.state.consume_base_recovery(&head));
17676 assert_eq!(runner.state.base_episode_used(), 3);
17677 runner.sync_to_base().await.unwrap();
17678 assert_eq!(runner.state.base_episode_used(), 4);
17679 runner.state.resource_block = Some(crate::run::ResourceBlock {
17680 node: "gate".into(),
17681 kind: "cache".into(),
17682 detail: "legacy wait".into(),
17683 at: Timestamp::now(),
17684 cause: crate::run::ResourceBlockCause::Unrecorded,
17685 });
17686 runner.state.save().unwrap();
17687 assert!(
17688 runner
17689 .state
17690 .base_sync_recovery
17691 .episodes
17692 .last()
17693 .unwrap()
17694 .boundary
17695 .is_none()
17696 );
17697 runner.state.resource_block = None;
17698 advance_base_fixture(&runner).await;
17699 runner.sync_to_base().await.unwrap();
17700 assert!(runner.state.base_sync_blocked());
17701 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17702 }
17703
17704 #[cfg(unix)]
17705 async fn advance_base_fixture(runner: &Runner) {
17706 let repo = &runner.state.repo;
17707 for args in [
17708 vec!["checkout", "main"],
17709 vec!["commit", "--allow-empty", "-m", "base advanced"],
17710 vec!["push", "origin", "main"],
17711 vec!["checkout", "fix/retained"],
17712 ] {
17713 git::git(repo, &args).await.unwrap();
17714 }
17715 }
17716
17717 #[cfg(unix)]
17718 fn fresh_base_interval_fixture(runner: &mut Runner) {
17719 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17720 runner.state.base_sync.as_mut().unwrap().conflict = None;
17721 runner.state.base_sync_recovery = Default::default();
17722 runner.state.status = RunStatus::Voting;
17723 }
17724
17725 #[cfg(unix)]
17726 #[tokio::test]
17727 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17728 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17729 fresh_base_interval_fixture(&mut runner);
17730 runner.state.begin_pass();
17731 runner.sync_to_base().await.unwrap();
17732 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17733 runner.state.resource_block = Some(crate::run::ResourceBlock {
17734 node: "gate".into(),
17735 kind: "build cache".into(),
17736 detail: "busy".into(),
17737 at: Timestamp::now(),
17738 cause: crate::run::ResourceBlockCause::Timeout {
17739 waited_secs: 1,
17740 limit_secs: 1,
17741 },
17742 });
17743 runner.state.save().unwrap();
17744 let queue = crate::queue::Queue::open();
17745 let mut owner = queue.get(&task).unwrap();
17746 runner.state.task_instruction = Some(owner.instruction.clone());
17747 owner.release_operator("fixture-owner", None);
17748 queue.put(&mut owner).unwrap();
17749 let approval = owner.resumption.clone().unwrap();
17750 runner.state.resource_block = None;
17751 runner.state.save().unwrap();
17752 runner.state = RunState::load(&runner.state.id).unwrap();
17753 advance_base_fixture(&runner).await;
17754 runner.sync_to_base().await.unwrap();
17755 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17756 assert_eq!(runner.state.base_episode_used(), 1);
17757 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17758 assert!(
17759 runner.state.base_sync_recovery.episodes[0]
17760 .boundary
17761 .as_ref()
17762 .unwrap()
17763 .consumed
17764 );
17765 assert!(
17766 !runner
17767 .state
17768 .renew_base_episode(&task, &approval, &head, "base")
17769 );
17770 for _ in 0..3 {
17771 owner.release_operator("fixture-owner", None);
17772 queue.put(&mut owner).unwrap();
17773 runner.state = RunState::load(&runner.state.id).unwrap();
17774 runner.state.begin_pass();
17775 runner.sync_to_base().await.unwrap();
17776 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17777 assert_eq!(runner.state.base_episode_used(), 1);
17778 }
17779 runner.state.resource_block = Some(crate::run::ResourceBlock {
17780 node: "gate".into(),
17781 kind: "disk pressure".into(),
17782 detail: "free space required".into(),
17783 at: Timestamp::now(),
17784 cause: crate::run::ResourceBlockCause::Other,
17785 });
17786 runner.state.end_pass_parked(ParkCause::Interrupt {
17787 reason: "disk pressure".into(),
17788 });
17789 runner.state.save().unwrap();
17790 owner.release_operator("fixture-owner-after-disk", None);
17791 queue.put(&mut owner).unwrap();
17792 runner.state.resource_block = None;
17793 advance_base_fixture(&runner).await;
17794 runner.sync_to_base().await.unwrap();
17795 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17796 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17797 assert_eq!(runner.state.base_episode_used(), 1);
17798 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17799 }
17800
17801 #[cfg(unix)]
17802 #[tokio::test]
17803 async fn base_recovery_continuous_advancement_stops_at_four() {
17804 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17805 fresh_base_interval_fixture(&mut runner);
17806 for spent in 1..=BASE_SYNC_ROUNDS {
17807 runner.sync_to_base().await.unwrap();
17808 assert!(!runner.state.base_sync_blocked());
17809 assert_eq!(runner.state.base_episode_used(), spent);
17810 advance_base_fixture(&runner).await;
17811 }
17812 runner.sync_to_base().await.unwrap();
17813 assert_eq!(
17814 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17815 crate::base_sync_recovery::StopKind::Exhausted
17816 );
17817 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17818 assert!(stop.unverified_sync);
17819 let action = runner.state.base_sync_next_action();
17820 assert!(action.contains("never reached its gate"));
17821 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17822 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17823 runner.state = RunState::load(&runner.state.id).unwrap();
17824 runner.sync_to_base().await.unwrap();
17825 assert_eq!(runner.state.base_episode_used(), 4);
17826 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17827 }
17828
17829 #[cfg(unix)]
17830 #[tokio::test]
17831 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17832 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17833 runner.state.ensure_base_episode();
17834 runner.state.record_base_progress(&head);
17836 runner.state.base_sync_recovery.stop = None;
17837 runner.state.record_base_sync_stop(&head);
17838 assert!(
17839 !runner
17840 .state
17841 .base_sync_recovery
17842 .stop
17843 .as_ref()
17844 .unwrap()
17845 .unverified_sync
17846 );
17847
17848 runner.state.finish_base_operation(Some(&head));
17849 runner.state.gate_ran = true;
17851 runner.state.gate_head = Some("older-head".into());
17852 runner.state.base_sync_recovery.stop = None;
17853 runner.state.record_base_sync_stop(&head);
17854 runner.state.save().unwrap();
17855 runner.state = RunState::load(&runner.state.id).unwrap();
17856 assert!(
17857 runner
17858 .state
17859 .base_sync_recovery
17860 .stop
17861 .as_ref()
17862 .unwrap()
17863 .unverified_sync
17864 );
17865
17866 runner.state.gate_head = Some(head.clone());
17867 runner.state.base_sync_recovery.stop = None;
17868 runner.state.record_base_sync_stop(&head);
17869 assert!(
17870 !runner
17871 .state
17872 .base_sync_recovery
17873 .stop
17874 .as_ref()
17875 .unwrap()
17876 .unverified_sync
17877 );
17878 assert!(
17879 !runner
17880 .state
17881 .base_sync_next_action()
17882 .contains("never reached its gate")
17883 );
17884
17885 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17887 legacy["base_sync_recovery"]["stop"]
17888 .as_object_mut()
17889 .unwrap()
17890 .remove("unverified_sync");
17891 legacy["base_sync_recovery"]["episodes"][0]
17892 .as_object_mut()
17893 .unwrap()
17894 .remove("last_sync_head");
17895 runner.state = serde_json::from_value(legacy).unwrap();
17896 assert!(
17897 !runner
17898 .state
17899 .base_sync_recovery
17900 .stop
17901 .as_ref()
17902 .unwrap()
17903 .unverified_sync
17904 );
17905 assert!(
17906 runner.state.base_sync_recovery.episodes[0]
17907 .last_sync_head
17908 .is_none()
17909 );
17910 }
17911
17912 #[cfg(unix)]
17913 #[tokio::test]
17914 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17915 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17916 runner.state.authorize_base_sync(&task).unwrap();
17917 assert!(
17918 runner
17919 .state
17920 .base_sync_next_action()
17921 .contains("Approval recorded")
17922 );
17923 assert!(
17924 runner
17925 .state
17926 .base_sync_next_action()
17927 .contains("4/4 in episode 0, 4 lifetime")
17928 );
17929 assert!(runner.state.consume_base_recovery(&head));
17930 assert!(
17931 runner
17932 .state
17933 .base_sync_next_action()
17934 .contains("0/4 in episode 1, 4 lifetime")
17935 );
17936 }
17937
17938 #[cfg(unix)]
17939 #[tokio::test]
17940 async fn base_recovery_changed_head_invalidates_without_spending() {
17941 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17942 runner.state.authorize_base_sync(&task).unwrap();
17943 git::git(
17944 &runner.state.repo,
17945 &["commit", "--allow-empty", "-m", "external head change"],
17946 )
17947 .await
17948 .unwrap();
17949 runner.sync_to_base().await.unwrap();
17950 let a = runner
17951 .state
17952 .base_sync_recovery
17953 .authorizations
17954 .last()
17955 .unwrap();
17956 assert!(a.consumed);
17957 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17958 runner.state = RunState::load(&runner.state.id).unwrap();
17959 assert!(
17960 runner
17961 .state
17962 .base_sync_next_action()
17963 .contains("Approval invalidated")
17964 );
17965 assert!(!runner.state.base_recovery_pending());
17966 assert!(runner.state.base_sync_blocked());
17967 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17968 }
17969
17970 #[cfg(unix)]
17971 #[tokio::test]
17972 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17973 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17974 runner.state.driver_exited = true;
17975 runner.state.save().unwrap();
17976 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17977 .await
17978 .unwrap();
17979 runner.state = RunState::load(&runner.state.id).unwrap();
17980 advance_base_fixture(&runner).await;
17981 runner.sync_to_base().await.unwrap();
17982 let a = runner
17983 .state
17984 .base_sync_recovery
17985 .authorizations
17986 .last()
17987 .unwrap();
17988 assert!(
17989 a.consumed
17990 && a.invalidated
17991 .as_deref()
17992 .unwrap()
17993 .contains("fetched base changed")
17994 );
17995 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17996 assert_eq!(
17997 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17998 head
17999 );
18000 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18001 .await
18002 .unwrap();
18003 runner.state = RunState::load(&runner.state.id).unwrap();
18004 runner.sync_to_base().await.unwrap();
18005 assert!(!runner.state.base_sync_blocked());
18006 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18007 }
18008
18009 #[cfg(unix)]
18010 #[tokio::test]
18011 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
18012 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18013 runner.state.authorize_base_sync(&task).unwrap();
18014 assert!(runner.state.consume_base_recovery(&head));
18015 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18016 runner.state.reserve_base_operation(&head, &tip);
18017 runner.state.save().unwrap();
18018 let scratch = runner.state.dir().join("base-sync");
18019 assert_eq!(
18020 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18021 .await
18022 .unwrap(),
18023 crate::rebase::Rebased::Applied
18024 );
18025 runner.state = RunState::load(&runner.state.id).unwrap();
18026 runner.sync_to_base().await.unwrap();
18027 assert!(
18028 runner
18029 .state
18030 .base_sync_recovery
18031 .operations
18032 .last()
18033 .unwrap()
18034 .finished
18035 );
18036 assert_eq!(runner.state.base_episode_used(), 1);
18037 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18038 assert!(!runner.state.base_sync_blocked());
18039 }
18040
18041 #[cfg(unix)]
18042 #[tokio::test]
18043 async fn base_recovery_crash_preserves_untracked_files() {
18044 for ignored in [false, true] {
18045 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18046 runner.state.authorize_base_sync(&task).unwrap();
18047 assert!(runner.state.consume_base_recovery(&head));
18048 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18049 runner.state.reserve_base_operation(&head, &tip);
18050 runner.state.save().unwrap();
18051 let scratch = runner.state.dir().join("base-sync");
18052 assert_eq!(
18053 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18054 .await
18055 .unwrap(),
18056 crate::rebase::Rebased::Applied
18057 );
18058 let repo = &runner.state.repo;
18059 let result = git::rev_parse(repo, "HEAD").await.unwrap();
18060 assert_ne!(result, head);
18061 if ignored {
18062 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
18063 .await
18064 .unwrap();
18065 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
18066 }
18067 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
18068 assert!(
18069 git::git_raw(repo, &["diff", "--quiet", &head])
18070 .await
18071 .unwrap()
18072 .ok()
18073 );
18074 runner.state = RunState::load(&runner.state.id).unwrap();
18076 runner.sync_to_base().await.unwrap();
18077 assert!(runner.state.base_sync_blocked());
18078 assert!(!runner.state.base_sync_recovery.operations[0].finished);
18079 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18080 assert_eq!(
18081 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18082 result
18083 );
18084 assert_eq!(
18085 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
18086 "retained work\n"
18087 );
18088 assert!(
18089 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
18090 .await
18091 .unwrap()
18092 .ok()
18093 );
18094 }
18095 }
18096
18097 #[cfg(unix)]
18098 #[tokio::test]
18099 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
18100 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18101 runner.state.authorize_base_sync(&task).unwrap();
18102 assert!(runner.state.consume_base_recovery(&head));
18103 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18104 runner.state.reserve_base_operation(&head, &tip);
18105 runner.state.save().unwrap();
18106 let scratch = runner.state.dir().join("base-sync");
18107 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18108 .await
18109 .unwrap();
18110 git::sync_to_head(&runner.state.repo).await.unwrap();
18111 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
18112 git::git(&runner.state.repo, &["add", "foreign"])
18113 .await
18114 .unwrap();
18115 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
18116 .await
18117 .unwrap();
18118 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18119 runner.state = RunState::load(&runner.state.id).unwrap();
18120 runner.sync_to_base().await.unwrap();
18121 assert!(runner.state.base_sync_blocked());
18122 assert!(
18123 !runner
18124 .state
18125 .base_sync_recovery
18126 .operations
18127 .last()
18128 .unwrap()
18129 .finished
18130 );
18131 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18132 assert_eq!(
18133 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18134 foreign_head
18135 );
18136 }
18137
18138 #[cfg(unix)]
18139 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
18140 {
18141 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18142 fresh_base_interval_fixture(&mut runner);
18143 let repo = &runner.state.repo;
18144 git::git(repo, &["checkout", "main"]).await.unwrap();
18145 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
18146 git::git(repo, &["add", "retained"]).await.unwrap();
18147 git::git(repo, &["commit", "-m", "upstream retained change"])
18148 .await
18149 .unwrap();
18150 git::git(repo, &["push", "origin", "main"]).await.unwrap();
18151 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
18152 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
18153 let scratch = runner.state.dir().join("base-sync");
18154 assert!(matches!(
18155 git::rebase_start(repo, &scratch, "fix/retained", &tip)
18156 .await
18157 .unwrap(),
18158 git::RebaseStart::Conflicted(_)
18159 ));
18160 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
18161 agent: "fixture".into(),
18162 paths: vec!["retained".into()],
18163 from: Some(head.clone()),
18164 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
18165 index_outside: None,
18166 finished: false,
18167 error: None,
18168 launcher: None,
18169 infra: None,
18170 outage: None,
18171 });
18172 (tmp, runner, head, tip, scratch)
18173 }
18174
18175 #[cfg(unix)]
18176 #[tokio::test]
18177 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
18178 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18179 std::fs::write(
18182 scratch.join("retained"),
18183 "upstream and retained resolution\n",
18184 )
18185 .unwrap();
18186 runner.state.resource_block = Some(crate::run::ResourceBlock {
18187 node: "rebase".into(),
18188 kind: "repository coordination".into(),
18189 detail: "busy".into(),
18190 at: Timestamp::now(),
18191 cause: crate::run::ResourceBlockCause::Other,
18192 });
18193 runner.state.save().unwrap();
18194 let newer = git::git(
18197 &runner.state.repo,
18198 &[
18199 "commit-tree",
18200 "main^{tree}",
18201 "-p",
18202 "main",
18203 "-m",
18204 "base advanced during wait",
18205 ],
18206 )
18207 .await
18208 .unwrap();
18209 git::git(
18210 &runner.state.repo,
18211 &[
18212 "push",
18213 "origin",
18214 &format!("{}:refs/heads/main", newer.trim()),
18215 ],
18216 )
18217 .await
18218 .unwrap();
18219 runner.state = RunState::load(&runner.state.id).unwrap();
18220 runner.state.resource_block = None;
18221 runner.sync_to_base().await.unwrap();
18222 assert!(!runner.state.base_sync_blocked());
18223 let op = &runner.state.base_sync_recovery.operations[0];
18224 assert_eq!(op.head, head);
18225 assert_eq!(op.tip, tip);
18226 assert!(op.finished);
18227 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18228 assert_eq!(runner.state.rebase_fixes.len(), 1);
18229 let latest = git::rev_parse(&runner.state.repo, "origin/main")
18230 .await
18231 .unwrap();
18232 assert_ne!(latest, tip);
18233 runner.state = RunState::load(&runner.state.id).unwrap();
18234 runner.sync_to_base().await.unwrap();
18235 assert!(!runner.state.base_sync_blocked());
18236 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
18237 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
18238 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18239 }
18240
18241 #[cfg(unix)]
18242 #[tokio::test]
18243 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
18244 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18245 runner.state.reserve_base_operation(&head, &tip);
18246 runner.state.save().unwrap();
18247 std::fs::write(
18248 scratch.join("retained"),
18249 "upstream and retained resolution\n",
18250 )
18251 .unwrap();
18252 git::git(&scratch, &["add", "retained"]).await.unwrap();
18253 git::git(
18254 &scratch,
18255 &["-c", "core.editor=true", "rebase", "--continue"],
18256 )
18257 .await
18258 .unwrap();
18259 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18260 assert!(
18261 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18262 .await
18263 .unwrap()
18264 .is_empty()
18265 );
18266 assert_eq!(
18267 crate::rebase::finish(
18268 &mut runner.state,
18269 &scratch,
18270 "fix/retained",
18271 &head,
18272 &tip,
18273 &["retained".into()],
18274 ""
18275 )
18276 .await
18277 .unwrap(),
18278 crate::rebase::Rebased::Applied
18279 );
18280 assert!(!scratch.exists());
18281 runner.state = RunState::load(&runner.state.id).unwrap();
18283 let op = &runner.state.base_sync_recovery.operations[0];
18284 assert!(!op.finished);
18285 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18286 runner.sync_to_base().await.unwrap();
18287 assert!(!runner.state.base_sync_blocked());
18288 assert!(runner.state.base_sync_recovery.operations[0].finished);
18289 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18290 assert_eq!(runner.state.rebase_fixes.len(), 1);
18291 let winner = runner.state.winner().unwrap().clone();
18292 let wait = runner.repository_wait();
18293 assert!(
18294 verify_current_head(&mut runner.state, &winner, &wait)
18295 .await
18296 .unwrap()
18297 );
18298 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18299 }
18300
18301 #[cfg(unix)]
18302 #[tokio::test]
18303 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18304 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18305 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18306 runner.state.reserve_base_operation(&head, &tip);
18307 runner.state.save().unwrap();
18308 std::fs::write(
18309 scratch.join("retained"),
18310 "upstream and retained resolution\n",
18311 )
18312 .unwrap();
18313 git::git(&scratch, &["add", "retained"]).await.unwrap();
18314 git::git(
18315 &scratch,
18316 &["-c", "core.editor=true", "rebase", "--continue"],
18317 )
18318 .await
18319 .unwrap();
18320 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18321 assert!(!git::rebase_in_progress(&scratch).await);
18322 assert!(
18323 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18324 .await
18325 .unwrap()
18326 .is_empty()
18327 );
18328 match damage {
18329 Some("record") => {
18330 let path =
18331 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18332 .await
18333 .unwrap();
18334 std::fs::write(scratch.join(path), "").unwrap();
18335 }
18336 Some("foreign") => {
18337 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18338 git::git(&scratch, &["add", "foreign"]).await.unwrap();
18339 git::git(&scratch, &["commit", "-m", "foreign work"])
18340 .await
18341 .unwrap();
18342 }
18343 Some("dirty") => {
18344 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18345 }
18346 _ => {}
18347 }
18348 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18349 runner.state = RunState::load(&runner.state.id).unwrap();
18351 assert!(
18352 runner.state.base_sync_recovery.operations[0]
18353 .validated_head
18354 .is_none()
18355 );
18356 runner.sync_to_base().await.unwrap();
18357 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18358 assert_eq!(runner.state.base_episode_used(), 1);
18359 assert_eq!(runner.state.rebase_fixes.len(), 1);
18360 assert_eq!(
18361 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18362 retained_head
18363 );
18364 if damage.is_some() {
18365 assert!(runner.state.base_sync_blocked());
18366 assert!(scratch.exists());
18367 assert!(!runner.state.base_sync_recovery.operations[0].finished);
18368 assert!(
18369 runner.state.base_sync_recovery.operations[0]
18370 .validated_head
18371 .is_none()
18372 );
18373 } else {
18374 assert!(!runner.state.base_sync_blocked());
18375 assert!(!scratch.exists());
18376 assert!(runner.state.base_sync_recovery.operations[0].finished);
18377 assert_eq!(
18378 runner.state.base_sync_recovery.operations[0]
18379 .validated_head
18380 .as_deref(),
18381 Some(result.as_str())
18382 );
18383 let winner = runner.state.winner().unwrap().clone();
18384 let wait = runner.repository_wait();
18385 assert!(
18386 verify_current_head(&mut runner.state, &winner, &wait)
18387 .await
18388 .unwrap()
18389 );
18390 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18391 runner.state = RunState::load(&runner.state.id).unwrap();
18392 runner.sync_to_base().await.unwrap();
18393 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18394 }
18395 }
18396 }
18397
18398 #[cfg(unix)]
18399 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18400 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18401 runner.state.prep_complete = Some(true);
18403 runner.state.base_sync = None;
18404 runner.state.base_sync_recovery = Default::default();
18405 runner.state.config.graph.gate_fix_rounds = 1;
18406 let mut review = review_round(true, 0, 1, 1, false, true);
18407 review.head = head.clone();
18408 review.verified_head = Some(head.clone());
18409 runner.state.reviews = vec![review];
18410 runner.state.gate = vec![CommandOutcome {
18411 command: "old gate".into(),
18412 code: Some(1),
18413 output_tail: "old failure".into(),
18414 duration_ms: 0,
18415 resource_blocked: false,
18416 }];
18417 runner.state.gate_head = Some(head.clone());
18418 runner.state.gate_ran = true;
18419 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18420 agent: "fixture".into(),
18421 failed: runner.state.gate.clone(),
18422 notes: "spent".into(),
18423 committed: false,
18424 error: None,
18425 }];
18426 runner.state.land_approval = Some(crate::run::LandApproval {
18427 question: "old approval".into(),
18428 head: head.clone(),
18429 });
18430 runner.state.status = RunStatus::Blocked;
18431 (tmp, runner, head)
18432 }
18433
18434 #[cfg(unix)]
18435 #[tokio::test]
18436 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18437 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18438 let (tmp, mut runner, old) = resumed_gate_fixture().await;
18439 let phases = tmp.path().join("phases");
18440 runner.state.config.verify.e2e = vec![format!(
18441 "echo e2e >> '{}'; {}",
18442 phases.display(),
18443 if e2e_ok {
18444 "true"
18445 } else {
18446 "echo new-e2e-failure; false"
18447 }
18448 )];
18449 runner.state.config.verify.gate = vec![format!(
18450 "echo gate >> '{}'; {}",
18451 phases.display(),
18452 if gate_ok {
18453 "true"
18454 } else {
18455 "echo new-gate-failure; false"
18456 }
18457 )];
18458 runner.state.save().unwrap();
18459 runner.state = RunState::load(&runner.state.id).unwrap();
18460 runner.sync_to_base().await.unwrap();
18461 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18462 assert_ne!(head, old);
18463 runner.state = RunState::load(&runner.state.id).unwrap();
18465 runner.execute_graph().await.unwrap();
18466 assert_eq!(runner.state.gate_fixes.len(), 1);
18467 assert_eq!(runner.state.reviews.len(), 1);
18468 assert_eq!(runner.state.reviews[0].head, old);
18469 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18470 assert_eq!(runner.state.verification_history.len(), 1);
18471 let history = &runner.state.verification_history[0];
18472 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18473 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18474 assert!(history.archived_at.is_some());
18475 assert_eq!(history.fixes_spent, Some(1));
18476 assert_eq!(history.gate[0].output_tail, "old failure");
18477 let archive_events: Vec<_> = runner
18478 .state
18479 .events
18480 .iter()
18481 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18482 .collect();
18483 assert_eq!(archive_events.len(), 1);
18484 assert!(archive_events[0].message.contains(&short(&old)));
18485 assert!(archive_events[0].message.contains(&short(&head)));
18486 assert!(
18487 archive_events[0]
18488 .message
18489 .contains("1 gate-fix round(s) already spent stay spent")
18490 );
18491 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18492 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18493 if e2e_ok {
18494 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18495 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18496 } else {
18497 assert!(!runner.state.gate_ran);
18498 }
18499 assert_eq!(
18500 runner.state.status,
18501 if e2e_ok && gate_ok {
18502 RunStatus::Ready
18503 } else {
18504 RunStatus::Blocked
18505 }
18506 );
18507 if !e2e_ok || !gate_ok {
18508 let cause = runner.state.verification_blocking_cause().unwrap();
18509 assert!(cause.contains(&head));
18510 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18511 runner.state.event("land", "rebased successfully");
18513 let notice = crate::notices::run_ended(&runner.state).unwrap();
18514 assert!(!format!("{notice:?}").contains("rebased successfully"));
18515 assert!(runner.state.merge.is_none());
18516 }
18517 let before = std::fs::read_to_string(&phases).unwrap();
18518 runner.state.save().unwrap();
18519 runner.state = RunState::load(&runner.state.id).unwrap();
18520 runner.review_loop().await.unwrap();
18521 runner.gate().await.unwrap();
18522 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18523 assert_eq!(runner.state.verification_history.len(), 1);
18524 let history = &runner.state.verification_history[0];
18525 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18526 assert!(history.archived_at.is_some());
18527 assert_eq!(history.fixes_spent, Some(1));
18528 }
18529 }
18530
18531 #[cfg(unix)]
18532 #[tokio::test]
18533 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18534 let (tmp, mut runner, head) = resumed_gate_fixture().await;
18535 let marker = tmp.path().join("gate-ran");
18536 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18537 runner.review_loop().await.unwrap();
18538 runner.gate().await.unwrap();
18539 assert_eq!(runner.state.status, RunStatus::Blocked);
18540 assert!(!marker.exists());
18541 assert!(runner.state.verification_history.is_empty());
18542 runner.state.gate_head = None;
18543 runner.gate().await.unwrap();
18544 assert!(marker.exists());
18545 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18546 assert_eq!(runner.state.verification_history.len(), 1);
18547 assert_eq!(runner.state.gate_fixes.len(), 1);
18548 }
18549
18550 #[cfg(unix)]
18551 #[tokio::test]
18552 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18553 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18554 runner.sync_to_base().await.unwrap();
18555 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18556 runner.state.reconcile_verification_head(&head);
18557 for outcomes in [
18559 Vec::new(),
18560 vec![CommandOutcome {
18561 command: "true".into(),
18562 code: None,
18563 output_tail: "cache unavailable".into(),
18564 duration_ms: 0,
18565 resource_blocked: true,
18566 }],
18567 ] {
18568 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18569 head: head.clone(),
18570 passed: false,
18571 outcomes,
18572 });
18573 runner.state.gate_ran = false;
18574 runner.state.gate.clear();
18575 runner.state.save().unwrap();
18576 runner.state = RunState::load(&runner.state.id).unwrap();
18577 runner.merge().await.unwrap();
18578 assert!(runner.state.merge.is_none());
18579 runner.review_loop().await.unwrap();
18580 runner.gate().await.unwrap();
18581 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18582 assert!(runner.state.gate_allows_progress());
18583 assert_eq!(runner.state.gate_fixes.len(), 1);
18584 assert!(runner.state.merge.is_none());
18585 }
18586 git::git(
18588 &runner.state.repo,
18589 &["commit", "--allow-empty", "-m", "external update"],
18590 )
18591 .await
18592 .unwrap();
18593 runner.merge().await.unwrap();
18594 assert!(runner.state.merge.is_none());
18595 assert_eq!(runner.state.status, RunStatus::Blocked);
18596 }
18597
18598 #[cfg(unix)]
18599 #[tokio::test]
18600 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18601 for waiting_on_gate in [false, true] {
18602 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18603 runner.sync_to_base().await.unwrap();
18604 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18605 runner.state.reconcile_verification_head(&head);
18606 if waiting_on_gate {
18607 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18608 head: head.clone(),
18609 passed: true,
18610 outcomes: Vec::new(),
18611 });
18612 }
18613 let cache = tmp.path().join("cache");
18614 runner.state.config.verify.e2e =
18615 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18616 runner.state.config.verify.gate = vec!["true".into()];
18617 runner.state.config.graph.timeout_resource_wait = 1;
18618 let owner =
18619 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18620 let guard =
18621 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18622 crate::cache::AcquireOutcome::Acquired(g) => g,
18623 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18624 };
18625 runner.review_loop().await.unwrap();
18626 runner.gate().await.unwrap();
18627 assert!(!runner.state.gate_ran);
18628 assert_eq!(runner.state.gate_fixes.len(), 1);
18629 assert!(runner.state.resource_block.is_some());
18630 runner.merge().await.unwrap();
18631 assert!(runner.state.merge.is_none());
18632 drop(guard);
18633 runner.state = RunState::load(&runner.state.id).unwrap();
18634 runner.state.resource_block = None;
18636 runner.review_loop().await.unwrap();
18637 runner.gate().await.unwrap();
18638 assert!(runner.state.gate_allows_progress());
18639 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18640 assert_eq!(runner.state.gate_fixes.len(), 1);
18641 }
18642 }
18643
18644 #[cfg(unix)]
18645 #[tokio::test]
18646 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18647 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18648 runner.sync_to_base().await.unwrap();
18649 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18650 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18651 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18653 runner.review_loop().await.unwrap();
18654 runner.gate().await.unwrap();
18655 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18656 assert_ne!(before, after);
18657 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18658 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18659 assert!(!runner.state.gate_allows_progress());
18660 runner.merge().await.unwrap();
18661 assert!(runner.state.merge.is_none());
18662 assert!(
18663 runner
18664 .state
18665 .verification_history
18666 .iter()
18667 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18668 );
18669
18670 runner.state.reviews[0].clean = false;
18672 runner.state.reviews[0].answered = 0;
18673 runner.state.config.verify.e2e = vec!["true".into()];
18674 runner.review_loop().await.unwrap();
18675 runner.gate().await.unwrap();
18676 runner.merge().await.unwrap();
18677 assert_eq!(runner.state.status, RunStatus::Blocked);
18678 assert!(runner.state.merge.is_none());
18679 assert!(
18680 runner
18681 .state
18682 .verification_blocking_cause()
18683 .unwrap()
18684 .contains("review blocked")
18685 );
18686 }
18687
18688 #[cfg(unix)]
18689 #[tokio::test]
18690 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18691 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18692 runner.state.gate_head = None;
18693 runner.state.config.verify.gate =
18694 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18695 runner.review_loop().await.unwrap();
18696 runner.gate().await.unwrap();
18697 assert_ne!(
18698 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18699 old
18700 );
18701 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18702 assert!(!runner.state.gate_allows_progress());
18703 assert_eq!(runner.state.status, RunStatus::Blocked);
18704 runner.merge().await.unwrap();
18705 assert!(runner.state.merge.is_none());
18706 }
18707
18708 #[cfg(unix)]
18709 #[tokio::test]
18710 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18711 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18712 let mut old_review = review_round(true, 0, 1, 1, false, true);
18713 old_review.head = head.clone();
18714 old_review.verified_head = Some(head.clone());
18715 runner.state.reviews = vec![old_review];
18716 runner.state.gate = vec![CommandOutcome {
18717 command: "old gate".into(),
18718 code: None,
18719 output_tail: "timed out after 2400s".into(),
18720 duration_ms: 2_400_000,
18721 resource_blocked: false,
18722 }];
18723 runner.state.gate_ran = true;
18724 runner.state.gate_head = Some(head.clone());
18725 runner.state.land_approval = Some(crate::run::LandApproval {
18726 question: "old-head-approval".into(),
18727 head: head.clone(),
18728 });
18729 runner.state.authorize_base_sync(&task).unwrap();
18730 runner.sync_to_base().await.unwrap();
18731 assert!(!runner.state.base_sync_blocked());
18732 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18733 assert_ne!(head, fresh);
18734 assert_eq!(runner.state.reviews.len(), 1);
18735 assert!(!runner.state.gate_ran);
18736 let saved = &runner.state.base_sync_recovery.authorizations[0];
18737 assert_eq!(saved.reviews[0].head, head);
18738 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18739 assert!(saved.gate_ran);
18740 assert!(!saved.gate[0].ok());
18741 runner.review_loop().await.unwrap();
18742 runner.gate().await.unwrap();
18743 assert_eq!(runner.state.reviews[0].head, head);
18744 assert_eq!(
18745 runner.state.reviews[0].verified_head.as_deref(),
18746 Some(head.as_str())
18747 );
18748 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18749 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18750 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18751 assert!(runner.state.gate_ran);
18752 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18753 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18754 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18755 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18756 assert_eq!(
18757 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18758 "retained change"
18759 );
18760 }
18761
18762 #[cfg(unix)]
18763 #[tokio::test]
18764 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18765 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18766 let queue = crate::queue::Queue::open();
18767 let mut owner = queue.get(&task).unwrap();
18768 owner.release_operator("test-resume", None);
18769 queue.put(&mut owner).unwrap();
18770 let generation = owner.resumption.as_ref().unwrap().generation;
18771 runner.state.task_instruction = Some(owner.instruction.clone());
18772 runner.state.driver_exited = true;
18773 runner.state.save().unwrap();
18774 let home = crate::run::home();
18775 let _claim = queue.claim(&task).unwrap();
18776 owner.resumption.as_mut().unwrap().at =
18777 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18778 queue.put(&mut owner).unwrap();
18779 assert!(
18780 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18781 .await
18782 .is_err(),
18783 "an approval from before the stop grants nothing"
18784 );
18785 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18786 queue.put(&mut owner).unwrap();
18787 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18788 .await
18789 .unwrap();
18790 assert!(
18791 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18792 .await
18793 .is_err()
18794 );
18795 runner.state = RunState::load(&runner.state.id).unwrap();
18796 assert_eq!(
18797 runner.state.base_sync_recovery.authorizations[0].generation,
18798 Some(generation)
18799 );
18800 runner.sync_to_base().await.unwrap();
18801 let repo = runner.state.repo.clone();
18802 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18803 git::git(&repo, &["checkout", "main"]).await.unwrap();
18804 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18805 git::git(&repo, &["add", "."]).await.unwrap();
18806 git::git(&repo, &["commit", "-m", "move base again"])
18807 .await
18808 .unwrap();
18809 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18810 git::git(&repo, &["checkout", "fix/retained"])
18811 .await
18812 .unwrap();
18813 runner.sync_to_base().await.unwrap();
18814 assert!(!runner.state.base_sync_blocked());
18815 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18816 assert_eq!(runner.state.base_episode_used(), 2);
18817 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18818 assert_ne!(fresh, head);
18819 assert!(!runner.state.base_recovery_pending());
18820 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18822 queue.put(&mut owner).unwrap();
18823 assert!(
18824 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18825 .await
18826 .is_err()
18827 );
18828 runner.state = RunState::load(&runner.state.id).unwrap();
18829 runner.sync_to_base().await.unwrap();
18830 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18831 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18832 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18833 }
18834
18835 #[cfg(unix)]
18836 #[tokio::test]
18837 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18838 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18839 runner.state.base_sync = None;
18840 runner.state.status = RunStatus::Blocked;
18841 let mut second = runner.state.candidates[0].clone();
18842 second.label = 'B';
18843 runner.state.candidates.push(second);
18844 assert!(runner.state.votes.is_empty());
18845 assert!(runner.state.deliberation.is_empty());
18846 runner.deliberate().await.unwrap();
18847 runner.vote().await.unwrap();
18848 assert_eq!(runner.state.status, RunStatus::Blocked);
18849 assert!(runner.state.votes.is_empty());
18850 assert!(runner.state.deliberation.is_empty());
18851 }
18852
18853 fn runner_at(status: RunStatus) -> Runner {
18856 let mut state = RunState::new(
18857 PathBuf::from("/nonexistent/repo"),
18858 "main".to_owned(),
18859 "deadbeef".to_owned(),
18860 "task".to_owned(),
18861 Config::default(),
18862 );
18863 state.config.disk.min_free_bytes = 0;
18865 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18868 state.id = format!(
18869 "20000101-000000-{:04x}",
18870 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18871 );
18872 state.status = status;
18873 Runner {
18874 state,
18875 roles: ResolvedRoles {
18876 implementers: Vec::new(),
18877 judges: Vec::new(),
18878 reviewers: Vec::new(),
18879 fixer: None,
18880 conductor: conductor(),
18881 implementer_roster: Vec::new(),
18882 judge_roster: Vec::new(),
18883 reviewer_roster: Vec::new(),
18884 },
18885 sem: Arc::new(Semaphore::new(1)),
18886 pause: Pause::new(),
18887 interrupt: Pause::new(),
18888 }
18889 }
18890
18891 #[tokio::test]
18892 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18893 crate::run::pin_test_home();
18894 let temp = tempfile::tempdir().unwrap();
18895 let root = temp.path().canonicalize().unwrap();
18896 let repo = root.join("repo");
18897 std::fs::create_dir(&repo).unwrap();
18898 init_repo(&repo);
18899 git::git(&repo, &["config", "core.autocrlf", "false"])
18900 .await
18901 .unwrap();
18902 let mut runner = runner_at(RunStatus::Blocked);
18903 runner.state.repo = repo.clone();
18904 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18905 runner.state.config.agents = vec![conductor()];
18906 runner.state.config.graph.selected_implementer = Some("conductor".into());
18907 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18908 let (shell, command, setup_output) = if cfg!(windows) {
18909 (
18910 vec!["cmd".into(), "/C".into()],
18911 "echo ready>seat-ready",
18912 "ready\r\n",
18913 )
18914 } else {
18915 (
18916 vec!["sh".into(), "-c".into()],
18917 "printf ready > seat-ready",
18918 "ready",
18919 )
18920 };
18921 runner.state.config.verify.shell = Some(shell);
18922 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18923 run: Some(command.into()),
18924 ..Default::default()
18925 }];
18926 runner.state.save().unwrap();
18927 let path = runner.state.worktree_root().join("judge-1");
18928 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18929 .await
18930 .unwrap();
18931 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18932 git::git(&path, &["add", "."]).await.unwrap();
18933 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18934 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18935 .await
18936 .unwrap();
18937 git::git(&path, &["checkout", "--detach", &head])
18938 .await
18939 .unwrap();
18940 let evidence = crate::fold_evidence::snapshot(
18941 &runner.state,
18942 &crate::run::home().canonicalize().unwrap(),
18943 &head,
18944 64 * 1024 * 1024,
18945 )
18946 .await
18947 .unwrap();
18948 let mut contents = BTreeMap::new();
18949 contents.insert(
18950 repo.join("logs/HEAD"),
18951 format!("{head} {head} fixture\n").into_bytes(),
18952 );
18953 let record = serde_json::json!({
18954 "version":1, "run":runner.state.id, "source":path, "head":head,
18955 "evidence":evidence, "admin":{}, "admin_contents":contents,
18956 "history":{head.clone():evidence}, "phase":"evicted"
18957 });
18958 std::fs::write(
18959 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18960 serde_json::to_vec(&record).unwrap(),
18961 )
18962 .unwrap();
18963 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18964 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18965 .await
18966 .unwrap();
18967 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18968 assert!(
18969 !git::git_raw(&repo, &["cat-file", "-e", &head])
18970 .await
18971 .unwrap()
18972 .ok()
18973 );
18974 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18975 let error = resumed.execute().await.unwrap_err();
18976 assert!(
18977 error
18978 .to_string()
18979 .contains("selected implementation is held"),
18980 "{error:#}"
18981 );
18982 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18983 assert_eq!(
18984 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18985 setup_output
18986 );
18987 assert_eq!(
18988 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18989 "unrelated seat"
18990 );
18991 let error = resumed.execute().await.unwrap_err();
18992 assert!(
18993 error
18994 .to_string()
18995 .contains("selected implementation is held"),
18996 "{error:#}"
18997 );
18998 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18999 git::git(&path, &["add", "."]).await.unwrap();
19000 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
19001 let next = git::git(
19002 &path,
19003 &[
19004 "commit-tree",
19005 &next_tree,
19006 "-p",
19007 &head,
19008 "-m",
19009 "next review checkpoint",
19010 ],
19011 )
19012 .await
19013 .unwrap();
19014 git::git(&path, &["checkout", "--detach", &next])
19015 .await
19016 .unwrap();
19017 resumed.state.reviews.push(
19018 serde_json::from_value(serde_json::json!({
19019 "round":1, "head":next, "reviews":[]
19020 }))
19021 .unwrap(),
19022 );
19023 resumed.state.save().unwrap();
19024 let error = resumed.execute().await.unwrap_err();
19025 assert!(
19026 error
19027 .to_string()
19028 .contains("selected implementation is held"),
19029 "{error:#}"
19030 );
19031 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
19032 }
19033
19034 #[tokio::test]
19035 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
19036 crate::run::pin_test_home();
19037 let mut runner = runner_at(RunStatus::Gating);
19038 runner.state.save().unwrap();
19039 let id = runner.state.id.clone();
19040 let passes = runner.state.passes.len();
19041 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19042 let mut claiming = Box::pin(runner.claim_driver());
19043 std::future::poll_fn(|cx| {
19044 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
19045 std::task::Poll::Ready(())
19046 })
19047 .await;
19048 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
19049 drop(owner);
19050 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
19051 .await
19052 .unwrap()
19053 .unwrap();
19054 assert_eq!(execution.runner.state.passes.len(), passes + 1);
19055 assert!(execution.runner.state.resource_block.is_none());
19056 assert_eq!(
19057 execution.runner.state.liveness(false),
19058 crate::run::Liveness::Live
19059 );
19060 drop(execution);
19061 assert!(RunState::load(&id).unwrap().driver_exited);
19062 }
19063
19064 #[tokio::test]
19065 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
19066 crate::run::pin_test_home();
19067 let mut runner = runner_at(RunStatus::Gating);
19068 runner.state.save().unwrap();
19069 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
19072 assert!(runner.claim_driver().await.is_err());
19073 assert!(runner.state.driver_exited);
19074 assert_eq!(runner.state.driver_pid, None);
19075 let disk = RunState::load(&runner.state.id).unwrap();
19076 assert!(disk.driver_exited);
19077 assert_eq!(disk.driver_pid, None);
19078 assert_eq!(disk.status, RunStatus::Gating);
19079 }
19080
19081 #[tokio::test]
19082 async fn an_early_graph_error_releases_its_driver() {
19083 crate::run::pin_test_home();
19084 let mut runner = runner_at(RunStatus::Blocked);
19085 runner.state.config.graph.selected_implementer = Some("selected".into());
19086 runner.state.save().unwrap();
19087 assert!(runner.execute().await.is_err());
19088 let disk = RunState::load(&runner.state.id).unwrap();
19089 assert!(disk.driver_exited);
19090 assert_eq!(disk.driver_pid, None);
19091 assert_eq!(disk.status, RunStatus::Blocked);
19092 }
19093
19094 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
19095 crate::run::MergeRepair {
19096 question: "repair-question".into(),
19097 head: "old-head".into(),
19098 pr: "https://github.com/o/r/pull/42".into(),
19099 reviews: vec![review_round(false, 1, 1, 1, false, true)],
19100 gate: vec![outcome(Some(0), false)],
19101 gate_head: Some("old-head".into()),
19102 gate_fixes: Vec::new(),
19103 pre_gate: Vec::new(),
19104 phase,
19105 failure: None,
19106 }
19107 }
19108
19109 #[tokio::test]
19110 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
19111 crate::run::pin_test_home();
19112 let mut runner = runner_at(RunStatus::Landing);
19113 runner.state.merge_repairs.push(merge_repair_request(
19114 crate::run::MergeRepairPhase::Requested,
19115 ));
19116 runner.state.save().unwrap();
19117 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
19118 assert!(
19119 runner
19120 .execute()
19121 .await
19122 .unwrap_err()
19123 .to_string()
19124 .contains("already running")
19125 );
19126 assert_eq!(
19127 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19128 crate::run::MergeRepairPhase::Requested
19129 );
19130 drop(claim);
19131 let mut disk = runner.state.clone();
19132 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
19133 disk.save().unwrap();
19134 assert!(
19135 runner
19136 .execute()
19137 .await
19138 .unwrap_err()
19139 .to_string()
19140 .contains("decision changed")
19141 );
19142 assert_eq!(
19143 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19144 crate::run::MergeRepairPhase::Complete
19145 );
19146 }
19147
19148 #[test]
19149 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
19150 crate::run::pin_test_home();
19151 let mut runner = runner_at(RunStatus::Landing);
19152 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
19153 runner.state.reviews = request.reviews.clone();
19154 runner.state.gate = request.gate.clone();
19155 runner.state.gate_ran = true;
19156 runner.state.land_approval = Some(crate::run::LandApproval {
19157 question: "old".into(),
19158 head: "old-head".into(),
19159 });
19160 runner.state.merge_repairs.push(request);
19161 runner.begin_merge_repair_review(0).unwrap();
19162 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
19163 assert!(!runner.state.gate_status().ok());
19164 assert!(runner.state.land_approval.is_none());
19165 let disk = RunState::load(&runner.state.id).unwrap();
19166 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
19167 assert!(disk.merge_repairs[0].gate[0].ok());
19168 assert_eq!(
19169 disk.merge_repairs[0].phase,
19170 crate::run::MergeRepairPhase::Reviewing
19171 );
19172 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
19173 assert!(disk.reviews.is_empty());
19174 }
19175
19176 #[tokio::test]
19177 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
19178 crate::run::pin_test_home();
19179 for phase in [
19180 crate::run::MergeRepairPhase::Fixing,
19181 crate::run::MergeRepairPhase::Requested,
19182 ] {
19183 let mut runner = runner_at(RunStatus::Landing);
19184 runner.state.config.graph.land_rounds = 0;
19185 runner.state.merge_repairs.push(merge_repair_request(phase));
19186 runner.state.save().unwrap();
19187 assert!(!runner.repair_merge_review().await.unwrap());
19188 assert_eq!(runner.state.status, RunStatus::Blocked);
19189 assert_eq!(
19190 runner.state.merge_repairs[0].phase,
19191 crate::run::MergeRepairPhase::Failed
19192 );
19193 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
19194 assert!(
19195 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
19196 "interrupted"
19197 } else {
19198 "budget exhausted"
19199 })
19200 );
19201 assert!(runner.state.merge.is_none());
19202 }
19203 }
19204
19205 #[test]
19209 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
19210 crate::run::pin_test_home();
19211 let mut runner = runner_at(RunStatus::Implementing);
19212 let interrupt = Pause::new();
19213 runner.watch_interrupt(interrupt.clone());
19214
19215 interrupt.park_because("task a1b2 asked to run first");
19216
19217 assert!(runner.park_here().expect("park_here"));
19218 assert!(runner.state.parked);
19219 let last = runner.state.events.last().expect("a park event");
19220 assert_eq!(last.node, "park");
19221 assert!(
19222 last.message.contains("task a1b2 asked to run first"),
19223 "expected the interrupt reason in {:?}",
19224 last.message
19225 );
19226 }
19227
19228 #[test]
19231 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
19232 crate::run::pin_test_home();
19233 let mut runner = runner_at(RunStatus::Implementing);
19234 runner.state.begin_pass();
19235 let shutdown = Pause::new();
19236 runner.on_pause(shutdown.clone());
19237 let upgrade = ParkCause::Upgrade {
19238 from: "0.121.0".to_owned(),
19239 to: Some("0.121.1".to_owned()),
19240 };
19241 shutdown.park_for(upgrade.clone());
19242 shutdown.park_for(ParkCause::Operator);
19243
19244 assert!(runner.park_here().expect("park_here"));
19245 assert_eq!(
19246 runner.state.passes[0].end,
19247 Some(crate::run::PassEnd::Parked { cause: upgrade })
19248 );
19249 let last = runner.state.events.last().expect("a park event");
19250 assert!(last.message.contains("0.121.0"), "{}", last.message);
19251 }
19252
19253 #[tokio::test]
19254 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
19255 crate::run::pin_test_home();
19256 let mut runner = runner_at(RunStatus::Reviewing);
19257 runner.state.save().unwrap();
19258 let mut execution = runner.claim_driver().await.expect("first execution");
19259 assert_eq!(execution.runner.state.passes.len(), 1);
19260 execution
19261 .driver
19262 .finish(&mut execution.runner.state)
19263 .await
19264 .unwrap();
19265 }
19266
19267 #[tokio::test]
19268 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19269 crate::run::pin_test_home();
19270 let mut runner = runner_at(RunStatus::Reviewing);
19271 runner.state.config.agents = vec![conductor()];
19272 runner.state.config.graph.timeout_resource_wait = 0;
19273 runner.state.save().unwrap();
19274 let id = runner.state.id.clone();
19275 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19276 assert!(runner.claim_driver().await.is_err());
19277 drop(owner);
19278 drop(runner);
19279 let mut resumed = Runner::resume(&id).unwrap();
19280 assert_eq!(resumed.state.driver_claimed, Some(false));
19281 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19282 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19283 let mut execution = resumed
19284 .claim_driver()
19285 .await
19286 .expect("retry initial claim after reload");
19287 assert_eq!(execution.runner.state.driver_claimed, Some(true));
19288 assert_eq!(execution.runner.state.passes.len(), 1);
19289 execution
19290 .driver
19291 .finish(&mut execution.runner.state)
19292 .await
19293 .unwrap();
19294 }
19295
19296 #[tokio::test]
19297 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19298 crate::run::pin_test_home();
19299 let mut runner = runner_at(RunStatus::Reviewing);
19300 runner.state.config.agents = vec![conductor()];
19301 runner.state.save().unwrap();
19302 let id = runner.state.id.clone();
19303 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19304 let mut resumed = Runner::resume(&id).unwrap();
19305 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19306 assert!(resumed.claim_driver().await.is_err());
19307 assert!(RunState::load(&id).unwrap().passes.is_empty());
19308 drop(lease);
19309 let mut execution = resumed.claim_driver().await.unwrap();
19310 execution
19311 .driver
19312 .finish(&mut execution.runner.state)
19313 .await
19314 .unwrap();
19315 }
19316
19317 #[tokio::test]
19318 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19319 crate::run::pin_test_home();
19320 let mut runner = runner_at(RunStatus::Reviewing);
19321 let state = &mut runner.state;
19322 state.config.agents = vec![conductor()];
19323 state.driver_claimed = None;
19324 state.driver_pid = Some(std::process::id());
19325 state.driver_started_at = None;
19326 state.begin_pass();
19327 state.save().unwrap();
19328 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19329 std::fs::write(
19330 state.dir().join("driver.json"),
19331 serde_json::to_vec(&serde_json::json!({
19332 "pid": state.driver_pid, "started_at": null,
19333 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19334 }))
19335 .unwrap(),
19336 )
19337 .unwrap();
19338 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19339 drop(lease);
19340 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19341 let mut resumed = Runner::resume(&state.id).unwrap();
19342 assert!(!resumed.state.driver_exited);
19343 let mut execution = resumed
19344 .claim_driver()
19345 .await
19346 .expect("lease proves previous execution ended");
19347 assert_eq!(execution.runner.state.passes.len(), 2);
19348 execution
19349 .driver
19350 .finish(&mut execution.runner.state)
19351 .await
19352 .unwrap();
19353 }
19354
19355 #[tokio::test]
19356 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19357 crate::run::pin_test_home();
19358 let mut runner = runner_at(RunStatus::Reviewing);
19359 let mut json = serde_json::to_value(&runner.state).unwrap();
19360 json.as_object_mut().unwrap().remove("driver_claimed");
19361 runner.state = serde_json::from_value(json).unwrap();
19362 assert_eq!(runner.state.driver_claimed, None);
19363 runner.state.save().unwrap();
19364 let mut execution = runner
19365 .claim_driver()
19366 .await
19367 .expect("the execution lease, not a missing marker, excludes a second driver");
19368 assert_eq!(execution.runner.state.passes.len(), 1);
19369 execution
19370 .driver
19371 .finish(&mut execution.runner.state)
19372 .await
19373 .unwrap();
19374 }
19375
19376 #[tokio::test]
19377 async fn driver_admission_refuses_a_proven_live_driver() {
19378 crate::run::pin_test_home();
19379 let mut runner = runner_at(RunStatus::Reviewing);
19380 let pid = std::process::id();
19381 runner.state.driver_pid = Some(pid);
19382 runner.state.driver_started_at = crate::proc::process_started_at(pid);
19383 assert!(runner.state.driver_started_at.is_some());
19384 runner.state.save().unwrap();
19385 let error = runner
19386 .claim_driver()
19387 .await
19388 .err()
19389 .expect("a live driver must refuse");
19390 assert!(error.to_string().contains("live driver"), "{error:#}");
19391 let saved = RunState::load(&runner.state.id).unwrap();
19392 assert_eq!(saved.driver_pid, Some(pid));
19393 assert_eq!(saved.status, RunStatus::Reviewing);
19394 assert!(saved.passes.is_empty());
19395 }
19396
19397 #[test]
19398 fn parking_resource_halt_persists_the_pass_once() {
19399 crate::run::pin_test_home();
19400 let mut runner = runner_at(RunStatus::Reviewing);
19401 runner.state.begin_pass();
19402 runner.state.resource_block = Some(crate::run::ResourceBlock {
19403 node: "fix".into(),
19404 kind: "shared build cache".into(),
19405 detail: "cancelled".into(),
19406 at: Timestamp::now(),
19407 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19408 });
19409 let cause = ParkCause::Upgrade {
19410 from: "old".into(),
19411 to: Some("new".into()),
19412 };
19413 runner.pause.park_for(cause.clone());
19414 runner.stop_for_resource().unwrap();
19417 assert!(runner.resource_halt().unwrap());
19418 assert!(runner.park_here().unwrap());
19419 let saved = RunState::load(&runner.state.id).unwrap();
19420 assert!(saved.parked);
19421 assert_eq!(saved.status, RunStatus::Reviewing);
19422 assert!(saved.passes[0].ended_at.is_some());
19423 assert_eq!(
19424 saved.passes[0].end,
19425 Some(crate::run::PassEnd::Parked {
19426 cause: cause.clone()
19427 })
19428 );
19429 assert_eq!(
19430 saved.resource_block.unwrap().cause,
19431 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19432 );
19433 assert_eq!(
19434 saved
19435 .events
19436 .iter()
19437 .filter(|event| event.node == "park")
19438 .count(),
19439 1
19440 );
19441 }
19442
19443 #[test]
19451 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19452 crate::run::pin_test_home();
19453 let mut runner = runner_at(RunStatus::Implementing);
19454 let shutdown = Pause::new();
19455 runner.on_pause(shutdown.clone());
19456 let interrupt = Pause::new();
19457 runner.watch_interrupt(interrupt.clone());
19458
19459 assert!(!runner.park_here().expect("park_here"));
19461 assert!(!runner.state.parked);
19462
19463 interrupt.park_because("test");
19465 assert!(!shutdown.parked());
19466 assert!(runner.park_here().expect("park_here"));
19467 }
19468
19469 #[tokio::test]
19483 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19484 crate::run::pin_test_home();
19485 let mut runner = runner_at(RunStatus::Implementing);
19486 let interrupt = Pause::new();
19487 runner.watch_interrupt(interrupt.clone());
19488
19489 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19490 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19491
19492 let node = async move {
19496 started_tx.send(()).expect("send started");
19497 finish_rx.await.expect("recv finish");
19498 "node finished"
19499 };
19500
19501 let interrupter = async move {
19502 started_rx.await.expect("recv started");
19503 interrupt.park_because("higher-priority task waiting");
19505 tokio::task::yield_now().await;
19509 finish_tx.send(()).expect("send finish");
19510 };
19511
19512 let (node_result, ()) = tokio::join!(node, interrupter);
19513 assert_eq!(
19514 node_result, "node finished",
19515 "the in-flight call ran to completion"
19516 );
19517
19518 assert!(runner.park_here().expect("park_here"));
19521 assert!(runner.state.parked);
19522 }
19523
19524 #[test]
19530 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19531 crate::run::pin_test_home();
19532 let mut runner = runner_at(RunStatus::Judging);
19533 runner.state.config.agents = vec![conductor()];
19537 runner.state.candidates = vec![Candidate {
19538 index: 0,
19539 label: 'A',
19540 agent: "alpha".to_owned(),
19541 branch: "magi/x/A".to_owned(),
19542 worktree: PathBuf::from("/nonexistent/worktree"),
19543 summary: "did the thing".to_owned(),
19544 stat: "1 file changed".to_owned(),
19545 files: 1,
19546 commits: 1,
19547 empty: false,
19548 failed: None,
19549 verified_noop: None,
19550 duration_ms: 1234,
19551 folded: false,
19552 }];
19553 let run_id = runner.state.id.clone();
19554
19555 let interrupt = Pause::new();
19556 runner.watch_interrupt(interrupt.clone());
19557 interrupt.park_because("task c3d4 asked to run first");
19558 assert!(runner.park_here().expect("park_here"));
19559
19560 let resumed = Runner::resume(&run_id).expect("resume");
19561 assert_eq!(resumed.state.candidates.len(), 1);
19562 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19563 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19564 assert_eq!(resumed.state.status, runner.state.status);
19565 assert!(
19566 resumed.state.parked,
19567 "still parked until `execute` actually walks the graph again"
19568 );
19569 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19570 }
19571
19572 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19574 let mut q = ask::Question::new(
19575 run.to_owned(),
19576 "implement".to_owned(),
19577 "impl-A".to_owned(),
19578 "Which storage backend should the cache use?".to_owned(),
19579 String::new(),
19580 vec!["SQLite".to_owned(), "Redis".to_owned()],
19581 );
19582 store.put(&mut q).unwrap();
19583 q
19584 }
19585
19586 #[test]
19587 fn a_failed_runs_open_question_is_abandoned() {
19588 ask_test_home();
19589 let store = ask::Questions::open();
19590 let mut runner = runner_at(RunStatus::Failed);
19591 let run = runner.state.id.clone();
19592 let q = ask_open_question(&store, &run);
19593
19594 runner.settle_questions();
19595
19596 let back = store.get(&q.id).unwrap();
19597 assert!(
19598 !back.status.open(),
19599 "the seat that asked died with the run; nobody is left to read an answer"
19600 );
19601 assert!(
19602 back.detail.contains(&run) && back.detail.contains("failed"),
19603 "the reason names what the run became, not just that it is gone: {}",
19604 back.detail
19605 );
19606 }
19607
19608 #[test]
19609 fn a_merged_runs_open_question_is_abandoned_too() {
19610 ask_test_home();
19611 let store = ask::Questions::open();
19612 for status in [RunStatus::Merged, RunStatus::Ready] {
19615 let mut runner = runner_at(status);
19616 let run = runner.state.id.clone();
19617 let q = ask_open_question(&store, &run);
19618
19619 runner.settle_questions();
19620
19621 let back = store.get(&q.id).unwrap();
19622 assert!(
19623 !back.status.open(),
19624 "{status:?} run's question must not outlive the run"
19625 );
19626 }
19627 }
19628
19629 #[test]
19630 fn a_still_resumable_runs_open_question_is_left_alone() {
19631 ask_test_home();
19632 let store = ask::Questions::open();
19633 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19639 let mut runner = runner_at(status);
19640 let run = runner.state.id.clone();
19641 let q = ask_open_question(&store, &run);
19642
19643 runner.settle_questions();
19644
19645 let back = store.get(&q.id).unwrap();
19646 assert!(
19647 back.status.open(),
19648 "{status:?} is still alive; the question must still be waiting"
19649 );
19650 }
19651 }
19652
19653 #[test]
19654 fn settle_questions_never_touches_an_already_answered_question() {
19655 ask_test_home();
19656 let store = ask::Questions::open();
19657 let mut runner = runner_at(RunStatus::Failed);
19658 let run = runner.state.id.clone();
19659 let mut q = ask_open_question(&store, &run);
19660 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19661 .unwrap();
19662 store.put(&mut q).unwrap();
19663
19664 runner.settle_questions();
19669 runner.settle_questions();
19670
19671 let back = store.get(&q.id).unwrap();
19672 assert_eq!(
19673 back.status,
19674 ask::QuestionStatus::Answered,
19675 "a real answer is a decision on record, never overwritten by a sweep"
19676 );
19677 }
19678
19679 #[tokio::test]
19690 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19691 crate::run::pin_test_home();
19692 let tmp = tempfile::tempdir().expect("tempdir");
19693 let repo = tmp.path().join("repo");
19694 std::fs::create_dir_all(&repo).unwrap();
19695 init_repo(&repo);
19696
19697 let mut config = Config::default();
19698 config.graph.worktree_root = Some(tmp.path().join("wt"));
19699
19700 let mut state = RunState::new(
19701 repo.clone(),
19702 "main".to_owned(),
19703 "deadbeef".to_owned(),
19704 "task".to_owned(),
19705 config,
19706 );
19707 let root = state.worktree_root();
19708 let wt_a = root.join("cand-A");
19709 let wt_b = root.join("cand-B");
19710 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19711 .await
19712 .expect("worktree A");
19713 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19714 .await
19715 .expect("worktree B");
19716
19717 state.candidates = vec![
19718 Candidate {
19719 index: 0,
19720 label: 'A',
19721 agent: "alpha".to_owned(),
19722 branch: "magi/x/A".to_owned(),
19723 worktree: wt_a.clone(),
19724 summary: String::new(),
19725 stat: String::new(),
19726 files: 0,
19727 commits: 0,
19728 empty: false,
19729 failed: None,
19730 verified_noop: None,
19731 duration_ms: 0,
19732 folded: false,
19733 },
19734 Candidate {
19735 index: 1,
19736 label: 'B',
19737 agent: "beta".to_owned(),
19738 branch: "magi/x/B".to_owned(),
19739 worktree: wt_b.clone(),
19740 summary: String::new(),
19741 stat: String::new(),
19742 files: 0,
19743 commits: 0,
19744 empty: false,
19745 failed: None,
19746 verified_noop: None,
19747 duration_ms: 0,
19748 folded: false,
19749 },
19750 ];
19751 state.tally = Some(Tally {
19752 first_choice: BTreeMap::from([('A', 1)]),
19753 borda: BTreeMap::new(),
19754 winner: 'A',
19755 rankings: 1,
19756 unanimous_initial: true,
19757 deliberated: false,
19758 changed_votes: 0,
19759 unanimous_final: true,
19760 tie_break: None,
19761 judges: 1,
19762 present: 1,
19763 quorum: 1,
19764 met_quorum: true,
19765 uncontested: None,
19766 });
19767 state.status = RunStatus::Ready;
19768
19769 fold_run(&mut state, false, &crate::run::home())
19770 .await
19771 .expect("fold_run");
19772
19773 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19774 assert!(
19775 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19776 "the unmerged winner's branch survives"
19777 );
19778 assert!(
19779 !state.candidates[0].folded,
19780 "the winner is not marked folded"
19781 );
19782
19783 assert!(!wt_b.exists(), "the loser's worktree is removed");
19784 assert!(
19785 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19786 "the loser's branch is removed"
19787 );
19788 assert!(state.candidates[1].folded, "the loser is marked folded");
19789 }
19790
19791 #[tokio::test]
19792 #[cfg(unix)]
19793 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19794 use std::os::fd::AsRawFd as _;
19795 use std::time::Duration;
19796
19797 let tmp = tempfile::tempdir().unwrap();
19798 let repo = tmp.path().join("repo");
19799 std::fs::create_dir_all(&repo).unwrap();
19800 init_repo(&repo);
19801 let common = repo.join(".git");
19802 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19803 let home = tmp.path().join("home");
19804 let mut config = Config::default();
19805 config.coordination = crate::config::test_coordination();
19806 config.graph.worktree_root = Some(tmp.path().join("wt"));
19807 config.graph.timeout_resource_wait = 1;
19808 let mut state = RunState::new(
19809 repo.clone(),
19810 "main".into(),
19811 "deadbeef".into(),
19812 "task".into(),
19813 config,
19814 );
19815 let root = state.worktree_root();
19816 let candidate = root.join("cand-A");
19817 let reviewer = root.join("review-1");
19818 let branch = state.branch_for('A');
19819 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19820 .await
19821 .unwrap();
19822 git::worktree_add_detached(&repo, &reviewer, "main")
19823 .await
19824 .unwrap();
19825 git::acquire_worktree_config(&repo).await.unwrap();
19826 state.enabled_worktree_config = true;
19827 state.status = RunStatus::Ready;
19828 state.candidates.push(Candidate {
19829 index: 0,
19830 label: 'A',
19831 agent: "implement".into(),
19832 branch: branch.clone(),
19833 worktree: candidate.clone(),
19834 summary: String::new(),
19835 stat: String::new(),
19836 files: 0,
19837 commits: 0,
19838 empty: false,
19839 failed: None,
19840 verified_noop: None,
19841 duration_ms: 0,
19842 folded: false,
19843 });
19844 state.save_under(&home).unwrap();
19845 let config_before = std::fs::read(common.join("config")).unwrap();
19846 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19847 .await
19848 .unwrap();
19849 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19852 assert_eq!(
19853 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19854 0
19855 );
19856 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19857 assert!(
19858 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19859 .await
19860 .is_err()
19861 );
19862 assert!(candidate.exists());
19863 assert!(reviewer.exists());
19864 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19865 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19866 assert_eq!(
19867 git::git(&repo, &["worktree", "list", "--porcelain"])
19868 .await
19869 .unwrap(),
19870 worktrees_before
19871 );
19872 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19873 .await
19874 .unwrap()
19875 .unwrap_err();
19876 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19877 drop(fold);
19878 assert!(candidate.exists() && reviewer.exists());
19879 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19880 assert_eq!(
19881 state.resource_block.as_ref().unwrap().kind,
19882 "repository coordination"
19883 );
19884 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19885 drop(external);
19886 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19887 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19888 .await
19889 .unwrap()
19890 .unwrap();
19891 drop(fold);
19892 assert!(removed.contains(&branch));
19893 assert!(!root.exists());
19894 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19895 assert!(state.candidates[0].folded);
19896 assert!(!state.enabled_worktree_config);
19897 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19898 let enabled = git::git_raw(
19899 &repo,
19900 &["config", "--local", "--get", "extensions.worktreeConfig"],
19901 )
19902 .await
19903 .unwrap();
19904 assert!(!enabled.ok(), "the last config reference was released");
19905 }
19906
19907 #[tokio::test]
19910 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19911 let tmp = tempfile::tempdir().expect("tempdir");
19912 let repo = tmp.path().join("repo");
19913 std::fs::create_dir_all(&repo).unwrap();
19914 init_repo(&repo);
19915 let home = tmp.path().join("home");
19916
19917 let mut config = Config::default();
19918 config.graph.worktree_root = Some(tmp.path().join("wt"));
19919 let mut state = RunState::new(
19920 repo.clone(),
19921 "main".to_owned(),
19922 "deadbeef".to_owned(),
19923 "task".to_owned(),
19924 config,
19925 );
19926 git::git(&repo, &["branch", "magi/x/A", "main"])
19928 .await
19929 .expect("branch");
19930 state.candidates = vec![Candidate {
19931 index: 0,
19932 label: 'A',
19933 agent: "alpha".to_owned(),
19934 branch: "magi/x/A".to_owned(),
19935 worktree: state.worktree_root().join("cand-A"),
19936 summary: String::new(),
19937 stat: String::new(),
19938 files: 0,
19939 commits: 0,
19940 empty: false,
19941 failed: None,
19942 verified_noop: None,
19943 duration_ms: 0,
19944 folded: true,
19945 }];
19946 state.released_to = Some("20260901-000000-new1".to_owned());
19947 state.released_branches = vec!["magi/x/A".to_owned()];
19948
19949 fold_run(&mut state, true, &home).await.expect("fold_run");
19950 state.status = RunStatus::Merged;
19951 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19952 state.merge = Some(crate::run::MergeOutcome {
19953 mode: MergeMode::Pr,
19954 ok: true,
19955 detail: "confirmed merged".into(),
19956 empty: false,
19957 });
19958 fold_run(&mut state, true, &home)
19959 .await
19960 .expect("merged fold");
19961
19962 assert!(
19963 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19964 "the handed-over branch survives a fold"
19965 );
19966 }
19967
19968 #[tokio::test]
19972 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19973 let tmp = tempfile::tempdir().expect("tempdir");
19974 let repo = tmp.path().join("repo");
19975 std::fs::create_dir_all(&repo).unwrap();
19976 init_repo(&repo);
19977 let run = |args: &[&str]| {
19978 let out = std::process::Command::new("git")
19979 .quiet()
19980 .args(args)
19981 .current_dir(&repo)
19982 .output()
19983 .expect("spawn git");
19984 assert!(out.status.success(), "git {args:?}");
19985 };
19986 run(&["branch", "magi/x/A"]);
19987 run(&["checkout", "-q", "-b", "magi/x/B"]);
19988 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19989 run(&["add", "-A"]);
19990 run(&["commit", "-q", "-m", "work"]);
19991 run(&["checkout", "-q", "main"]);
19992
19993 let probe = RunState::new(
19997 repo.clone(),
19998 "main".to_owned(),
19999 "deadbeef".to_owned(),
20000 "task".to_owned(),
20001 Config::default(),
20002 );
20003 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
20004 let bare = tmp.path().join("origin.git");
20005 let out = std::process::Command::new("git")
20006 .quiet()
20007 .args(["init", "-q", "--bare"])
20008 .arg(&bare)
20009 .output()
20010 .expect("spawn git");
20011 assert!(out.status.success(), "git init --bare");
20012 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
20013 run(&["push", "-q", "origin", "main"]);
20014
20015 let mut state = RunState::new(
20016 repo.clone(),
20017 "main".to_owned(),
20018 "deadbeef".to_owned(),
20019 "task".to_owned(),
20020 Config::default(),
20021 );
20022 state.seeds = vec![refs::Seed {
20023 token: "magi/27b2/A".to_owned(),
20024 kind: refs::SeedKind::Unresolved,
20025 sha: String::new(),
20026 branch: true,
20027 detail: "no branch or commit named magi/27b2/A".to_owned(),
20028 }];
20029
20030 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
20031 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
20032 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
20033 let detail = empty_candidate_detail(&state, "main");
20034 assert!(detail.starts_with("empty candidate"), "{detail}");
20035 assert!(!detail.contains("magi/27b2/A"), "{detail}");
20036 assert!(detail.contains("--start-from"), "{detail}");
20037 }
20038
20039 #[tokio::test]
20048 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
20049 let tmp = tempfile::tempdir().expect("tempdir");
20050 let repo = tmp.path().join("repo");
20051 std::fs::create_dir_all(&repo).unwrap();
20052 init_repo(&repo);
20053
20054 let mut config = Config::default();
20055 config.merge.mode = MergeMode::Local;
20056
20057 let mut state = RunState::new(
20058 repo.clone(),
20059 "main".to_owned(),
20060 "deadbeef".to_owned(),
20061 "task".to_owned(),
20062 config,
20063 );
20064 state.candidates = vec![Candidate {
20065 index: 0,
20066 label: 'A',
20067 agent: "alpha".to_owned(),
20068 branch: "does-not-exist".to_owned(),
20069 worktree: repo.clone(),
20070 summary: String::new(),
20071 stat: String::new(),
20072 files: 0,
20073 commits: 0,
20074 empty: false,
20075 failed: None,
20076 verified_noop: None,
20077 duration_ms: 0,
20078 folded: false,
20079 }];
20080 state.tally = Some(Tally {
20081 first_choice: BTreeMap::from([('A', 1)]),
20082 borda: BTreeMap::new(),
20083 winner: 'A',
20084 rankings: 1,
20085 unanimous_initial: true,
20086 deliberated: false,
20087 changed_votes: 0,
20088 unanimous_final: true,
20089 tie_break: None,
20090 judges: 0,
20091 present: 0,
20092 quorum: 0,
20093 met_quorum: true,
20094 uncontested: Some("only candidate A produced a change".to_owned()),
20095 });
20096 state.reviews = vec![ReviewRound {
20097 round: 1,
20098 head: "deadbeef".to_owned(),
20099 verified_head: None,
20100 verified_at: None,
20101 reviews: Vec::new(),
20102 e2e: Vec::new(),
20103 fix: None,
20104 blocking: 0,
20105 answered: 0,
20106 expected: 0,
20107 clean: true,
20108 verify_retried: false,
20109 e2e_deferred: false,
20110 e2e_defer_reason: None,
20111 progressed: false,
20112 vote_split: false,
20113 reconsideration: Vec::new(),
20114 verdict: None,
20115 }];
20116 state.gate = vec![CommandOutcome {
20117 command: "test".to_owned(),
20118 code: Some(0),
20119 output_tail: String::new(),
20120 duration_ms: 0,
20121 resource_blocked: false,
20122 }];
20123 state.gate_ran = true;
20124 state.status = RunStatus::Ready;
20129 state.merge = Some(MergeOutcome {
20130 mode: MergeMode::Local,
20131 ok: false,
20132 detail: "already concluded".to_owned(),
20133 empty: false,
20134 });
20135
20136 let mut runner = Runner {
20137 state,
20138 roles: ResolvedRoles {
20139 implementers: Vec::new(),
20140 judges: Vec::new(),
20141 reviewers: Vec::new(),
20142 fixer: None,
20143 conductor: conductor(),
20144 implementer_roster: Vec::new(),
20145 judge_roster: Vec::new(),
20146 reviewer_roster: Vec::new(),
20147 },
20148 sem: Arc::new(Semaphore::new(1)),
20149 pause: Pause::new(),
20150 interrupt: Pause::new(),
20151 };
20152
20153 runner.merge().await.expect("merge");
20154
20155 assert_eq!(
20156 runner.state.status,
20157 RunStatus::Ready,
20158 "a concluded run's status must not change on reentry"
20159 );
20160 assert_eq!(
20161 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20162 Some("already concluded"),
20163 "merge must not run again once the node already recorded an outcome"
20164 );
20165 }
20166
20167 #[tokio::test]
20176 async fn merge_refuses_a_gate_that_has_not_actually_run() {
20177 let tmp = tempfile::tempdir().expect("tempdir");
20178 let repo = tmp.path().join("repo");
20179 std::fs::create_dir_all(&repo).unwrap();
20180 init_repo(&repo);
20181
20182 let mut config = Config::default();
20183 config.merge.mode = MergeMode::Local;
20184
20185 let mut state = RunState::new(
20186 repo.clone(),
20187 "main".to_owned(),
20188 "deadbeef".to_owned(),
20189 "task".to_owned(),
20190 config,
20191 );
20192 state.candidates = vec![Candidate {
20193 index: 0,
20194 label: 'A',
20195 agent: "alpha".to_owned(),
20196 branch: "does-not-exist".to_owned(),
20197 worktree: repo.clone(),
20198 summary: String::new(),
20199 stat: String::new(),
20200 files: 0,
20201 commits: 0,
20202 empty: false,
20203 failed: None,
20204 verified_noop: None,
20205 duration_ms: 0,
20206 folded: false,
20207 }];
20208 state.tally = Some(Tally {
20209 first_choice: BTreeMap::from([('A', 1)]),
20210 borda: BTreeMap::new(),
20211 winner: 'A',
20212 rankings: 1,
20213 unanimous_initial: true,
20214 deliberated: false,
20215 changed_votes: 0,
20216 unanimous_final: true,
20217 tie_break: None,
20218 judges: 0,
20219 present: 0,
20220 quorum: 0,
20221 met_quorum: true,
20222 uncontested: Some("only candidate A produced a change".to_owned()),
20223 });
20224 state.reviews = vec![ReviewRound {
20225 round: 1,
20226 head: "deadbeef".to_owned(),
20227 verified_head: None,
20228 verified_at: None,
20229 reviews: Vec::new(),
20230 e2e: Vec::new(),
20231 fix: None,
20232 blocking: 0,
20233 answered: 0,
20234 expected: 0,
20235 clean: true,
20236 verify_retried: false,
20237 e2e_deferred: false,
20238 e2e_defer_reason: None,
20239 progressed: false,
20240 vote_split: false,
20241 reconsideration: Vec::new(),
20242 verdict: None,
20243 }];
20244 state.gate = Vec::new();
20246 state.gate_ran = false;
20247 state.status = RunStatus::Gating;
20248
20249 let mut runner = Runner {
20250 state,
20251 roles: ResolvedRoles {
20252 implementers: Vec::new(),
20253 judges: Vec::new(),
20254 reviewers: Vec::new(),
20255 fixer: None,
20256 conductor: conductor(),
20257 implementer_roster: Vec::new(),
20258 judge_roster: Vec::new(),
20259 reviewer_roster: Vec::new(),
20260 },
20261 sem: Arc::new(Semaphore::new(1)),
20262 pause: Pause::new(),
20263 interrupt: Pause::new(),
20264 };
20265
20266 runner.merge().await.expect("merge");
20267
20268 assert!(
20269 runner.state.merge.is_none(),
20270 "an empty gate must never be read as a passing one: {:?}",
20271 runner.state.merge
20272 );
20273 }
20274
20275 #[tokio::test]
20282 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20283 let tmp = tempfile::tempdir().expect("tempdir");
20284 let repo = tmp.path().join("repo");
20285 std::fs::create_dir_all(&repo).unwrap();
20286 init_repo(&repo);
20287
20288 let config = Config::default();
20290
20291 let mut state = RunState::new(
20292 repo.clone(),
20293 "main".to_owned(),
20294 "deadbeef".to_owned(),
20295 "task".to_owned(),
20296 config,
20297 );
20298 state.candidates = vec![Candidate {
20299 index: 0,
20300 label: 'A',
20301 agent: "alpha".to_owned(),
20302 branch: "does-not-exist".to_owned(),
20303 worktree: repo.clone(),
20304 summary: String::new(),
20305 stat: String::new(),
20306 files: 0,
20307 commits: 0,
20308 empty: false,
20309 failed: None,
20310 verified_noop: None,
20311 duration_ms: 0,
20312 folded: false,
20313 }];
20314 state.tally = Some(Tally {
20315 first_choice: BTreeMap::from([('A', 1)]),
20316 borda: BTreeMap::new(),
20317 winner: 'A',
20318 rankings: 1,
20319 unanimous_initial: true,
20320 deliberated: false,
20321 changed_votes: 0,
20322 unanimous_final: true,
20323 tie_break: None,
20324 judges: 0,
20325 present: 0,
20326 quorum: 0,
20327 met_quorum: true,
20328 uncontested: Some("only candidate A produced a change".to_owned()),
20329 });
20330 state.reviews = vec![ReviewRound {
20331 round: 1,
20332 head: "deadbeef".to_owned(),
20333 verified_head: None,
20334 verified_at: None,
20335 reviews: Vec::new(),
20336 e2e: Vec::new(),
20337 fix: None,
20338 blocking: 0,
20339 answered: 0,
20340 expected: 0,
20341 clean: true,
20342 verify_retried: false,
20343 e2e_deferred: false,
20344 e2e_defer_reason: None,
20345 progressed: false,
20346 vote_split: false,
20347 reconsideration: Vec::new(),
20348 verdict: None,
20349 }];
20350
20351 let mut runner = Runner {
20352 state,
20353 roles: ResolvedRoles {
20354 implementers: Vec::new(),
20355 judges: Vec::new(),
20356 reviewers: Vec::new(),
20357 fixer: None,
20358 conductor: conductor(),
20359 implementer_roster: Vec::new(),
20360 judge_roster: Vec::new(),
20361 reviewer_roster: Vec::new(),
20362 },
20363 sem: Arc::new(Semaphore::new(1)),
20364 pause: Pause::new(),
20365 interrupt: Pause::new(),
20366 };
20367
20368 runner.gate().await.expect("gate");
20369 assert!(
20370 runner.state.gate_ran,
20371 "zero configured commands is still a real attempt, not an unrun gate"
20372 );
20373 assert!(runner.state.gate.is_empty());
20374 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20375 assert_ne!(
20376 runner.state.status,
20377 RunStatus::Blocked,
20378 "a gate with nothing to check must not read as failed"
20379 );
20380
20381 runner.merge().await.expect("merge");
20382 assert_eq!(
20383 runner.state.status,
20384 RunStatus::Ready,
20385 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20386 );
20387 }
20388
20389 #[tokio::test]
20400 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20401 crate::run::pin_test_home();
20402 let home = crate::run::home();
20403
20404 let tmp = tempfile::tempdir().expect("tempdir");
20405 let repo = tmp.path().join("repo");
20406 std::fs::create_dir_all(&repo).unwrap();
20407 init_repo(&repo);
20408 let cache_dir = tmp.path().join("target");
20411
20412 let mut config = Config::default();
20413 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20414 config.graph.timeout_verify = Some(2);
20418 config.graph.timeout_resource_wait = 0;
20419
20420 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20421 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20422 .expect("no io error acquiring directly")
20423 {
20424 crate::cache::AcquireOutcome::Acquired(g) => g,
20425 crate::cache::AcquireOutcome::Busy(b) => {
20426 panic!("expected the direct acquire to win the lease first: {b:?}")
20427 }
20428 };
20429
20430 let mut state = RunState::new(
20431 repo.clone(),
20432 "main".to_owned(),
20433 "deadbeef".to_owned(),
20434 "task".to_owned(),
20435 config,
20436 );
20437 state.candidates = vec![Candidate {
20438 index: 0,
20439 label: 'A',
20440 agent: "alpha".to_owned(),
20441 branch: "does-not-exist".to_owned(),
20442 worktree: repo.clone(),
20443 summary: String::new(),
20444 stat: String::new(),
20445 files: 0,
20446 commits: 0,
20447 empty: false,
20448 failed: None,
20449 verified_noop: None,
20450 duration_ms: 0,
20451 folded: false,
20452 }];
20453 state.tally = Some(Tally {
20454 first_choice: BTreeMap::from([('A', 1)]),
20455 borda: BTreeMap::new(),
20456 winner: 'A',
20457 rankings: 1,
20458 unanimous_initial: true,
20459 deliberated: false,
20460 changed_votes: 0,
20461 unanimous_final: true,
20462 tie_break: None,
20463 judges: 0,
20464 present: 0,
20465 quorum: 0,
20466 met_quorum: true,
20467 uncontested: Some("only candidate A produced a change".to_owned()),
20468 });
20469 state.reviews = vec![ReviewRound {
20470 round: 1,
20471 head: "deadbeef".to_owned(),
20472 verified_head: None,
20473 verified_at: None,
20474 reviews: Vec::new(),
20475 e2e: Vec::new(),
20476 fix: None,
20477 blocking: 0,
20478 answered: 0,
20479 expected: 0,
20480 clean: true,
20481 verify_retried: false,
20482 e2e_deferred: false,
20483 e2e_defer_reason: None,
20484 progressed: false,
20485 vote_split: false,
20486 reconsideration: Vec::new(),
20487 verdict: None,
20488 }];
20489
20490 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20491 state.reviews[0].e2e = vec![CommandOutcome {
20492 command: state.config.verify.e2e[0].clone(),
20493 code: Some(0),
20494 output_tail: String::new(),
20495 duration_ms: 0,
20496 resource_blocked: false,
20497 }];
20498 let mut runner = Runner {
20499 state,
20500 roles: ResolvedRoles {
20501 implementers: Vec::new(),
20502 judges: Vec::new(),
20503 reviewers: Vec::new(),
20504 fixer: None,
20505 conductor: conductor(),
20506 implementer_roster: Vec::new(),
20507 judge_roster: Vec::new(),
20508 reviewer_roster: Vec::new(),
20509 },
20510 sem: Arc::new(Semaphore::new(1)),
20511 pause: Pause::new(),
20512 interrupt: Pause::new(),
20513 };
20514
20515 runner.gate().await.expect("gate");
20516 assert!(runner.state.resource_block.is_none());
20517 assert!(
20518 !runner
20519 .state
20520 .events
20521 .iter()
20522 .any(|event| { event.message.contains("waiting for the shared build cache") }),
20523 "a gate with nothing to run must never request the held cache lease"
20524 );
20525 assert!(
20526 runner.state.gate_ran,
20527 "zero commands is still a real, immediate attempt"
20528 );
20529 assert!(runner.state.gate.is_empty());
20530 assert_ne!(
20531 runner.state.status,
20532 RunStatus::Blocked,
20533 "must not read as resource-blocked on a lease it never asked for"
20534 );
20535 }
20536
20537 #[cfg(unix)]
20540 #[tokio::test]
20541 async fn verification_is_standalone_inside_a_live_worker() {
20542 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20543 if std::env::var_os(CHILD).is_none() {
20544 let home = tempfile::tempdir().unwrap();
20545 let generation = crate::worker::pin_current(home.path()).unwrap();
20546 let output = std::process::Command::new(std::env::current_exe().unwrap())
20547 .args([
20548 "--exact",
20549 "graph::tests::verification_is_standalone_inside_a_live_worker",
20550 "--nocapture",
20551 ])
20552 .env(CHILD, "1")
20553 .env("MAGI_HOME", home.path())
20554 .env("MAGI_WORKER_HOME", home.path())
20555 .env("MAGI_WORKER_ID", "abcdef")
20556 .env("MAGI_WORKER_GENERATION", &generation.id)
20557 .env("MAGI_WORKER_EXE", &generation.executable)
20558 .env("MAGI_RUN", "verification-parent")
20559 .env("MAGI_NODE", "gate")
20560 .env("MAGI_SEAT", "worker-fixture")
20561 .env("MAGI_PROCESS_OWNER", "fixture-owner")
20562 .env("MAGI_NO_AUTOUPDATE", "0")
20563 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20564 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20565 .env(
20566 "VERIFY_EXPECT_TOOLCHAIN",
20567 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20568 )
20569 .env(
20570 "VERIFY_EXPECT_CACHE",
20571 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20572 )
20573 .output()
20574 .unwrap();
20575 assert!(
20576 output.status.success()
20577 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20578 "{}\n{}",
20579 String::from_utf8_lossy(&output.stdout),
20580 String::from_utf8_lossy(&output.stderr)
20581 );
20582 return;
20583 }
20584 let home = crate::run::home();
20585 let directory = home.join("workers/jobs/abcdef");
20586 std::fs::create_dir_all(&directory).unwrap();
20587 let job = crate::worker::Job {
20588 id: "abcdef".into(),
20589 sequence: 1,
20590 kind: "task".into(),
20591 key: "task:verification".into(),
20592 payload: serde_json::json!({}),
20593 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20594 version: Some(env!("CARGO_PKG_VERSION").into()),
20595 status: crate::worker::Status::Running,
20596 result: None,
20597 pid: Some(std::process::id()),
20598 created_at: 0,
20599 finished_at: None,
20600 };
20601 std::fs::write(
20602 directory.join("job.json"),
20603 serde_json::to_vec(&job).unwrap(),
20604 )
20605 .unwrap();
20606 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20607 let assert_worker = || {
20608 assert_eq!(
20609 crate::worker::current_job(&home).unwrap().unwrap().id,
20610 job.id
20611 );
20612 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20613 assert_eq!(
20614 crate::worker::helper_executable(&home).unwrap(),
20615 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20616 );
20617 assert!(
20618 crate::updater::Checker::new(&crate::config::Update {
20619 mode: crate::config::UpdateMode::Notify,
20620 interval: None,
20621 auto_restart: false,
20622 })
20623 .is_none(),
20624 "the driver remains pinned, even while tests are standalone"
20625 );
20626 };
20627 assert_worker();
20628 let repo = home.join("repo");
20629 std::fs::create_dir_all(&repo).unwrap();
20630 init_repo(&repo);
20631 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20634 test -z "${MAGI_WORKER_GENERATION+x}" &&
20635 test -z "${MAGI_WORKER_HOME+x}" &&
20636 test -z "${MAGI_WORKER_EXE+x}" &&
20637 test "$MAGI_HOME/repo" -ef . &&
20638 test "$MAGI_RUN" = verification-parent &&
20639 test "$MAGI_NODE" = gate &&
20640 test "$MAGI_SEAT" = worker-fixture &&
20641 test -n "$MAGI_PROCESS_OWNER" &&
20642 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20643 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20644 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20645 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20646 assert!(
20648 !std::process::Command::new("sh")
20649 .args(["-c", probe])
20650 .current_dir(&repo)
20651 .status()
20652 .unwrap()
20653 .success()
20654 );
20655 let unrelated_home = tempfile::tempdir().unwrap();
20656 assert!(
20657 crate::worker::current_job(unrelated_home.path()).is_err(),
20658 "an inherited worker id cannot be resolved in a unit fixture's home"
20659 );
20660 let mut runner = runner_at(RunStatus::Gating);
20661 runner.state.repo = repo.clone();
20662 runner.state.config.disk.min_free_bytes = 0;
20663 runner.state.config.coordination = crate::config::test_coordination();
20664 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20665 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20666 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20667 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20668 let winner = Candidate {
20669 index: 0,
20670 label: 'A',
20671 agent: "fixture".into(),
20672 branch: "main".into(),
20673 worktree: repo.clone(),
20674 summary: String::new(),
20675 stat: String::new(),
20676 files: 0,
20677 commits: 0,
20678 empty: false,
20679 failed: None,
20680 verified_noop: None,
20681 duration_ms: 0,
20682 folded: false,
20683 };
20684 let (e2e, retried, pids) = run_e2e_with_retry(
20685 &mut runner.state,
20686 &["sh".into(), "-c".into()],
20687 &[format!("{probe} && printf initial")],
20688 &repo,
20689 Duration::from_secs(10),
20690 "initial",
20691 )
20692 .await;
20693 assert!(!retried && pids.is_empty());
20694 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20695 assert_eq!(e2e[0].output_tail, "initial");
20696 let gate = runner.run_gate(&winner).await.unwrap();
20697 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20698 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20701 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20702 passed: false,
20703 outcomes: Vec::new(),
20704 });
20705 runner.state.save().unwrap();
20706 let mut resumed = RunState::load(&runner.state.id).unwrap();
20707 let wait = crate::repo_lock::WaitPolicy::default();
20708 assert!(matches!(
20709 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20710 .await
20711 .unwrap(),
20712 GateFix::Retry
20713 ));
20714 let saved = RunState::load(&resumed.id).unwrap();
20715 let repair = saved.gate_repair_e2e.unwrap();
20716 assert!(repair.passed);
20717 assert_eq!(repair.outcomes.len(), 1);
20718 assert!(repair.outcomes[0].ok());
20719 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20720 .await
20721 .unwrap();
20722 assert_eq!(landing.len(), 1);
20723 assert!(landing[0].ok(), "{landing:?}");
20724 assert_eq!(
20725 std::fs::read_to_string(repo.join("phases")).unwrap(),
20726 "pre\ngate\ne2e\npre\ngate\n"
20727 );
20728 assert_worker();
20729 }
20730
20731 #[cfg(unix)]
20732 #[tokio::test]
20733 async fn verification_keeps_the_panic_and_the_full_output() {
20734 crate::run::pin_test_home();
20735 let tmp = tempfile::tempdir().unwrap();
20736 let mut config = Config::default();
20737 config.disk.min_free_bytes = 0;
20738 let mut state = RunState::new(
20739 tmp.path().to_owned(),
20740 "main".into(),
20741 "base".into(),
20742 "verification evidence".into(),
20743 config,
20744 );
20745 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20746 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20747 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20748 let mut names = Vec::new();
20749 for _ in 0..2 {
20750 let (outcomes, _) = run_commands(
20751 &mut state,
20752 "gate",
20753 "gate",
20754 0,
20755 &["sh".into(), "-c".into()],
20756 &[script.into()],
20757 tmp.path(),
20758 Duration::from_secs(60),
20759 )
20760 .await;
20761 let tail = &outcomes[0].output_tail;
20762 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20763 assert!(tail.contains("boom assertion"));
20764 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20765 let line = tail.lines().next().unwrap();
20766 names.push(line.to_owned());
20767 }
20768 assert_ne!(names[0], names[1], "the second run must not overwrite");
20769 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20770 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20771 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20772
20773 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20775 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20776 let (outcomes, _) = run_commands(
20777 &mut state,
20778 "gate",
20779 "gate",
20780 0,
20781 &["sh".into(), "-c".into()],
20782 &["echo failure-output; exit 7".into()],
20783 tmp.path(),
20784 Duration::from_secs(10),
20785 )
20786 .await;
20787 assert_eq!(outcomes[0].code, Some(7));
20788 assert!(state.events.iter().any(|e| {
20789 e.node == "gate" && e.message.contains("could not save the full gate output:")
20790 }));
20791 }
20792
20793 #[cfg(unix)]
20794 #[tokio::test]
20795 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20796 crate::run::pin_test_home();
20797 let tmp = tempfile::tempdir().unwrap();
20798 let mut config = Config::default();
20799 config.disk.min_free_bytes = 0;
20800 let mut state = RunState::new(
20801 tmp.path().to_owned(),
20802 "main".into(),
20803 "base".into(),
20804 "verification evidence".into(),
20805 config,
20806 );
20807 let (outcomes, pids) = run_commands(
20808 &mut state,
20809 "gate",
20810 "gate",
20811 1,
20812 &["sh".into(), "-c".into()],
20813 &[
20814 "printf before-timeout; exec sleep 30".into(),
20815 "kill -TERM $$".into(),
20816 "exit 7".into(),
20817 "printf success".into(),
20818 ],
20819 tmp.path(),
20820 Duration::from_secs(1),
20821 )
20822 .await;
20823 wait_for_timed_out_children_to_die(&pids).await;
20824 assert_eq!(
20825 pids.len(),
20826 1,
20827 "unleased timeout retains its death-check PID"
20828 );
20829 assert_eq!(outcomes.len(), 4);
20830 assert_eq!(outcomes[0].code, None);
20831 assert!(outcomes[0].output_tail.contains("before-timeout"));
20832 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20833 assert_eq!(outcomes[1].code, None);
20834 assert!(outcomes[1].output_tail.contains("command terminated:"));
20835 assert!(!outcomes[1].output_tail.contains("timed out"));
20836 assert_eq!(outcomes[2].code, Some(7));
20837 assert!(!outcomes[2].output_tail.contains("timed out"));
20838 assert_eq!(outcomes[3].code, Some(0));
20839 assert_eq!(outcomes[3].output_tail, "success");
20840 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20841 }
20842
20843 #[tokio::test]
20853 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20854 crate::run::pin_test_home();
20855
20856 let tmp = tempfile::tempdir().expect("tempdir");
20857 let repo = tmp.path().join("repo");
20858 std::fs::create_dir_all(&repo).unwrap();
20859 init_repo(&repo);
20860
20861 let mut config = Config::default();
20862 config.disk.min_free_bytes = 0;
20863 config.verify.gate = vec![
20864 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20865 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20866 .to_owned(),
20867 ];
20868
20869 let mut state = RunState::new(
20870 repo.clone(),
20871 "main".to_owned(),
20872 "deadbeef".to_owned(),
20873 "task".to_owned(),
20874 config,
20875 );
20876 let run_id = state.id.clone();
20877 state.candidates = vec![Candidate {
20878 index: 0,
20879 label: 'A',
20880 agent: "alpha".to_owned(),
20881 branch: "does-not-exist".to_owned(),
20882 worktree: repo.clone(),
20883 summary: String::new(),
20884 stat: String::new(),
20885 files: 0,
20886 commits: 0,
20887 empty: false,
20888 failed: None,
20889 verified_noop: None,
20890 duration_ms: 0,
20891 folded: false,
20892 }];
20893 state.tally = Some(Tally {
20894 first_choice: BTreeMap::from([('A', 1)]),
20895 borda: BTreeMap::new(),
20896 winner: 'A',
20897 rankings: 1,
20898 unanimous_initial: true,
20899 deliberated: false,
20900 changed_votes: 0,
20901 unanimous_final: true,
20902 tie_break: None,
20903 judges: 0,
20904 present: 0,
20905 quorum: 0,
20906 met_quorum: true,
20907 uncontested: Some("only candidate A produced a change".to_owned()),
20908 });
20909 state.reviews = vec![ReviewRound {
20910 round: 1,
20911 head: "deadbeef".to_owned(),
20912 verified_head: None,
20913 verified_at: None,
20914 reviews: Vec::new(),
20915 e2e: Vec::new(),
20916 fix: None,
20917 blocking: 0,
20918 answered: 0,
20919 expected: 0,
20920 clean: true,
20921 verify_retried: false,
20922 e2e_deferred: false,
20923 e2e_defer_reason: None,
20924 progressed: false,
20925 vote_split: false,
20926 reconsideration: Vec::new(),
20927 verdict: None,
20928 }];
20929
20930 let mut runner = Runner {
20931 state,
20932 roles: ResolvedRoles {
20933 implementers: Vec::new(),
20934 judges: Vec::new(),
20935 reviewers: Vec::new(),
20936 fixer: None,
20937 conductor: conductor(),
20938 implementer_roster: Vec::new(),
20939 judge_roster: Vec::new(),
20940 reviewer_roster: Vec::new(),
20941 },
20942 sem: Arc::new(Semaphore::new(1)),
20943 pause: Pause::new(),
20944 interrupt: Pause::new(),
20945 };
20946
20947 let started_marker = repo.join("started.marker");
20948 let release_marker = repo.join("release.marker");
20949 let poller = tokio::spawn(async move {
20950 for _ in 0..100 {
20955 if started_marker.exists()
20956 && let Ok(s) = crate::run::RunState::load(&run_id)
20957 && let Some(a) = s.active.get("gate")
20958 {
20959 std::fs::write(&release_marker, b"go").expect("release marker");
20960 return Some(a.clone());
20961 }
20962 tokio::time::sleep(Duration::from_millis(50)).await;
20963 }
20964 None
20965 });
20966
20967 runner.gate().await.expect("gate");
20968 let captured = poller.await.expect("poller task");
20969 let captured = captured.expect(
20970 "the poller never saw a `gate` task entry in run.json while the command was \
20971 still blocked on its own release marker",
20972 );
20973
20974 assert_eq!(captured.task.as_deref(), Some("gate"));
20975 assert_eq!(captured.node, "gate");
20976 assert_eq!(captured.index, Some(1));
20977 assert_eq!(captured.total, Some(1));
20978 assert!(
20979 captured
20980 .command
20981 .as_deref()
20982 .is_some_and(|c| c.contains("started.marker")),
20983 "{captured:?}"
20984 );
20985
20986 assert!(
20987 runner.state.active.is_empty(),
20988 "the entry must be cleared once the command actually finished: {:?}",
20989 runner.state.active
20990 );
20991 assert!(runner.state.gate_ran);
20992 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20993 }
20994
20995 #[tokio::test]
20998 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20999 use crate::verdict::{Finding, ReviewVote, Severity};
21000 crate::run::pin_test_home();
21001 let tmp = tempfile::tempdir().expect("tempdir");
21002 let repo = tmp.path().join("repo");
21003 std::fs::create_dir_all(&repo).unwrap();
21004 init_repo(&repo);
21005
21006 for (severity, vote, expect) in [
21007 (Severity::Major, ReviewVote::Reject, true),
21008 (Severity::Minor, ReviewVote::Reject, false),
21009 (Severity::Major, ReviewVote::Approve, false),
21010 ] {
21011 let mut round = review_round(false, 1, 1, 1, false, true);
21012 round.reviews = vec![ReviewRecord {
21013 reviewer: 1,
21014 agent: "alpha".to_owned(),
21015 summary: String::new(),
21016 findings: vec![Finding {
21017 id: "R1-1-1".to_owned(),
21018 severity,
21019 file: None,
21020 line: None,
21021 title: "t".to_owned(),
21022 detail: String::new(),
21023 }],
21024 vote: Some(vote),
21025 failed: None,
21026 duration_ms: 0,
21027 attempts: 0,
21028 }];
21029 let mut state = RunState::new(
21030 repo.clone(),
21031 "main".to_owned(),
21032 "deadbeef".to_owned(),
21033 "task".to_owned(),
21034 Config::default(),
21035 );
21036 state.reviews = vec![round];
21037 let mut runner = Runner {
21038 state,
21039 roles: ResolvedRoles {
21040 implementers: Vec::new(),
21041 judges: Vec::new(),
21042 reviewers: Vec::new(),
21043 fixer: None,
21044 conductor: conductor(),
21045 implementer_roster: Vec::new(),
21046 judge_roster: Vec::new(),
21047 reviewer_roster: Vec::new(),
21048 },
21049 sem: Arc::new(Semaphore::new(1)),
21050 pause: Pause::new(),
21051 interrupt: Pause::new(),
21052 };
21053 let shell = runner.state.config.shell();
21054 runner
21055 .stop_reviewing("round budget spent", &shell, &repo)
21056 .await
21057 .expect("stop_reviewing");
21058 assert_eq!(runner.state.status, RunStatus::Gating);
21059 assert_eq!(
21060 runner.state.contested_handoff.is_some(),
21061 expect,
21062 "{severity:?} + {vote:?}"
21063 );
21064 }
21065 }
21066
21067 #[tokio::test]
21080 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
21081 crate::run::pin_test_home();
21082 let home = crate::run::home();
21083
21084 let tmp = tempfile::tempdir().expect("tempdir");
21085 let repo = tmp.path().join("repo");
21086 std::fs::create_dir_all(&repo).unwrap();
21087 init_repo(&repo);
21088 let head = crate::git::rev_parse(&repo, "HEAD")
21089 .await
21090 .expect("rev-parse");
21091 let cache_dir = tmp.path().join("target");
21094
21095 let mut config = Config::default();
21096 config.verify.e2e = vec![format!(
21097 "CARGO_TARGET_DIR='{}' test -f README.md",
21098 cache_dir.display()
21099 )];
21100 config.graph.review_rounds = 1;
21101 config.graph.timeout_verify = Some(2);
21104 config.graph.timeout_resource_wait = 2;
21105
21106 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21107 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21108 .expect("no io error acquiring directly")
21109 {
21110 crate::cache::AcquireOutcome::Acquired(g) => g,
21111 crate::cache::AcquireOutcome::Busy(b) => {
21112 panic!("expected the direct acquire to win the lease first: {b:?}")
21113 }
21114 };
21115
21116 let mut state = RunState::new(
21117 repo.clone(),
21118 "main".to_owned(),
21119 head.clone(),
21120 "task".to_owned(),
21121 config,
21122 );
21123 state.candidates = vec![Candidate {
21124 index: 0,
21125 label: 'A',
21126 agent: "alpha".to_owned(),
21127 branch: "does-not-exist".to_owned(),
21128 worktree: repo.clone(),
21129 summary: String::new(),
21130 stat: String::new(),
21131 files: 0,
21132 commits: 0,
21133 empty: false,
21134 failed: None,
21135 verified_noop: None,
21136 duration_ms: 0,
21137 folded: false,
21138 }];
21139 state.tally = Some(Tally {
21140 first_choice: BTreeMap::from([('A', 1)]),
21141 borda: BTreeMap::new(),
21142 winner: 'A',
21143 rankings: 1,
21144 unanimous_initial: true,
21145 deliberated: false,
21146 changed_votes: 0,
21147 unanimous_final: true,
21148 tie_break: None,
21149 judges: 0,
21150 present: 0,
21151 quorum: 0,
21152 met_quorum: true,
21153 uncontested: Some("only candidate A produced a change".to_owned()),
21154 });
21155 state.reviews = vec![ReviewRound {
21159 round: 1,
21160 head: head.clone(),
21161 verified_head: None,
21162 verified_at: None,
21163 reviews: Vec::new(),
21164 e2e: Vec::new(),
21165 fix: None,
21166 blocking: 1,
21167 answered: 1,
21168 expected: 1,
21169 clean: false,
21170 verify_retried: false,
21171 e2e_deferred: true,
21172 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
21173 progressed: false,
21174 vote_split: false,
21175 reconsideration: Vec::new(),
21176 verdict: None,
21177 }];
21178
21179 let mut runner = Runner {
21180 state,
21181 roles: ResolvedRoles {
21182 implementers: Vec::new(),
21183 judges: Vec::new(),
21184 reviewers: Vec::new(),
21185 fixer: None,
21186 conductor: conductor(),
21187 implementer_roster: Vec::new(),
21188 judge_roster: Vec::new(),
21189 reviewer_roster: Vec::new(),
21190 },
21191 sem: Arc::new(Semaphore::new(1)),
21192 pause: Pause::new(),
21193 interrupt: Pause::new(),
21194 };
21195
21196 let shell = runner.state.config.shell();
21197 runner
21198 .stop_reviewing("round budget spent", &shell, &repo)
21199 .await
21200 .expect("stop_reviewing");
21201
21202 let last = runner.state.reviews.last().expect("round record");
21203 assert_eq!(
21204 last.e2e_status(),
21205 E2eStatus::ResourceBlocked,
21206 "the shared cache is still held; the attempt must read as blocked, not deferred or \
21207 failed: {last:?}"
21208 );
21209 assert_eq!(
21210 last.verified_head.as_deref(),
21211 Some(head.as_str()),
21212 "which commit this attempt targeted is known even though nothing finished checking \
21213 it"
21214 );
21215 let first_attempt_at = last
21216 .verified_at
21217 .expect("when this attempt ran is known too");
21218 assert_ne!(
21219 runner.state.status,
21220 RunStatus::Blocked,
21221 "contention is evidence about the machine, not the patch — it must not settle the \
21222 run as blocked: {:?}",
21223 runner.state.status
21224 );
21225 assert!(
21226 !runner
21227 .state
21228 .events
21229 .iter()
21230 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
21231 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
21232 runner.state.events
21233 );
21234
21235 runner
21240 .stop_reviewing("round budget spent", &shell, &repo)
21241 .await
21242 .expect("stop_reviewing retry");
21243 assert_eq!(
21244 runner.state.reviews.len(),
21245 1,
21246 "no new round was started: {:?}",
21247 runner.state.reviews
21248 );
21249 let last = runner.state.reviews.last().expect("round record");
21250 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
21251 assert!(
21252 last.verified_at.expect("still known") > first_attempt_at,
21253 "a second reentry must be a fresh attempt, not a stale copy of the first"
21254 );
21255 assert_ne!(runner.state.status, RunStatus::Blocked);
21256
21257 held.release();
21258 }
21259
21260 #[cfg(unix)]
21261 #[tokio::test]
21262 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
21263 ask_test_home();
21264 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21265 let tmp = tempfile::tempdir().unwrap();
21266 let repo = tmp.path().join("repo");
21267 std::fs::create_dir(&repo).unwrap();
21268 init_repo(&repo);
21269 let run_git = |args: &[&str]| {
21270 let out = std::process::Command::new("git")
21271 .args(args)
21272 .current_dir(&repo)
21273 .output()
21274 .unwrap();
21275 assert!(
21276 out.status.success(),
21277 "{args:?}: {}",
21278 String::from_utf8_lossy(&out.stderr)
21279 );
21280 };
21281 let origin = tmp.path().join("origin.git");
21282 run_git(&["init", "--bare", origin.to_str().unwrap()]);
21283 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21284 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21285 run_git(&["add", "spec.rs"]);
21286 run_git(&["commit", "-m", "initial spec"]);
21287 run_git(&["push", "origin", "main"]);
21288 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21289 run_git(&["checkout", "-b", "fix/review"]);
21290 std::fs::write(
21291 repo.join("check.rs"),
21292 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21293 )
21294 .unwrap();
21295 run_git(&["add", "check.rs"]);
21296 run_git(&["commit", "-m", "use spec"]);
21297 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21298 run_git(&["checkout", "main"]);
21299 std::fs::write(
21300 repo.join("spec.rs"),
21301 "struct AgentSpec { max_turn_secs: u64 }\n",
21302 )
21303 .unwrap();
21304 run_git(&["add", "spec.rs"]);
21305 run_git(&["commit", "-m", "extend spec"]);
21306 run_git(&["push", "origin", "main"]);
21307 run_git(&["checkout", "fix/review"]);
21308
21309 let mut runner = runner_at(RunStatus::Reviewing);
21310 runner.state.repo = repo.clone();
21311 runner.state.base_commit = base;
21312 runner.state.pr_head = Some("fix/review".into());
21313 runner.state.config.graph.review_rounds = max_rounds;
21314 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21315 runner.state.config.graph.sessions = false;
21316 runner.state.config.disk.min_free_bytes = 0;
21317 runner.state.config.verify.e2e =
21318 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21319 runner.state.candidates = vec![Candidate {
21320 index: 0,
21321 label: 'A',
21322 agent: "fixture".into(),
21323 branch: "fix/review".into(),
21324 worktree: repo.clone(),
21325 summary: String::new(),
21326 stat: String::new(),
21327 files: 1,
21328 commits: 1,
21329 empty: false,
21330 failed: None,
21331 verified_noop: None,
21332 duration_ms: 0,
21333 folded: false,
21334 }];
21335 runner.state.tally = Some(Tally {
21336 first_choice: BTreeMap::new(),
21337 borda: BTreeMap::new(),
21338 winner: 'A',
21339 rankings: 0,
21340 unanimous_initial: true,
21341 deliberated: false,
21342 changed_votes: 0,
21343 unanimous_final: true,
21344 tie_break: None,
21345 judges: 0,
21346 present: 0,
21347 quorum: 0,
21348 met_quorum: true,
21349 uncontested: Some("review-only".into()),
21350 });
21351 let mut last = review_round(false, 0, 1, 1, false, false);
21352 last.head = reviewed.clone();
21353 last.e2e_deferred = true;
21354 last.e2e[0].resource_blocked = true;
21355 last.e2e[0].output_tail = "waiting for the shared build cache".into();
21356 if max_rounds == 1 {
21357 last.e2e.clear();
21358 }
21359 runner.state.reviews = vec![last];
21360 let calls = tmp.path().join("fix-calls");
21361 let mut fixer = spec("repair");
21362 fixer
21363 .env
21364 .insert("FIX_CALLS".into(), calls.display().to_string());
21365 fixer.command = vec!["sh".into(), "-c".into(), r#"
21366 echo fix >> "$FIX_CALLS"
21367 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21368 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21369 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21370 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21371 "#.into()];
21372 let mut reviewer = spec("reviewer");
21373 reviewer.command = vec![
21374 "sh".into(),
21375 "-c".into(),
21376 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21377 .into(),
21378 ];
21379 runner.roles.fixer = Some(vec![fixer]);
21380 runner.roles.reviewers = vec![reviewer];
21381
21382 runner.sync_to_base().await.unwrap();
21383 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21384 assert_ne!(rebased, reviewed);
21385 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21386 if reentry {
21387 let shell = runner.state.config.shell();
21390 runner
21391 .stop_unresolved_review("deferred verification", &shell, &repo)
21392 .await
21393 .unwrap();
21394 assert!(runner.catchup_repairable());
21395 assert!(!calls.exists());
21396 runner.state = RunState::load(&runner.state.id).unwrap();
21397 assert!(runner.catchup_repairable());
21398 }
21399 runner.review_loop().await.unwrap();
21400 let first = &runner.state.reviews[0];
21401 assert_eq!(first.head, reviewed);
21402 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21403 assert!(
21404 first.verify_retried,
21405 "a reproduced build failure gets exactly one retry: {:?}",
21406 first.e2e
21407 );
21408 assert!(first.e2e[0].output_tail.contains("E0063"));
21409 assert!(!runner.catchup_repairable());
21410 if max_rounds == 2 {
21411 assert_eq!(
21412 runner.state.status,
21413 RunStatus::Gating,
21414 "{}",
21415 crate::report::run(&runner.state)
21416 );
21417 assert_eq!(runner.state.reviews.len(), 2);
21418 assert!(first.fix.as_ref().unwrap().committed);
21419 assert!(runner.state.reviews[1].clean);
21420 assert_eq!(
21421 runner
21422 .state
21423 .reviews
21424 .iter()
21425 .filter(|r| r.fix.is_some())
21426 .count(),
21427 1
21428 );
21429 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21430 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21431 } else {
21432 assert_eq!(runner.state.status, RunStatus::Blocked);
21433 assert!(first.fix.is_none());
21434 assert!(!calls.exists());
21435 let recovery = crate::recovery::assess(
21436 "prior",
21437 Some(&runner.state),
21438 &runner.landing_base(),
21439 crate::recovery::Relation::Diverged,
21440 );
21441 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21442 let reasons = recovery.reasons.join("\n");
21443 assert!(
21444 reasons.contains(&reviewed) && reasons.contains(&rebased),
21445 "{reasons}"
21446 );
21447 assert!(
21448 reasons.contains("base sync to")
21449 && reasons.contains("additional review is required"),
21450 "{reasons}"
21451 );
21452 }
21453 }
21454 }
21455
21456 #[tokio::test]
21468 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21469 crate::run::pin_test_home();
21470 let home = crate::run::home();
21471
21472 let tmp = tempfile::tempdir().expect("tempdir");
21473 let repo = tmp.path().join("repo");
21474 std::fs::create_dir_all(&repo).unwrap();
21475 init_repo(&repo);
21476 let head = crate::git::rev_parse(&repo, "HEAD")
21477 .await
21478 .expect("rev-parse");
21479 let cache_dir = tmp.path().join("target");
21480
21481 let mut config = Config::default();
21482 config.verify.e2e = vec![format!(
21483 "CARGO_TARGET_DIR='{}' test -f README.md",
21484 cache_dir.display()
21485 )];
21486 config.graph.review_rounds = 1;
21487 config.graph.timeout_verify = Some(2);
21488 config.graph.timeout_resource_wait = 2;
21489
21490 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21491 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21492 .expect("no io error acquiring directly")
21493 {
21494 crate::cache::AcquireOutcome::Acquired(g) => g,
21495 crate::cache::AcquireOutcome::Busy(b) => {
21496 panic!("expected the direct acquire to win the lease first: {b:?}")
21497 }
21498 };
21499
21500 let mut state = RunState::new(
21501 repo.clone(),
21502 "main".to_owned(),
21503 head.clone(),
21504 "task".to_owned(),
21505 config,
21506 );
21507 state.candidates = vec![Candidate {
21508 index: 0,
21509 label: 'A',
21510 agent: "alpha".to_owned(),
21511 branch: "does-not-exist".to_owned(),
21512 worktree: repo.clone(),
21513 summary: String::new(),
21514 stat: String::new(),
21515 files: 0,
21516 commits: 0,
21517 empty: false,
21518 failed: None,
21519 verified_noop: None,
21520 duration_ms: 0,
21521 folded: false,
21522 }];
21523 state.tally = Some(Tally {
21524 first_choice: BTreeMap::from([('A', 1)]),
21525 borda: BTreeMap::new(),
21526 winner: 'A',
21527 rankings: 1,
21528 unanimous_initial: true,
21529 deliberated: false,
21530 changed_votes: 0,
21531 unanimous_final: true,
21532 tie_break: None,
21533 judges: 0,
21534 present: 0,
21535 quorum: 0,
21536 met_quorum: true,
21537 uncontested: Some("only candidate A produced a change".to_owned()),
21538 });
21539 state.reviews = vec![ReviewRound {
21543 round: 1,
21544 head: head.clone(),
21545 verified_head: Some(head.clone()),
21546 verified_at: Some(jiff::Timestamp::now()),
21547 reviews: Vec::new(),
21548 e2e: vec![CommandOutcome {
21549 command: format!(
21550 "CARGO_TARGET_DIR='{}' test -f README.md",
21551 cache_dir.display()
21552 ),
21553 code: None,
21554 output_tail: "waiting for the shared build cache".to_owned(),
21555 duration_ms: 0,
21556 resource_blocked: true,
21557 }],
21558 fix: None,
21559 blocking: 1,
21560 answered: 1,
21561 expected: 1,
21562 clean: false,
21563 verify_retried: false,
21564 e2e_deferred: false,
21565 e2e_defer_reason: None,
21566 progressed: false,
21567 vote_split: false,
21568 reconsideration: Vec::new(),
21569 verdict: None,
21570 }];
21571
21572 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21573 let mut runner = Runner {
21574 state,
21575 roles: ResolvedRoles {
21576 implementers: Vec::new(),
21577 judges: Vec::new(),
21578 reviewers: Vec::new(),
21579 fixer: None,
21580 conductor: conductor(),
21581 implementer_roster: Vec::new(),
21582 judge_roster: Vec::new(),
21583 reviewer_roster: Vec::new(),
21584 },
21585 sem: Arc::new(Semaphore::new(1)),
21586 pause: Pause::new(),
21587 interrupt: Pause::new(),
21588 };
21589
21590 runner.review_loop().await.expect("review_loop");
21595
21596 assert_eq!(
21597 runner.state.reviews.len(),
21598 1,
21599 "no new round was started on top of the unresolved one: {:?}",
21600 runner.state.reviews
21601 );
21602 let last = &runner.state.reviews[0];
21603 assert_eq!(
21604 last.e2e_status(),
21605 E2eStatus::ResourceBlocked,
21606 "still contended: {last:?}"
21607 );
21608 assert!(
21609 last.verified_at.expect("still known") > first_attempt_at,
21610 "review_loop must have actually retried the check, not left it exactly as found"
21611 );
21612 assert_ne!(
21613 runner.state.status,
21614 RunStatus::Blocked,
21615 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21616 runner.state.status
21617 );
21618
21619 held.release();
21620 }
21621
21622 #[tokio::test]
21623 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21624 crate::run::pin_test_home();
21625 for status in [RunStatus::Blocked, RunStatus::Gating] {
21626 let mut runner = runner_at(status);
21627 runner.state.config.graph.land = true;
21628 runner.state.gate_ran = true;
21629 runner.state.merge = Some(MergeOutcome {
21630 mode: MergeMode::Pr,
21631 ok: false,
21632 detail: "the fixer produced no commit".into(),
21633 empty: false,
21634 });
21635 let url = "https://example.invalid/o/r/pull/1";
21636 runner.state.pr = Some(crate::run::PrRecord {
21637 url: url.into(),
21638 number: 1,
21639 state: "open".into(),
21640 checks: "red".into(),
21641 round: 1,
21642 rounds: 1,
21643 red_at_merge: Vec::new(),
21644 });
21645 runner.state.save().unwrap();
21646 runner.execute_graph().await.unwrap();
21650 assert_eq!(runner.state.status, RunStatus::Blocked);
21651 assert!(runner.state.land_sync_pending);
21652 assert!(runner.state.events.iter().any(|event|
21653 event.node == "land" && event.message == format!("watching {url}")));
21654 assert_eq!(
21655 runner.state.merge.as_ref().unwrap().detail,
21656 "the fixer produced no commit"
21657 );
21658 }
21659 }
21660
21661 #[tokio::test]
21662 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21663 crate::run::pin_test_home();
21664 let tmp = tempfile::tempdir().expect("tempdir");
21665 let repo = tmp.path().join("repo");
21666 std::fs::create_dir_all(&repo).unwrap();
21667 init_repo(&repo);
21668
21669 let mut config = Config::default();
21670 config.merge.mode = MergeMode::Pr;
21671 config.graph.land = true;
21672 config.graph.land_approval = false;
21673
21674 let mut state = RunState::new(
21675 repo.clone(),
21676 "main".to_owned(),
21677 "deadbeef".to_owned(),
21678 "task".to_owned(),
21679 config,
21680 );
21681 state.candidates = vec![Candidate {
21682 index: 0,
21683 label: 'A',
21684 agent: "alpha".to_owned(),
21685 branch: "does-not-exist".to_owned(),
21686 worktree: repo.clone(),
21687 summary: String::new(),
21688 stat: String::new(),
21689 files: 0,
21690 commits: 0,
21691 empty: false,
21692 failed: None,
21693 verified_noop: None,
21694 duration_ms: 0,
21695 folded: false,
21696 }];
21697 state.tally = Some(Tally {
21698 first_choice: BTreeMap::from([('A', 1)]),
21699 borda: BTreeMap::new(),
21700 winner: 'A',
21701 rankings: 1,
21702 unanimous_initial: true,
21703 deliberated: false,
21704 changed_votes: 0,
21705 unanimous_final: true,
21706 tie_break: None,
21707 judges: 0,
21708 present: 0,
21709 quorum: 0,
21710 met_quorum: true,
21711 uncontested: Some("only candidate A produced a change".to_owned()),
21712 });
21713 state.reviews = vec![ReviewRound {
21714 round: 1,
21715 head: "deadbeef".to_owned(),
21716 verified_head: None,
21717 verified_at: None,
21718 reviews: Vec::new(),
21719 e2e: Vec::new(),
21720 fix: None,
21721 blocking: 0,
21722 answered: 0,
21723 expected: 0,
21724 clean: true,
21725 verify_retried: false,
21726 e2e_deferred: false,
21727 e2e_defer_reason: None,
21728 progressed: false,
21729 vote_split: false,
21730 reconsideration: Vec::new(),
21731 verdict: None,
21732 }];
21733 state.gate = vec![CommandOutcome {
21734 command: "test".to_owned(),
21735 code: Some(0),
21736 output_tail: String::new(),
21737 duration_ms: 0,
21738 resource_blocked: false,
21739 }];
21740 state.gate_ran = true;
21741 state.status = RunStatus::Landing;
21745 state.merge = Some(MergeOutcome {
21746 mode: MergeMode::Pr,
21747 ok: true,
21748 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21749 empty: false,
21750 });
21751
21752 ask_test_home();
21756 state.save().expect("save landing run");
21759 let store = ask::Questions::open();
21760 let q = ask_open_question(&store, &state.id);
21761
21762 let mut runner = Runner {
21763 state,
21764 roles: ResolvedRoles {
21765 implementers: Vec::new(),
21766 judges: Vec::new(),
21767 reviewers: Vec::new(),
21768 fixer: None,
21769 conductor: conductor(),
21770 implementer_roster: Vec::new(),
21771 judge_roster: Vec::new(),
21772 reviewer_roster: Vec::new(),
21773 },
21774 sem: Arc::new(Semaphore::new(1)),
21775 pause: Pause::new(),
21776 interrupt: Pause::new(),
21777 };
21778
21779 runner.execute().await.expect("execute");
21784
21785 assert_eq!(
21786 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21787 Some("https://example.invalid/x/y/pull/1"),
21788 "reentry must not push again or open a second pull request over the \
21789 one `land` is already watching"
21790 );
21791 assert_ne!(
21792 runner.state.status,
21793 RunStatus::Landing,
21794 "land could not actually reach the fake pull request, so it must \
21795 have given up rather than left the run silently parked forever"
21796 );
21797 assert_eq!(runner.state.status, RunStatus::Blocked);
21801 assert!(
21802 store.get(&q.id).unwrap().status.open(),
21803 "Blocked is still alive; settle_questions must have been a no-op here"
21804 );
21805 }
21806
21807 fn state_with_round(round: ReviewRound) -> RunState {
21808 let mut s = RunState::new(
21809 PathBuf::from("/repo"),
21810 "main".to_owned(),
21811 "abc1234".to_owned(),
21812 "add retries".to_owned(),
21813 Config::default(),
21814 );
21815 s.reviews = vec![round];
21816 s
21817 }
21818
21819 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21820 crate::verdict::Finding {
21821 id: id.to_owned(),
21822 severity,
21823 file: None,
21824 line: None,
21825 title: title.to_owned(),
21826 detail: String::new(),
21827 }
21828 }
21829
21830 #[test]
21831 fn pr_body_names_open_findings_and_declined_ones() {
21832 let round = ReviewRound {
21833 round: 2,
21834 head: "deadbee".to_owned(),
21835 verified_head: None,
21836 verified_at: None,
21837 reviews: vec![ReviewRecord {
21838 attempts: 0,
21839 reviewer: 1,
21840 agent: "alpha".to_owned(),
21841 summary: String::new(),
21842 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21843 vote: None,
21844 failed: None,
21845 duration_ms: 0,
21846 }],
21847 e2e: vec![CommandOutcome {
21848 command: "cargo test".to_owned(),
21849 code: Some(0),
21850 output_tail: String::new(),
21851 duration_ms: 0,
21852 resource_blocked: false,
21853 }],
21854 verify_retried: false,
21855 e2e_deferred: false,
21856 e2e_defer_reason: None,
21857 fix: Some(FixRecord {
21858 agent: "alpha".to_owned(),
21859 addressed: Vec::new(),
21860 rejected: vec![crate::verdict::Rejection {
21861 id: "R1-1-1".to_owned(),
21862 why: "not reachable from any caller".to_owned(),
21863 }],
21864 notes: String::new(),
21865 committed: true,
21866 failed: None,
21867 duration_ms: 0,
21868 continuation: None,
21869 }),
21870 blocking: 0,
21871 answered: 1,
21872 expected: 1,
21873 clean: false,
21874 progressed: true,
21875 vote_split: false,
21876 reconsideration: Vec::new(),
21877 verdict: None,
21878 };
21879 let state = state_with_round(round);
21880 let body = pr_message(&state, 'A').body;
21881
21882 assert!(body.contains("add retries"), "the task must still be there");
21883 assert!(body.contains("R2-1-1"), "{body}");
21884 assert!(!body.contains("unused import"), "{body}");
21885 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21886 assert!(
21887 !body.contains("not reachable from any caller"),
21888 "the reason it was declined: {body}"
21889 );
21890 }
21891
21892 #[test]
21893 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21894 let round = ReviewRound {
21895 round: 1,
21896 head: "deadbee".to_owned(),
21897 verified_head: None,
21898 verified_at: None,
21899 reviews: vec![ReviewRecord {
21900 attempts: 0,
21901 reviewer: 1,
21902 agent: "alpha".to_owned(),
21903 summary: String::new(),
21904 findings: Vec::new(),
21905 vote: None,
21906 failed: None,
21907 duration_ms: 0,
21908 }],
21909 e2e: Vec::new(),
21910 verify_retried: false,
21911 e2e_deferred: false,
21912 e2e_defer_reason: None,
21913 fix: None,
21914 blocking: 0,
21915 answered: 1,
21916 expected: 1,
21917 clean: true,
21918 progressed: false,
21919 vote_split: false,
21920 reconsideration: Vec::new(),
21921 verdict: None,
21922 };
21923 let state = state_with_round(round);
21924 let body = pr_message(&state, 'A').body;
21925 assert!(!body.contains("Open review findings"), "{body}");
21926 assert!(!body.contains("Declined"), "{body}");
21927 }
21928
21929 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21930 let mut state = RunState::new(
21931 PathBuf::from("/repo"),
21932 "main".to_owned(),
21933 "abc1234".to_owned(),
21934 instruction.to_owned(),
21935 Config::default(),
21936 );
21937 state.config.disk.min_free_bytes = 0;
21939 state.candidates.push(Candidate {
21940 index: 0,
21941 label: 'A',
21942 agent: "alpha".to_owned(),
21943 branch: "magi/x/A".to_owned(),
21944 worktree: PathBuf::from("/wt"),
21945 summary: summary.to_owned(),
21946 stat: String::new(),
21947 files: 1,
21948 commits: 1,
21949 empty: false,
21950 failed: None,
21951 verified_noop: None,
21952 folded: false,
21953 duration_ms: 0,
21954 });
21955 state
21956 }
21957
21958 fn review_state(subjects: &[&str]) -> RunState {
21959 let mut state = state_with_summary(
21960 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21961 "",
21962 );
21963 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21964 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21965 state
21966 }
21967
21968 #[tokio::test]
21971 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21972 ask_test_home();
21973 let tmp = tempfile::tempdir().unwrap();
21974 let repo = tmp.path().join("repo");
21975 std::fs::create_dir_all(&repo).unwrap();
21976 init_repo(&repo);
21977 let origin = tmp.path().join("origin.git");
21978 let g = |dir: &Path, args: &[&str]| {
21979 let out = std::process::Command::new("git")
21980 .args(args)
21981 .current_dir(dir)
21982 .quiet()
21983 .output()
21984 .expect("spawn git");
21985 assert!(
21986 out.status.success(),
21987 "git {args:?}: {}",
21988 String::from_utf8_lossy(&out.stderr)
21989 );
21990 };
21991 g(
21992 tmp.path(),
21993 &[
21994 "clone",
21995 "--bare",
21996 "-q",
21997 repo.to_str().unwrap(),
21998 origin.to_str().unwrap(),
21999 ],
22000 );
22001 g(
22002 &repo,
22003 &["remote", "add", "origin", origin.to_str().unwrap()],
22004 );
22005 let c1 = git::rev_parse(&repo, "main").await.unwrap();
22006
22007 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
22009 g(&repo, &["add", "-A"]);
22010 g(
22011 &repo,
22012 &["commit", "-q", "-m", "chore(deps): update a crate"],
22013 );
22014 g(&repo, &["push", "-q", "origin", "main"]);
22015 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
22016 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22017 g(&repo, &["add", "-A"]);
22018 g(
22019 &repo,
22020 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
22021 );
22022 g(&repo, &["checkout", "-q", "main"]);
22023
22024 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
22025 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
22026 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
22028 assert_eq!(fallback, c1);
22029
22030 let mut state = review_state(&[
22031 "chore(deps): update a crate",
22032 "fix(daemon): apply a chosen action",
22033 ]);
22034 state.repo = repo.clone();
22035 state.base_branch = "main".to_owned();
22036 state.base_commit = c1;
22037 refresh_reviewed_commits(&mut state, "feat/own").await;
22038 assert_eq!(
22039 state.reviewed_commits,
22040 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
22041 );
22042 assert_eq!(
22043 review_title(&state).as_deref(),
22044 Some("fix(daemon): apply a chosen action")
22045 );
22046 assert_eq!(
22047 leaked_subjects(&state, "feat/own").await,
22048 Some(vec!["chore(deps): update a crate".to_owned()])
22049 );
22050 state.config.merge.remote = "nowhere".to_owned();
22053 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
22054 }
22055
22056 struct SummaryFixture {
22060 _tmp: tempfile::TempDir,
22061 repo: PathBuf,
22062 reply: PathBuf,
22063 runner: Runner,
22064 }
22065
22066 fn summary_fixture() -> SummaryFixture {
22067 ask_test_home();
22068 let tmp = tempfile::tempdir().unwrap();
22069 let repo = tmp.path().join("repo");
22070 std::fs::create_dir_all(&repo).unwrap();
22071 init_repo(&repo);
22072 let origin = tmp.path().join("origin.git");
22073 let g = |args: &[&str]| {
22074 let out = std::process::Command::new("git")
22075 .args(args)
22076 .current_dir(&repo)
22077 .quiet()
22078 .output()
22079 .expect("spawn git");
22080 assert!(
22081 out.status.success(),
22082 "git {args:?}: {}",
22083 String::from_utf8_lossy(&out.stderr)
22084 );
22085 };
22086 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
22087 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
22088 g(&["push", "-q", "origin", "main"]);
22089 g(&["checkout", "-q", "-b", "feat/own"]);
22090 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22091 g(&["add", "-A"]);
22092 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
22093 let reply = tmp.path().join("reply.txt");
22094 std::fs::write(
22095 &reply,
22096 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
22097 )
22098 .unwrap();
22099 let mut runner = runner_at(RunStatus::Gating);
22100 let mut state = review_state(&["feat(own): add the own file"]);
22101 state.repo = repo.clone();
22102 state.base_branch = "main".to_owned();
22103 state.candidates[0].branch = "feat/own".to_owned();
22104 state.candidates[0].worktree = repo.clone();
22105 let mut author = spec("alpha");
22106 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
22107 author
22108 .env
22109 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
22110 state.config.agents = vec![author];
22111 runner.state = state;
22112 SummaryFixture {
22113 _tmp: tmp,
22114 repo,
22115 reply,
22116 runner,
22117 }
22118 }
22119
22120 async fn posted(f: &mut SummaryFixture) -> PrMessage {
22121 let winner = f.runner.state.candidates[0].clone();
22122 f.runner
22123 .guarded_pr_message(&winner, None, true)
22124 .await
22125 .unwrap()
22126 .expect("an English description never parks")
22127 }
22128
22129 fn clean_round_with_minor_findings() -> ReviewRound {
22130 let mut round = passing_round(2, None);
22131 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
22132 round.reviews = vec![ReviewRecord {
22133 attempts: 0,
22134 reviewer: 1,
22135 agent: "alpha".into(),
22136 summary: "内部レビュー".into(),
22137 findings: vec![
22138 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
22139 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
22140 ],
22141 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
22142 failed: None,
22143 duration_ms: 0,
22144 }];
22145 round
22146 }
22147
22148 #[tokio::test]
22149 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
22150 for review_only in [false, true] {
22151 let mut f = summary_fixture();
22152 if !review_only {
22153 f.runner.state.candidates[0].agent = "alpha".into();
22154 }
22155 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
22156 assert!(f.runner.state.reviews[0].clean);
22157 assert!(f.runner.state.reviews[0].fix.is_none());
22158 assert!(f.runner.state.open_findings().is_empty());
22159 assert_eq!(
22160 crate::pr_outcome::required_finding_ids(&f.runner.state),
22161 ["R2-1-1", "R2-1-2"]
22162 );
22163 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22164 let evidence = crate::pr_outcome::summary_evidence(
22165 &f.runner.state,
22166 crate::pr_outcome::FinalHead::At(&head),
22167 );
22168 assert!(
22169 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
22170 );
22171 let omitted = posted(&mut f).await;
22173 assert!(omitted.body.contains("No description"));
22174 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
22175 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
22176 assert!(f.runner.state.change_summary.is_none());
22177 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
22178 let described = posted(&mut f).await;
22179 assert!(described.body.contains("improve documentation for callers"));
22180 assert!(described.body.contains("remaining edge case"));
22181 assert!(described.body.contains("## Open review findings"));
22182 assert!(!described.body.contains("未修正"));
22183 }
22184 }
22185
22186 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
22187 let mut f = summary_fixture();
22188 if !review_only {
22189 f.runner.state.candidates[0].agent = "alpha".into();
22190 }
22191 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22192 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
22193 fixed
22194 .fix
22195 .as_mut()
22196 .unwrap()
22197 .rejected
22198 .push(crate::verdict::Rejection {
22199 id: "R1-1-1".into(),
22200 why: "呼び出されないコードのため辞退".into(),
22201 });
22202 let mut clean = clean_round_with_minor_findings();
22203 clean.verified_head = Some(head.clone());
22204 f.runner.state.reviews = vec![fixed, clean];
22205 ok_gate(&mut f.runner.state, Some(&head));
22206 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
22207 f.runner.state.config.agents.push(scripted("beta", reply));
22208 f.runner.state.config.roles.language_judge =
22209 Some(crate::config::AgentChoice::One("beta".into()));
22210 f
22211 }
22212
22213 #[tokio::test]
22214 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
22215 for review_only in [false, true] {
22216 let mut f = rewrite_fixture_with_findings(review_only,
22217 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
22218 let m = posted(&mut f).await;
22219 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22220 let expected =
22221 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
22222 assert!(m.body.contains(&expected));
22223 assert!(m.body.contains("## Review rounds"));
22224 assert!(
22225 m.body
22226 .contains("Gate (`verify.gate`): passed on the final commit")
22227 );
22228 assert!(
22229 m.body
22230 .contains("Tests (`verify.e2e`): passed on the final commit")
22231 );
22232 let c = f.runner.state.change_summary.as_ref().unwrap();
22233 assert!(
22234 m.body
22235 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
22236 );
22237 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22238 assert!(crate::pr_outcome::covers_required_findings(
22239 &f.runner.state,
22240 &m.body
22241 ));
22242 assert!(m.body.contains("documentation remains incomplete"));
22243 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
22244 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
22245 assert!(f.runner.state.github_text.is_none());
22246 }
22247 }
22248
22249 #[tokio::test]
22250 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
22251 for review_only in [false, true] {
22252 for reply in [
22253 "TITLE: fix: public description\nA short English description.",
22254 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
22255 ] {
22256 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
22257 let winner = f.runner.state.candidates[0].clone();
22258 let posted = f
22259 .runner
22260 .guarded_pr_message(&winner, None, true)
22261 .await
22262 .unwrap();
22263 assert!(posted.is_none());
22264 assert!(f.runner.state.parked);
22265 let withheld = f.runner.state.github_text.as_ref().unwrap();
22266 assert!(
22267 withheld.rewrite.is_none(),
22268 "an incomplete rewrite must not be selectable"
22269 );
22270 let q = ask::Questions::open()
22271 .get(withheld.question.as_ref().unwrap())
22272 .unwrap();
22273 assert!(
22274 !q.choices
22275 .contains(&crate::github_text::USE_REWRITE.to_owned())
22276 );
22277 assert!(
22278 q.detail.contains("R1-1-1")
22279 && q.detail.contains("R2-1-1")
22280 && q.detail.contains("R2-1-2")
22281 );
22282 }
22283 }
22284 }
22285
22286 #[tokio::test]
22287 async fn japanese_reports_stay_local_in_both_public_run_modes() {
22288 for review_only in [false, true] {
22289 let mut f = summary_fixture();
22290 if !review_only {
22291 f.runner.state.candidates[0].agent = "alpha".into();
22292 }
22293 f.runner.state.instruction = "内部タスクの説明".into();
22294 f.runner.state.candidates[0].summary = "内部の候補報告".into();
22295 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22296 f.runner.state.reviews = vec![
22297 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22298 round_with_notes(2, Some("二回目の内部修正報告です。")),
22299 passing_round(3, Some(&head)),
22300 ];
22301 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22302 ok_gate(&mut f.runner.state, Some(&head));
22303 let captured = f.repo.join("public-prompt.txt");
22304 let author = &mut f.runner.state.config.agents[0];
22305 author.command = vec![
22306 "sh".into(),
22307 "-c".into(),
22308 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22309 ];
22310 author.env.insert(
22311 "PROMPT_FILE".into(),
22312 captured.to_string_lossy().into_owned(),
22313 );
22314 let m = posted(&mut f).await;
22315 assert!(m.body.contains("Adds `own.txt`"));
22316 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22317 assert!(m.body.contains("R1-1-1"));
22318 assert!(
22319 m.body
22320 .contains("Gate (`verify.gate`): passed on the final commit")
22321 );
22322 assert!(
22323 m.body
22324 .contains("Tests (`verify.e2e`): passed on the final commit")
22325 );
22326 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22327 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22328 let prompt = std::fs::read_to_string(captured).unwrap();
22329 assert!(prompt.contains(&format!(
22330 "git diff {} {head}",
22331 f.runner.state.change_summary.as_ref().unwrap().base
22332 )));
22333 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22334 assert_eq!(
22335 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22336 "二回目の内部修正報告です。"
22337 );
22338 }
22339 }
22340
22341 #[tokio::test]
22342 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22343 for review_only in [false, true] {
22344 let mut f = summary_fixture();
22345 if !review_only {
22346 f.runner.state.candidates[0].agent = "alpha".into();
22347 }
22348 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22349 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22350 std::fs::write(&f.reply, "no usable description").unwrap();
22351 let failed = posted(&mut f).await;
22352 assert!(
22353 failed
22354 .body
22355 .contains("No description of what this change does could be generated")
22356 );
22357 assert!(failed.body.is_ascii());
22358 assert!(f.runner.state.github_text.is_none());
22359 assert!(f.runner.state.change_summary.is_none());
22360 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22361 let evidence = crate::pr_outcome::summary_evidence(
22362 &f.runner.state,
22363 crate::pr_outcome::FinalHead::At(&head),
22364 );
22365 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22366 assert!(
22367 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22368 .exists()
22369 );
22370 f.runner.state =
22371 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22372 std::fs::write(
22373 &f.reply,
22374 "TITLE: feat(own): add the own file\nAdds the committed own file.",
22375 )
22376 .unwrap();
22377 let retried = posted(&mut f).await;
22378 assert!(retried.body.contains("Adds the committed own file."));
22379 assert_eq!(
22380 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22381 "日本語の修正報告"
22382 );
22383 }
22384 }
22385
22386 #[tokio::test]
22387 async fn same_head_verification_and_findings_invalidate_public_prose() {
22388 let mut f = summary_fixture();
22389 posted(&mut f).await;
22390 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22391 let original = f
22392 .runner
22393 .state
22394 .change_summary
22395 .as_ref()
22396 .unwrap()
22397 .evidence
22398 .clone();
22399 ok_gate(&mut f.runner.state, Some(&head));
22400 std::fs::write(
22401 &f.reply,
22402 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22403 )
22404 .unwrap();
22405 let verified = posted(&mut f).await;
22406 assert!(verified.body.contains("now verified"));
22407 assert_ne!(
22408 f.runner.state.change_summary.as_ref().unwrap().evidence,
22409 original
22410 );
22411 let mut round = round_with_notes(1, Some("内部報告"));
22412 round.reviews.push(ReviewRecord {
22413 attempts: 0,
22414 reviewer: 1,
22415 agent: "alpha".into(),
22416 summary: "内部レビュー".into(),
22417 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22418 vote: None,
22419 failed: None,
22420 duration_ms: 0,
22421 });
22422 round
22423 .fix
22424 .as_mut()
22425 .unwrap()
22426 .rejected
22427 .push(crate::verdict::Rejection {
22428 id: "R1-1-1".into(),
22429 why: "呼び出し元から到達できない".into(),
22430 });
22431 f.runner.state.reviews = vec![round];
22432 let omitted = posted(&mut f).await;
22434 assert!(omitted.body.contains("No description"));
22435 assert!(omitted.body.contains("R1-1-1"));
22436 assert!(!omitted.body.contains("到達"));
22437 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22438 let described = posted(&mut f).await;
22439 assert!(described.body.contains("unreachable from callers"));
22440 assert!(described.body.contains("R1-1-1"));
22441 assert!(!described.body.contains("到達"));
22442 let evidence = f
22443 .runner
22444 .state
22445 .change_summary
22446 .as_ref()
22447 .unwrap()
22448 .evidence
22449 .clone();
22450 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22451 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22452 assert!(posted(&mut f).await.body.contains("rationale updated"));
22453 assert_ne!(
22454 f.runner.state.change_summary.as_ref().unwrap().evidence,
22455 evidence
22456 );
22457 f.runner
22459 .state
22460 .change_summary
22461 .as_mut()
22462 .unwrap()
22463 .evidence
22464 .clear();
22465 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22466 assert!(posted(&mut f).await.body.contains("current specification"));
22467 }
22468
22469 #[tokio::test]
22470 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22471 let mut f = summary_fixture();
22472 let m = posted(&mut f).await;
22473 assert_eq!(m.title, "feat(own): add the own file");
22474 assert!(
22475 m.body
22476 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22477 "{}",
22478 m.body
22479 );
22480 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22481 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22482 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22483 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22484 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22485 let kept = f.runner.state.change_summary.as_ref().unwrap();
22486 assert_eq!(
22487 (kept.head.as_str(), kept.base.as_str()),
22488 (head.as_str(), base.as_str())
22489 );
22490 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22491 }
22492
22493 #[tokio::test]
22494 async fn review_only_summary_receives_the_configured_github_language() {
22495 let mut f = summary_fixture();
22496 f.runner.state.config.graph.github_language = "ja".into();
22497 let captured = f.repo.join("summary-prompt.txt");
22498 let author = &mut f.runner.state.config.agents[0];
22499 author.command = vec![
22500 "sh".into(),
22501 "-c".into(),
22502 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22503 ];
22504 author.env.insert(
22505 "PROMPT_FILE".into(),
22506 captured.to_string_lossy().into_owned(),
22507 );
22508 std::fs::write(
22509 &f.reply,
22510 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22511 )
22512 .unwrap();
22513 let m = posted(&mut f).await;
22514 let prompt = std::fs::read_to_string(captured).unwrap();
22515 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22516 assert!(!prompt.contains("in English"));
22517 assert_eq!(m.title, "feat(own): ファイルを追加");
22518 assert!(m.body.contains("own.txt を追加します。"));
22519 }
22520
22521 #[tokio::test]
22522 async fn review_fixes_come_after_the_whole_change_summary() {
22523 let mut f = summary_fixture();
22524 f.runner.state.reviews = vec![
22525 round_with_notes(1, Some("handled the PATH case")),
22526 round_with_notes(2, None),
22527 ];
22528 let m = posted(&mut f).await;
22529 let summary = m.body.find("## Summary").expect("summary");
22530 let fixes = m.body.find("## Review rounds").expect("review rounds");
22531 assert!(summary < fixes, "{}", m.body);
22532 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22533 assert!(m.body.contains("## Final verification"), "{}", m.body);
22534 }
22535
22536 #[tokio::test]
22537 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22538 let mut f = summary_fixture();
22539 let first = posted(&mut f).await;
22540 let first_head = f.runner.state.change_summary.clone().unwrap().head;
22541 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22543 let again = posted(&mut f).await;
22544 assert_eq!((again.title, again.body), (first.title, first.body));
22545
22546 let g = |args: &[&str]| {
22547 let out = std::process::Command::new("git")
22548 .args(args)
22549 .current_dir(&f.repo)
22550 .quiet()
22551 .output()
22552 .expect("spawn git");
22553 assert!(out.status.success(), "git {args:?}");
22554 };
22555 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22556 g(&["add", "-A"]);
22557 g(&["commit", "-q", "-m", "fix(own): handle more"]);
22558 let second = posted(&mut f).await;
22559 let kept = f.runner.state.change_summary.clone().unwrap();
22560 assert_ne!(kept.head, first_head);
22561 assert!(second.body.contains("Not read yet."), "{}", second.body);
22562 assert!(!second.body.contains("so the first summary is read"));
22563 assert_eq!(second.title, "feat(own): other");
22564 }
22565
22566 #[tokio::test]
22567 async fn an_unavailable_summary_is_stated_and_never_invented() {
22568 let mut f = summary_fixture();
22569 std::fs::write(&f.reply, "no title line here").unwrap();
22570 let m = posted(&mut f).await;
22571 assert!(
22572 m.body
22573 .contains("No description of what this change does could be generated"),
22574 "{}",
22575 m.body
22576 );
22577 assert!(m.body.contains("internal reports remain in local records"));
22578 assert!(f.runner.state.change_summary.is_none());
22579 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22580 let winner = f.runner.state.candidates[0].clone();
22582 let preview = f
22583 .runner
22584 .guarded_pr_message(&winner, None, false)
22585 .await
22586 .unwrap()
22587 .unwrap();
22588 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22589 }
22590
22591 #[test]
22592 fn pr_message_review_single_commit_uses_its_subject() {
22593 let state = review_state(&["feat(nats): per-role user"]);
22594 let m = pr_message(&state, 'A');
22595 assert_eq!(m.title, "feat(nats): per-role user");
22596 assert!(!m.body.contains("Review the work already"), "{}", m.body);
22597 assert!(m.body.contains("## Commits under review"), "{}", m.body);
22598 }
22599
22600 #[test]
22601 fn pr_message_review_multi_commit_takes_the_oldest() {
22602 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22603 let m = pr_message(&state, 'A');
22604 assert_eq!(m.title, "feat: the change");
22605 for s in ["feat: the change", "fix: typo", "fix: again"] {
22606 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22607 }
22608 }
22609
22610 #[test]
22611 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22612 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22613 let mut state = review_state(&[first, "fix: later fixup"]);
22614 state.candidates[0].branch = "機能/ログイン".to_owned();
22615 let m = pr_message(&state, 'A');
22616 assert!(
22617 m.title.starts_with("chore: land candidate A of run"),
22618 "{}",
22619 m.title
22620 );
22621 }
22622 }
22623
22624 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22625 BranchFacts {
22626 commits: commits
22627 .iter()
22628 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22629 .collect(),
22630 stat: stat.to_owned(),
22631 }
22632 }
22633
22634 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22635 let mut r = review_round(true, 0, 1, 1, true, true);
22636 r.round = round;
22637 r.fix = notes.map(|n| FixRecord {
22638 agent: "fixer".to_owned(),
22639 addressed: Vec::new(),
22640 rejected: Vec::new(),
22641 notes: n.to_owned(),
22642 committed: true,
22643 failed: None,
22644 duration_ms: 0,
22645 continuation: None,
22646 });
22647 r
22648 }
22649
22650 #[test]
22651 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22652 let state = review_state(&["ignored"]);
22653 let f = facts(
22654 &[
22655 (
22656 "fix(login): resolve PATH on macOS",
22657 "Login shells skip rc files.",
22658 ),
22659 ("fix: address review", ""),
22660 ],
22661 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22662 );
22663 let m = pr_message_with(&state, 'A', Some(&f));
22664 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22665 assert!(
22666 m.body
22667 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22668 "{}",
22669 m.body
22670 );
22671 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22672 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22673 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22674 for banned in [
22675 "Review the work already",
22676 "no task statement",
22677 "Original task",
22678 ] {
22679 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22680 }
22681 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22682 }
22683
22684 #[test]
22685 fn pr_message_review_truncates_a_huge_first_commit_body() {
22686 let state = review_state(&["ignored"]);
22687 let f = facts(
22688 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22689 "s",
22690 );
22691 let m = pr_message_with(&state, 'A', Some(&f));
22692 assert!(m.body.len() < 30_000, "{}", m.body.len());
22693 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22694 assert!(
22695 m.body.contains("- ... 1 more commit(s)"),
22696 "{}",
22697 m.body.len()
22698 );
22699 assert!(m.body.ends_with("magi:candidate-a\n"));
22700 }
22701
22702 #[test]
22703 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22704 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22705 assert_eq!(m.title, "feat: x");
22706 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22707 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22708 assert!(!m.body.contains("no task statement"), "{}", m.body);
22709 }
22710
22711 #[test]
22712 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22713 let mut state = review_state(&["日本語の件名"]);
22714 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22715 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22716 state.candidates[0].branch = "機能/ログイン".to_owned();
22717 assert!(
22718 pr_message(&state, 'A')
22719 .title
22720 .starts_with("chore: land candidate A")
22721 );
22722 }
22723
22724 #[test]
22725 fn pr_message_review_fixes_survive_a_clean_final_round() {
22726 let mut state = review_state(&["feat: x"]);
22727 state.reviews = vec![
22728 round_with_notes(1, Some("handled the PATH case")),
22729 round_with_notes(2, None),
22730 ];
22731 let body = pr_message(&state, 'A').body;
22732 assert!(
22733 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22734 "{body}"
22735 );
22736 assert!(body.contains("### Round 1"), "{body}");
22737
22738 state.reviews = vec![
22739 round_with_notes(1, Some("first fix")),
22740 round_with_notes(2, Some("")),
22741 round_with_notes(3, Some("second fix")),
22742 round_with_notes(4, None),
22743 ];
22744 let body = pr_message(&state, 'A').body;
22745 assert!(
22746 body.contains("### Round 1") && !body.contains("first fix"),
22747 "{body}"
22748 );
22749 assert!(
22750 body.contains("### Round 3") && !body.contains("second fix"),
22751 "{body}"
22752 );
22753 assert!(body.contains("### Round 2"), "{body}");
22754 }
22755
22756 const FINAL: &str = "abcdef1234567890";
22757
22758 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22759 let mut r = round_with_notes(round, None);
22760 r.verified_head = verified.map(str::to_owned);
22761 r
22762 }
22763
22764 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22765 state.gate = vec![CommandOutcome {
22766 command: "gate".to_owned(),
22767 code: Some(0),
22768 output_tail: String::new(),
22769 duration_ms: 0,
22770 resource_blocked: false,
22771 }];
22772 state.gate_ran = true;
22773 state.gate_head = head.map(str::to_owned);
22774 }
22775
22776 fn body_at(state: &RunState) -> String {
22777 pr_message_raw(
22778 state,
22779 'A',
22780 None,
22781 crate::pr_outcome::FinalHead::At(FINAL),
22782 ChangeNote::NotAsked,
22783 )
22784 .body
22785 }
22786
22787 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22788
22789 #[test]
22790 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22791 let mut state = review_state(&["feat: x"]);
22792 state.reviews = vec![
22793 round_with_notes(1, Some(SANDBOXED)),
22794 passing_round(2, Some(FINAL)),
22795 ];
22796 ok_gate(&mut state, Some(FINAL));
22797 let body = body_at(&state);
22798 assert!(!body.contains("Handled the PATH case."), "{body}");
22799 assert!(!body.contains("could not commit"), "{body}");
22800 assert!(!body.contains("could not run"), "{body}");
22801 assert!(
22802 body.contains("The fix is committed on the branch"),
22803 "{body}"
22804 );
22805 assert!(!body.contains("sandbox"), "{body}");
22806 assert!(
22807 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22808 "{body}"
22809 );
22810 assert!(
22811 body.contains("Gate (`verify.gate`): passed on the final commit"),
22812 "{body}"
22813 );
22814 assert!(
22816 state.reviews[0]
22817 .fix
22818 .as_ref()
22819 .unwrap()
22820 .notes
22821 .contains("could not commit")
22822 );
22823 }
22824
22825 #[test]
22826 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22827 let mut state = review_state(&["feat: x"]);
22828 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22829 r1.fix.as_mut().unwrap().committed = false;
22830 let mut r2 = passing_round(2, Some(FINAL));
22831 r2.e2e[0].code = Some(1);
22832 state.reviews = vec![r1, r2];
22833 let body = body_at(&state);
22834 assert!(!body.contains("could not commit"), "{body}");
22835 assert!(!body.contains("could not run"), "{body}");
22836 assert!(body.contains("This round produced no commit"), "{body}");
22837 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22838 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22839 assert!(!body.contains("passed"), "{body}");
22840 }
22841
22842 #[test]
22843 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22844 let mut state = review_state(&["feat: x"]);
22845 state.reviews = vec![
22846 round_with_notes(1, Some(SANDBOXED)),
22847 passing_round(2, Some("0123456789abcdef")),
22848 ];
22849 ok_gate(&mut state, Some("0123456789abcdef"));
22850 let body = body_at(&state);
22851 assert!(!body.contains("HTTP tests could not run"), "{body}");
22852 assert!(!body.contains("could not commit"), "{body}");
22854 assert!(
22855 body.contains("passed on an earlier commit (`0123456`)"),
22856 "{body}"
22857 );
22858 assert!(!body.contains("passed on the final commit"), "{body}");
22859
22860 ok_gate(&mut state, None);
22862 let body = body_at(&state);
22863 assert!(
22864 body.contains("passed on a commit that was not recorded"),
22865 "{body}"
22866 );
22867 }
22868
22869 #[test]
22870 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22871 let state = state_with_summary(
22872 "add retries to the client",
22873 "TITLE: feat: retries\n\nDid it.",
22874 );
22875 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22876 assert_eq!(m.title, "feat: retries");
22877 assert!(
22878 m.body.contains("<summary>Original task</summary>"),
22879 "{}",
22880 m.body
22881 );
22882 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22883 assert!(
22884 m.body
22885 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22886 "{}",
22887 m.body
22888 );
22889 }
22890
22891 #[test]
22892 fn pr_message_review_bounds_a_long_english_subject() {
22893 let long = format!("feat: {}", "word ".repeat(100));
22894 let m = pr_message(&review_state(&[&long]), 'A');
22895 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22896 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22897 }
22898
22899 #[test]
22900 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22901 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22902 let m = pr_message(&state, 'A');
22903 assert_eq!(m.title, "add retries");
22904 assert!(m.body.contains("<summary>Original task</summary>"));
22905 assert!(!m.body.contains("Commits under review"));
22906 assert_eq!(landing_subject_source(&state), state.instruction);
22907 }
22908
22909 #[test]
22910 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22911 let state = review_state(&["feat: the change", "fix: typo"]);
22912 let source = landing_subject_source(&state);
22913 assert_eq!(land::merge_subject("", &source), "feat: the change");
22914 assert_eq!(
22915 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22916 "feat: the change"
22917 );
22918 assert_eq!(
22920 land::merge_subject("feat: renamed by hand", &source),
22921 "feat: renamed by hand"
22922 );
22923 let blank = review_state(&["日本語"]);
22924 assert!(
22925 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22926 );
22927 }
22928
22929 #[test]
22930 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22931 let state = review_state(&["feat: the change"]);
22932 let source = landing_subject_source(&state);
22933 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22934 assert_eq!(
22935 land::merge_subject(landing_title(&state, old), &source),
22936 "feat: the change"
22937 );
22938 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22939 let task = state_with_summary("add retries", "");
22940 assert_eq!(landing_title(&task, old), old);
22941 }
22942
22943 #[test]
22944 fn pr_message_describes_the_change_not_the_task() {
22945 let state = state_with_summary(
22946 "今回やってほしいこと: results projector を直す",
22947 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22948 );
22949 let m = pr_message(&state, 'A');
22950 assert_eq!(m.title, "fix(web): batch the runs list reads");
22951 assert!(
22952 m.body.starts_with("## Summary\n\n- reads run.json once"),
22953 "{}",
22954 m.body
22955 );
22956 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22957 let task_at = m.body.find("今回やってほしいこと").unwrap();
22958 let details_at = m.body.find("<details>").unwrap();
22959 assert!(
22960 details_at < task_at,
22961 "the task lives inside <details>: {}",
22962 m.body
22963 );
22964 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22965 assert!(m.body.contains("magi:candidate-a"));
22966 }
22967
22968 #[test]
22969 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22970 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22971 let m = pr_message(&state, 'A');
22972 assert_eq!(m.title, "add retries");
22973 assert!(
22974 m.body.contains("## Summary\n\n- did some things"),
22975 "{}",
22976 m.body
22977 );
22978
22979 let none = RunState::new(
22980 PathBuf::from("/repo"),
22981 "main".to_owned(),
22982 "abc1234".to_owned(),
22983 "add retries".to_owned(),
22984 Config::default(),
22985 );
22986 let m = pr_message(&none, 'A');
22987 assert_eq!(m.title, "add retries");
22988 assert!(!m.body.contains("## Summary"), "{}", m.body);
22989 }
22990
22991 #[test]
22992 fn pr_message_refuses_the_candidate_commit_subject() {
22993 for bad in [
22994 "TITLE: magi: candidate A (uncommitted work)",
22995 "TITLE: chore: stuff (uncommitted work)",
22996 "TITLE: ",
22997 ] {
22998 let state = state_with_summary("add retries", bad);
22999 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
23000 }
23001 }
23002
23003 #[test]
23004 fn pr_message_bounds_a_very_long_task_and_title() {
23005 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
23006 let state = state_with_summary(&long, "- nothing");
23007 let m = pr_message(&state, 'A');
23008 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23009 assert!(!m.title.contains('\n'));
23010
23011 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
23012 let m = pr_message(&state, 'A');
23013 assert!(m.title.starts_with("feat: "));
23014 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23015 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
23016 }
23017
23018 fn long_title_of(instruction: &str) -> String {
23019 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
23020 }
23021
23022 #[test]
23023 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
23024 let first = "Make the landing path keep a readable title for long tasks";
23025 let line = format!(
23026 "{first}. {}",
23027 "And then keep going with more words ".repeat(20)
23028 );
23029 let t = long_title_of(&line);
23030 assert_eq!(t, first);
23031 assert!(!t.starts_with("chore: land"));
23032 }
23033
23034 #[test]
23035 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
23036 let line = "word ".repeat(200);
23037 let t = long_title_of(&line);
23038 assert!(t.ends_with("word..."), "{t}");
23039 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
23040 }
23041
23042 #[test]
23043 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
23044 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
23045 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
23046 }
23047 }
23048
23049 #[test]
23050 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
23051 let mut state = state_with_summary("再試行を追加する", "- nothing");
23052 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
23053 state.config.graph.github_language = "ja".into();
23054 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
23055 }
23056
23057 #[test]
23058 fn pr_message_title_limit_is_exact() {
23059 let at = "a".repeat(PR_TITLE_MAX);
23060 assert_eq!(long_title_of(&at), at);
23061 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
23062 assert!(over.ends_with("..."), "{over}");
23063 assert_eq!(over.chars().count(), PR_TITLE_MAX);
23064 }
23065
23066 #[test]
23067 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
23068 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
23069 let t = long_title_of(&line);
23070 assert!(t.ends_with("..."), "{t}");
23071 assert!(t.is_ascii(), "{t}");
23072 }
23073
23074 #[test]
23075 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
23076 let line = format!(
23077 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
23078 "plus more filler words ".repeat(20)
23079 );
23080 let t = long_title_of(&line);
23081 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
23082 }
23083
23084 #[test]
23085 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
23086 let line = format!("\n\n# {}", "title words ".repeat(40));
23087 let state = state_with_summary(&line, "- nothing");
23088 let m = pr_message(&state, 'A');
23089 assert!(m.title.starts_with("title words"), "{}", m.title);
23090 assert_eq!(
23091 land::merge_subject(&m.title, &landing_subject_source(&state)),
23092 m.title
23093 );
23094 assert_eq!(
23096 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
23097 "feat: renamed by hand"
23098 );
23099 }
23100
23101 #[tokio::test]
23102 async fn github_text_rewrite_is_bounded_and_falls_back() {
23103 crate::run::pin_test_home();
23104 for (reply, accepted) in [
23105 (
23106 "TITLE: fix: retries\nAdd retries for failed requests.",
23107 true,
23108 ),
23109 (
23110 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
23111 false,
23112 ),
23113 (
23114 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
23115 true,
23116 ),
23117 (
23118 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
23119 true,
23120 ),
23121 (
23122 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
23123 true,
23124 ),
23125 (
23126 "TITLE: fix: retries\n日本語の説明です。 token=secret",
23127 false,
23128 ),
23129 ("TITLE: fix: retries\n", false),
23130 ] {
23131 let dir = tempfile::tempdir().unwrap();
23132 let mut runner = runner_at(RunStatus::Gating);
23133 runner.state = state_with_summary(
23134 "add retries",
23135 "TITLE: fix: retries\n日本語の説明を書きます。",
23136 );
23137 runner.state.repo = dir.path().to_owned();
23138 runner.state.candidates[0].worktree = dir.path().to_owned();
23139 let mut author = spec("alpha");
23140 author.command = vec![
23141 "sh".into(),
23142 "-c".into(),
23143 "printf '%s' \"$REWRITE_REPLY\"".into(),
23144 ];
23145 author.env.insert("REWRITE_REPLY".into(), reply.into());
23146 runner.state.config.agents = vec![author];
23147 seed_public_summary(&mut runner);
23148 let winner = runner.state.candidates[0].clone();
23149 let first = runner
23150 .guarded_pr_message(&winner, None, true)
23151 .await
23152 .unwrap();
23153 let message = if accepted {
23154 first.expect("a passing rewrite never asks")
23155 } else {
23156 assert!(first.is_none());
23158 assert!(runner.state.parked);
23159 let store = ask::Questions::open();
23160 let asked: Vec<_> = store
23161 .list()
23162 .into_iter()
23163 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23164 .collect();
23165 assert_eq!(asked.len(), 1);
23166 assert!(!asked[0].detail.contains("secret"));
23167 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
23168 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
23169 assert_eq!(
23170 asked[0]
23171 .choices
23172 .contains(&crate::github_text::USE_REWRITE.to_owned()),
23173 rewrite.is_some()
23174 );
23175 store
23178 .update(&asked[0].id, |q| {
23179 q.abandon("test");
23180 Ok(())
23181 })
23182 .unwrap();
23183 let again = runner
23184 .guarded_pr_message(&winner, None, true)
23185 .await
23186 .unwrap()
23187 .expect("an abandoned question falls back");
23188 let n = store
23189 .list()
23190 .into_iter()
23191 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23192 .count();
23193 assert_eq!(n, 1, "asked once per rejected text");
23194 again
23195 };
23196 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
23197 assert!(!message.body.contains("secret"));
23198 assert!(!message.body.contains("prefix"));
23199 assert!(!message.title.contains("secret"));
23200 assert_eq!(
23201 message.body.contains("Add retries for failed requests."),
23202 accepted
23203 );
23204 assert_eq!(
23205 runner.state.seats["github-text-rewrite"].turns, 1,
23206 "only one rewrite invocation"
23207 );
23208 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
23209 if accepted {
23210 assert!(runner.state.github_text.is_none());
23211 assert!(!runner.state.parked);
23212 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
23213 assert!(message.body.contains("## Final verification"));
23214 assert!(message.body.contains("Evidence: final diff"));
23215 assert!(
23216 message
23217 .body
23218 .contains(&format!("magi:run/{}", runner.state.id))
23219 );
23220 }
23221 if !accepted {
23222 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23223 assert!(
23224 message
23225 .body
23226 .contains(&format!("magi:run/{}", runner.state.id))
23227 );
23228 }
23229 }
23230 }
23231
23232 fn scripted(id: &str, reply: &str) -> AgentSpec {
23233 let mut a = spec(id);
23234 a.command = vec![
23235 "sh".into(),
23236 "-c".into(),
23237 "printf '%s' \"$REWRITE_REPLY\"".into(),
23238 ];
23239 a.env.insert("REWRITE_REPLY".into(), reply.into());
23240 a
23241 }
23242
23243 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
23244 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
23245
23246 fn seed_public_summary(runner: &mut Runner) {
23248 let repo = &runner.state.repo;
23249 let g = |args: &[&str]| {
23250 let out = std::process::Command::new("git")
23251 .args(args)
23252 .current_dir(repo)
23253 .quiet()
23254 .output()
23255 .unwrap();
23256 assert!(out.status.success(), "git {args:?}");
23257 String::from_utf8_lossy(&out.stdout).trim().to_owned()
23258 };
23259 g(&["init", "-q", "-b", "main"]);
23260 g(&[
23261 "-c",
23262 "user.name=Test",
23263 "-c",
23264 "user.email=test@example.invalid",
23265 "commit",
23266 "-q",
23267 "--allow-empty",
23268 "-m",
23269 "feat: fixture",
23270 ]);
23271 let head = g(&["rev-parse", "HEAD"]);
23272 g(&["update-ref", "refs/remotes/origin/main", &head]);
23273 let evidence = crate::pr_outcome::summary_evidence(
23274 &runner.state,
23275 crate::pr_outcome::FinalHead::At(&head),
23276 );
23277 let generated = &runner.state.candidates[0].summary;
23278 runner.state.change_summary = Some(crate::run::ChangeSummary {
23279 base: head.clone(),
23280 head,
23281 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23282 title: summary_title(generated).unwrap(),
23283 body: summary_without_title(generated),
23284 });
23285 }
23286
23287 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23288 let mut runner = runner_at(RunStatus::Gating);
23289 runner.state = state_with_summary("add retries", JA_SUMMARY);
23290 if review_only {
23291 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23292 }
23293 runner.state.repo = dir.to_owned();
23294 runner.state.candidates[0].worktree = dir.to_owned();
23295 runner.state.config.agents = agents;
23296 seed_public_summary(&mut runner);
23297 runner
23298 }
23299
23300 fn gate_questions(runner: &Runner) -> usize {
23301 ask::Questions::open()
23302 .list()
23303 .into_iter()
23304 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23305 .count()
23306 }
23307
23308 #[tokio::test]
23309 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23310 crate::run::pin_test_home();
23311 let dir = tempfile::tempdir().unwrap();
23312 let mut runner = rewrite_runner(
23313 dir.path(),
23314 true,
23315 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23316 );
23317 runner.state.config.roles.language_judge =
23318 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23319 let winner = runner.state.candidates[0].clone();
23320 let msg = runner
23321 .guarded_pr_message(&winner, None, true)
23322 .await
23323 .unwrap()
23324 .expect("an English rewrite posts without asking");
23325 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23326 assert!(msg.body.contains("<details>kept</details>"));
23327 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23328 assert_eq!(gate_questions(&runner), 0);
23329 assert!(runner.state.github_text.is_none());
23330 assert!(!runner.state.seats.contains_key("impl-A"));
23331 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23332 }
23333
23334 #[tokio::test]
23335 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23336 crate::run::pin_test_home();
23337 let dir = tempfile::tempdir().unwrap();
23338 let mut runner = rewrite_runner(
23339 dir.path(),
23340 false,
23341 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23342 );
23343 runner.state.config.roles.language_judge =
23344 Some(crate::config::AgentChoice::One("beta".into()));
23345 runner.state.seats.insert(
23346 "impl-A".into(),
23347 SeatState::new("impl-A", "alpha", runner.state.seed),
23348 );
23349 let winner = runner.state.candidates[0].clone();
23350 let msg = runner
23351 .guarded_pr_message(&winner, None, true)
23352 .await
23353 .unwrap()
23354 .expect("rewritten");
23355 assert!(msg.body.contains("<details>kept</details>"));
23356 assert_eq!(runner.state.seats["impl-A"].turns, 0);
23357 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23358 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23359 }
23360
23361 #[tokio::test]
23362 async fn rewrite_reuses_a_resumable_implementer_session() {
23363 crate::run::pin_test_home();
23364 let dir = tempfile::tempdir().unwrap();
23365 let mut runner = rewrite_runner(
23366 dir.path(),
23367 false,
23368 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23369 );
23370 runner.state.config.roles.language_judge =
23371 Some(crate::config::AgentChoice::One("beta".into()));
23372 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23373 seat.turns = 1;
23374 runner.state.seats.insert("impl-A".into(), seat);
23375 let winner = runner.state.candidates[0].clone();
23376 runner
23377 .guarded_pr_message(&winner, None, true)
23378 .await
23379 .unwrap()
23380 .expect("rewritten");
23381 assert_eq!(runner.state.seats["impl-A"].turns, 2);
23382 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23383 }
23384
23385 #[tokio::test]
23386 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23387 crate::run::pin_test_home();
23390 let dir = tempfile::tempdir().unwrap();
23391 let mut exhausted = spec("alpha");
23392 exhausted.command = vec!["false".into()];
23393 let mut runner = rewrite_runner(
23394 dir.path(),
23395 false,
23396 vec![exhausted, scripted("beta", EN_REWRITE)],
23397 );
23398 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23400 "alpha".into(),
23401 "beta".into(),
23402 ]));
23403 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23404 seat.turns = 1;
23405 runner.state.seats.insert("impl-A".into(), seat);
23406 let winner = runner.state.candidates[0].clone();
23407 let msg = runner
23408 .guarded_pr_message(&winner, None, true)
23409 .await
23410 .unwrap()
23411 .expect("the role rewrites after the implementer fails");
23412 assert!(msg.body.contains("`src/net.rs`"));
23413 assert_eq!(
23414 runner.state.seats["impl-A"].turns, 2,
23415 "the implementer was tried once"
23416 );
23417 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23418 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23419 }
23420
23421 #[tokio::test]
23422 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23423 crate::run::pin_test_home();
23424 let dir = tempfile::tempdir().unwrap();
23425 let mut broken = spec("bad");
23426 broken.command = vec!["false".into()];
23427 let mut runner = rewrite_runner(
23428 dir.path(),
23429 true,
23430 vec![
23431 broken,
23432 scripted("noformat", "Add retries."),
23433 scripted("good", EN_REWRITE),
23434 ],
23435 );
23436 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23437 "bad".into(),
23438 "noformat".into(),
23439 "good".into(),
23440 ]));
23441 let winner = runner.state.candidates[0].clone();
23442 let msg = runner
23443 .guarded_pr_message(&winner, None, true)
23444 .await
23445 .unwrap()
23446 .expect("the third agent rewrites");
23447 assert!(msg.body.contains("`src/net.rs`"));
23448 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23449 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23450 }
23451
23452 #[tokio::test]
23453 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23454 crate::run::pin_test_home();
23455 for (agents, role, saved) in [
23457 (
23458 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23459 Some("beta"),
23460 true,
23461 ),
23462 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23463 (Vec::new(), None, false),
23464 ] {
23465 let dir = tempfile::tempdir().unwrap();
23466 let mut runner = rewrite_runner(dir.path(), true, agents);
23467 runner.state.config.roles.language_judge =
23468 role.map(|r| crate::config::AgentChoice::One(r.into()));
23469 let winner = runner.state.candidates[0].clone();
23470 let first = runner
23471 .guarded_pr_message(&winner, None, true)
23472 .await
23473 .unwrap();
23474 assert!(first.is_none(), "withheld text parks the run");
23475 assert!(runner.state.parked);
23476 assert_eq!(gate_questions(&runner), 1);
23477 let g = runner.state.github_text.as_ref().unwrap();
23478 assert_eq!(g.rewrite.is_some(), saved);
23479 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23480 }
23481 }
23482
23483 #[tokio::test]
23484 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23485 crate::run::pin_test_home();
23486 let dir = tempfile::tempdir().unwrap();
23487 let mut runner = runner_at(RunStatus::Gating);
23488 runner.state =
23489 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23490 runner.state.repo = dir.path().to_owned();
23491 runner.state.candidates[0].worktree = dir.path().to_owned();
23492 let mut author = spec("alpha");
23493 author.command = vec![
23494 "sh".into(),
23495 "-c".into(),
23496 "printf '%s' \"$REWRITE_REPLY\"".into(),
23497 ];
23498 author.env.insert(
23499 "REWRITE_REPLY".into(),
23500 format!(
23501 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23502 "English sentence. ".repeat(600)
23503 ),
23504 );
23505 runner.state.config.agents = vec![author];
23506 seed_public_summary(&mut runner);
23507 let winner = runner.state.candidates[0].clone();
23508 assert!(
23509 runner
23510 .guarded_pr_message(&winner, None, true)
23511 .await
23512 .unwrap()
23513 .is_none()
23514 );
23515 let record = runner.state.github_text.clone().unwrap();
23516 let rewrite = record.rewrite.clone().unwrap();
23517 assert_eq!(rewrite.failed, ["body-language"]);
23518 assert!(
23519 rewrite
23520 .body
23521 .ends_with(&format!("magi:run/{}", runner.state.id))
23522 );
23523 let store = ask::Questions::open();
23524 let q = store.get(record.question.as_ref().unwrap()).unwrap();
23525 assert!(
23526 q.choices
23527 .contains(&crate::github_text::USE_REWRITE.to_owned())
23528 );
23529 for label in [
23530 "ORIGINAL",
23531 "FALLBACK",
23532 "REWRITE",
23533 "Rules still failed: body-language",
23534 "truncated",
23535 ] {
23536 assert!(q.detail.contains(label), "{label}");
23537 }
23538 let artifact = std::fs::read_to_string(crate::run::artifact_path(
23539 &runner.state,
23540 &crate::github_text::artifact_name(&record.fingerprint),
23541 ))
23542 .unwrap();
23543 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23544 runner.state =
23546 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23547 store
23548 .update(&q.id, |q| {
23549 q.say("まだ日本語")?;
23550 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23551 })
23552 .unwrap();
23553 assert!(
23554 runner
23555 .guarded_pr_message(&winner, None, true)
23556 .await
23557 .unwrap()
23558 .is_none()
23559 );
23560 let retry = runner.state.github_text.as_ref().unwrap();
23561 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23562 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23563 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23564 assert!(
23565 q.posting_comparison
23566 .as_ref()
23567 .unwrap()
23568 .matches_detail(&q.detail)
23569 );
23570 store
23571 .update(&q.id, |q| {
23572 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23573 })
23574 .unwrap();
23575 let message = runner
23576 .guarded_pr_message(&winner, None, true)
23577 .await
23578 .unwrap()
23579 .unwrap();
23580 assert_eq!(message.title, rewrite.title);
23581 assert_eq!(message.body, rewrite.body);
23582 assert_eq!(
23583 runner.state.github_text.as_ref().unwrap().choice,
23584 Some(GithubTextChoice::Rewrite)
23585 );
23586 let again = runner
23587 .guarded_pr_message(&winner, None, true)
23588 .await
23589 .unwrap()
23590 .unwrap();
23591 assert_eq!(again.title, message.title);
23592 assert_eq!(again.body, message.body);
23593 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23594 assert!(
23595 !runner
23596 .state
23597 .events
23598 .iter()
23599 .any(|e| e.message.contains(&rewrite.title)
23600 || e.message.contains("English sentence."))
23601 );
23602 }
23603
23604 #[tokio::test]
23605 async fn historical_posting_reply_never_expires_into_fallback() {
23606 crate::run::pin_test_home();
23607 let dir = tempfile::tempdir().unwrap();
23608 let mut runner = runner_at(RunStatus::Gating);
23609 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23610 runner.state.repo = dir.path().to_owned();
23611 runner.state.candidates[0].worktree = dir.path().to_owned();
23612 seed_public_summary(&mut runner);
23613 let winner = runner.state.candidates[0].clone();
23614 assert!(
23615 runner
23616 .guarded_pr_message(&winner, None, true)
23617 .await
23618 .unwrap()
23619 .is_none()
23620 );
23621 let id = runner
23622 .state
23623 .github_text
23624 .as_ref()
23625 .unwrap()
23626 .question
23627 .clone()
23628 .unwrap();
23629 let store = ask::Questions::open();
23630 let mut q = store.get(&id).unwrap();
23631 q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23632 q.answer(ask::Answer::Choice(
23633 crate::github_text::POST_ORIGINAL.into(),
23634 ))
23635 .unwrap();
23636 q.answer_delivered = true;
23637 store.put(&mut q).unwrap();
23638 let source = std::fs::read(store.path_of(&id)).unwrap();
23639 let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23640 std::fs::create_dir_all(&migration).unwrap();
23641 std::fs::write(
23642 migration.join("state.json"),
23643 serde_json::to_vec(&serde_json::json!({
23644 "schema": 1, "sources": [id], "pending": [id]
23645 }))
23646 .unwrap(),
23647 )
23648 .unwrap();
23649 for _ in 0..2 {
23650 assert!(
23651 runner
23652 .guarded_pr_message(&winner, None, true)
23653 .await
23654 .unwrap()
23655 .is_none()
23656 );
23657 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23658 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23659 }
23660 let home = crate::run::home();
23661 crate::task_chat::sweep(
23662 &crate::queue::Queue::at(home.join("queue")),
23663 &crate::talk::Talks::at(home.join("talks")),
23664 &store,
23665 &home,
23666 );
23667 let outbox = crate::task_inquiry::Outbox::at(&home);
23668 outbox
23669 .update_decision(&id, |d| {
23670 d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23671 d.source_answer_at = q.answered_at;
23672 Ok(true)
23673 })
23674 .unwrap();
23675 outbox
23676 .settle(
23677 &id,
23678 "owner-chat",
23679 "Owner confirms no publication remains necessary",
23680 None,
23681 )
23682 .unwrap();
23683 assert!(
23684 runner
23685 .guarded_pr_message(&winner, None, true)
23686 .await
23687 .unwrap()
23688 .is_none()
23689 );
23690 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23691 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23692 }
23693
23694 #[tokio::test]
23695 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23696 crate::run::pin_test_home();
23697 let dir = tempfile::tempdir().unwrap();
23698 let mut runner = runner_at(RunStatus::Gating);
23699 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23700 runner.state.repo = dir.path().to_owned();
23701 runner.state.candidates[0].worktree = dir.path().to_owned();
23702 seed_public_summary(&mut runner);
23703 let winner = runner.state.candidates[0].clone();
23704 assert!(
23705 runner
23706 .guarded_pr_message(&winner, None, true)
23707 .await
23708 .unwrap()
23709 .is_none()
23710 );
23711 let record = runner.state.github_text.as_ref().unwrap();
23712 assert!(record.rewrite.is_none());
23713 let mut old = serde_json::to_value(record).unwrap();
23715 old.as_object_mut().unwrap().remove("rewrite");
23716 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23717 assert!(old.rewrite.is_none());
23718 let id = record.question.clone().unwrap();
23719 ask::Questions::open()
23720 .update(&id, |q| {
23721 q.choices.push(crate::github_text::USE_REWRITE.into());
23722 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23723 })
23724 .unwrap();
23725 let message = runner
23726 .guarded_pr_message(&winner, None, true)
23727 .await
23728 .unwrap()
23729 .unwrap();
23730 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23731 assert_eq!(
23732 runner.state.github_text.as_ref().unwrap().choice,
23733 Some(GithubTextChoice::Fallback)
23734 );
23735 }
23736
23737 #[tokio::test]
23738 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23739 crate::run::pin_test_home();
23740 let dir = tempfile::tempdir().unwrap();
23741 let mut runner = runner_at(RunStatus::Gating);
23742 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23743 runner.state.repo = dir.path().to_owned();
23744 runner.state.candidates[0].worktree = dir.path().to_owned();
23745 seed_public_summary(&mut runner);
23746 let winner = runner.state.candidates[0].clone();
23747 assert!(
23748 runner
23749 .guarded_pr_message(&winner, None, true)
23750 .await
23751 .unwrap()
23752 .is_none()
23753 );
23754 let store = ask::Questions::open();
23755 let find = |run: &str| {
23756 store
23757 .list()
23758 .into_iter()
23759 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23760 .unwrap()
23761 };
23762 let run = runner.state.id.clone();
23763 let q = find(&run);
23764 let fp = runner
23765 .state
23766 .github_text
23767 .as_ref()
23768 .unwrap()
23769 .fingerprint
23770 .clone();
23771 let name = crate::github_text::artifact_name(&fp);
23772 let path = crate::run::artifact_path(&runner.state, &name);
23773 assert!(q.detail.contains(&path.display().to_string()));
23774 assert!(q.detail.contains("title-language: title"));
23775 let saved = std::fs::read_to_string(&path).expect("artifact written");
23776 assert!(saved.contains("日本語のタイトル"));
23777 assert!(q.cwd.is_none());
23778 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23779 let say = |id: &str, text: &str, choice: &str| {
23781 store
23782 .update(id, |q| {
23783 q.thread.push(ask::Turn {
23784 who: ask::Who::Operator,
23785 body: text.to_owned(),
23786 at: jiff::Timestamp::now(),
23787 note: None,
23788 });
23789 q.answer(ask::Answer::Choice(choice.to_owned()))
23790 })
23791 .unwrap();
23792 };
23793 say(&q.id, "まだ日本語", "use my text");
23794 assert!(
23795 runner
23796 .guarded_pr_message(&winner, None, true)
23797 .await
23798 .unwrap()
23799 .is_none()
23800 );
23801 let second = find(&run);
23802 assert_ne!(second.id, q.id);
23803 assert!(second.detail.contains("did not pass"));
23804 assert!(second.detail.contains("日本語のタイトル"));
23807 assert!(!second.detail.contains("まだ日本語"));
23808 assert!(second.detail.contains(&path.display().to_string()));
23809 say(&second.id, "fix: retry failed requests", "use my text");
23810 let message = runner
23811 .guarded_pr_message(&winner, None, true)
23812 .await
23813 .unwrap()
23814 .expect("a vetted replacement is posted");
23815 assert_eq!(message.title, "fix: retry failed requests");
23816 assert_eq!(
23817 runner
23818 .state
23819 .github_text
23820 .as_ref()
23821 .unwrap()
23822 .chosen_title
23823 .as_deref(),
23824 Some("fix: retry failed requests")
23825 );
23826 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23827 let again = runner
23829 .guarded_pr_message(&winner, None, true)
23830 .await
23831 .unwrap()
23832 .unwrap();
23833 assert_eq!(again.title, message.title);
23834 assert_eq!(
23835 store
23836 .list()
23837 .into_iter()
23838 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23839 .count(),
23840 2
23841 );
23842 }
23843
23844 #[test]
23845 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23846 let mut state = state_with_summary(
23850 "add retries",
23851 "TITLE: fix(web): batch reads\n- reads run.json once",
23852 );
23853 state.config.graph.language = "ja".to_owned();
23854 let m = pr_message(&state, 'A');
23855 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23856 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23857
23858 let task = "今回やってほしいこと: results projector を直す";
23861 let mut state = state_with_summary(task, "- no title line");
23862 state.config.graph.language = "ja".to_owned();
23863 let m = pr_message(&state, 'A');
23864 assert_eq!(
23865 m.title,
23866 format!("chore: land candidate A of run {}", state.id)
23867 );
23868 assert!(
23869 m.body.contains(&format!(
23870 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23871 )),
23872 "{}",
23873 m.body
23874 );
23875 }
23876
23877 #[test]
23878 fn pr_message_scrubs_home_paths_and_addresses() {
23879 let state = state_with_summary(
23880 "fix it in /Users/someone/src/x",
23881 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23882 );
23883 let m = pr_message(&state, 'A');
23884 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23885 assert!(!m.body.contains(leak), "{}", m.body);
23886 }
23887 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23888 }
23889
23890 #[test]
23891 fn pr_message_survives_a_task_that_closes_details() {
23892 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23893 let m = pr_message(&state, 'A');
23894 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23895 }
23896
23897 #[test]
23898 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23899 let cmd = manual_merge_command(
23900 MergeStyle::Squash,
23901 Path::new("/repo"),
23902 "b",
23903 "fix: \"quoted\" $(x) `y`\n\nbody",
23904 );
23905 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23906 }
23907
23908 #[test]
23909 fn manual_merge_command_matches_the_configured_style() {
23910 let repo = Path::new("/repo");
23911 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23912
23913 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23914 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23915
23916 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23917 assert_eq!(
23918 squash,
23919 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23920 \"Merge magi run 0832 (candidate A)\""
23921 );
23922
23923 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23924 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23925 }
23926
23927 #[test]
23928 fn a_nudge_gets_a_quarter_of_the_budget() {
23929 assert_eq!(retry_budget(secs(1200), true), secs(300));
23931 assert_eq!(retry_budget(secs(3600), true), secs(900));
23932 }
23933
23934 #[test]
23935 fn a_resent_prompt_keeps_the_whole_budget() {
23936 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23939 assert_eq!(retry_budget(secs(60), false), secs(60));
23940 }
23941
23942 #[test]
23943 fn the_floor_never_exceeds_the_original_budget() {
23944 assert_eq!(retry_budget(secs(60), true), secs(60));
23948 assert_eq!(retry_budget(secs(480), true), secs(120));
23949 assert_eq!(retry_budget(secs(0), true), secs(0));
23950 }
23951
23952 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23953 agent::CommandEvidence {
23954 id: "item1".to_owned(),
23955 description: "cargo test".to_owned(),
23956 exit_code,
23957 result_summary: String::new(),
23958 source: "codex".to_owned(),
23959 }
23960 }
23961
23962 #[test]
23963 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23964 assert!(!has_unconfirmed_command(&[]));
23968 }
23969
23970 #[test]
23971 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23972 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23976 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23977 assert!(!has_unconfirmed_command(&[
23978 evidence(Some(0)),
23979 evidence(Some(101))
23980 ]));
23981 }
23982
23983 #[test]
23984 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23985 assert!(has_unconfirmed_command(&[
23986 evidence(Some(0)),
23987 evidence(None)
23988 ]));
23989 }
23990
23991 #[test]
23992 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23993 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23994 assert_eq!(
23995 verified_noop_claim(true, &[], text).as_deref(),
23996 Some("already fixed by b32cfc4, on main.")
23997 );
23998 }
23999
24000 #[test]
24001 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
24002 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24005 assert!(verified_noop_claim(false, &[], text).is_none());
24006 }
24007
24008 #[test]
24009 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
24010 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24011 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
24012 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
24014 }
24015
24016 #[test]
24017 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
24018 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
24019 }
24020
24021 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
24024 runner.state.candidates = shape
24025 .iter()
24026 .enumerate()
24027 .map(|(i, &(empty, verified))| Candidate {
24028 index: i,
24029 label: (b'A' + i as u8) as char,
24030 agent: "sonnet".to_owned(),
24031 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
24032 worktree: PathBuf::from(format!("/wt/{i}")),
24033 summary: String::new(),
24034 stat: String::new(),
24035 files: 0,
24036 commits: 0,
24037 empty,
24038 failed: None,
24039 verified_noop: verified.map(str::to_owned),
24040 duration_ms: 0,
24041 folded: false,
24042 })
24043 .collect();
24044 }
24045
24046 #[test]
24047 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
24048 ask_test_home();
24049 let mut runner = runner_at(RunStatus::Implementing);
24050 set_candidates(
24051 &mut runner,
24052 &[
24053 (true, Some("already on main at b32cfc4")),
24054 (true, Some("same fix, see the existing test")),
24055 ],
24056 );
24057
24058 runner
24059 .after_implement()
24060 .expect("a verified no-op is not an error");
24061
24062 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
24063 }
24064
24065 #[test]
24066 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
24067 ask_test_home();
24068 let mut runner = runner_at(RunStatus::Implementing);
24069 set_candidates(
24073 &mut runner,
24074 &[(true, Some("already on main at b32cfc4")), (true, None)],
24075 );
24076
24077 let err = runner
24078 .after_implement()
24079 .expect_err("an unverified empty candidate must still fail the run");
24080
24081 assert!(
24082 err.to_string().contains("no candidate produced a change"),
24083 "{err}"
24084 );
24085 assert_eq!(runner.state.status, RunStatus::Failed);
24086 }
24087
24088 #[test]
24089 fn after_implement_still_fails_an_ordinary_all_empty_run() {
24090 ask_test_home();
24091 let mut runner = runner_at(RunStatus::Implementing);
24092 set_candidates(&mut runner, &[(true, None), (true, None)]);
24093
24094 let err = runner
24095 .after_implement()
24096 .expect_err("no candidate declared anything; this is an ordinary failure");
24097
24098 assert!(
24099 err.to_string().contains("no candidate produced a change"),
24100 "{err}"
24101 );
24102 assert_eq!(runner.state.status, RunStatus::Failed);
24103 }
24104}
24105
24106#[derive(Debug)]
24109struct GraftRestoreFailed(String);
24110
24111impl std::fmt::Display for GraftRestoreFailed {
24112 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24113 write!(
24114 f,
24115 "could not restore the winner to its pre-graft commit: {}",
24116 self.0
24117 )
24118 }
24119}
24120
24121impl std::error::Error for GraftRestoreFailed {}
24122
24123#[cfg(test)]
24127mod resource_wait_tests {
24128 use super::*;
24129 use std::collections::BTreeMap;
24130
24131 fn sh(id: &str, script: &str) -> AgentSpec {
24132 AgentSpec {
24133 id: id.to_owned(),
24134 kind: crate::config::AgentKind::Command,
24135 model: None,
24136 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
24137 extra_args: Vec::new(),
24138 env: BTreeMap::new(),
24139 prompt_delivery: None,
24140 max_turn_secs: None,
24141 }
24142 }
24143
24144 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
24145 SeatJob {
24146 seat: SeatState::new("impl-A", &spec.id, 1),
24147 spec,
24148 cwd: dir.to_path_buf(),
24149 prompt: "do it".to_owned(),
24150 timeout,
24151 allow_write: true,
24152 sessions: false,
24153 artifacts: dir.join("art"),
24154 stem: "impl-A".to_owned(),
24155 handover: None,
24156 }
24157 }
24158
24159 fn state(resource_wait_secs: u64) -> RunState {
24160 let mut config = Config::default();
24161 config.graph.timeout_resource_wait = resource_wait_secs;
24162 RunState::new(
24163 PathBuf::from("/repo"),
24164 "main".to_owned(),
24165 "abc1234".to_owned(),
24166 "task".to_owned(),
24167 config,
24168 )
24169 }
24170
24171 fn hold(cache: &Path) -> crate::cache::Guard {
24172 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
24173 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
24174 crate::cache::AcquireOutcome::Acquired(g) => g,
24175 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
24176 }
24177 }
24178
24179 async fn run(
24180 jobs: Vec<SeatJob>,
24181 sem: Arc<Semaphore>,
24182 cache: Option<&Path>,
24183 st: &mut RunState,
24184 ) -> Vec<(usize, SeatState, AgentOutcome)> {
24185 let prompts = Prompts::default();
24186 let ctx = WaveCtx {
24187 run: "r",
24188 node: "implement",
24189 prompts: &prompts,
24190 cache,
24191 round: None,
24192 carry_seats: false,
24193 pauses: None,
24194 };
24195 wave(jobs, sem, &ctx, st, 0).await
24196 }
24197
24198 #[tokio::test]
24199 async fn verification_cache_wait_preserves_the_command_budget() {
24200 crate::run::pin_test_home();
24201 let tmp = tempfile::tempdir().unwrap();
24202 let cache = tmp.path().join("target");
24203 let guard = hold(&cache);
24204 let release = tokio::spawn(async move {
24205 tokio::time::sleep(Duration::from_millis(100)).await;
24206 guard.release();
24207 });
24208 let mut st = state(10);
24209 let budget = Duration::from_millis(50);
24210 let wait = repository_wait(&st, None);
24211 let (outcomes, _) = with_cache_lease(
24212 &mut st,
24213 Some(&cache),
24214 "gate",
24215 "gate",
24216 tmp.path(),
24217 "abc1234",
24218 budget,
24219 "test gate",
24220 &wait,
24221 |_, available| async move {
24222 assert_eq!(
24223 available, budget,
24224 "contention must not shorten verification"
24225 );
24226 (Vec::new(), false, Vec::new())
24227 },
24228 )
24229 .await
24230 .unwrap();
24231 release.await.unwrap();
24232 assert!(outcomes.is_empty(), "{outcomes:?}");
24233 assert!(
24234 st.events
24235 .iter()
24236 .any(|event| event.message.contains("waiting for"))
24237 );
24238 assert!(st.resource_wait.is_none());
24239 hold(&cache).release();
24241 }
24242
24243 #[tokio::test]
24244 async fn verification_cache_wait_expires_without_starting_commands() {
24245 crate::run::pin_test_home();
24246 let tmp = tempfile::tempdir().unwrap();
24247 let cache = tmp.path().join("target");
24248 let _held = hold(&cache);
24249 let mut st = state(0);
24250 let wait = repository_wait(&st, None);
24251 let (outcomes, _) = with_cache_lease(
24252 &mut st,
24253 Some(&cache),
24254 "gate",
24255 "gate",
24256 tmp.path(),
24257 "abc1234",
24258 Duration::from_secs(60),
24259 "test gate",
24260 &wait,
24261 |_, _| async { panic!("commands must not start without the cache lease") },
24262 )
24263 .await
24264 .unwrap();
24265 assert_eq!(outcomes.len(), 1);
24266 assert!(outcomes[0].resource_blocked);
24267 assert!(st.resource_wait.is_none());
24268 assert!(matches!(
24269 st.resource_block.unwrap().cause,
24270 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24271 ));
24272 }
24273
24274 #[tokio::test]
24275 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24276 crate::run::pin_test_home();
24277 let tmp = tempfile::tempdir().unwrap();
24278 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24279 let cache = tmp.path().join("cache");
24280 let mut state = state(21600);
24281 let wait = repository_wait(&state, None);
24282 let (outcomes, _) = with_cache_lease(
24283 &mut state,
24284 Some(&cache),
24285 "gate",
24286 "gate",
24287 tmp.path(),
24288 "head",
24289 Duration::from_secs(60),
24290 "gate",
24291 &wait,
24292 |_, _| async { panic!("freshness failure must not start verification") },
24293 )
24294 .await
24295 .unwrap();
24296 assert_eq!(outcomes.len(), 1);
24297 assert!(outcomes[0].resource_blocked);
24298 let block = state.resource_block.unwrap();
24299 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24300 assert!(!block.description().contains("timed out"));
24301 }
24302
24303 #[tokio::test]
24304 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24305 crate::run::pin_test_home();
24306 let tmp = tempfile::tempdir().unwrap();
24307 let cache = tmp.path().join("cache");
24308 let _held = hold(&cache);
24309 let mut state = state(21600);
24310 let pause = Pause::new();
24311 let interrupt = Pause::new();
24312 let cause = ParkCause::Operator;
24313 pause.park_for(cause.clone());
24314 let prompts = Prompts::default();
24315 let ctx = WaveCtx {
24316 run: "test",
24317 node: "fix",
24318 prompts: &prompts,
24319 cache: Some(&cache),
24320 round: None,
24321 carry_seats: false,
24322 pauses: Some((&pause, &interrupt)),
24323 };
24324 let result = wave(
24325 vec![job(
24326 sh("writer", "exit 99"),
24327 tmp.path(),
24328 Duration::from_secs(1),
24329 )],
24330 Arc::new(Semaphore::new(1)),
24331 &ctx,
24332 &mut state,
24333 0,
24334 )
24335 .await;
24336 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24337 assert_eq!(result[0].1.turns, 0);
24338 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24339 assert_eq!(
24340 state.resource_block.unwrap().cause,
24341 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24342 );
24343 assert!(state.active.is_empty());
24344 }
24345
24346 #[tokio::test]
24347 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24348 crate::run::pin_test_home();
24349 let tmp = tempfile::tempdir().unwrap();
24350 let cache = tmp.path().join("target-a");
24351 let guard = hold(&cache);
24352 tokio::spawn(async move {
24354 tokio::time::sleep(Duration::from_millis(1500)).await;
24355 guard.release();
24356 });
24357 let mut st = state(60);
24358 let results = run(
24359 vec![job(
24360 sh("alpha", "echo ok"),
24361 tmp.path(),
24362 Duration::from_secs(1),
24363 )],
24364 Arc::new(Semaphore::new(1)),
24365 Some(&cache),
24366 &mut st,
24367 )
24368 .await;
24369 assert!(
24370 matches!(results[0].2, AgentOutcome::Ok(_)),
24371 "the wait must not be charged to the agent"
24372 );
24373 assert!(st.resource_block.is_none());
24374 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24375 assert!(st.resource_wait.is_none() && st.active.is_empty());
24376 }
24377
24378 #[tokio::test]
24379 #[cfg(unix)]
24380 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24381 use std::os::fd::AsRawFd as _;
24382 crate::run::pin_test_home();
24383 let tmp = tempfile::tempdir().unwrap();
24384 crate::git::git(tmp.path(), &["init"]).await.unwrap();
24385 let common = tmp.path().join(".git");
24386 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24387 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24388 assert_eq!(
24389 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24390 0
24391 );
24392 let mut st = state(1);
24393 st.config.coordination = crate::config::test_coordination();
24394 let mut reader = job(
24395 sh("reader", "echo must-not-run"),
24396 tmp.path(),
24397 Duration::from_secs(30),
24398 );
24399 reader.allow_write = false;
24400 let results = run(
24401 vec![reader.clone()],
24402 Arc::new(Semaphore::new(1)),
24403 None,
24404 &mut st,
24405 )
24406 .await;
24407 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24408 assert_eq!(results[0].1.turns, 0);
24409 assert!(FailClass::of(&results[0].2).is_none());
24410 assert_eq!(
24411 st.resource_block.as_ref().unwrap().kind,
24412 "repository coordination"
24413 );
24414 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24415 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24416 drop(external);
24417 let editor = crate::repo_lock::SeatLease::for_checkout(
24418 tmp.path(),
24419 true,
24420 crate::config::test_coordination().as_ref(),
24421 )
24422 .await
24423 .unwrap();
24424 let pause = Pause::new();
24425 let interrupt = Pause::new();
24426 let prompts = Prompts::default();
24427 let ctx = WaveCtx {
24428 run: "r",
24429 node: "review",
24430 prompts: &prompts,
24431 cache: None,
24432 round: None,
24433 carry_seats: false,
24434 pauses: Some((&pause, &interrupt)),
24435 };
24436 let (results, ()) = tokio::join!(
24437 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24438 async {
24439 tokio::time::sleep(Duration::from_millis(50)).await;
24440 pause.park();
24441 }
24442 );
24443 assert!(
24444 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24445 );
24446 assert_eq!(results[0].1.turns, 0);
24447 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24448 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24449 drop(editor);
24450 }
24451
24452 #[tokio::test]
24453 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24454 crate::run::pin_test_home();
24455 let tmp = tempfile::tempdir().unwrap();
24456 let cache = tmp.path().join("target-b");
24457 let _held = hold(&cache);
24458 let mut st = state(1);
24459 let results = run(
24460 vec![job(
24461 sh("alpha", "echo ok"),
24462 tmp.path(),
24463 Duration::from_secs(1),
24464 )],
24465 Arc::new(Semaphore::new(1)),
24466 Some(&cache),
24467 &mut st,
24468 )
24469 .await;
24470 let (_, seat, out) = &results[0];
24471 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24472 assert!(wave_blocked(&results));
24473 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24474 assert_eq!(seat.turns, 0, "nothing was asked");
24475 assert!(
24476 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24477 "no agent was spawned"
24478 );
24479 assert!(FailClass::of(out).is_none(), "not a failure class");
24480 let block = st.resource_block.as_ref().expect("recorded for the resume");
24481 assert_eq!(block.kind, "shared build cache");
24482 assert!(matches!(
24483 block.cause,
24484 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24485 ));
24486 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24487 assert!(st.active.is_empty());
24488 }
24489
24490 #[tokio::test]
24491 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24492 crate::run::pin_test_home();
24493 let tmp = tempfile::tempdir().unwrap();
24494 let mut st = state(1);
24495 let results = run(
24496 vec![job(
24497 sh("alpha", "echo ok"),
24498 tmp.path(),
24499 Duration::from_secs(1),
24500 )],
24501 Arc::new(Semaphore::new(0)),
24502 None,
24503 &mut st,
24504 )
24505 .await;
24506 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24507 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24508 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24509 }
24510
24511 #[tokio::test]
24512 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24513 crate::run::pin_test_home();
24514 let tmp = tempfile::tempdir().unwrap();
24515 let mut st = state(60);
24516 let results = run(
24517 vec![job(
24518 sh("alpha", "sleep 30"),
24519 tmp.path(),
24520 Duration::from_secs(1),
24521 )],
24522 Arc::new(Semaphore::new(1)),
24523 None,
24524 &mut st,
24525 )
24526 .await;
24527 let out = &results[0].2;
24528 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24529 let class = FailClass::of(out).expect("a failure");
24530 assert_eq!(class, FailClass::Timeout);
24531 assert!(should_hand_over(None, &class), "existing handover policy");
24532 assert!(st.resource_block.is_none());
24533 }
24534}