1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
776 state.origin = Some(origin);
777 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778 state.status = RunStatus::Prep;
779 state.event("start", format!("run {} created", state.id));
780 startup.finish(&mut state, Ok(()))?;
781 Ok(Self {
782 state,
783 roles,
784 sem: Arc::new(Semaphore::new(max_parallel)),
785 pause: Pause::new(),
786 interrupt: Pause::new(),
787 })
788 }
789
790 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804 Self::review_taking_over(repo, branch, config, None, origin).await
805 }
806
807 pub async fn review_taking_over(
814 repo: &Path,
815 branch: &str,
816 config: Config,
817 takeover: Option<crate::handover::Takeover>,
818 origin: Origin,
819 ) -> Result<Self> {
820 let repo = git::toplevel(repo).await?;
821 let missing = agent::missing_programs(&config.agents);
822 if !missing.is_empty() {
823 bail!(
824 "these agent programs are not on PATH: {}. Fix the roster in \
825 magi.toml or install them.",
826 missing.join(", ")
827 );
828 }
829 let base_branch =
830 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831 .await?;
832 if base_branch == branch {
833 bail!("`{branch}` is the base branch; there is nothing to review against");
834 }
835 let fetched_base = resolve_base(
836 &repo,
837 &base_branch,
838 &config.merge.remote,
839 config.coordination.as_ref(),
840 )
841 .await?;
842 let base_commit = fetched_base.commit.clone();
843
844 let roles = config.resolve_roles()?;
845 let max_parallel = config.graph.max_parallel.max(1);
846 let mut state = RunState::new(
847 repo.clone(),
848 base_branch,
849 base_commit.clone(),
850 format!("Review the work already on branch `{branch}`."),
851 config,
852 );
853 state.fetched_base = Some(fetched_base);
854 state.origin = Some(origin);
855 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857 let opened = async {
858 startup.stage(&mut state, "handover")?;
859
860 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867 earlier: Vec::new(),
868 home: crate::run::home(),
869 choice: None,
870 task_instruction: None,
871 });
872 let wait = repository_wait(&state, None);
873 let coordination_config = state.config.coordination.clone();
874 let coordination =
875 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876 .await?;
877 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878 if let Some(released) = &released {
879 state.event(
880 "release",
881 format!(
882 "took `{branch}` over from run {}: its worktree was released: {}",
883 crate::run::short_of(&released.old_id),
884 released.audit
885 ),
886 );
887 }
888 let opened = async {
889 startup.stage(&mut state, "review-preflight")?;
890 state.foreign_checkout = crate::handover::preflight_review(
894 &repo,
895 branch,
896 &takeover.home,
897 &state.config.merge.remote,
898 )
899 .await?;
900 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902 }
903 if let Some(choice) = takeover.choice.as_ref()
904 && let Err(e) =
905 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906 .await
907 {
908 return Err(e.context("applying the owner's answer about the diverged branch"));
909 }
910 for id in takeover.earlier.iter().rev() {
916 if let Ok(old) = RunState::load_under(id, &takeover.home)
917 && let Some(head) = old.pr_head.clone()
918 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919 {
920 state.pr_head = Some(head);
921 break;
922 }
923 }
924 drop(coordination);
925 startup.stage(&mut state, "open-review")?;
926 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927 }
928 .await;
929 if opened.is_err()
930 && let Some(released) = &released
931 {
932 match crate::repo_lock::SeatLease::operation_with(
935 &repo,
936 coordination_config.as_ref(),
937 &wait,
938 )
939 .await
940 {
941 Ok(_coordination) => {
942 released.restore(&repo, branch).await;
943 let outcome = RunState::load_under(&released.old_id, &takeover.home)
944 .map(|old| {
945 format!(
946 "predecessor released_to={:?}; candidate path exists={}",
947 old.released_to,
948 old.candidates.iter().any(|c| {
949 c.branch == branch && c.worktree.exists()
950 }),
951 )
952 })
953 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955 }
956 Err(error) => {
957 tracing::warn!(
958 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959 state.id
960 );
961 state.event(
962 "startup-restore",
963 format!("restoration not attempted: {error:#}"),
964 );
965 }
966 }
967 }
968 opened
969 }
970 .await;
971 startup.finish(&mut state, opened)?;
972 Ok(Self {
973 state,
974 roles,
975 sem: Arc::new(Semaphore::new(max_parallel)),
976 pause: Pause::new(),
977 interrupt: Pause::new(),
978 })
979 }
980
981 pub async fn recover_pr(
983 repo: &Path,
984 task: &str,
985 selection: crate::pr_recovery::Selection,
986 mut config: Config,
987 ) -> Result<Self> {
988 selection.configure(&mut config)?;
989 selection.check_task(task, &selection.instruction)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.origin = Some(crate::run::Origin::queue(task));
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true).await
1357 }
1358
1359 async fn authorize_base_recovery_request(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 resumption: bool,
1364 ) -> Result<()> {
1365 let _owner = crate::pr_head::owner_lock(home, id)?;
1366 let mut state = RunState::load_under(id, home)?;
1367 crate::pr_head::ensure_owner(&state)?;
1368 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369 anyhow::ensure!(
1370 queued.runs.last().is_some_and(|run| run == id)
1371 && queued.status != crate::queue::TaskStatus::Done
1372 && !queued.fresh_start
1373 && queued.review_branch.is_none(),
1374 "run task ownership changed"
1375 );
1376 anyhow::ensure!(
1377 state
1378 .origin
1379 .as_ref()
1380 .and_then(|o| o.task.as_deref())
1381 .is_none_or(|owner| owner == task),
1382 "run belongs to another task"
1383 );
1384 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385 anyhow::ensure!(
1386 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387 "run is live or its ownership is unknown"
1388 );
1389 let _execution = crate::driver::maintenance(home, id)?;
1392 anyhow::ensure!(
1393 state.status.resumable()
1394 && state.released_to.is_none()
1395 && state.retired_to.is_none()
1396 && state.pr_taken_over_by.is_none(),
1397 "run ownership is final"
1398 );
1399 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400 anyhow::ensure!(
1401 !state.pr_publication_pending
1402 && !crate::handover::landing_checkpoint(&state)
1403 && state.merge_decision.is_none(),
1404 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405 );
1406 let winner = state.winner().context("run has no retained winner")?;
1407 anyhow::ensure!(
1408 git::is_clean(&winner.worktree).await?,
1409 "retained worktree has uncommitted changes"
1410 );
1411 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412 anyhow::ensure!(
1413 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414 "retained worktree does not match the winner branch"
1415 );
1416 state.ensure_base_episode();
1417 state.record_base_sync_stop(&head);
1418 state.save()?;
1419 let fetched =
1420 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421 anyhow::ensure!(
1422 fetched.ok(),
1423 "cannot authorize base recovery without a fetched base"
1424 );
1425 let approved_tip = git::rev_parse(
1426 &state.repo,
1427 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428 )
1429 .await?;
1430 if resumption {
1431 anyhow::ensure!(
1432 crate::daemon::compatible_run_instruction(&state, &queued),
1433 "resumption instruction does not match the retained run"
1434 );
1435 let approval = queued
1436 .resumption
1437 .as_ref()
1438 .context("no operator resumption")?;
1439 anyhow::ensure!(
1440 approval.instruction == queued.instruction,
1441 "resumption approval does not match the current task instruction"
1442 );
1443 state.authorize_base_sync_resumption(task, approval)?;
1444 } else {
1445 state.authorize_base_sync(task)?;
1446 }
1447 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448 state
1449 .base_sync_recovery
1450 .authorizations
1451 .last_mut()
1452 .unwrap()
1453 .scope = Some(crate::base_sync_recovery::Scope {
1454 tip: approved_tip,
1455 instruction: queued.instruction.clone(),
1456 episode,
1457 });
1458 state.event(
1459 "base-recovery",
1460 "owner authorized a bounded base synchronization interval; fresh verification required",
1461 );
1462 state.save()?;
1463 Ok(())
1464 }
1465
1466 pub fn resume(id: &str) -> Result<Self> {
1468 let state = RunState::load(id)?;
1469 if state.startup.as_ref().is_some_and(|s| !s.complete)
1470 || (state.status == RunStatus::Prep
1471 && state.instruction.trim().is_empty()
1472 && state.candidates.is_empty()
1473 && state.prep_complete == Some(false))
1474 {
1475 bail!(
1476 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477 );
1478 }
1479 crate::pr_head::ensure_owner(&state)?;
1480 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481 bail!(
1482 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483 );
1484 }
1485 if let Some(to) = &state.released_to {
1486 bail!(
1487 "run {} cannot be resumed: its worktree was released to run {}",
1488 state.short(),
1489 crate::run::short_of(to)
1490 );
1491 }
1492 let roles = state.config.resolve_roles()?;
1493 if let Some(id) = &state.config.graph.selected_implementer
1494 && state.candidates.iter().any(|c| c.agent != *id)
1495 {
1496 bail!(
1497 "replacement implementer pin does not match the saved candidates; task must stay held"
1498 );
1499 }
1500 let max_parallel = state.config.graph.max_parallel.max(1);
1501 Ok(Self {
1502 state,
1503 roles,
1504 sem: Arc::new(Semaphore::new(max_parallel)),
1505 pause: Pause::new(),
1506 interrupt: Pause::new(),
1507 })
1508 }
1509
1510 pub async fn execute(&mut self) -> Result<()> {
1517 anyhow::ensure!(
1518 self.state.startup.as_ref().is_none_or(|s| s.complete),
1519 "startup incomplete; inspect retained evidence before explicit recovery"
1520 );
1521 if let Some(selection) = &self.state.pr_recovery
1522 && (selection.source_proof.is_some() || selection.source_run.is_some())
1523 {
1524 Box::pin(selection.verify_source(&self.state.repo)).await?;
1525 }
1526 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529 || self.state.land_approval.is_some()
1530 || self.state.merge_decision.is_some()
1531 || !self.state.merge_repairs.is_empty()
1532 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533 {
1534 let claim = FixClaim::acquire(&self.state.dir())?;
1535 let disk = RunState::load(&self.state.id)?;
1536 let checkpoint = |s: &RunState| {
1537 (
1538 s.merge_repairs.len(),
1539 s.merge_repairs
1540 .last()
1541 .map(|r| (r.question.clone(), r.phase)),
1542 s.land_approval
1543 .as_ref()
1544 .map(|a| (a.question.clone(), a.head.clone())),
1545 )
1546 };
1547 if checkpoint(&disk) != checkpoint(&self.state) {
1548 bail!(
1549 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550 );
1551 }
1552 Some(claim)
1553 } else {
1554 None
1555 };
1556 let mut execution = self.claim_driver().await?;
1557 let runner = &mut *execution.runner;
1558 let wait = runner.repository_wait();
1559 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562 let result = match result {
1563 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564 record_repository_block(&mut runner.state, &e);
1565 runner.park_here()?;
1566 Ok(())
1567 }
1568 other => other,
1569 };
1570 execution.driver.finish(&mut runner.state).await?;
1571 let ended = if result.is_err() {
1572 Some({
1573 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574 if let Err(error) = &result {
1575 notice.message = format!(
1576 "Run {} stopped with an error: {}",
1577 runner.state.id,
1578 error.to_string().lines().next().unwrap_or("unknown error")
1579 );
1580 }
1581 notice
1582 })
1583 } else {
1584 crate::notices::run_ended(&runner.state)
1585 };
1586 if let Some(notice) = ended {
1587 crate::notices::raise(notice);
1588 }
1589 result
1590 }
1591
1592 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594 }
1595
1596 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597 let wait = self.repository_wait();
1598 coordinate_operation(&mut self.state, cwd, &wait).await
1599 }
1600
1601 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603 let owner = match crate::pr_head::owner_lock_wait(
1604 &crate::run::home(),
1605 &self.state.id,
1606 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607 )
1608 .await
1609 {
1610 Ok(owner) => owner,
1611 Err(error) => {
1612 let detail = format!("worktree maintenance or ownership update: {error:#}");
1613 self.state.resource_block = Some(crate::run::ResourceBlock {
1617 cause: crate::run::ResourceBlockCause::Other,
1618 node: "start".into(),
1619 kind: "worktree maintenance".into(),
1620 detail: detail.clone(),
1621 at: Timestamp::now(),
1622 });
1623 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624 }
1625 };
1626 let disk = RunState::load(&self.state.id)?;
1627 crate::pr_head::ensure_owner(&disk)?;
1628 let claimed =
1629 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630 anyhow::ensure!(
1635 disk.liveness(claimed) != crate::run::Liveness::Live,
1636 "run already has a live driver; wait for its execution to end"
1637 );
1638 if let Some(to) = &disk.released_to {
1639 bail!(
1640 "run {} cannot continue: its worktree was released to run {}",
1641 self.state.short(),
1642 crate::run::short_of(to)
1643 );
1644 }
1645 anyhow::ensure!(
1646 serde_json::to_value(&disk.base_sync_recovery)?
1647 == serde_json::to_value(&self.state.base_sync_recovery)?,
1648 "base recovery changed since this run was loaded; reload the saved run"
1649 );
1650 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651 let mut execution = DriverExecution {
1652 runner: self,
1653 driver,
1654 };
1655 let _owner = owner;
1658 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659 .await?;
1660 let state = &mut execution.runner.state;
1661 state.parked = false;
1666 state.resource_wait = None;
1667 state.resource_block = None;
1668 state.clear_active();
1675 let pid = std::process::id();
1690 state.driver_pid = Some(pid);
1691 state.driver_started_at = crate::proc::process_started_at(pid);
1692 state.driver_exited = false;
1693 state.driver_claimed = Some(true);
1694 state.begin_pass();
1695 execution.driver.bind(state);
1696 state.save()?;
1697 execution.driver.record(state)?;
1698 Ok(execution)
1699 }
1700
1701 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1704 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1705 if let Err(error) = &result
1706 && !error.is::<crate::repo_lock::ResourceBlocked>()
1707 && self
1708 .state
1709 .repair_grants
1710 .last()
1711 .is_some_and(|g| g.conflict.is_none())
1712 {
1713 self.state.repair_grants.last_mut().unwrap().phase =
1714 crate::repair_budget::Phase::Complete;
1715 self.state.status = RunStatus::Blocked;
1716 self.state.save()?;
1717 bail!(
1718 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1719 self.state.passes.last().map_or(0, |p| p.index)
1720 );
1721 }
1722 result
1723 }
1724
1725 async fn execute_granted_repair(&mut self) -> Result<()> {
1728 use crate::repair_budget::{Kind, Phase};
1729 let index = self.state.repair_grants.len() - 1;
1730 let winner = self
1731 .state
1732 .winner()
1733 .cloned()
1734 .context("repair winner missing")?;
1735 loop {
1736 let grant = self.state.repair_grants[index].clone();
1737 match grant.phase {
1738 Phase::Pending => {
1739 if grant.spent >= grant.receipt.request.rounds {
1740 self.state.repair_grants[index].phase = Phase::Complete;
1741 self.state.status = RunStatus::Blocked;
1742 self.state.save()?;
1743 return Ok(());
1744 }
1745 self.state.repair_grants[index].spent += 1;
1748 self.state.repair_grants[index].phase = Phase::Fixing;
1749 self.state.save()?;
1750 match grant.receipt.request.kind {
1751 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1752 Kind::Gate => {
1753 let failed = self.state.gate.clone();
1754 let _ = self.gate_fix_round(&winner, &failed).await?;
1755 }
1756 Kind::Review => {
1757 let mut record = self
1758 .state
1759 .reviews
1760 .last()
1761 .cloned()
1762 .context("review failure missing")?;
1763 let round = record.round;
1764 let step = self.run_fix_round(&winner, &mut record).await?;
1765 self.state.reviews[round - 1] = record;
1766 if matches!(step, FixStep::Recorded { report_lost: true }) {
1767 self.state.repair_grants[index].phase = Phase::Complete;
1768 self.state.status = RunStatus::Blocked;
1769 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1770 self.state.save()?;
1771 return Ok(());
1772 }
1773 if matches!(step, FixStep::Resource) {
1774 self.stop_for_resource()?;
1775 return Ok(());
1776 }
1777 self.state.save()?;
1778 }
1779 }
1780 if self.resource_halt()? {
1781 return Ok(());
1782 }
1783 }
1784 Phase::Fixing => {
1785 if grant.receipt.request.kind == Kind::BaseConflict {
1786 self.repair_base_conflict(false).await?;
1787 }
1788 let _coordination = self.coordinate(&winner.worktree).await?;
1791 let rescued = git::rescue_commit(
1792 &winner.worktree,
1793 "magi: preserve interrupted granted repair",
1794 )
1795 .await?;
1796 self.state.note_withheld("repair-grant", &rescued.withheld);
1797 self.state
1800 .verification_history
1801 .push(crate::run::VerificationEvidence {
1802 superseded_by: None,
1803 archived_at: Some(Timestamp::now()),
1804 fixes_spent: Some(self.state.gate_fixes.len()),
1805 gate_head: self.state.gate_head.take(),
1806 gate_ran: std::mem::take(&mut self.state.gate_ran),
1807 gate: std::mem::take(&mut self.state.gate),
1808 e2e: self.state.head_e2e.take(),
1809 });
1810 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1811 self.state
1812 .verification_history
1813 .push(crate::run::VerificationEvidence {
1814 superseded_by: None,
1815 archived_at: Some(Timestamp::now()),
1816 fixes_spent: Some(self.state.gate_fixes.len()),
1817 gate_head: None,
1818 gate_ran: false,
1819 gate: Vec::new(),
1820 e2e: Some(e2e),
1821 });
1822 }
1823 self.state.repair_grants[index].review_start = self.state.reviews.len();
1824 self.state.repair_grants[index].head =
1825 git::rev_parse(&winner.worktree, "HEAD").await?;
1826 self.state.repair_grants[index].phase = Phase::Reviewing;
1827 if grant.receipt.request.kind == Kind::BaseConflict {
1828 self.state.base_sync_recovery.stop = None;
1829 if let Some(sync) = &mut self.state.base_sync {
1830 sync.tip = grant
1831 .receipt
1832 .request
1833 .tip
1834 .clone()
1835 .context("repair base tip missing")?;
1836 sync.conflict = None;
1837 sync.behind = 0;
1838 }
1839 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1840 }
1841 self.state.save()?;
1842 }
1843 Phase::Reviewing => {
1844 self.review_loop_pass().await?;
1845 if self.resource_halt()? {
1846 return Ok(());
1847 }
1848 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1849 let clean = self.state.reviews.len() > grant.review_start
1850 && self.state.reviews.last().is_some_and(|r| {
1851 r.head == head
1852 && r.clean
1853 && r.expected > 0
1854 && r.answered == r.expected
1855 && matches!(
1856 r.e2e_status(),
1857 E2eStatus::Passed | E2eStatus::NotConfigured
1858 )
1859 });
1860 if !clean {
1861 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1863 == Kind::Review
1864 && grant.spent < grant.receipt.request.rounds
1865 && self.state.reviews.last().is_some_and(|r| {
1866 r.fix.is_none()
1867 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1868 }) {
1869 Phase::Pending
1870 } else {
1871 Phase::Complete
1872 };
1873 self.state.status = RunStatus::Blocked;
1874 self.state.event(
1875 "repair-grant",
1876 "fresh review did not clear the repaired head; no publication",
1877 );
1878 self.state.save()?;
1879 if self.state.repair_grants[index].phase == Phase::Pending {
1880 continue;
1881 }
1882 return Ok(());
1883 }
1884 self.state.repair_grants[index].phase = Phase::Gating;
1885 self.state.save()?;
1886 }
1887 Phase::Gating => {
1888 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1889 self.state.reconcile_verification_head(&head);
1890 let wait = self.repository_wait();
1891 if self
1892 .state
1893 .reviews
1894 .last()
1895 .is_none_or(|r| r.head != head || !r.clean)
1896 {
1897 self.state.status = RunStatus::Blocked;
1898 self.state.repair_grants[index].phase = Phase::Complete;
1899 self.state.event(
1900 "repair-grant",
1901 "head changed since fresh review; repair held",
1902 );
1903 self.state.save()?;
1904 return Ok(());
1905 }
1906 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1907 return Ok(());
1908 }
1909 let outcomes = self.run_gate(&winner).await?;
1910 if verify_inconclusive(&outcomes) {
1911 return Ok(());
1912 }
1913 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1914 if after != head {
1915 self.state.repair_grants[index].review_start = self.state.reviews.len();
1916 self.state.repair_grants[index].head = after;
1917 self.state.repair_grants[index].phase = Phase::Reviewing;
1918 self.state.status = RunStatus::Blocked;
1919 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1920 self.state.save()?;
1921 return Ok(());
1922 }
1923 let passed = outcomes.iter().all(CommandOutcome::ok);
1924 if passed
1927 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1928 {
1929 return Ok(());
1930 }
1931 self.state.repair_grants[index].succeeded = passed;
1932 self.state.gate = outcomes;
1933 self.state.gate_ran = true;
1934 self.state.status = if passed {
1935 RunStatus::Gating
1936 } else {
1937 RunStatus::Blocked
1938 };
1939 self.state.repair_grants[index].phase = if !passed
1940 && grant.receipt.request.kind == Kind::Gate
1941 && grant.spent < grant.receipt.request.rounds
1942 {
1943 Phase::Pending
1944 } else {
1945 Phase::Complete
1946 };
1947 if passed && self.state.config.merge.mode == MergeMode::Pr {
1948 self.state.pr_publication_pending = true;
1950 } else if passed {
1951 self.state.status = RunStatus::Ready;
1952 }
1953 self.state.save()?;
1954 if !passed {
1955 continue;
1956 }
1957 if self.state.config.merge.mode != MergeMode::Pr {
1961 self.state.status = RunStatus::Ready;
1962 self.state.event(
1963 "repair-grant",
1964 "repair verified locally; grant carries no merge authorization",
1965 );
1966 self.state.save()?;
1967 return Ok(());
1968 }
1969 return self.merge().await;
1970 }
1971 Phase::Complete => return Ok(()),
1972 }
1973 }
1974 }
1975
1976 async fn execute_graph(&mut self) -> Result<()> {
1977 let archive_home = crate::run::home().canonicalize()?;
1981 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
1982 let repo = self.state.repo.clone();
1983 let _coordination = self.coordinate(&repo).await?;
1984 ensure_setup_config(&mut self.state, &repo).await?;
1985 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
1986 }
1987 self.state.configure_recovery_resume()?;
1988 if self
1989 .state
1990 .repair_grants
1991 .last()
1992 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1993 {
1994 return self.execute_granted_repair().await;
1995 }
1996 if self
1997 .state
1998 .repair_grants
1999 .last()
2000 .is_some_and(|g| !g.succeeded)
2001 {
2002 self.state.status = RunStatus::Blocked;
2003 self.state.save()?;
2004 return Ok(());
2005 }
2006 self.persist_candidates().await?;
2009 if self.state.base_sync_blocked() {
2012 self.sync_to_base().await?;
2013 if !self.state.base_sync_blocked()
2014 && self.state.status != RunStatus::AlreadyInBase
2015 && !self.resource_halt()?
2016 {
2017 self.finish_after_tally().await?;
2018 }
2019 return Ok(());
2020 }
2021 if self
2022 .state
2023 .merge_decision
2024 .as_ref()
2025 .is_some_and(|decision| !decision.closed)
2026 && self.state.status != RunStatus::Landing
2027 {
2028 if crate::merge_decision::resumable_hold(&self.state) {
2029 self.state.merge = Some(MergeOutcome {
2030 mode: MergeMode::Pr,
2031 ok: false,
2032 detail: self
2033 .state
2034 .merge_decision
2035 .as_ref()
2036 .unwrap()
2037 .target
2038 .pr
2039 .clone(),
2040 empty: false,
2041 });
2042 self.run_land().await?;
2043 self.settle_questions();
2044 return Ok(());
2045 }
2046 bail!(
2047 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2048 );
2049 }
2050 if self.state.config.graph.selected_implementer.is_some()
2051 && self.state.status == RunStatus::Blocked
2052 && self.state.tally.is_none()
2053 && self.state.candidate_preservation.is_empty()
2054 && self.state.interrupted_implementations.is_empty()
2055 {
2056 bail!(
2057 "selected implementation is held; explicitly release the owning task to retry it"
2058 );
2059 }
2060
2061 if self.state.merge_repairs.last().is_some_and(|r| {
2062 matches!(
2063 r.phase,
2064 crate::run::MergeRepairPhase::Requested
2065 | crate::run::MergeRepairPhase::Fixing
2066 | crate::run::MergeRepairPhase::Reviewing
2067 )
2068 }) {
2069 if self.repair_merge_review().await? {
2070 self.run_land().await?;
2071 }
2072 self.settle_questions();
2073 return Ok(());
2074 }
2075 if self.state.pr_publication_pending {
2078 self.merge().await?;
2079 if self.resource_halt()? {
2080 return Ok(());
2081 }
2082 self.state.save()?;
2083 self.settle_questions();
2084 return Ok(());
2085 }
2086 if self.state.status == RunStatus::Stalled {
2099 if self.recover_stall().await? {
2100 self.finish_after_tally().await?;
2101 } else {
2102 self.state.save()?;
2104 }
2105 return Ok(());
2106 }
2107 if self.state.status == RunStatus::Landing
2117 || crate::handover::landing_checkpoint(&self.state)
2118 {
2119 self.state.land_sync_pending = true;
2120 self.state.save()?;
2121 self.run_land().await?;
2122 self.settle_questions();
2127 return Ok(());
2128 }
2129 if self.state.github_text.is_some()
2134 && self.state.merge.is_none()
2135 && self.state.status == RunStatus::Gating
2136 {
2137 self.merge().await?;
2138 if self.resource_halt()? {
2139 return Ok(());
2140 }
2141 self.state.save()?;
2142 return Ok(());
2143 }
2144 if let Err(e) = self.prep().await {
2145 if e.is::<crate::repo_lock::ResourceBlocked>() {
2146 return Err(e);
2147 }
2148 self.state.status = RunStatus::Blocked;
2149 self.state
2150 .prep_failure
2151 .get_or_insert_with(|| format!("{e:#}"));
2152 self.state
2153 .event("prep", format!("preparation stopped: {e:#}"));
2154 self.state.save()?;
2155 return Err(e);
2156 }
2157 if self.park_here()? {
2158 return Ok(());
2159 }
2160 self.advise().await?;
2161 if self.park_here()? {
2162 return Ok(());
2163 }
2164 self.implement().await?;
2165 if self.park_here()? {
2166 return Ok(());
2167 }
2168 if self.state.status == RunStatus::VerifiedNoop {
2172 return Ok(());
2173 }
2174 self.judge().await?;
2175 if self.park_here()? {
2176 return Ok(());
2177 }
2178 self.deliberate().await?;
2179 if self.park_here()? {
2180 return Ok(());
2181 }
2182 self.vote().await?;
2183 if self.park_here()? {
2184 return Ok(());
2185 }
2186 self.tally()?;
2187 if self.state.status == RunStatus::Stalled {
2192 self.state.save()?;
2196 return Ok(());
2197 }
2198 self.finish_after_tally().await?;
2199 Ok(())
2200 }
2201
2202 fn stop_for_resource(&mut self) -> Result<()> {
2207 if self.pause.parked() || self.interrupt.parked() {
2210 self.park_here()?;
2211 return Ok(());
2212 }
2213 if self.state.resource_block.is_some() {
2214 self.state.status = RunStatus::Blocked;
2215 }
2216 self.state.save()
2217 }
2218
2219 fn resource_halt(&mut self) -> Result<bool> {
2221 if self.state.resource_block.is_none() {
2222 return Ok(false);
2223 }
2224 self.stop_for_resource()?;
2225 Ok(true)
2226 }
2227
2228 fn park_here(&mut self) -> Result<bool> {
2235 let parking = self.pause.parked() || self.interrupt.parked();
2236 if !parking && self.state.resource_block.is_some() {
2237 self.stop_for_resource()?;
2238 return Ok(true);
2239 }
2240 if !self.pause.parked() && !self.interrupt.parked() {
2245 return Ok(false);
2246 }
2247 if self.state.parked {
2248 return Ok(true);
2249 }
2250 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2251 if let Some(block) = &mut self.state.resource_block
2252 && matches!(
2253 block.cause,
2254 crate::run::ResourceBlockCause::Cancelled { .. }
2255 )
2256 {
2257 block.cause = crate::run::ResourceBlockCause::Cancelled {
2258 park: cause.clone(),
2259 };
2260 }
2261 let why = match cause.as_ref().map(ParkCause::label) {
2262 Some(reason) => format!(
2263 "parked after `{}` ({reason}) — resume to carry on from here",
2264 self.state.status.as_str()
2265 ),
2266 None => format!(
2267 "parked after `{}` — resume to carry on from here",
2268 self.state.status.as_str()
2269 ),
2270 };
2271 self.state.event("park", why);
2272 self.state
2273 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2274 self.state.parked = true;
2275 self.state.save()?;
2276 Ok(true)
2277 }
2278
2279 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2281 if self.state.prep_failure.is_none() {
2282 return Ok(());
2283 }
2284 let failed_at = self
2285 .state
2286 .events
2287 .iter()
2288 .rev()
2289 .find(|event| event.node == "prep")
2290 .map_or(self.state.updated_at, |event| event.at);
2291 anyhow::ensure!(
2292 approval_at > failed_at,
2293 "preparation retry requires a new explicit release after the failure"
2294 );
2295 self.state.prep_failure = None;
2296 self.state.prep_complete = Some(false);
2297 self.state
2298 .event("prep-retry", "operator approved retry of preparation");
2299 Ok(())
2300 }
2301
2302 pub fn on_pause(&mut self, pause: Pause) {
2304 self.pause = pause;
2305 }
2306
2307 pub fn watch_interrupt(&mut self, pause: Pause) {
2313 self.interrupt = pause;
2314 }
2315
2316 fn settle_questions(&mut self) {
2336 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2337 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2338 }
2339 }
2340
2341 async fn finish_after_tally(&mut self) -> Result<()> {
2344 self.graft().await?;
2347 if self.resource_halt()? {
2348 return Ok(());
2349 }
2350 self.fold_losers().await?;
2351 self.sync_to_base().await?;
2356 if self.hold_base_sync().await? || self.resource_halt()? {
2357 return Ok(());
2358 }
2359 if self.state.status == RunStatus::AlreadyInBase {
2360 return Ok(());
2361 }
2362 self.review_loop().await?;
2363 if self.resource_halt()? {
2364 return Ok(());
2365 }
2366 self.sync_to_base().await?;
2367 if self.hold_base_sync().await? || self.resource_halt()? {
2368 return Ok(());
2369 }
2370 if self.state.status == RunStatus::AlreadyInBase {
2371 return Ok(());
2372 }
2373 self.gate().await?;
2374 self.merge().await?;
2375 if self.resource_halt()? {
2376 return Ok(());
2377 }
2378 self.state.save()?;
2379 Ok(())
2380 }
2381
2382 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2389 let result = match ensure_setup_config(&mut self.state, repo).await {
2390 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2391 Err(e) => Err(e),
2392 };
2393 if let Err(e) = result {
2394 git::worktree_remove_clean(repo, wt).await.ok();
2395 if in_prep {
2396 self.rollback_prep(repo).await;
2397 }
2398 self.state.status = RunStatus::Blocked;
2399 self.state.event(
2400 "setup",
2401 format!("worktree setup failed in {}: {e:#}", wt.display()),
2402 );
2403 self.state.save()?;
2404 return Err(e);
2405 }
2406 Ok(())
2407 }
2408
2409 async fn rollback_prep(&mut self, repo: &Path) {
2411 let root = self.state.worktree_root();
2412 for c in std::mem::take(&mut self.state.candidates) {
2413 if matches!(
2414 git::worktree_remove_clean(repo, &c.worktree).await,
2415 Ok(true)
2416 ) {
2417 git::branch_delete(repo, &c.branch).await.ok();
2418 } else {
2419 self.state.candidates.push(c);
2420 }
2421 }
2422 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2423 for j in 1..=seats {
2424 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2425 .await
2426 .ok();
2427 }
2428 for k in 1..=self.state.config.graph.advisors {
2429 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2430 .await
2431 .ok();
2432 }
2433 }
2434
2435 async fn prep(&mut self) -> Result<()> {
2436 if let Some(reason) = &self.state.prep_failure {
2437 bail!(
2438 "candidate preparation previously failed: {reason}; correct the source before retrying"
2439 );
2440 }
2441 if self
2442 .state
2443 .prep_complete
2444 .unwrap_or(!self.state.candidates.is_empty())
2445 {
2446 return Ok(());
2447 }
2448 let repo = self.state.repo.clone();
2449 let _coordination = self.coordinate(&repo).await?;
2452 let base = self.state.base_commit.clone();
2453 let plan = self.state.sources.plan(&repo, &base).await?;
2454 self.rollback_prep(&repo).await;
2455 if !self.state.candidates.is_empty() {
2456 bail!(
2457 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2458 );
2459 }
2460 self.state.prep_complete = Some(false);
2461 self.state.status = RunStatus::Prep;
2462 self.state.save()?;
2463 let repo = self.state.repo.clone();
2464 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2465 let root = self.state.worktree_root();
2466 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2467
2468 let hooks_dir = self.state.dir().join("hooks");
2471 if self.state.config.blind.commit_msg_hook {
2472 std::fs::create_dir_all(&hooks_dir)
2473 .with_context(|| format!("create {}", hooks_dir.display()))?;
2474 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2475 let path = hooks_dir.join("commit-msg");
2476 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2477 make_executable(&path)?;
2478 git::acquire_worktree_config(&repo).await?;
2486 self.state.enabled_worktree_config = true;
2487 }
2488
2489 ensure_setup_config(&mut self.state, &repo).await?;
2492
2493 for (index, (spec, label)) in self
2494 .roles
2495 .implementers
2496 .clone()
2497 .into_iter()
2498 .zip(labels)
2499 .enumerate()
2500 {
2501 let branch = self.state.branch_for(label);
2502 let worktree = root.join(format!("cand-{label}"));
2503 self.state.event(
2507 "prep-intent",
2508 serde_json::json!({
2509 "branch": branch,
2510 "worktree": worktree,
2511 "start": start,
2512 "picks": plan.picks,
2513 })
2514 .to_string(),
2515 );
2516 self.state.save()?;
2517 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2518 if self.state.config.blind.commit_msg_hook {
2519 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2520 }
2521 git::local_exclude(&worktree, "/.magi/").await?;
2522 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2523 git::worktree_remove_clean(&repo, &worktree).await.ok();
2526 git::branch_delete(&repo, &branch).await.ok();
2527 self.rollback_prep(&repo).await;
2528 self.state.status = RunStatus::Blocked;
2529 self.state
2530 .event("prep", format!("worktree setup failed: {e:#}"));
2531 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2532 self.state.save()?;
2533 return Err(e);
2534 }
2535 self.state.candidates.push(Candidate {
2536 index,
2537 label,
2538 agent: spec.id.clone(),
2539 branch,
2540 worktree,
2541 summary: String::new(),
2542 stat: String::new(),
2543 files: 0,
2544 commits: 0,
2545 empty: false,
2546 failed: None,
2547 verified_noop: None,
2548 duration_ms: 0,
2549 folded: false,
2550 });
2551 self.state.save()?;
2552 for pick in &plan.picks {
2553 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2554 {
2555 self.state.status = RunStatus::Blocked;
2556 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2557 self.state
2558 .event("prep", format!("cannot apply referenced commit: {e}"));
2559 self.state.save()?;
2560 return Err(e);
2561 }
2562 }
2563 }
2564
2565 for j in 1..=self.roles.judges.len() {
2566 let wt = root.join(format!("judge-{j}"));
2567 if !wt.exists() {
2568 git::worktree_add_detached(&repo, &wt, &base).await?;
2569 self.setup_seat_worktree(&repo, &wt, true).await?;
2570 }
2571 }
2572
2573 if self.state.config.graph.advise {
2581 for k in 1..=self.state.config.graph.advisors {
2582 let wt = root.join(format!("advisor-{k}"));
2583 if !wt.exists() {
2584 git::worktree_add_detached(&repo, &wt, &base).await?;
2585 self.setup_seat_worktree(&repo, &wt, true).await?;
2586 }
2587 }
2588 }
2589
2590 let authors: Vec<&str> = self
2595 .roles
2596 .implementers
2597 .iter()
2598 .map(|a| a.id.as_str())
2599 .collect();
2600 let overlap: Vec<String> = self
2601 .roles
2602 .judges
2603 .iter()
2604 .enumerate()
2605 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2606 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2607 .collect();
2608 if !overlap.is_empty() {
2609 let note = format!(
2610 "{} also authored a candidate; blind, but the panel is less \
2611 independent than {} distinct agents would be",
2612 overlap.join(", "),
2613 self.roles.judges.len()
2614 );
2615 self.state.event("prep", note);
2616 }
2617
2618 self.state.event(
2619 "prep",
2620 format!(
2621 "{} candidates, {} judges, base {} ({})",
2622 self.state.candidates.len(),
2623 self.roles.judges.len(),
2624 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2625 self.state.base_branch
2626 ),
2627 );
2628 self.state.prep_complete = Some(true);
2629 self.state.status = RunStatus::Implementing;
2630 self.state.save()?;
2631 Ok(())
2632 }
2633
2634 async fn advise(&mut self) -> Result<()> {
2667 let implement_untouched = self
2668 .state
2669 .candidates
2670 .iter()
2671 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2672 if !self.state.config.graph.advise || self.state.advise_attempted {
2673 return Ok(());
2674 }
2675 if !implement_untouched {
2676 self.state.event(
2677 "advise",
2678 "skipping the design-deliberation stage: at least one \
2679 candidate already shows implementation progress, so this \
2680 run is past the point the stage exists to run before"
2681 .to_owned(),
2682 );
2683 self.state.advise_attempted = true;
2684 self.state.save()?;
2685 return Ok(());
2686 }
2687 let run_id = self.state.id.clone();
2688 let prompts = self.state.config.prompts.clone();
2689 let instruction = self.state.instruction.clone();
2690 let language = self.state.config.graph.language.clone();
2691 let root = self.state.worktree_root();
2692 let n = self.state.config.graph.advisors;
2693 let where_recorded = self.state.dir().join("run.json");
2694
2695 let seats = match self.state.config.advisors() {
2696 Ok(seats) if !seats.is_empty() => seats,
2697 Ok(_) => {
2698 self.state.event(
2699 "advise",
2700 format!(
2701 "[graph] advisors is 0; skipping the design-deliberation \
2702 stage and continuing without a synthesis brief (see {})",
2703 where_recorded.display()
2704 ),
2705 );
2706 self.state.advise_attempted = true;
2707 self.state.save()?;
2708 return Ok(());
2709 }
2710 Err(e) => {
2711 self.state.event(
2712 "advise",
2713 format!(
2714 "could not resolve advisor seats ({e:#}); continuing \
2715 without a design-deliberation brief (see {})",
2716 where_recorded.display()
2717 ),
2718 );
2719 self.state.advise_attempted = true;
2720 self.state.save()?;
2721 return Ok(());
2722 }
2723 };
2724
2725 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2726 let artifacts = agent::artifacts_dir(&self.state.dir());
2727 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2728
2729 let mut jobs = Vec::new();
2730 for (i, spec) in seats.iter().cloned().enumerate() {
2731 let seat_key = format!("advisor-{}", i + 1);
2732 let seat = self.seat(&seat_key, &spec.id);
2733 jobs.push(SeatJob {
2734 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2735 spec,
2736 seat,
2737 cwd: worktrees[i % worktrees.len()].clone(),
2738 timeout,
2739 allow_write: false,
2740 sessions: false,
2741 artifacts: artifacts.clone(),
2742 stem: seat_key,
2743 handover: None,
2744 });
2745 }
2746
2747 self.state.event(
2748 "advise",
2749 format!(
2750 "{} advisor seat(s) sketching a design in parallel",
2751 jobs.len()
2752 ),
2753 );
2754 let mut quota_losses = Vec::new();
2755 let cache = self.state.config.cache_dir();
2756 let ctx = WaveCtx {
2757 carry_seats: false,
2758 pauses: Some((&self.pause, &self.interrupt)),
2759 run: &run_id,
2760 node: "advise",
2761 prompts: &prompts,
2762 cache: cache.as_deref(),
2763 round: None,
2764 };
2765 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2766 let results = ask_json_wave::<Proposal>(
2767 jobs,
2768 Arc::clone(&self.sem),
2769 self.state.config.graph.retries,
2770 &advisor_roster,
2771 &ctx,
2772 &mut quota_losses,
2773 &mut self.state,
2774 &|p: &Proposal| p.validate(),
2775 )
2776 .await;
2777 if self.state.resource_block.is_some() {
2778 return self.stop_for_resource();
2779 }
2780 self.state.quota.extend(quota_losses);
2781
2782 let mut records = Vec::with_capacity(results.len());
2783 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2784 let agent_id = seat.agent.clone();
2785 self.state.seats.insert(seat.key.clone(), seat);
2786 match res {
2787 Ok((proposal, out)) => {
2788 self.state
2789 .event("advise", format!("advisor-{} proposed a design", i + 1));
2790 records.push(advise::AdvisorRecord::proposed(
2791 i + 1,
2792 agent_id,
2793 proposal,
2794 out.duration_ms,
2795 ));
2796 }
2797 Err(e) => {
2798 self.state.event(
2799 "advise",
2800 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2801 );
2802 records.push(advise::AdvisorRecord::failed(
2803 i + 1,
2804 agent_id,
2805 e.to_string(),
2806 ));
2807 }
2808 }
2809 }
2810
2811 let mut advice = advise::Advice {
2812 records,
2813 synthesis: None,
2814 };
2815 if advice.proposals().is_empty() {
2816 self.state.event(
2817 "advise",
2818 "no advisor produced a usable proposal; continuing without a \
2819 synthesis brief"
2820 .to_owned(),
2821 );
2822 } else {
2823 match self
2824 .synthesize_brief(
2825 &advice,
2826 &instruction,
2827 &language,
2828 &worktrees[0],
2829 &artifacts,
2830 &run_id,
2831 &prompts,
2832 cache.as_deref(),
2833 )
2834 .await
2835 {
2836 Ok(Some(text)) => {
2837 self.state.event(
2838 "advise",
2839 "synthesized a design brief for the implementer".to_owned(),
2840 );
2841 advice.synthesis = Some(text);
2842 }
2843 Ok(None) => {
2844 self.state.event(
2845 "advise",
2846 "the synthesis seat produced nothing usable; continuing \
2847 without a design brief"
2848 .to_owned(),
2849 );
2850 }
2851 Err(e) => {
2852 if e.is::<crate::repo_lock::ResourceBlocked>() {
2853 return Err(e);
2854 }
2855 self.state.event(
2856 "advise",
2857 format!("could not synthesize a design brief: {e:#}"),
2858 );
2859 }
2860 }
2861 }
2862 advise::apply_reflection(&mut advice);
2863
2864 self.state.advice = Some(advice);
2865 self.state.advise_attempted = true;
2866 self.state.save()?;
2867 Ok(())
2868 }
2869
2870 #[allow(clippy::too_many_arguments)]
2882 async fn synthesize_brief(
2883 &mut self,
2884 advice: &advise::Advice,
2885 instruction: &str,
2886 language: &str,
2887 cwd: &Path,
2888 artifacts: &Path,
2889 run_id: &str,
2890 prompts: &Prompts,
2891 cache: Option<&Path>,
2892 ) -> Result<Option<String>> {
2893 let chain = agent::pick_chain(
2894 &self.state.config.agents,
2895 self.state.config.roles.synthesizer.as_ref(),
2896 &agent::installed,
2897 "synthesizer",
2898 )?;
2899 let proposals = advice.proposals();
2900 let mut prompt = prompt::with_overlay(
2901 prompt::synthesize_brief(instruction, &proposals, language),
2902 prompts.overlay("advise"),
2903 );
2904 if cache.is_some() {
2905 prompt.push('\n');
2910 prompt.push_str(&prompt::build_cache_note("advise", false));
2911 }
2912 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2913 let mut last = None;
2918 for (n, spec) in chain.iter().enumerate() {
2919 if n > 0 {
2920 self.state
2921 .event("advise", format!("synthesis falling back to {}", spec.id));
2922 }
2923 let mut seat = self.seat("advise-synthesis", &spec.id);
2924 let wait = self.repository_wait();
2925 let outcome = wait
2926 .scope(agent::invoke(
2927 spec,
2928 &mut seat,
2929 &Invocation {
2930 coordination: self.state.config.coordination.as_ref(),
2931 fetched_base: None,
2932 cwd,
2933 prompt: &prompt,
2934 timeout,
2935 allow_write: false,
2936 sessions: false,
2937 artifacts,
2938 stem: &if n == 0 {
2939 "advise-synthesis".to_owned()
2940 } else {
2941 format!("advise-synthesis-{}", spec.id)
2942 },
2943 run: run_id,
2944 node: "advise",
2945 cache_dir: None,
2946 attachments: &[],
2947 writable: &[],
2948 },
2949 ))
2950 .await;
2951 if outcome.is_ok() {
2952 self.state.seats.insert(seat.key.clone(), seat);
2953 }
2954 let advance = agent::chain_advances(&outcome);
2955 last = Some(outcome);
2956 if !advance {
2957 break;
2958 }
2959 }
2960 let out = last.expect("a chain holds at least one agent")?;
2963 if !out.usable() {
2964 return Ok(None);
2965 }
2966 let text =
2967 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2968 Ok((!text.trim().is_empty()).then_some(text))
2969 }
2970
2971 async fn implement(&mut self) -> Result<()> {
2974 self.persist_candidates().await?;
2975 let run_id = self.state.id.clone();
2980 let prompts = self.state.config.prompts.clone();
2981 let todo: Vec<usize> = self
2982 .state
2983 .candidates
2984 .iter()
2985 .enumerate()
2986 .filter(|(_, c)| {
2987 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2988 && c.failed.is_none()
2989 && !c.empty
2990 })
2991 .map(|(i, _)| i)
2992 .collect();
2993 if todo.is_empty() {
2994 return self.after_implement();
2995 }
2996 self.state.status = RunStatus::Implementing;
2997
2998 let language = self.state.config.graph.language.clone();
2999 let github_language = self.state.config.graph.github_language.clone();
3000 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3001 let sessions = self.state.config.graph.sessions;
3002 let artifacts = agent::artifacts_dir(&self.state.dir());
3003 let brief = self
3007 .state
3008 .advice
3009 .as_ref()
3010 .and_then(|a| a.synthesis.as_deref())
3011 .map(str::to_owned);
3012 let attachments = self.state.attachments.clone();
3013
3014 let mut jobs = Vec::new();
3015 for &i in &todo {
3016 let (index, label, worktree) = {
3017 let c = &self.state.candidates[i];
3018 (c.index, c.label, c.worktree.clone())
3019 };
3020 let spec = self.roles.implementers[index].clone();
3021 let seat_key = format!("impl-{label}");
3022 let seat = self.seat(&seat_key, &spec.id);
3023 let instruction = seeded_instruction(&self.state);
3024 jobs.push(SeatJob {
3025 spec,
3026 seat,
3027 prompt: prompt::implement_in(
3028 &instruction,
3029 &worktree.to_string_lossy(),
3030 &language,
3031 &github_language,
3032 brief.as_deref(),
3033 &attachments,
3034 ),
3035 cwd: worktree,
3036 timeout,
3037 allow_write: true,
3038 sessions,
3039 artifacts: artifacts.clone(),
3040 stem: format!("impl-{label}"),
3041 handover: None,
3042 });
3043 }
3044
3045 self.state.event(
3046 "implement",
3047 format!("{} candidates in parallel", jobs.len()),
3048 );
3049 let mut sent = jobs.clone();
3055 let cache = self.state.config.cache_dir();
3056 let ctx = WaveCtx {
3057 carry_seats: false,
3058 pauses: Some((&self.pause, &self.interrupt)),
3059 run: &run_id,
3060 node: "implement",
3061 prompts: &prompts,
3062 cache: cache.as_deref(),
3063 round: None,
3064 };
3065 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3066 if !wave_blocked(&results) {
3067 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3068 .await;
3069 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3070 .await?;
3071 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3072 .await;
3073 }
3074 if !results.is_empty()
3077 && results
3078 .iter()
3079 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3080 {
3081 let losses: Vec<_> = results
3082 .iter()
3083 .map(|(_, seat, out)| {
3084 let AgentOutcome::Quota(o) = out else {
3085 unreachable!()
3086 };
3087 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3088 })
3089 .collect();
3090 self.state.quota.extend(losses.clone());
3091 for (_, seat, _) in results {
3092 self.state.seats.insert(seat.key.clone(), seat);
3093 }
3094 for &i in &todo {
3095 let label = self.state.candidates[i].label;
3096 self.state.interrupted_implementations.insert(label);
3097 self.state
3098 .candidate_preservation
3099 .insert(label, "machine outage awaiting persistence".into());
3100 }
3101 block_machine_outage(&mut self.state, "implement", &losses);
3104 self.state.save()?;
3105 self.persist_candidates().await?;
3106 for &i in &todo {
3107 self.state
3108 .interrupted_implementations
3109 .insert(self.state.candidates[i].label);
3110 }
3111 return self.stop_for_resource();
3112 }
3113 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3114
3115 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3116 && results
3117 .iter()
3118 .any(|(_, _, out)| FailClass::of(out).is_some());
3119 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3120 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3121 if self
3125 .state
3126 .candidate_preservation
3127 .contains_key(&self.state.candidates[i].label)
3128 {
3129 self.state.seats.insert(seat.key.clone(), seat);
3130 }
3131 continue;
3132 }
3133 let seat_key = seat.key.clone();
3134 let agent = seat.agent.clone();
3143 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3144 self.state.seats.insert(seat.key.clone(), seat);
3145 let label = self.state.candidates[i].label;
3146 self.state.interrupted_implementations.remove(&label);
3147
3148 let (summary, duration, failed, verified_claim) = match out {
3149 AgentOutcome::Ok(o) => {
3150 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3151 let failed = (!o.usable()).then(|| {
3152 if o.timed_out {
3153 "agent timed out".to_owned()
3154 } else {
3155 format!(
3156 "agent exited with {}",
3157 crate::notices::exit_status(o.exit_code)
3158 )
3159 }
3160 });
3161 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3162 (text, o.duration_ms, failed, verified_claim)
3163 }
3164 AgentOutcome::Dropped(o) => {
3170 let why = o
3171 .dropped
3172 .as_ref()
3173 .map(|d| d.why.as_str())
3174 .unwrap_or("the CLI ended the stream without delivering its answer");
3175 (
3176 String::new(),
3177 o.duration_ms,
3178 Some(format!("the CLI dropped the stream ({why})")),
3179 None,
3180 )
3181 }
3182 AgentOutcome::Quota(o) => {
3183 self.state.quota.push(QuotaLoss::from_report(
3184 seat_key,
3185 "implement",
3186 o.quota.as_ref(),
3187 ));
3188 (
3189 String::new(),
3190 o.duration_ms,
3191 Some("rate limited (quota); produced no change".to_owned()),
3192 None,
3193 )
3194 }
3195 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3196 (String::new(), 0, Some(e), None)
3197 }
3198 };
3199
3200 let c = &mut self.state.candidates[i];
3203 if !exhausted_the_fallback_chain {
3204 c.agent = agent;
3205 }
3206 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3207 c.duration_ms = duration;
3208 c.failed = failed;
3209 c.verified_noop = verified_claim;
3210 self.state
3211 .candidate_preservation
3212 .insert(label, "awaiting persistence".to_owned());
3213 }
3214 self.state.save()?;
3215 self.persist_candidates().await?;
3216
3217 if resource_blocked {
3218 return self.stop_for_resource();
3219 }
3220 if selected_failed {
3221 self.state.status = RunStatus::Blocked;
3222 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3223 self.state.save()?;
3224 bail!(
3225 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3226 );
3227 }
3228 self.after_implement()
3229 }
3230
3231 async fn persist_candidates(&mut self) -> Result<()> {
3233 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3234 for label in labels {
3235 let i = self
3236 .state
3237 .candidates
3238 .iter()
3239 .position(|c| c.label == label)
3240 .context("pending candidate is missing")?;
3241 let c = self.state.candidates[i].clone();
3242 let interrupted = self.state.interrupted_implementations.contains(&label);
3243 let result = async {
3244 let wait = crate::repo_lock::WaitPolicy::new(
3245 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3246 || false,
3247 );
3248 let _coordination =
3249 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3250 if let Err(e) = git::snapshot_uncommitted(
3253 &c.worktree,
3254 &self
3255 .state
3256 .dir()
3257 .join("artifacts")
3258 .join(format!("candidate-{label}-recovery")),
3259 )
3260 .await
3261 {
3262 self.state.event(
3263 "preserve",
3264 format!("candidate {label} snapshot incomplete: {e:#}"),
3265 );
3266 }
3267 let rescued = git::rescue_commit(
3268 &c.worktree,
3269 &format!("magi: candidate {label} (uncommitted work)"),
3270 )
3271 .await?;
3272 self.state.note_withheld("implement", &rescued.withheld);
3273 let base = &self.state.base_commit;
3274 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3275 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3276 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3277 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3278 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3279 if c.empty && commits > 0 && !patch.trim().is_empty()
3280 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3281 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3282 || self.state.tally.is_some())
3283 {
3284 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3288 }
3289 let c = &mut self.state.candidates[i];
3290 c.commits = commits;
3291 c.stat = stat;
3292 c.files = files;
3293 let no_changes = commits == 0 || patch.trim().is_empty();
3294 c.empty = no_changes && !interrupted;
3295 if interrupted && no_changes {
3296 c.failed = None;
3297 c.verified_noop = None;
3298 }
3299 if !c.empty {
3300 c.failed = None;
3301 c.verified_noop = None;
3302 }
3303 self.state.event(
3304 "preserve",
3305 format!("candidate {label}: {files} files, {commits} commits"),
3306 );
3307 Ok::<_, anyhow::Error>(())
3308 }
3309 .await;
3310 match result {
3311 Ok(()) => {
3312 self.state.candidate_preservation.remove(&label);
3313 if !interrupted || self.state.candidates[i].files > 0 {
3314 self.state.interrupted_implementations.remove(&label);
3315 }
3316 }
3317 Err(e) => {
3318 let reason = format!("candidate {label} preservation failed: {e:#}");
3319 self.state.event("preserve", &reason);
3320 self.state.candidate_preservation.insert(label, reason);
3321 }
3322 }
3323 if self.state.candidate_preservation.is_empty()
3324 && self.state.status == RunStatus::Blocked
3325 && self.state.candidates.iter().all(|c| c.failed.is_none())
3326 {
3327 self.state.status = RunStatus::Implementing;
3331 self.state.event(
3332 "preserve",
3333 "candidate persistence recovered; implementation can continue",
3334 );
3335 }
3336 self.state.save()?;
3337 }
3338 if !self.state.candidate_preservation.is_empty() {
3339 self.state.status = RunStatus::Blocked;
3340 self.state.save()?;
3341 bail!(
3342 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3343 );
3344 }
3345 Ok(())
3346 }
3347
3348 async fn resume_undelivered(
3376 &mut self,
3377 results: &mut [(usize, SeatState, AgentOutcome)],
3378 sent: &[SeatJob],
3379 prompts: &Prompts,
3380 run_id: &str,
3381 ) {
3382 for (wi, seat, out) in results.iter_mut() {
3383 let Some(dropped) = (match &*out {
3384 AgentOutcome::Dropped(o) => o.dropped.clone(),
3385 _ => None,
3386 }) else {
3387 continue;
3388 };
3389 let Some(job) = sent.get(*wi) else { continue };
3390 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3392 self.state.event(
3393 "implement",
3394 format!(
3395 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3396 work is in the tree",
3397 seat.key, dropped.output_tokens, dropped.why
3398 ),
3399 );
3400 continue;
3401 }
3402 if !has_context(&job.spec, seat, job.sessions) {
3410 self.state.event(
3411 "implement",
3412 format!(
3413 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3414 is no session left to resume",
3415 seat.key, dropped.output_tokens, dropped.why
3416 ),
3417 );
3418 continue;
3419 }
3420 self.state.event(
3421 "implement",
3422 format!(
3423 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3424 conversation",
3425 seat.key, dropped.output_tokens, dropped.why
3426 ),
3427 );
3428 let mut retry = job.clone();
3429 retry.seat = seat.clone();
3430 retry.prompt = prompt::resume_after_drop(&dropped.why);
3431 retry.timeout = retry_budget(job.timeout, true);
3432 retry.stem = format!("{}-resume", job.stem);
3433 let cache = self.state.config.cache_dir();
3434 let ctx = WaveCtx {
3435 carry_seats: false,
3436 pauses: Some((&self.pause, &self.interrupt)),
3437 run: run_id,
3438 node: "implement",
3439 prompts,
3440 cache: cache.as_deref(),
3441 round: None,
3442 };
3443 let (resumed_seat, resumed) =
3444 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3445 *seat = resumed_seat;
3446 *out = resumed;
3447 }
3448 }
3449
3450 async fn resume_seat_handovers(
3515 &mut self,
3516 results: &mut [(usize, SeatState, AgentOutcome)],
3517 sent: &mut [SeatJob],
3518 prompts: &Prompts,
3519 run_id: &str,
3520 ) -> Result<()> {
3521 let instruction = seeded_instruction(&self.state);
3522 let language = self.state.config.graph.language.clone();
3523 let github_language = self.state.config.graph.github_language.clone();
3524 let brief = self
3525 .state
3526 .advice
3527 .as_ref()
3528 .and_then(|a| a.synthesis.as_deref())
3529 .map(str::to_owned);
3530 let attachments = self.state.attachments.clone();
3531 for (wi, seat, out) in results.iter_mut() {
3532 let others: BTreeSet<String> = sent
3535 .iter()
3536 .enumerate()
3537 .filter(|(j, _)| j != wi)
3538 .map(|(_, j)| j.spec.id.clone())
3539 .collect();
3540 let Some(job) = sent.get_mut(*wi) else {
3541 continue;
3542 };
3543 let start = self
3548 .roles
3549 .implementer_roster
3550 .iter()
3551 .position(|s| s.id == job.spec.id)
3552 .unwrap_or(0);
3553 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3554 let mut fallback_attempt = 0usize;
3555 let mut prev: Option<FailClass> = None;
3556 while let Some(cur) = FailClass::of(&*out) {
3557 if !should_hand_over(prev.as_ref(), &cur) {
3558 break;
3559 }
3560 let Some(next) =
3561 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3562 .cloned()
3563 else {
3564 break;
3565 };
3566 tried.insert(next.id.clone());
3567 fallback_attempt += 1;
3568
3569 let coordination = self.coordinate(&job.cwd).await?;
3570 if let Ok(r) = git::rescue_commit(
3571 &job.cwd,
3572 &format!(
3573 "magi: candidate {} (uncommitted work before {} fallback)",
3574 seat.key,
3575 if cur == FailClass::Quota {
3576 "quota"
3577 } else {
3578 "handover"
3579 }
3580 ),
3581 )
3582 .await
3583 {
3584 self.state.note_withheld("implement", &r.withheld);
3585 }
3586 drop(coordination);
3587
3588 record_handover(
3589 &mut self.state,
3590 "implement",
3591 &seat.key,
3592 &seat.agent,
3593 &next.id,
3594 &cur,
3595 &fail_reason(&*out),
3596 );
3597 prev = Some(cur.clone());
3598
3599 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3600 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3601 job.spec = next.clone();
3609 let mut retry = job.clone();
3610 retry.seat = new_seat;
3611 retry.prompt = prompt::implement_in(
3612 &instruction,
3613 &job.cwd.to_string_lossy(),
3614 &language,
3615 &github_language,
3616 brief.as_deref(),
3617 &attachments,
3618 );
3619 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3620 let cache = self.state.config.cache_dir();
3621 let ctx = WaveCtx {
3622 carry_seats: false,
3623 pauses: Some((&self.pause, &self.interrupt)),
3624 run: run_id,
3625 node: "implement",
3626 prompts,
3627 cache: cache.as_deref(),
3628 round: None,
3629 };
3630 let (fallback_seat, fallback_out) = run_one(
3631 retry,
3632 Arc::clone(&self.sem),
3633 &ctx,
3634 &mut self.state,
3635 fallback_attempt,
3636 )
3637 .await;
3638 *seat = fallback_seat;
3639 *out = fallback_out;
3640 }
3641 }
3642 Ok(())
3643 }
3644
3645 async fn resume_unconfirmed_commands(
3669 &mut self,
3670 results: &mut [(usize, SeatState, AgentOutcome)],
3671 sent: &[SeatJob],
3672 prompts: &Prompts,
3673 run_id: &str,
3674 ) {
3675 for (wi, seat, out) in results.iter_mut() {
3676 let AgentOutcome::Ok(o) = &*out else {
3677 continue;
3678 };
3679 if !has_unconfirmed_command(&o.commands) {
3680 continue;
3681 }
3682 let Some(job) = sent.get(*wi) else { continue };
3683 if !has_context(&job.spec, seat, job.sessions) {
3684 self.state.event(
3685 "implement",
3686 format!(
3687 "{}: the reply named a command whose own CLI never confirmed the exit \
3688 status of, but there is no session left to resume",
3689 seat.key
3690 ),
3691 );
3692 continue;
3693 }
3694 self.state.event(
3695 "implement",
3696 format!(
3697 "{}: the reply named a command whose own CLI never confirmed the exit \
3698 status of; resuming the conversation",
3699 seat.key
3700 ),
3701 );
3702 let mut retry = job.clone();
3703 retry.seat = seat.clone();
3704 retry.prompt = prompt::resume_incomplete(
3705 "a command in your last reply had no confirmed exit status",
3706 );
3707 retry.timeout = retry_budget(job.timeout, true);
3708 retry.stem = format!("{}-confirm", job.stem);
3709 let cache = self.state.config.cache_dir();
3710 let ctx = WaveCtx {
3711 carry_seats: false,
3712 pauses: Some((&self.pause, &self.interrupt)),
3713 run: run_id,
3714 node: "implement",
3715 prompts,
3716 cache: cache.as_deref(),
3717 round: None,
3718 };
3719 let (resumed_seat, resumed) =
3720 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3721 *seat = resumed_seat;
3722 *out = resumed;
3723 }
3724 }
3725
3726 async fn continue_fix_report(
3747 &mut self,
3748 mut seat: SeatState,
3749 parse_err: String,
3750 job: &SeatJob,
3751 prompts: &Prompts,
3752 run_id: &str,
3753 round: usize,
3754 ) -> (
3755 SeatState,
3756 Option<FixReport>,
3757 Option<String>,
3758 ContinuationRecord,
3759 ) {
3760 let mut last_err = parse_err;
3761 let mut cumulative_wait_ms = 0u64;
3762 let mut attempts = 0usize;
3763 loop {
3764 if !has_context(&job.spec, &seat, job.sessions) {
3765 self.state.event(
3766 "fix",
3767 format!(
3768 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3769 session left to resume into"
3770 ),
3771 );
3772 let outcome = if attempts == 0 {
3773 ContinuationOutcome::NoSession
3774 } else {
3775 ContinuationOutcome::Exhausted
3776 };
3777 return (
3778 seat,
3779 None,
3780 Some(format!("unparsable fix report: {last_err}")),
3781 ContinuationRecord {
3782 attempts,
3783 cumulative_wait_ms,
3784 outcome,
3785 },
3786 );
3787 }
3788 if attempts >= MAX_FIX_CONTINUATIONS {
3789 self.state.event(
3790 "fix",
3791 format!(
3792 "round {round}: fixer's reply still had no adoption report after \
3793 {attempts} continuation(s) ({last_err}); giving up"
3794 ),
3795 );
3796 return (
3797 seat,
3798 None,
3799 Some(format!(
3800 "unparsable fix report after {attempts} continuation(s): {last_err}"
3801 )),
3802 ContinuationRecord {
3803 attempts,
3804 cumulative_wait_ms,
3805 outcome: ContinuationOutcome::Exhausted,
3806 },
3807 );
3808 }
3809 attempts += 1;
3810 self.state.event(
3811 "fix",
3812 format!(
3813 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3814 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3815 ),
3816 );
3817 let mut retry = job.clone();
3818 retry.seat = seat.clone();
3819 retry.prompt = prompt::resume_incomplete(&last_err);
3820 retry.timeout = retry_budget(job.timeout, true);
3821 retry.stem = format!("{}-continue{attempts}", job.stem);
3822 let cache = self.state.config.cache_dir();
3823 let ctx = WaveCtx {
3824 carry_seats: false,
3825 pauses: Some((&self.pause, &self.interrupt)),
3826 run: run_id,
3827 node: "fix",
3828 prompts,
3829 cache: cache.as_deref(),
3830 round: Some(round),
3831 };
3832 let (resumed_seat, resumed_out) = run_one(
3833 retry,
3834 Arc::clone(&self.sem),
3835 &ctx,
3836 &mut self.state,
3837 attempts,
3838 )
3839 .await;
3840 seat = resumed_seat;
3841 match resumed_out {
3842 AgentOutcome::Ok(o) => {
3843 cumulative_wait_ms += o.duration_ms;
3844 match verdict::extract_json::<FixReport>(&o.text) {
3845 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3846 self.state.event(
3847 "fix",
3848 format!(
3849 "round {round}: fixer's adoption report recovered after \
3850 {attempts} continuation(s)"
3851 ),
3852 );
3853 return (
3854 seat,
3855 Some(report),
3856 None,
3857 ContinuationRecord {
3858 attempts,
3859 cumulative_wait_ms,
3860 outcome: ContinuationOutcome::Resumed,
3861 },
3862 );
3863 }
3864 Ok(_) => {
3872 last_err = "the reply parsed, but it reported a command whose own CLI \
3873 never confirmed an exit status"
3874 .to_owned();
3875 }
3876 Err(e) => last_err = e.to_string(),
3877 }
3878 }
3879 AgentOutcome::Quota(o) => {
3880 cumulative_wait_ms += o.duration_ms;
3881 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3882 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3883 self.state.quota.push(loss);
3884 self.state.event(
3885 "fix",
3886 format!(
3887 "round {round}: continuation rate limited (quota); not retrying now"
3888 ),
3889 );
3890 return (
3891 seat,
3892 None,
3893 Some("rate limited (quota) while recovering the fix report".to_owned()),
3894 ContinuationRecord {
3895 attempts,
3896 cumulative_wait_ms,
3897 outcome: ContinuationOutcome::QuotaLost,
3898 },
3899 );
3900 }
3901 AgentOutcome::Dropped(o) => {
3902 cumulative_wait_ms += o.duration_ms;
3903 let why = o
3904 .dropped
3905 .as_ref()
3906 .map(|d| d.why.as_str())
3907 .unwrap_or("the CLI ended the stream without delivering its answer");
3908 last_err = format!("the CLI dropped the stream ({why})");
3909 }
3910 AgentOutcome::Failed(e) => last_err = e,
3911 AgentOutcome::ResourceBlocked(e) => {
3914 return (
3915 seat,
3916 None,
3917 Some(format!("the continuation was not started: {e}")),
3918 ContinuationRecord {
3919 attempts: attempts.saturating_sub(1),
3920 cumulative_wait_ms,
3921 outcome: ContinuationOutcome::NoSession,
3922 },
3923 );
3924 }
3925 }
3926 }
3927 }
3928
3929 fn after_implement(&mut self) -> Result<()> {
3930 if !self.state.candidate_preservation.is_empty() {
3931 bail!("candidate preservation incomplete; judging is unsafe");
3932 }
3933 if self.state.leaks.is_empty() {
3935 let cfg = self.state.config.blind.clone();
3936 let mut leaks = Vec::new();
3937 for c in &self.state.candidates {
3938 let Some(patch) =
3939 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3940 else {
3941 continue;
3942 };
3943 leaks.extend(blind::scan(
3944 &format!("candidate {} patch", c.label),
3945 &patch,
3946 &cfg.vendor_tokens,
3947 ));
3948 }
3949 if !leaks.is_empty() {
3950 let summary = leaks
3951 .iter()
3952 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3953 .collect::<Vec<_>>()
3954 .join(", ");
3955 match cfg.on_leak {
3956 LeakPolicy::Fail => {
3957 self.state.status = RunStatus::Failed;
3958 self.state
3959 .event("blind", format!("vendor text in a patch: {summary}"));
3960 self.state.leaks = leaks;
3961 self.state.save()?;
3962 self.settle_questions();
3963 bail!(
3964 "blind.on_leak = \"fail\" and vendor text reached a \
3965 judged patch: {summary}"
3966 );
3967 }
3968 LeakPolicy::Redact => self.state.event(
3969 "blind",
3970 format!("redacting vendor text for judging: {summary}"),
3971 ),
3972 LeakPolicy::Warn => self.state.event(
3973 "blind",
3974 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3975 ),
3976 }
3977 self.state.leaks = leaks;
3978 }
3979 }
3980
3981 if self.state.viable().is_empty() {
3982 if self.state.all_candidates_verified_noop() {
3983 self.state.status = RunStatus::VerifiedNoop;
3994 self.state.save()?;
3995 self.settle_questions();
3996 return Ok(());
3997 }
3998 self.state.status = RunStatus::Failed;
3999 self.state.save()?;
4000 self.settle_questions();
4001 bail!("no candidate produced a change; nothing to judge");
4002 }
4003 self.state.status = RunStatus::Judging;
4004 self.state.save()?;
4005 Ok(())
4006 }
4007
4008 async fn judge(&mut self) -> Result<()> {
4011 let run_id = self.state.id.clone();
4016 let prompts = self.state.config.prompts.clone();
4017 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4018 return Ok(());
4019 }
4020 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4021 if viable.len() == 1 {
4022 self.state.judge_skipped = true;
4029 self.state.event(
4030 "judge",
4031 format!(
4032 "only candidate {} produced a change; judging skipped",
4033 viable[0].label
4034 ),
4035 );
4036 self.state.save()?;
4037 return Ok(());
4038 }
4039 self.state.status = RunStatus::Judging;
4040
4041 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4042 let language = self.state.config.graph.language.clone();
4043 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4044 let sessions = self.state.config.graph.sessions;
4045 let artifacts = agent::artifacts_dir(&self.state.dir());
4046 let root = self.state.worktree_root();
4047 let base_short = short(&self.state.base_commit);
4048
4049 let mut jobs = Vec::new();
4050 let mut orders = Vec::new();
4051 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4052 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4053 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4054 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4055 let seat_key = format!("judge-{}", j + 1);
4056 let seat = self.seat(&seat_key, &spec.id);
4057 jobs.push(SeatJob {
4058 prompt: prompt::judge(
4059 &self.state.instruction,
4060 &views,
4061 self.roles.judges.len(),
4062 &base_short,
4063 &language,
4064 ),
4065 spec,
4066 seat,
4067 cwd: root.join(format!("judge-{}", j + 1)),
4068 timeout,
4069 allow_write: false,
4070 sessions,
4071 artifacts: artifacts.clone(),
4072 stem: format!("judge-{}", j + 1),
4073 handover: None,
4074 });
4075 }
4076
4077 self.state.event(
4078 "judge",
4079 format!(
4080 "{} judges ranking {} candidates blind",
4081 jobs.len(),
4082 viable.len()
4083 ),
4084 );
4085 let labels_for_check = labels.clone();
4086 let mut quota_losses = Vec::new();
4087 let cache = self.state.config.cache_dir();
4088 let ctx = WaveCtx {
4089 carry_seats: false,
4090 pauses: Some((&self.pause, &self.interrupt)),
4091 run: &run_id,
4092 node: "judge",
4093 prompts: &prompts,
4094 cache: cache.as_deref(),
4095 round: None,
4096 };
4097 let results = ask_json_wave::<Ranking>(
4098 jobs,
4099 Arc::clone(&self.sem),
4100 self.state.config.graph.retries,
4101 &self.roles.judge_roster,
4102 &ctx,
4103 &mut quota_losses,
4104 &mut self.state,
4105 &move |r: &Ranking| r.validate(&labels_for_check),
4106 )
4107 .await;
4108 if self.state.resource_block.is_some() {
4109 return self.stop_for_resource();
4110 }
4111 self.state.quota.extend(quota_losses);
4112
4113 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4114 let agent_id = seat.agent.clone();
4115 self.state.seats.insert(seat.key.clone(), seat);
4116 let mut record = Judgement {
4117 judge: j + 1,
4118 seat: format!("judge-{}", j + 1),
4119 agent: agent_id,
4120 ranking: Vec::new(),
4121 reasons: BTreeMap::new(),
4122 confidence: None,
4123 strengths: BTreeMap::new(),
4124 order: orders[j].clone(),
4125 failed: None,
4126 duration_ms: 0,
4127 };
4128 match res {
4129 Ok((ranking, out)) => {
4130 record.ranking = ranking.normalized();
4131 record.reasons = ranking.reasons;
4132 record.strengths = ranking.strengths;
4133 record.confidence = ranking.confidence;
4134 record.duration_ms = out.duration_ms;
4135 self.state.event(
4136 "judge",
4137 format!(
4138 "judge {} ranked {}",
4139 j + 1,
4140 record.ranking.iter().collect::<String>()
4141 ),
4142 );
4143 }
4144 Err(e) => {
4145 record.failed = Some(e.to_string());
4146 self.state
4147 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4148 }
4149 }
4150 self.state.judgements.push(record);
4151 self.state.save()?;
4152 }
4153 Ok(())
4154 }
4155
4156 async fn deliberate(&mut self) -> Result<()> {
4159 if self.hold_base_sync().await? {
4160 return Ok(());
4161 }
4162 let run_id = self.state.id.clone();
4167 let prompts = self.state.config.prompts.clone();
4168 let rounds = self.state.config.graph.deliberate_rounds;
4169 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4171 return Ok(());
4172 }
4173 let tops: Vec<char> = self
4174 .state
4175 .judgements
4176 .iter()
4177 .filter_map(|j| j.ranking.first().copied())
4178 .collect();
4179 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4180 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4181 self.state.event(
4182 "deliberate",
4183 format!("judges agreed on {} outright; no deliberation", tops[0]),
4184 );
4185 }
4186 self.state.status = RunStatus::Voting;
4187 self.state.save()?;
4188 return Ok(());
4189 }
4190
4191 self.state.status = RunStatus::Deliberating;
4192 self.state.event(
4193 "deliberate",
4194 format!(
4195 "split: first choices were {} — opening {rounds} round(s)",
4196 tops.iter().collect::<String>()
4197 ),
4198 );
4199
4200 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4201 let language = self.state.config.graph.language.clone();
4202 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4203 let sessions = self.state.config.graph.sessions;
4204 let artifacts = agent::artifacts_dir(&self.state.dir());
4205 let root = self.state.worktree_root();
4206 let base_short = short(&self.state.base_commit);
4207
4208 for round in self.state.deliberation.len() + 1..=rounds {
4212 let mut turns: Vec<DeliberationTurn> = Vec::new();
4213 let mut round_losses = Vec::new();
4214 let mut asked = 0;
4215 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4216 if self.state.judgements[j].failed.is_some() {
4217 continue;
4218 }
4219 let seat_key = format!("judge-{}", j + 1);
4220 let spec = self.occupant(&seat_key, spec);
4221 let mut seat = self.seat(&seat_key, &spec.id);
4222 let transcript = self.transcript(&turns, j);
4223 let build = |context: Option<&str>| {
4224 prompt::deliberate(
4225 &self.state.instruction,
4226 context,
4227 &transcript,
4228 round,
4229 rounds,
4230 &language,
4231 )
4232 };
4233 let block = self.candidate_block(&viable, &base_short);
4234 let full = build(Some(&block));
4235 let text = if has_context(&spec, &seat, sessions) {
4236 build(None)
4237 } else {
4238 full.clone()
4239 };
4240 let job = SeatJob {
4241 spec,
4242 seat: seat.clone(),
4243 prompt: text,
4244 cwd: root.join(format!("judge-{}", j + 1)),
4245 timeout,
4246 allow_write: false,
4247 sessions,
4248 artifacts: artifacts.clone(),
4249 stem: format!("delib-{round}-judge-{}", j + 1),
4250 handover: Some(full),
4251 };
4252 let cache = self.state.config.cache_dir();
4253 let ctx = WaveCtx {
4254 carry_seats: false,
4255 pauses: Some((&self.pause, &self.interrupt)),
4256 run: &run_id,
4257 node: "deliberate",
4258 prompts: &prompts,
4259 cache: cache.as_deref(),
4260 round: None,
4261 };
4262 let mut losses = Vec::new();
4266 asked += 1;
4267 let mut results = ask_wave_with::<String>(
4268 vec![job],
4269 Arc::clone(&self.sem),
4270 0,
4271 &self.roles.judge_roster,
4272 &ctx,
4273 &mut losses,
4274 &mut self.state,
4275 &|text: &str| {
4276 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4277 },
4278 )
4279 .await;
4280 if self.state.resource_block.is_some() {
4281 return self.stop_for_resource();
4282 }
4283 round_losses.extend(losses.iter().cloned());
4284 self.state.quota.extend(losses);
4285 let (updated, res, _) = results.pop().expect("one job in, one result out");
4286 seat = updated;
4287 let agent_id = seat.agent.clone();
4288 self.state.seats.insert(seat.key.clone(), seat);
4289 let body = match res {
4290 Ok((body, _)) => body,
4291 Err(e) => {
4294 self.state
4295 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4296 continue;
4297 }
4298 };
4299 let tentative = verdict::extract_json::<Position>(&body)
4300 .ok()
4301 .and_then(|p| p.tentative)
4302 .and_then(|s| s.trim().chars().next())
4303 .map(|c| c.to_ascii_uppercase());
4304 self.state.event(
4305 "deliberate",
4306 format!(
4307 "round {round}: judge {} now favours {}",
4308 j + 1,
4309 tentative.map_or("—".to_owned(), |c| c.to_string())
4310 ),
4311 );
4312 turns.push(DeliberationTurn {
4313 judge: j + 1,
4314 agent: agent_id,
4315 body: blind::sanitize_prose(&body, &self.state.config.blind),
4316 tentative,
4317 });
4318 }
4319 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4320 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4321 return self.stop_for_resource();
4322 }
4323 self.state
4324 .deliberation
4325 .push(DeliberationRound { round, turns });
4326 self.state.save()?;
4327 }
4328
4329 self.state.status = RunStatus::Voting;
4330 self.state.save()?;
4331 Ok(())
4332 }
4333
4334 async fn vote(&mut self) -> Result<()> {
4337 if self.hold_base_sync().await? {
4338 return Ok(());
4339 }
4340 let run_id = self.state.id.clone();
4345 let prompts = self.state.config.prompts.clone();
4346 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4347 return Ok(());
4348 }
4349 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4350 if viable.len() == 1 {
4351 return Ok(());
4352 }
4353 self.state.status = RunStatus::Voting;
4354
4355 let language = self.state.config.graph.language.clone();
4356 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4357 let sessions = self.state.config.graph.sessions;
4358 let artifacts = agent::artifacts_dir(&self.state.dir());
4359 let root = self.state.worktree_root();
4360 let base_short = short(&self.state.base_commit);
4361 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4362
4363 let mut jobs = Vec::new();
4364 let mut seats_at = Vec::new();
4365 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4366 if self
4367 .state
4368 .judgements
4369 .get(j)
4370 .is_some_and(|r| r.failed.is_some())
4371 {
4372 continue;
4373 }
4374 let seat_key = format!("judge-{}", j + 1);
4375 let spec = self.occupant(&seat_key, spec);
4376 let seat = self.seat(&seat_key, &spec.id);
4377 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4378 let text = if has_context(&spec, &seat, sessions) {
4379 prompt::final_vote(&viable, &language)
4380 } else {
4381 full.clone()
4382 };
4383 jobs.push(SeatJob {
4384 spec,
4385 seat,
4386 prompt: text,
4387 cwd: root.join(format!("judge-{}", j + 1)),
4388 timeout,
4389 allow_write: false,
4390 sessions,
4391 artifacts: artifacts.clone(),
4392 stem: format!("vote-judge-{}", j + 1),
4393 handover: Some(full),
4394 });
4395 seats_at.push(j);
4396 }
4397
4398 self.state.event(
4399 "vote",
4400 format!(
4401 "collecting {} final votes one by one, privately",
4402 jobs.len()
4403 ),
4404 );
4405 let allowed = viable.clone();
4406 let mut quota_losses = Vec::new();
4407 let cache = self.state.config.cache_dir();
4408 let ctx = WaveCtx {
4409 carry_seats: false,
4410 pauses: Some((&self.pause, &self.interrupt)),
4411 run: &run_id,
4412 node: "vote",
4413 prompts: &prompts,
4414 cache: cache.as_deref(),
4415 round: None,
4416 };
4417 let results = ask_json_wave::<FinalVote>(
4418 jobs,
4419 Arc::clone(&self.sem),
4420 self.state.config.graph.retries,
4421 &self.roles.judge_roster,
4422 &ctx,
4423 &mut quota_losses,
4424 &mut self.state,
4425 &move |v: &FinalVote| match v.label() {
4426 Some(c) if allowed.contains(&c) => Ok(()),
4427 other => bail!("vote {other:?} is not one of {allowed:?}"),
4428 },
4429 )
4430 .await;
4431 if self.state.resource_block.is_some() {
4432 return self.stop_for_resource();
4433 }
4434 self.state.quota.extend(quota_losses);
4435
4436 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4437 let agent_id = seat.agent.clone();
4438 self.state.seats.insert(seat.key.clone(), seat);
4439 let initial = self
4440 .state
4441 .judgements
4442 .get(j)
4443 .and_then(|r| r.ranking.first().copied());
4444 let mut record = VoteRecord {
4445 judge: j + 1,
4446 agent: agent_id,
4447 vote: None,
4448 reason: String::new(),
4449 changed: false,
4450 };
4451 match res {
4452 Ok((v, _)) => {
4453 record.vote = v.label();
4454 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4455 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4456 self.state.event(
4457 "vote",
4458 format!(
4459 "judge {} voted {}{}",
4460 j + 1,
4461 record.vote.unwrap_or('?'),
4462 if record.changed { " (changed)" } else { "" }
4463 ),
4464 );
4465 }
4466 Err(e) => {
4467 self.state
4468 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4469 }
4470 }
4471 self.state.votes.push(record);
4472 self.state.save()?;
4473 }
4474 Ok(())
4475 }
4476
4477 fn tally(&mut self) -> Result<()> {
4480 if self.state.tally.is_some() {
4481 return Ok(());
4482 }
4483 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4484 let tops: Vec<char> = self
4485 .state
4486 .judgements
4487 .iter()
4488 .filter_map(|j| j.ranking.first().copied())
4489 .collect();
4490 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4491
4492 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4495 let mut cast: Vec<char> = Vec::new();
4496 for (i, j) in self.state.judgements.iter().enumerate() {
4497 let vote = self
4498 .state
4499 .votes
4500 .iter()
4501 .find(|v| v.judge == i + 1)
4502 .and_then(|v| v.vote)
4503 .or_else(|| j.ranking.first().copied());
4504 if let Some(v) = vote {
4505 *first_choice.entry(v).or_insert(0) += 1;
4506 cast.push(v);
4507 }
4508 }
4509
4510 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4511 for j in &self.state.judgements {
4512 let n = j.ranking.len();
4513 for (pos, label) in j.ranking.iter().enumerate() {
4514 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4515 }
4516 }
4517
4518 let best = first_choice.values().copied().max().unwrap_or(0);
4519 let mut leaders: Vec<char> = first_choice
4520 .iter()
4521 .filter(|(_, v)| **v == best)
4522 .map(|(k, _)| *k)
4523 .collect();
4524 let mut tie_break = None;
4525 if leaders.len() > 1 {
4526 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4527 let borda_leaders: Vec<char> = leaders
4528 .iter()
4529 .copied()
4530 .filter(|l| borda[l] == top_borda)
4531 .collect();
4532 tie_break = Some(if borda_leaders.len() == 1 {
4533 format!(
4534 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4535 leaders.len()
4536 )
4537 } else {
4538 format!(
4539 "{} way tie on both first-choice votes and Borda points, broken by label order",
4540 leaders.len()
4541 )
4542 });
4543 leaders = borda_leaders;
4544 leaders.sort_unstable();
4545 }
4546 let winner = *leaders
4547 .first()
4548 .or(viable.first())
4549 .context("no candidate to declare a winner from")?;
4550
4551 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4552 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4553 let deliberated = !self.state.deliberation.is_empty();
4554
4555 let quota_seats: std::collections::BTreeSet<&str> =
4559 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4560 let mut present = 0usize;
4561 for (i, j) in self.state.judgements.iter().enumerate() {
4562 if quota_seats.contains(j.seat.as_str()) {
4563 continue;
4564 }
4565 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4566 let voted = self
4567 .state
4568 .votes
4569 .iter()
4570 .any(|v| v.judge == i + 1 && v.vote.is_some());
4571 if ranked || voted {
4572 present += 1;
4573 }
4574 }
4575 let needs_quorum = viable.len() > 1;
4581 let judges_total = if needs_quorum {
4582 self.roles.judges.len()
4583 } else {
4584 0
4585 };
4586 let quorum = if needs_quorum {
4587 judges_total / 2 + 1
4588 } else {
4589 0
4590 };
4591 let met_quorum = !needs_quorum || present >= quorum;
4592 let uncontested = (!needs_quorum).then(|| {
4593 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4594 });
4595
4596 self.state.event(
4597 "tally",
4598 match &uncontested {
4599 Some(reason) => format!("winner {winner} — {reason}"),
4600 None => format!(
4601 "winner {winner} — votes {} | initial {} | {} changed | \
4602 {present}/{judges_total} judges{}",
4603 first_choice
4604 .iter()
4605 .map(|(k, v)| format!("{k}:{v}"))
4606 .collect::<Vec<_>>()
4607 .join(" "),
4608 if unanimous_initial {
4609 "unanimous"
4610 } else {
4611 "split"
4612 },
4613 changed_votes,
4614 if met_quorum {
4615 String::new()
4616 } else {
4617 format!(" — below quorum ({quorum} required)")
4618 },
4619 ),
4620 },
4621 );
4622 if !met_quorum {
4623 self.state.event(
4624 "stall",
4625 format!(
4626 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4627 the run stops here, resumable"
4628 ),
4629 );
4630 }
4631 self.state.tally = Some(Tally {
4632 first_choice,
4633 borda,
4634 winner,
4635 rankings: tops.len(),
4636 unanimous_initial,
4637 deliberated,
4638 changed_votes,
4639 unanimous_final,
4640 tie_break,
4641 judges: judges_total,
4642 present,
4643 quorum,
4644 met_quorum,
4645 uncontested,
4646 });
4647 self.state.status = if met_quorum {
4648 RunStatus::Reviewing
4649 } else {
4650 RunStatus::Stalled
4651 };
4652 self.state.save()?;
4653 Ok(())
4654 }
4655
4656 #[allow(clippy::too_many_lines)]
4677 async fn recover_stall(&mut self) -> Result<bool> {
4678 let run_id = self.state.id.clone();
4683 let prompts = self.state.config.prompts.clone();
4684 let quota_seats: BTreeSet<&str> =
4689 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4690 let absent: Vec<String> = self
4691 .state
4692 .judgements
4693 .iter()
4694 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4695 .map(|j| j.seat.clone())
4696 .collect();
4697 if absent.is_empty() {
4698 return Ok(false);
4699 }
4700 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4701 if viable.len() <= 1 {
4702 return Ok(false);
4703 }
4704 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4705 let language = self.state.config.graph.language.clone();
4706 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4707 let sessions = self.state.config.graph.sessions;
4708 let artifacts = agent::artifacts_dir(&self.state.dir());
4709 let root = self.state.worktree_root();
4710 let base_short = short(&self.state.base_commit);
4711 let candidates: Vec<Candidate> = viable.clone();
4712
4713 let mut positions: Vec<usize> = absent
4715 .iter()
4716 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4717 .collect();
4718 if positions.is_empty() {
4719 return Ok(false);
4720 }
4721 positions.sort_unstable();
4722 positions.dedup();
4723
4724 let mut judge_jobs = Vec::new();
4726 for &j in &positions {
4727 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4728 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4729 let seat_key = format!("judge-{}", j + 1);
4730 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4731 let seat = self.seat(&seat_key, &spec.id);
4732 judge_jobs.push(SeatJob {
4733 spec,
4734 seat,
4735 prompt: prompt::judge(
4736 &self.state.instruction,
4737 &views,
4738 self.roles.judges.len(),
4739 &base_short,
4740 &language,
4741 ),
4742 cwd: root.join(seat_key),
4743 timeout,
4744 allow_write: false,
4745 sessions,
4746 artifacts: artifacts.clone(),
4747 stem: format!("judge-{}-recover", j + 1),
4748 handover: None,
4749 });
4750 }
4751
4752 let labels_for_check = labels.clone();
4753 let mut judge_losses = Vec::new();
4754 let retries = self.state.config.graph.retries;
4755 let cache = self.state.config.cache_dir();
4756 let ctx = WaveCtx {
4757 carry_seats: false,
4758 pauses: Some((&self.pause, &self.interrupt)),
4759 run: &run_id,
4760 node: "judge",
4761 prompts: &prompts,
4762 cache: cache.as_deref(),
4763 round: None,
4764 };
4765 let results = ask_json_wave::<Ranking>(
4766 judge_jobs,
4767 Arc::clone(&self.sem),
4768 retries,
4769 &self.roles.judge_roster,
4770 &ctx,
4771 &mut judge_losses,
4772 &mut self.state,
4773 &move |r: &Ranking| r.validate(&labels_for_check),
4774 )
4775 .await;
4776 if self.state.resource_block.is_some() {
4777 self.stop_for_resource()?;
4778 return Ok(false);
4779 }
4780
4781 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4783 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4784 let agent_id = seat.agent.clone();
4785 self.state.seats.insert(seat.key.clone(), seat);
4786 let record = &mut self.state.judgements[j];
4787 match res {
4788 Ok((ranking, out)) => {
4789 record.agent = agent_id;
4790 record.ranking = ranking.normalized();
4791 record.reasons = ranking.reasons;
4792 record.strengths = ranking.strengths;
4793 record.confidence = ranking.confidence;
4794 record.failed = None;
4795 record.duration_ms = out.duration_ms;
4796 recovered.insert(j);
4797 self.state.event(
4798 "recover",
4799 format!("judge {} ranked again after the limit", j + 1),
4800 );
4801 }
4802 Err(e) => {
4803 self.state
4804 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4805 }
4806 }
4807 }
4808
4809 let mut vote_jobs = Vec::new();
4811 let mut vote_pos: Vec<usize> = Vec::new();
4812 for &j in &recovered {
4813 let seat_key = format!("judge-{}", j + 1);
4814 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4815 let seat = self.seat(&seat_key, &spec.id);
4816 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4817 let text = if has_context(&spec, &seat, sessions) {
4818 prompt::final_vote(&labels, &language)
4819 } else {
4820 full.clone()
4821 };
4822 vote_jobs.push(SeatJob {
4823 spec,
4824 seat,
4825 prompt: text,
4826 cwd: root.join(seat_key),
4827 timeout,
4828 allow_write: false,
4829 sessions,
4830 artifacts: artifacts.clone(),
4831 stem: format!("vote-judge-{}-recover", j + 1),
4832 handover: Some(full),
4833 });
4834 vote_pos.push(j);
4835 }
4836 let allowed = labels.clone();
4837 let mut vote_losses = Vec::new();
4838 let vote_retries = self.state.config.graph.retries;
4839 let vote_cache = self.state.config.cache_dir();
4840 let ctx = WaveCtx {
4841 carry_seats: false,
4842 pauses: Some((&self.pause, &self.interrupt)),
4843 run: &run_id,
4844 node: "vote",
4845 prompts: &prompts,
4846 cache: vote_cache.as_deref(),
4847 round: None,
4848 };
4849 let votes = ask_json_wave::<FinalVote>(
4850 vote_jobs,
4851 Arc::clone(&self.sem),
4852 vote_retries,
4853 &self.roles.judge_roster,
4854 &ctx,
4855 &mut vote_losses,
4856 &mut self.state,
4857 &move |v: &FinalVote| match v.label() {
4858 Some(c) if allowed.contains(&c) => Ok(()),
4859 other => bail!("vote {other:?} is not one of {allowed:?}"),
4860 },
4861 )
4862 .await;
4863 if self.state.resource_block.is_some() {
4864 self.stop_for_resource()?;
4865 return Ok(false);
4866 }
4867 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4868 let agent_id = seat.agent.clone();
4869 self.state.seats.insert(seat.key.clone(), seat);
4870 match res {
4871 Ok((v, _)) => {
4872 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4873 rec.vote = v.label();
4874 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4875 } else {
4876 self.state.votes.push(VoteRecord {
4877 judge: j + 1,
4878 agent: agent_id,
4879 vote: v.label(),
4880 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4881 changed: false,
4882 });
4883 }
4884 self.state.event(
4885 "recover",
4886 format!("judge {} voted again after the limit", j + 1),
4887 );
4888 }
4889 Err(e) => {
4890 self.state
4891 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4892 }
4893 }
4894 }
4895
4896 let recovered_keys: BTreeSet<String> = recovered
4900 .iter()
4901 .map(|&j| format!("judge-{}", j + 1))
4902 .collect();
4903 self.state
4904 .quota
4905 .retain(|q| !recovered_keys.contains(&q.seat));
4906 for loss in judge_losses.into_iter().chain(vote_losses) {
4910 if recovered_keys.contains(&loss.seat) {
4911 continue;
4912 }
4913 self.state.quota.retain(|q| q.seat != loss.seat);
4914 self.state.quota.push(loss);
4915 }
4916
4917 self.state.tally = None;
4919 self.tally()?;
4920 Ok(self
4921 .state
4922 .tally
4923 .as_ref()
4924 .map(|t| t.met_quorum)
4925 .unwrap_or(false))
4926 }
4927
4928 async fn graft(&mut self) -> Result<()> {
4944 let Err(e) = self.graft_inner().await else {
4945 return Ok(());
4946 };
4947 if e.is::<crate::repo_lock::ResourceBlocked>() {
4948 return Err(e);
4949 }
4950 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4951 self.state.status = RunStatus::Blocked;
4952 self.state.event("graft", format!("blocked: {e:#}"));
4953 self.state.save()?;
4954 return Err(e);
4955 }
4956 if let Some(rec) = self.state.graft.clone()
4961 && !rec.outcome.settled()
4962 {
4963 if let Some(winner) = self.state.winner().cloned() {
4964 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4965 if re.is::<crate::repo_lock::ResourceBlocked>() {
4966 return Err(re);
4967 }
4968 self.state.status = RunStatus::Blocked;
4969 self.state.event("graft", format!("blocked: {re:#}"));
4970 self.state.save()?;
4971 return Err(re);
4972 }
4973 }
4974 if let Some(g) = self.state.graft.as_mut() {
4975 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4976 }
4977 }
4978 self.state.event("graft", format!("skipped: {e:#}"));
4979 if let Err(e) = self.state.save() {
4980 tracing::warn!("save after graft: {e:#}");
4981 }
4982 Ok(())
4983 }
4984
4985 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4988 let _coordination = self.coordinate(wt).await?;
4989 crate::recovery::restore_repair(wt, pre)
4990 .await
4991 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4992 }
4993
4994 fn graft_skipped(&mut self, why: &str) {
4996 let message = format!("skipped: {why}");
4997 if !self
4998 .state
4999 .events
5000 .iter()
5001 .any(|e| e.node == "graft" && e.message == message)
5002 {
5003 self.state.event("graft", message);
5004 }
5005 }
5006
5007 async fn graft_inner(&mut self) -> Result<()> {
5008 if let Some(rec) = self.state.graft.clone()
5011 && !rec.outcome.settled()
5012 {
5013 if self.state.reviews.is_empty()
5017 && let Some(winner) = self.state.winner().cloned()
5018 {
5019 self.graft_restore(&winner.worktree, &rec.pre).await?;
5020 }
5021 if rec.outcome == GraftOutcome::WaitingForRestore {
5022 self.state.graft = None;
5023 self.state.save()?;
5024 } else {
5025 if let Some(g) = self.state.graft.as_mut() {
5026 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5027 }
5028 self.state.event(
5029 "graft",
5030 "rolled back: the previous round was interrupted; the winner is untouched",
5031 );
5032 self.state.save()?;
5033 return Ok(());
5034 }
5035 }
5036 let plan = match graft::plan(&self.state) {
5037 Ok(plan) => plan,
5038 Err(skip) => {
5039 if skip != graft::Skip::AlreadyDone {
5040 self.graft_skipped(skip.reason());
5041 self.state.save()?;
5042 }
5043 return Ok(());
5044 }
5045 };
5046 let Some(winner) = self.state.winner().cloned() else {
5047 return Ok(());
5048 };
5049 let wt = winner.worktree.clone();
5050 if !git::is_clean(&wt).await? {
5051 self.graft_skipped("the winner's worktree has uncommitted changes");
5052 self.state.save()?;
5053 return Ok(());
5054 }
5055 let pre = git::rev_parse(&wt, "HEAD").await?;
5056 self.state.graft = Some(GraftRecord {
5059 pre: pre.clone(),
5060 post: None,
5061 lines: None,
5062 limit: None,
5063 agent: None,
5064 outcome: GraftOutcome::Running,
5065 requested: graft::requested(&plan),
5066 adoption: None,
5067 adoption_error: None,
5068 });
5069 self.state.event(
5070 "graft",
5071 format!(
5072 "porting strengths of candidate(s) {} onto {}",
5073 plan.strengths
5074 .iter()
5075 .map(|(l, _, _)| l.to_string())
5076 .collect::<Vec<_>>()
5077 .join(", "),
5078 plan.winner
5079 ),
5080 );
5081 self.state.save()?;
5082
5083 let fix_prompt = prompt::graft(
5084 &self.state.instruction,
5085 plan.winner,
5086 &plan.strengths,
5087 &self.state.config.graph.language,
5088 );
5089 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5090 let sessions = self.state.config.graph.sessions;
5091 let artifacts = agent::artifacts_dir(&self.state.dir());
5092 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5093 let (_, seat, out) = self
5094 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5095 SeatJob {
5096 prompt: fix_prompt.clone(),
5097 spec,
5098 seat,
5099 cwd: wt.clone(),
5100 timeout,
5101 allow_write: true,
5102 sessions,
5103 artifacts: artifacts.clone(),
5104 stem: "graft".to_owned(),
5105 handover: None,
5106 }
5107 })
5108 .await;
5109 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5110 self.state.graft.as_mut().expect("graft record").outcome =
5113 GraftOutcome::WaitingForRestore;
5114 return self.stop_for_resource();
5115 }
5116 let mut why: Option<String> = None;
5117 match &out {
5118 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5119 why = Some("the fixer produced nothing usable".to_owned());
5120 }
5121 AgentOutcome::Ok(_) => {}
5122 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5123 AgentOutcome::Quota(o) => {
5124 self.state.quota.push(QuotaLoss::from_report(
5125 seat.key.clone(),
5126 "graft",
5127 o.quota.as_ref(),
5128 ));
5129 why = Some("rate limited (quota)".to_owned());
5130 }
5131 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5132 why = Some(format!(
5133 "the fixer failed: {}",
5134 e.lines()
5135 .next()
5136 .unwrap_or("")
5137 .chars()
5138 .take(160)
5139 .collect::<String>()
5140 ));
5141 }
5142 }
5143 let agent_id = seat.agent.clone();
5144 self.state.seats.insert(seat.key.clone(), seat);
5145
5146 let mut post = None;
5147 let mut lines = None;
5148 let _coordination = self.coordinate(&wt).await?;
5149 if why.is_none() {
5150 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5151 if let Ok(r) = &rescued {
5152 self.state.note_withheld("graft", &r.withheld);
5153 }
5154 let head = git::rev_parse(&wt, "HEAD").await?;
5155 let dirty = match &rescued {
5159 Ok(r) => git::status_porcelain(&wt)
5160 .await
5161 .map(|st| {
5162 st.lines().any(|l| {
5163 !r.withheld
5164 .iter()
5165 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5166 })
5167 })
5168 .unwrap_or(true),
5169 Err(_) => true,
5170 };
5171 if dirty {
5172 why = Some("the fixer left changes that could not be committed".to_owned());
5173 } else if head == pre {
5174 why = Some("the fixer changed nothing".to_owned());
5175 } else {
5176 lines = git::changed_lines(&wt, &pre, &head).await;
5177 post = Some(head);
5178 }
5179 }
5180 let (adoption, adoption_error) = match &out {
5181 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5182 _ => (None, None),
5183 };
5184 let requested = graft::requested(&plan);
5185 let outcome = match why {
5186 None => {
5187 self.state.event(
5188 "graft",
5189 format!(
5190 "applied: {}",
5191 lines.map_or_else(
5192 || "changed lines unavailable".to_owned(),
5193 |n| format!("{n} changed lines")
5194 )
5195 ),
5196 );
5197 GraftOutcome::Applied
5198 }
5199 Some(why) => {
5200 crate::recovery::restore_repair(&wt, &pre)
5203 .await
5204 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5205 self.state.event(
5206 "graft",
5207 format!("rolled back: {why}; the winner is untouched"),
5208 );
5209 GraftOutcome::RolledBack(why)
5210 }
5211 };
5212 self.state.graft = Some(GraftRecord {
5213 pre,
5214 post,
5215 lines,
5216 limit: None,
5217 agent: Some(agent_id),
5218 outcome,
5219 requested,
5220 adoption,
5221 adoption_error,
5222 });
5223 self.state.save()?;
5224 Ok(())
5225 }
5226
5227 async fn fold_losers(&mut self) -> Result<()> {
5230 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5231 return Ok(());
5232 };
5233 if !self.state.candidate_preservation.is_empty() {
5234 bail!("candidate preservation incomplete; refusing cleanup");
5235 }
5236 let repo = self.state.repo.clone();
5237 let _coordination = self.coordinate(&repo).await?;
5238 let mut folded = Vec::new();
5239 for i in 0..self.state.candidates.len() {
5240 let c = &self.state.candidates[i];
5241 if c.label == winner || c.folded {
5242 continue;
5243 }
5244 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5245 remove_preserved_worktree(&repo, &wt).await?;
5246 if git::branch_exists(&repo, &branch).await?
5247 && !git::branch_delete(&repo, &branch).await?
5248 {
5249 bail!("candidate branch removal refused");
5250 }
5251 self.state.candidates[i].folded = true;
5252 self.state.save()?;
5253 folded.push(label.to_string());
5254 }
5255 let root = self.state.worktree_root();
5257 for j in 1..=self.roles.judges.len() {
5258 let wt = root.join(format!("judge-{j}"));
5259 if wt.exists() {
5260 remove_preserved_worktree(&repo, &wt).await?;
5261 }
5262 }
5263 if self.state.config.graph.advise {
5266 for k in 1..=self.state.config.graph.advisors {
5267 let wt = root.join(format!("advisor-{k}"));
5268 if wt.exists() {
5269 remove_preserved_worktree(&repo, &wt).await?;
5270 }
5271 }
5272 }
5273 if !folded.is_empty() {
5274 self.state
5275 .event("fold", format!("folded candidates {}", folded.join(", ")));
5276 self.state.save()?;
5277 }
5278 Ok(())
5279 }
5280
5281 async fn hold_base_sync(&mut self) -> Result<bool> {
5284 self.hold_base_sync_as(None).await
5285 }
5286
5287 async fn hold_base_sync_kind(
5288 &mut self,
5289 kind: crate::base_sync_recovery::StopKind,
5290 ) -> Result<()> {
5291 self.hold_base_sync_as(Some(kind)).await?;
5292 Ok(())
5293 }
5294
5295 async fn hold_base_sync_as(
5296 &mut self,
5297 kind: Option<crate::base_sync_recovery::StopKind>,
5298 ) -> Result<bool> {
5299 if !self.state.base_sync_blocked() {
5300 self.state.base_sync_recovery.stop = None;
5301 return Ok(false);
5302 }
5303 let head = match self.state.winner() {
5304 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5305 .await
5306 .unwrap_or_default(),
5307 None => String::new(),
5308 };
5309 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5310 && !authorization.consumed
5311 && authorization.stop.head != head
5312 {
5313 authorization.refuse(format!(
5314 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5315 authorization.stop.head
5316 ));
5317 }
5318 self.state.record_base_sync_stop(&head);
5319 if let Some(kind) = kind {
5320 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5321 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5322 stop.kind = kind;
5323 }
5324 }
5325 self.state.save()?;
5326 Ok(true)
5327 }
5328
5329 async fn sync_to_base(&mut self) -> Result<()> {
5371 if self.state.status == RunStatus::AlreadyInBase {
5372 return Ok(());
5373 }
5374 self.hold_base_sync().await?;
5375 let conflicted = self
5376 .state
5377 .base_sync
5378 .as_ref()
5379 .is_some_and(|s| s.conflict.is_some());
5380 let Some(winner) = self.state.winner().cloned() else {
5381 return Ok(());
5382 };
5383
5384 let repo = self.state.repo.clone();
5385 let remote = self.state.config.merge.remote.clone();
5386 let base_branch = self.state.base_branch.clone();
5387 let tracking = format!("{remote}/{base_branch}");
5388
5389 let fetched_base =
5390 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5391 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5396 return Ok(());
5397 };
5398
5399 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5400 self.state.ensure_base_episode();
5401 let mut pending = self
5402 .state
5403 .base_sync_recovery
5404 .operations
5405 .last()
5406 .filter(|o| !o.finished)
5407 .cloned();
5408 let scratch = self.state.dir().join("base-sync");
5409 if git::rebase_in_progress(&scratch).await {
5410 if pending.is_none()
5413 && self.state.base_sync_recovery.operations.is_empty()
5414 && !conflicted
5415 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5416 .await
5417 .ok()
5418 .as_deref()
5419 == Some(head.as_str())
5420 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5421 && git::is_clean(&winner.worktree).await?
5422 {
5423 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5424 if self.state.base_sync.is_none() {
5425 self.state.base_sync = Some(BaseSync {
5426 tip: saved_tip.clone(),
5427 behind: 0,
5428 attempts: 0,
5429 conflict: None,
5430 already_in: None,
5431 });
5432 }
5433 self.state.reserve_base_operation(&head, &saved_tip);
5434 self.state.event(
5435 "base-recovery",
5436 "imported standing legacy rebase as a spent synchronization reservation",
5437 );
5438 self.state.save()?;
5439 pending = self.state.base_sync_recovery.operations.last().cloned();
5440 }
5441 let expected = pending.as_ref().map(|op| op.tip.as_str());
5442 if expected.is_none()
5443 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5444 {
5445 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5446 if self.state.base_sync.is_none() {
5447 self.state.base_sync = Some(BaseSync {
5448 tip: tip.clone(),
5449 behind: 0,
5450 attempts: 0,
5451 conflict: None,
5452 already_in: None,
5453 });
5454 }
5455 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5456 self.state.event("land", why);
5457 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5458 .await?;
5459 return Ok(());
5460 }
5461 }
5462 if let Some(op) = &pending {
5463 let mut replay_proven =
5465 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5466 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5467 let wait = repository_wait(&self.state, None);
5468 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5469 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5470 Ok(true) => {
5471 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5472 self.state.save()?;
5473 git::worktree_remove(&repo, &scratch).await.ok();
5474 replay_proven = true;
5475 }
5476 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5477 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5478 }
5479 }
5480 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5481 if self.state.foreign_checkout.is_some() {
5482 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5483 .await
5484 .ok()
5485 .as_deref()
5486 == Some(op.head.as_str())
5487 && git::rev_parse(&repo, &format!("{head}^2"))
5488 .await
5489 .ok()
5490 .as_deref()
5491 == Some(op.tip.as_str());
5492 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5493 .await
5494 .is_ok_and(|m| m.is_empty())
5495 && let Ok(added) =
5496 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5497 {
5498 replay_proven = true;
5499 for commit in added {
5500 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5501 replay_proven = false;
5502 break;
5503 }
5504 }
5505 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5506 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5507 }
5508 }
5509 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5513 || (head != op.head
5514 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5515 .await?
5516 .is_empty())
5517 || (!git::is_clean(&winner.worktree).await?
5518 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5519 .await
5520 .is_ok_and(|o| o.ok()))
5521 || !replay_proven
5522 {
5523 let why = format!(
5524 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5525 op.id
5526 );
5527 let sync = self
5528 .state
5529 .base_sync
5530 .as_mut()
5531 .context("reserved synchronization has no lifetime accounting")?;
5532 sync.conflict = Some(why.clone());
5533 self.state.event("land", why);
5534 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535 .await?;
5536 return Ok(());
5537 }
5538 if head != op.head {
5539 git::sync_to_head(&winner.worktree).await?;
5540 }
5541 tip = op.tip.clone();
5542 }
5543 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5544 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5545 if pending.is_none()
5546 && !conflicted
5547 && fetched_base
5548 && self
5549 .state
5550 .base_sync_recovery
5551 .episodes
5552 .last()
5553 .is_some_and(|e| e.boundary.is_some())
5554 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5555 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5556 && owner.runs.last() == Some(&self.state.id)
5557 && owner.status != crate::queue::TaskStatus::Done
5558 && !owner.fresh_start
5559 && owner.review_branch.is_none()
5560 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5561 && let Some(approval) = owner.resumption.as_ref()
5562 && approval.instruction == owner.instruction
5563 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5564 && git::is_clean(&winner.worktree).await?
5565 && self.state.renew_base_episode(&task, approval, &head, &tip)
5566 {
5567 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5568 self.state.save()?;
5569 }
5570
5571 if (behind > 0 || conflicted || head == tip)
5580 && self
5581 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5582 .await?
5583 {
5584 return Ok(());
5585 }
5586 if conflicted && self.state.base_recovery_pending() {
5587 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5588 let owner =
5589 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5590 let owner_valid = owner.as_ref().is_ok_and(|t| {
5591 t.runs.last() == Some(&self.state.id)
5592 && t.status != crate::queue::TaskStatus::Done
5593 && !t.fresh_start
5594 && t.review_branch.is_none()
5595 });
5596 let invalidation = if !owner_valid {
5597 Some("latest task ownership changed")
5598 } else if !owner.as_ref().is_ok_and(|t| {
5599 authorization
5600 .generation
5601 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5602 }) {
5603 Some("owner approval generation changed")
5604 } else if let Some(scope) = &authorization.scope {
5605 if scope.tip != tip {
5606 Some("fetched base changed")
5607 } else if !self
5608 .state
5609 .base_sync_recovery
5610 .episodes
5611 .last()
5612 .is_some_and(|e| e.id == scope.episode)
5613 {
5614 Some("active synchronization interval changed")
5615 } else if !owner
5616 .as_ref()
5617 .is_ok_and(|t| t.instruction == scope.instruction)
5618 {
5619 Some("task instruction changed")
5620 } else {
5621 None
5622 }
5623 } else {
5624 None
5625 };
5626 if let Some(reason) = invalidation {
5627 let authorization = self
5628 .state
5629 .base_sync_recovery
5630 .authorizations
5631 .last_mut()
5632 .unwrap();
5633 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5634 self.state.save()?;
5635 return Ok(());
5636 }
5637 if !fetched_base
5638 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5639 || !git::is_clean(&winner.worktree).await?
5640 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5641 {
5642 return Ok(());
5643 }
5644 }
5645 let recovery = conflicted && self.state.consume_base_recovery(&head);
5646 if conflicted {
5647 self.state.save()?;
5649 if !recovery {
5650 return Ok(());
5651 }
5652 }
5653
5654 if behind == 0 {
5655 if let Some(from) = self
5664 .state
5665 .rebase_fixes
5666 .iter()
5667 .rev()
5668 .find_map(|r| r.from.clone())
5669 && from != head
5670 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5671 .await
5672 .is_ok_and(|o| o.ok())
5673 {
5674 git::sync_to_head(&winner.worktree).await?;
5675 }
5676 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5682 if let Some(why) = &conflict {
5683 self.state.status = RunStatus::Blocked;
5684 self.state.event("land", why.clone());
5685 }
5686 self.state.base_sync = Some(BaseSync {
5687 tip,
5688 behind: 0,
5689 attempts,
5690 conflict,
5691 already_in: None,
5692 });
5693 if pending.is_some() {
5694 self.state.finish_base_operation(Some(&head));
5695 }
5696 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5697 .await?;
5698 self.state.save()?;
5699 return Ok(());
5700 }
5701
5702 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5703 let why = format!(
5704 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5705 synchronization(s); synchronizing again would only race it",
5706 winner.branch
5707 );
5708 self.state.status = RunStatus::Blocked;
5709 self.state.base_sync = Some(BaseSync {
5710 tip,
5711 behind,
5712 attempts,
5713 conflict: Some(why.clone()),
5714 already_in: None,
5715 });
5716 self.state.event("land", why);
5717 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5718 .await?;
5719 self.state.save()?;
5720 return Ok(());
5721 }
5722
5723 if self.state.foreign_checkout.is_some() {
5724 let wait = repository_wait(&self.state, None);
5727 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5728 if self.state.base_sync.is_none() {
5729 self.state.base_sync = Some(BaseSync {
5730 tip: tip.clone(),
5731 behind,
5732 attempts: 0,
5733 conflict: None,
5734 already_in: None,
5735 });
5736 }
5737 self.state.reserve_base_operation(&head, &tip);
5738 self.state.save()?;
5739 let merged = git::git_raw(
5740 &winner.worktree,
5741 &[
5742 "-c",
5743 "core.hooksPath=/dev/null",
5744 "merge",
5745 "--no-ff",
5746 "--no-autostash",
5747 "--no-edit",
5748 "-m",
5749 &format!("Merge {tracking} before verifying"),
5750 &tip,
5751 ],
5752 )
5753 .await?;
5754 let conflict = if merged.ok() {
5755 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5756 self.state.event(
5757 "land",
5758 format!("merged {tracking} into {} before verifying", winner.branch),
5759 );
5760 None
5761 } else {
5762 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5763 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5764 }
5765 let why = format!(
5766 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5767 winner.branch,
5768 merged.stderr.chars().take(600).collect::<String>()
5769 );
5770 self.state.status = RunStatus::Blocked;
5771 self.state.event("land", &why);
5772 Some(why)
5773 };
5774 let synced = conflict.is_none();
5775 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5776 self.state
5777 .finish_base_operation(synced.then_some(fresh.as_str()));
5778 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5779 self.state.base_sync = Some(BaseSync {
5780 tip,
5781 behind: if synced { 0 } else { behind },
5782 attempts,
5783 conflict,
5784 already_in: None,
5785 });
5786 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5787 .await?;
5788 self.state.save()?;
5789 return Ok(());
5790 }
5791
5792 self.state.event(
5793 "land",
5794 format!(
5795 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5796 winner.branch
5797 ),
5798 );
5799 self.state.save()?;
5800
5801 let branch_tracking = format!("{remote}/{}", winner.branch);
5807 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5808 let remote_tip = if self.state.task_pr_branch().is_none()
5809 && matches!(&fetched_branch, Ok(o) if o.ok())
5810 {
5811 git::rev_parse(&repo, &branch_tracking).await.ok()
5812 } else {
5813 None
5814 };
5815 if let Some(theirs) = &remote_tip
5819 && !git::is_ancestor(&repo, theirs, &head).await
5820 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5821 .await
5822 .is_ok_and(|missing| missing.is_empty())
5823 {
5824 let why = format!(
5825 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5826 them",
5827 short(theirs),
5828 winner.branch
5829 );
5830 self.state.status = RunStatus::Blocked;
5831 self.state.base_sync = Some(BaseSync {
5832 tip,
5833 behind,
5834 attempts,
5835 conflict: Some(why.clone()),
5836 already_in: None,
5837 });
5838 self.state.event("land", why);
5839 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5840 .await?;
5841 self.state.save()?;
5842 return Ok(());
5843 }
5844
5845 if self.state.base_sync.is_none() {
5846 self.state.base_sync = Some(BaseSync {
5847 tip: tip.clone(),
5848 behind,
5849 attempts: 0,
5850 conflict: None,
5851 already_in: None,
5852 });
5853 }
5854 self.state.reserve_base_operation(&head, &tip);
5855 self.state.save()?;
5856 let scratch = self.state.dir().join("base-sync");
5857 let rebased =
5858 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5859 .await
5860 {
5861 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5862 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5863 Err(e) => Err(e),
5864 };
5865 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5866 let stop_kind = match &rebased {
5867 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5868 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5869 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5870 };
5871 match rebased {
5872 Ok(None) => {
5873 git::sync_to_head(&winner.worktree).await?;
5877 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5878 self.state.finish_base_operation(Some(&fresh));
5879 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5880 let mut conflict = None;
5881 if let Some(pinned) = &remote_tip {
5882 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5883 match pushed {
5884 Ok(o) if o.ok() => self.state.event(
5885 "land",
5886 format!("pushed rebased {} to {remote}", winner.branch),
5887 ),
5888 Ok(o) => {
5889 conflict = Some(format!(
5890 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5891 winner.branch,
5892 short(pinned),
5893 o.stderr.chars().take(600).collect::<String>()
5894 ));
5895 }
5896 Err(e) => {
5897 conflict = Some(format!(
5898 "rebased {} locally but could not push it: {e:#}",
5899 winner.branch
5900 ));
5901 }
5902 }
5903 }
5904 if let Some(why) = &conflict {
5905 self.state.status = RunStatus::Blocked;
5906 self.state.event("land", why.clone());
5907 }
5908 self.state.base_sync = Some(BaseSync {
5909 tip: tip.clone(),
5910 behind: 0,
5911 attempts,
5912 conflict,
5913 already_in: None,
5914 });
5915 self.state
5916 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5917 }
5918 Ok(Some(conflict)) => {
5919 self.state.finish_base_operation(None);
5920 let why = format!(
5921 "{} conflicts with {tracking} and did not rebase: {}",
5922 winner.branch,
5923 conflict.chars().take(600).collect::<String>()
5924 );
5925 self.state.status = RunStatus::Blocked;
5926 self.state.base_sync = Some(BaseSync {
5927 tip,
5928 behind,
5929 attempts,
5930 conflict: Some(why.clone()),
5931 already_in: None,
5932 });
5933 self.state.event("land", why);
5934 }
5935 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5939 self.state.save()?;
5940 return Ok(());
5941 }
5942 Err(e) => {
5943 self.state.finish_base_operation(None);
5944 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5945 self.state.status = RunStatus::Blocked;
5946 self.state.base_sync = Some(BaseSync {
5947 tip,
5948 behind,
5949 attempts,
5950 conflict: Some(why.clone()),
5951 already_in: None,
5952 });
5953 self.state.event("land", why);
5954 }
5955 }
5956 if !self.state.base_sync_blocked() {
5957 self.state.base_sync_recovery.stop = None;
5958 }
5959 self.hold_base_sync_kind(stop_kind).await?;
5960 self.state.save()?;
5961 Ok(())
5962 }
5963
5964 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5970 if self.state.task_pr_branch().is_some() {
5973 return None;
5974 }
5975 let repo = self.state.repo.clone();
5976 let remote = self.state.config.merge.remote.clone();
5977 let fetched = git::fetch(&repo, &remote, branch).await;
5978 if !matches!(&fetched, Ok(o) if o.ok()) {
5979 return None;
5980 }
5981 let branch_tracking = format!("{remote}/{branch}");
5982 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5983 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5984 return None;
5985 }
5986 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5987 .await
5988 .is_ok_and(|missing| missing.is_empty())
5989 {
5990 return Some(format!(
5991 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5992 them",
5993 short(&theirs)
5994 ));
5995 }
5996 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5997 Ok(o) if o.ok() => {
5998 self.state
5999 .event("land", format!("pushed rebased {branch} to {remote}"));
6000 None
6001 }
6002 Ok(o) => Some(format!(
6003 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6004 someone may have pushed): {}",
6005 short(&theirs),
6006 o.stderr.chars().take(600).collect::<String>()
6007 )),
6008 Err(e) => Some(format!(
6009 "{branch} is rebased locally but could not be pushed: {e:#}"
6010 )),
6011 }
6012 }
6013
6014 async fn settle_already_in(
6031 &mut self,
6032 branch: &str,
6033 tip: &str,
6034 head: &str,
6035 attempts: usize,
6036 behind: usize,
6037 ) -> Result<bool> {
6038 let repo = self.state.repo.clone();
6039 let remote = self.state.config.merge.remote.clone();
6040 let start = self.state.base_commit.clone();
6041 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6042 Ok(Some(e)) => e,
6043 Ok(None) => return Ok(false),
6044 Err(e) => {
6045 tracing::warn!("already-in-base check for {branch}: {e:#}");
6046 return Ok(false);
6047 }
6048 };
6049 let mut verified = vec![head.to_owned()];
6050 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6051 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6052 if matches!(&fetched, Ok(o) if o.ok())
6053 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6054 && theirs != head
6055 {
6056 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6057 Ok(Some(_)) => verified.push(theirs),
6058 _ => return Ok(false),
6059 }
6060 }
6061 let base_branch = self.state.base_branch.clone();
6062 let message = format!(
6063 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6064 land",
6065 evidence.names(),
6066 evidence.proof.as_str()
6067 );
6068 let closed =
6069 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6070 .await;
6071 match closed {
6072 Ok(Ok(url)) => self
6073 .state
6074 .event("land", format!("closed {url}: superseded on {base_branch}")),
6075 Ok(Err(why)) => self
6076 .state
6077 .event("land", format!("did not close a pull request: {why}")),
6078 Err(e) => {
6079 let why = format!(
6080 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6081 not be closed ({e:#}); resume to retry"
6082 );
6083 self.state.status = RunStatus::Blocked;
6084 self.state.base_sync = Some(BaseSync {
6085 tip: tip.to_owned(),
6086 behind,
6087 attempts,
6088 conflict: Some(why.clone()),
6089 already_in: None,
6090 });
6091 self.state.event("land", why);
6092 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6093 .await?;
6094 self.state.save()?;
6095 return Ok(true);
6096 }
6097 }
6098 self.state.status = RunStatus::AlreadyInBase;
6099 self.state.base_sync_recovery.stop = None;
6100 self.state.base_sync = Some(BaseSync {
6101 tip: tip.to_owned(),
6102 behind,
6103 attempts,
6104 conflict: None,
6105 already_in: Some(evidence),
6106 });
6107 self.state.event("land", message);
6108 self.state.save()?;
6109 self.settle_questions();
6110 Ok(true)
6111 }
6112
6113 fn landing_base(&self) -> String {
6123 self.state
6124 .base_sync
6125 .as_ref()
6126 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6127 }
6128
6129 pub async fn fix_selected(
6162 &mut self,
6163 ids: &[String],
6164 reason: &str,
6165 allow_stale: bool,
6166 ) -> Result<()> {
6167 let reason = reason.trim();
6168 if reason.is_empty() {
6169 bail!("a fix request needs a reason — that is the operator's own record of why");
6170 }
6171 if ids.is_empty() {
6172 bail!("no finding id given");
6173 }
6174 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6175 bail!(
6176 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6177 has already concluded — can be given a targeted fix. A run still \
6178 in progress should simply be resumed; a `merged` run's branch has \
6179 already landed, so its answer is a fresh `magi review <branch>`, \
6180 not reopening this run's own record",
6181 self.state.id,
6182 self.state.status.as_str()
6183 );
6184 }
6185 let Some(winner) = self.state.winner().cloned() else {
6186 bail!("run {} has no winning candidate to fix", self.state.id);
6187 };
6188 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6189 bail!(
6190 "branch `{}` no longer exists; this run cannot be extended",
6191 winner.branch
6192 );
6193 }
6194 let home = crate::run::home();
6195 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6196 bail!(
6197 "run {} is currently being worked on by another magi process",
6198 self.state.id
6199 );
6200 }
6201 let _claim = FixClaim::acquire(&self.state.dir())?;
6207
6208 let mut seen = BTreeSet::new();
6212 let mut findings = Vec::new();
6213 let mut missing = Vec::new();
6214 for id in ids {
6215 if !seen.insert(id.clone()) {
6216 continue;
6217 }
6218 match self.state.finding(id) {
6219 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6220 id: f.id.clone(),
6221 severity: f.severity,
6222 reviewer_vote: rec.vote,
6223 round: round.round,
6224 round_head: round.head.clone(),
6225 reviewer: rec.reviewer,
6226 agent: rec.agent.clone(),
6227 file: f.file.clone(),
6228 line: f.line,
6229 title: f.title.clone(),
6230 detail: f.detail.clone(),
6231 outcome: OperatorFixOutcome::Pending,
6232 }),
6233 None => missing.push(id.clone()),
6234 }
6235 }
6236 if !missing.is_empty() {
6237 bail!(
6238 "unknown finding id(s): {}; nothing was changed",
6239 missing.join(", ")
6240 );
6241 }
6242
6243 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6244 let stale_details: Vec<(String, String)> = findings
6245 .iter()
6246 .filter(|f| f.round_head != head_at_request)
6247 .map(|f| (f.id.clone(), f.round_head.clone()))
6248 .collect();
6249 let stale = !stale_details.is_empty();
6250 if stale && !allow_stale {
6251 bail!(
6252 "the branch has moved since some finding(s) were raised — {} — now \
6253 at {}; pass --allow-stale to fix anyway, or re-run review first",
6254 stale_details
6255 .iter()
6256 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6257 .collect::<Vec<_>>()
6258 .join(", "),
6259 short(&head_at_request)
6260 );
6261 }
6262
6263 let request = OperatorFixRequest {
6264 requested_at: Timestamp::now(),
6265 reason: reason.to_owned(),
6266 findings,
6267 head_at_request: head_at_request.clone(),
6268 allow_stale,
6269 stale,
6270 fix: None,
6271 result_head: None,
6272 follow_up_review_run: None,
6273 };
6274 self.state.event(
6275 "fix",
6276 format!(
6277 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6278 request.findings.len(),
6279 request
6280 .findings
6281 .iter()
6282 .map(|f| f.id.as_str())
6283 .collect::<Vec<_>>()
6284 .join(", "),
6285 ),
6286 );
6287 self.state.operator_fixes.push(request);
6294 self.state.save()?;
6295 let request_index = self.state.operator_fixes.len() - 1;
6296
6297 let repo = self.state.repo.clone();
6306 let coordination = self.coordinate(&repo).await?;
6307 if winner.worktree.exists() {
6308 let dirty = git::git(
6311 &winner.worktree,
6312 &["status", "--porcelain", "--untracked-files=all"],
6313 )
6314 .await?;
6315 let only_withheld = dirty.lines().all(|l| {
6316 l.strip_prefix("?? ")
6317 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6318 });
6319 if !only_withheld {
6320 bail!(
6321 "`{}` has uncommitted changes; refusing to touch it — commit or \
6322 discard them first",
6323 winner.worktree.display()
6324 );
6325 }
6326 git::worktree_remove(&self.state.repo, &winner.worktree)
6327 .await
6328 .ok();
6329 }
6330 let fix_worktree = self.state.worktree_root().join("operator-fix");
6331 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6332 git::git(
6333 &self.state.repo,
6334 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6335 )
6336 .await
6337 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6338 if !git::is_clean(&fix_worktree).await? {
6339 git::worktree_remove(&self.state.repo, &fix_worktree)
6340 .await
6341 .ok();
6342 bail!(
6343 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6344 winner.branch
6345 );
6346 }
6347 let repo_for_setup = self.state.repo.clone();
6348 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6349 Ok(()) => {
6350 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6351 }
6352 Err(e) => Err(e),
6353 };
6354 if let Err(e) = setup {
6355 git::worktree_remove(&self.state.repo, &fix_worktree)
6356 .await
6357 .ok();
6358 return Err(e);
6359 }
6360
6361 self.state.resource_block = None;
6364
6365 let run_id = self.state.id.clone();
6366 let prompts = self.state.config.prompts.clone();
6367 let language = self.state.config.graph.language.clone();
6368 let sessions = self.state.config.graph.sessions;
6369 let artifacts = agent::artifacts_dir(&self.state.dir());
6370 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6371 .findings
6372 .iter()
6373 .map(|f| Finding {
6374 id: f.id.clone(),
6375 severity: f.severity,
6376 file: f.file.clone(),
6377 line: f.line,
6378 title: f.title.clone(),
6379 detail: f.detail.clone(),
6380 })
6381 .collect();
6382 let fix_prompt = prompt::operator_fix_in(
6383 &self.state.instruction,
6384 &finding_list,
6385 reason,
6386 &stale_details,
6387 &head_at_request,
6388 &language,
6389 &self.state.config.graph.github_language,
6390 );
6391 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6392 drop(coordination);
6393 let (job, seat, out) = self
6394 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6395 prompt: fix_prompt.clone(),
6396 spec,
6397 seat,
6398 cwd: fix_worktree.clone(),
6399 timeout,
6400 allow_write: true,
6401 sessions,
6402 artifacts: artifacts.clone(),
6403 stem: "operator-fix".to_owned(),
6404 handover: None,
6405 })
6406 .await;
6407 let agent_id = seat.agent.clone();
6408
6409 let mut fix = FixRecord {
6410 agent: agent_id,
6411 addressed: Vec::new(),
6412 rejected: Vec::new(),
6413 notes: String::new(),
6414 committed: false,
6415 failed: None,
6416 duration_ms: 0,
6417 continuation: None,
6418 };
6419 let mut final_seat = seat.clone();
6420 match out {
6421 AgentOutcome::Ok(o) => {
6422 fix.duration_ms = o.duration_ms;
6423 let parsed = verdict::extract_json::<FixReport>(&o.text);
6424 let incomplete_reason = match &parsed {
6425 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6426 "the reply parsed, but it reported a command whose own CLI \
6427 never confirmed an exit status"
6428 .to_owned(),
6429 ),
6430 Ok(_) => None,
6431 Err(e) => Some(e.to_string()),
6432 };
6433 match incomplete_reason {
6434 None => {
6435 let report = parsed.expect("checked Ok above");
6436 fix.addressed = report.addressed;
6437 fix.rejected = report.rejected;
6438 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6439 }
6440 Some(reason) => {
6441 let (resumed_seat, resolved, failure, cont) = self
6442 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6443 .await;
6444 fix.duration_ms += cont.cumulative_wait_ms;
6445 fix.continuation = Some(cont);
6446 final_seat = resumed_seat;
6447 match resolved {
6448 Some(report) => {
6449 fix.addressed = report.addressed;
6450 fix.rejected = report.rejected;
6451 fix.notes =
6452 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6453 }
6454 None => fix.failed = failure,
6455 }
6456 }
6457 }
6458 }
6459 AgentOutcome::Dropped(o) => {
6460 fix.duration_ms = o.duration_ms;
6461 let why = o
6462 .dropped
6463 .as_ref()
6464 .map(|d| d.why.as_str())
6465 .unwrap_or("the CLI ended the stream without delivering its answer");
6466 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6467 }
6468 AgentOutcome::Quota(o) => {
6469 self.state.quota.push(QuotaLoss::from_report(
6470 final_seat.key.clone(),
6471 "fix",
6472 o.quota.as_ref(),
6473 ));
6474 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6475 }
6476 AgentOutcome::Failed(e) => fix.failed = Some(e),
6477 AgentOutcome::ResourceBlocked(e) => {
6478 fix.failed = Some(format!("the fixer was not started: {e}"));
6479 }
6480 }
6481 if fix.continuation.is_none() {
6484 fix.continuation = Some(ContinuationRecord::not_needed());
6485 }
6486 self.state.seats.insert(final_seat.key.clone(), final_seat);
6487
6488 let rescue_message = format!(
6489 "magi: operator-selected fix ({}) (uncommitted work)",
6490 self.state.operator_fixes[request_index]
6491 .findings
6492 .iter()
6493 .map(|f| f.id.as_str())
6494 .collect::<Vec<_>>()
6495 .join(", ")
6496 );
6497 let repo = self.state.repo.clone();
6498 let coordination = self.coordinate(&repo).await?;
6499 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6500 self.state.note_withheld("fix", &r.withheld);
6501 }
6502 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6503 fix.committed = after != head_at_request;
6504 git::worktree_remove(&self.state.repo, &fix_worktree)
6505 .await
6506 .ok();
6507 drop(coordination);
6508
6509 self.state.event(
6510 "fix",
6511 match &fix.failed {
6512 Some(reason) => format!(
6513 "operator fix: adoption report was lost ({reason}); {}",
6514 if fix.committed {
6515 "committed"
6516 } else {
6517 "NO new commit"
6518 }
6519 ),
6520 None => format!(
6521 "operator fix: {} addressed, {} rejected, {}",
6522 fix.addressed.len(),
6523 fix.rejected.len(),
6524 if fix.committed {
6525 "committed"
6526 } else {
6527 "NO new commit"
6528 }
6529 ),
6530 },
6531 );
6532
6533 for f in &mut self.state.operator_fixes[request_index].findings {
6540 f.outcome = if fix.failed.is_some() {
6541 OperatorFixOutcome::Unreported
6542 } else if fix.addressed.contains(&f.id) {
6543 OperatorFixOutcome::Addressed
6544 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6545 OperatorFixOutcome::Rejected { why: r.why.clone() }
6546 } else {
6547 OperatorFixOutcome::Unreported
6548 };
6549 }
6550
6551 let committed = fix.committed;
6552 if committed {
6553 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6554 }
6555 self.state.operator_fixes[request_index].fix = Some(fix);
6556 self.state.save()?;
6559 if self.state.resource_block.is_some() {
6560 return self.stop_for_resource();
6561 }
6562
6563 if committed {
6564 self.state.event(
6565 "fix",
6566 format!(
6567 "operator fix committed {}; opening a follow-up review-only run",
6568 short(&after)
6569 ),
6570 );
6571 let origin =
6574 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6575 match Self::review(
6576 &self.state.repo,
6577 &winner.branch,
6578 self.state.config.clone(),
6579 origin,
6580 )
6581 .await
6582 {
6583 Ok(mut follow_up) => {
6584 follow_up.state.event(
6585 "start",
6586 format!(
6587 "requested by an operator fix on run {} for finding(s) {}",
6588 self.state.id,
6589 self.state.operator_fixes[request_index]
6590 .findings
6591 .iter()
6592 .map(|f| f.id.as_str())
6593 .collect::<Vec<_>>()
6594 .join(", "),
6595 ),
6596 );
6597 follow_up.state.save()?;
6598 let follow_up_id = follow_up.state.id.clone();
6599 let adopted = match crate::direct::adopt(
6602 &follow_up.state,
6603 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6604 ) {
6605 Ok(a) => a,
6606 Err(e) => {
6607 self.state.event(
6610 "fix",
6611 format!(
6612 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6613 ),
6614 );
6615 self.state.save()?;
6616 return Ok(());
6617 }
6618 };
6619 let executed = follow_up.execute().await;
6620 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6621 if let Some(a) = adopted {
6622 a.finish(&follow_up.state, executed);
6623 }
6624 if let Some(e) = failure {
6625 self.state.event(
6626 "fix",
6627 format!(
6628 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6629 ),
6630 );
6631 }
6632 self.state.operator_fixes[request_index].follow_up_review_run =
6633 Some(follow_up_id);
6634 }
6635 Err(e) => {
6636 self.state.event(
6637 "fix",
6638 format!("committed the fix but could not open a follow-up review: {e:#}"),
6639 );
6640 }
6641 }
6642 self.state.save()?;
6643 }
6644
6645 Ok(())
6646 }
6647
6648 async fn ask_fixer(
6668 &mut self,
6669 winner: &Candidate,
6670 node: &'static str,
6671 round: Option<usize>,
6672 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6673 ) -> (SeatJob, SeatState, AgentOutcome) {
6674 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6675 self.ask_fixer_among(attempts, node, round, None, build)
6676 .await
6677 }
6678
6679 async fn ask_fixer_among(
6684 &mut self,
6685 attempts: Vec<(AgentSpec, String)>,
6686 node: &'static str,
6687 round: Option<usize>,
6688 restore: Option<(&Path, &str)>,
6689 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6690 ) -> (SeatJob, SeatState, AgentOutcome) {
6691 let run_id = self.state.id.clone();
6692 let prompts = self.state.config.prompts.clone();
6693 let cache = self.state.config.cache_dir();
6694 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6695 let mut last = None;
6696 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6697 let seat = self.seat(&key, &spec.id);
6698 let mut job = build(spec.clone(), seat);
6699 if i > 0 {
6700 job.stem = format!("{}-{}", job.stem, spec.id);
6701 }
6702 let ctx = WaveCtx {
6703 carry_seats: false,
6704 pauses: Some((&self.pause, &self.interrupt)),
6705 run: &run_id,
6706 node,
6707 prompts: &prompts,
6708 cache: cache.as_deref(),
6709 round,
6710 };
6711 let (seat, out) =
6712 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6713 let advances = match &out {
6716 AgentOutcome::Ok(o) => agent::output_advances(o),
6717 AgentOutcome::ResourceBlocked(_) => false,
6718 _ => true,
6719 };
6720 if let Some(next) = ids.get(i + 1)
6721 && advances
6722 {
6723 self.state.seats.insert(seat.key.clone(), seat.clone());
6724 let class =
6725 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6726 let reason = match &out {
6727 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6728 other => fail_reason(other),
6729 };
6730 if let Some((wt, rev)) = restore
6734 && let Err(e) = self.graft_restore(wt, rev).await
6735 {
6736 let why =
6737 format!("could not restore the worktree before the next fixer: {e:#}");
6738 tracing::warn!("{why}");
6739 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6740 AgentOutcome::ResourceBlocked(why)
6741 } else {
6742 AgentOutcome::Failed(why)
6743 };
6744 last = Some((job, seat, out));
6745 break;
6746 }
6747 record_handover(
6748 &mut self.state,
6749 node,
6750 &seat.key,
6751 &spec.id,
6752 next,
6753 &class,
6754 &reason,
6755 );
6756 continue;
6757 }
6758 last = Some((job, seat, out));
6759 break;
6760 }
6761 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6762 if let AgentOutcome::Quota(output) = &out
6763 && node != "graft"
6764 {
6765 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6770 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6771 self.state.quota.push(loss);
6772 self.state.seats.insert(seat.key.clone(), seat.clone());
6773 self.state.status = RunStatus::Blocked;
6774 return (
6775 job,
6776 seat,
6777 AgentOutcome::ResourceBlocked(
6778 "fixer unavailable; restore agent availability and resume".into(),
6779 ),
6780 );
6781 }
6782 (job, seat, out)
6783 }
6784
6785 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6796 use crate::recovery::{Mode, Repair};
6797 if !self.state.reviews.is_empty() {
6798 return Ok(());
6799 }
6800 let Some(mut rec) = self.state.recovery.clone() else {
6801 return Ok(());
6802 };
6803 if rec.mode != Mode::VerifyRepair {
6804 return Ok(());
6805 }
6806 let wt = winner.worktree.clone();
6807 let mut head = git::rev_parse(&wt, "HEAD").await?;
6808 match rec.repair.clone() {
6809 Repair::Done { .. } => return Ok(()),
6810 Repair::WaitingForRestore { before } => {
6811 self.graft_restore(&wt, &before).await?;
6812 head = before;
6813 rec.repair = Repair::NotStarted;
6814 self.state.recovery = Some(rec.clone());
6815 self.state.save()?;
6816 }
6817 Repair::Started { before } => {
6818 if let Err(e) = self.graft_restore(&wt, &before).await {
6822 if e.is::<crate::repo_lock::ResourceBlocked>() {
6823 return Err(e);
6824 }
6825 return self
6826 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6827 .await;
6828 }
6829 rec.repair = Repair::Done {
6830 before: before.clone(),
6831 after: before,
6832 committed: false,
6833 failed: Some("interrupted; rolled back".to_owned()),
6834 };
6835 self.state
6836 .event("recovery", "the repair was interrupted and rolled back");
6837 self.state.recovery = Some(rec);
6838 self.state.save()?;
6839 return Ok(());
6840 }
6841 Repair::NotStarted => {}
6842 }
6843 if crate::recovery::recheck(&mut rec, &head) {
6844 self.state.event("recovery", rec.summary());
6845 self.state.recovery = Some(rec);
6846 self.state.save()?;
6847 return Ok(());
6848 }
6849 rec.repair = Repair::Started {
6850 before: head.clone(),
6851 };
6852 self.state.recovery = Some(rec.clone());
6853 self.state.save()?;
6854
6855 let language = self.state.config.graph.language.clone();
6856 let sessions = self.state.config.graph.sessions;
6857 let artifacts = agent::artifacts_dir(&self.state.dir());
6858 let fix_prompt = prompt::recovery_fix(
6859 &self.state.instruction,
6860 &rec,
6861 self.state.config.graph.review_rounds,
6862 &language,
6863 );
6864 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6865 self.state.event(
6866 "recovery",
6867 format!(
6868 "repair: the fixer starts from the failed verification on {}",
6869 short(&head)
6870 ),
6871 );
6872 let (_job, seat, out) = self
6873 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6874 prompt: fix_prompt.clone(),
6875 spec,
6876 seat,
6877 cwd: wt.clone(),
6878 timeout,
6879 allow_write: true,
6880 sessions,
6881 artifacts: artifacts.clone(),
6882 stem: "fix-recovery".to_owned(),
6883 handover: None,
6884 })
6885 .await;
6886 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6887 rec.repair = Repair::WaitingForRestore { before: head };
6888 self.state.recovery = Some(rec);
6889 return self.stop_for_resource();
6890 }
6891 let failed = match &out {
6892 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6893 AgentOutcome::Quota(o) => {
6894 self.state.quota.push(QuotaLoss::from_report(
6895 seat.key.clone(),
6896 "fix",
6897 o.quota.as_ref(),
6898 ));
6899 Some("rate limited (quota); the fixer could not run".to_owned())
6900 }
6901 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6902 AgentOutcome::Failed(e) => Some(e.clone()),
6903 };
6904 self.state.seats.insert(seat.key.clone(), seat);
6905 let coordination = self.coordinate(&wt).await?;
6906 let failed = match failed {
6907 Some(why) => {
6909 if let Err(e) = git::restore_to(&wt, &head).await {
6910 return self
6911 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6912 .await;
6913 }
6914 Some(why)
6915 }
6916 None => {
6917 if let Ok(r) = git::rescue_commit(
6918 &wt,
6919 "magi: recovery repair of the failed verification (uncommitted work)",
6920 )
6921 .await
6922 {
6923 self.state.note_withheld("fix", &r.withheld);
6924 }
6925 None
6926 }
6927 };
6928 let after = git::rev_parse(&wt, "HEAD").await?;
6929 drop(coordination);
6930 self.state.event(
6931 "recovery",
6932 match &failed {
6933 Some(why) => format!("repair: nothing kept ({why})"),
6934 None if after != head => {
6935 format!("repair: committed {}", short(&after))
6936 }
6937 None => "repair: the fixer made no commit".to_owned(),
6938 },
6939 );
6940 rec.repair = Repair::Done {
6941 before: head.clone(),
6942 committed: after != head,
6943 after,
6944 failed,
6945 };
6946 self.state.recovery = Some(rec);
6947 self.state.save()?;
6948 Ok(())
6949 }
6950
6951 async fn stop_recovery(
6954 &mut self,
6955 mut rec: crate::recovery::RecoveryRecord,
6956 head: &str,
6957 why: String,
6958 ) -> Result<()> {
6959 rec.repair = crate::recovery::Repair::Started {
6961 before: head.to_owned(),
6962 };
6963 self.state.recovery = Some(rec);
6964 self.state
6965 .event("recovery", format!("repair could not be undone: {why}"));
6966 self.state.status = RunStatus::Blocked;
6967 self.state.save()?;
6968 bail!("recovery repair could not be rolled back: {why}")
6969 }
6970
6971 async fn review_loop(&mut self) -> Result<()> {
6972 if self.hold_base_sync().await? {
6973 return Ok(());
6974 }
6975 loop {
6976 if self
6977 .state
6978 .base_sync
6979 .as_ref()
6980 .is_some_and(|s| s.conflict.is_some())
6981 {
6982 return Ok(());
6983 }
6984 if self.catchup_repairable() {
6985 let record = self.state.reviews.last().expect("repairable round").clone();
6986 let winner = self
6987 .state
6988 .winner()
6989 .cloned()
6990 .context("catch-up repair has no winner")?;
6991 if !self.fix_review_round(&winner, record).await? {
6992 return Ok(());
6993 }
6994 }
6995 self.review_loop_pass().await?;
6996 if !self.catchup_repairable() {
6997 return Ok(());
6998 }
6999 }
7000 }
7001
7002 fn catchup_repairable(&self) -> bool {
7005 self.state.reviews.len() < self.state.config.graph.review_rounds
7006 && self.state.reviews.last().is_some_and(|r| {
7007 !r.clean
7008 && r.blocking == 0
7009 && r.fix.is_none()
7010 && r.e2e_status() == E2eStatus::Failed
7011 })
7012 }
7013
7014 async fn review_loop_pass(&mut self) -> Result<()> {
7015 if self
7020 .state
7021 .base_sync
7022 .as_ref()
7023 .is_some_and(|s| s.conflict.is_some())
7024 {
7025 return Ok(());
7026 }
7027 let run_id = self.state.id.clone();
7032 let prompts = self.state.config.prompts.clone();
7033 let Some(winner) = self.state.winner().cloned() else {
7034 return Ok(());
7035 };
7036 let granted_review = self
7037 .state
7038 .repair_grants
7039 .last()
7040 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7041 .cloned();
7042 let fresh_review = granted_review
7043 .as_ref()
7044 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7045 let max_rounds = granted_review
7046 .as_ref()
7047 .map_or(self.state.config.graph.review_rounds, |g| {
7048 g.review_start.saturating_add(1)
7049 });
7050 if let Some(status) =
7060 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7061 {
7062 if status == RunStatus::Gating {
7065 self.record_contested_handoff();
7066 }
7067 self.state.status = status;
7068 self.state.save()?;
7069 return Ok(());
7070 }
7071 self.state.status = RunStatus::Reviewing;
7072 if !fresh_review
7082 && self.state.reviews.last().is_some_and(|r| {
7083 r.e2e_status() == E2eStatus::ResourceBlocked
7084 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7085 })
7086 {
7087 let shell = self.state.config.shell();
7088 return self
7089 .stop_unresolved_review(
7090 "the last round's own verification never resolved",
7091 &shell,
7092 &winner.worktree,
7093 )
7094 .await;
7095 }
7096
7097 if granted_review.is_none() {
7098 self.recovery_repair(&winner).await?;
7099 }
7100 if self.resource_halt()? {
7101 return Ok(());
7102 }
7103
7104 let repo = self.state.repo.clone();
7105 let root = self.state.worktree_root();
7106 let language = self.state.config.graph.language.clone();
7107 let sessions = self.state.config.graph.sessions;
7108 let artifacts = self.merge_repair_artifacts();
7109 let base = self.landing_base();
7110 let base_short = short(&base);
7111 let reviewers = self.roles.reviewers.clone();
7112 let shell = self.state.config.shell();
7113
7114 for round in (self.state.reviews.len() + 1)..=max_rounds {
7115 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7116 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7117 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7118 let prev_verification = self
7127 .state
7128 .reviews
7129 .last()
7130 .and_then(|r| r.verification_summary(&head));
7131 let prior_section = match (&self.state.recovery, round) {
7134 (Some(rec), 1) if self.state.reviews.is_empty() => {
7135 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7136 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7137 Some(crate::recovery::head_of(
7138 &git::diff(&winner.worktree, reviewed, "HEAD")
7139 .await
7140 .unwrap_or_default(),
7141 crate::recovery::SINCE_DIFF_MAX,
7142 ))
7143 }
7144 _ => None,
7145 };
7146 Some(prompt::recovery_review(rec, since.as_deref()))
7147 }
7148 _ => None,
7149 };
7150
7151 let mut jobs = Vec::new();
7155 let coordination = self.coordinate(&repo).await?;
7156 for (r, spec) in reviewers.iter().cloned().enumerate() {
7157 let wt = root.join(format!("review-{}", r + 1));
7158 if wt.exists() {
7161 git::reset_detached(&wt, &head).await?;
7162 } else {
7163 git::worktree_add_detached(&repo, &wt, &head).await?;
7164 }
7165 self.setup_seat_worktree(&repo, &wt, false).await?;
7166 let seat_key = format!("review-{}", r + 1);
7167 let spec = pick_start_spec(
7170 &self.roles.reviewer_roster,
7171 spec,
7172 self.state.seat_history.get(&seat_key),
7173 );
7174 let seat = self.seat(&seat_key, &spec.id);
7175 jobs.push(SeatJob {
7176 prompt: prompt::review(&prompt::ReviewCtx {
7177 instruction: &self.state.instruction,
7178 branch: &winner.branch,
7179 base_short: &base_short,
7180 stat: &stat,
7181 patch: &patch,
7182 verification: prev_verification.as_ref(),
7183 prior: prior_section.as_deref(),
7184 reviewers: reviewers.len(),
7185 round,
7186 rounds: max_rounds,
7187 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7190 lens: Lens::for_seat(r),
7191 language: &language,
7192 github_language: &self.state.config.graph.github_language,
7193 }),
7194 spec,
7195 seat,
7196 cwd: wt,
7197 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7198 allow_write: false,
7199 sessions,
7200 artifacts: artifacts.clone(),
7201 stem: format!("review-{round}-{}", r + 1),
7202 handover: None,
7203 });
7204 }
7205
7206 drop(coordination);
7207 self.state.event(
7208 "review",
7209 format!(
7210 "round {round}: {} reviewers on {}",
7211 jobs.len(),
7212 short(&head)
7213 ),
7214 );
7215 let mut quota_losses = Vec::new();
7216 let review_retries = self.state.config.graph.retries;
7217 let review_cache = self.state.config.cache_dir();
7218 let ctx = WaveCtx {
7219 carry_seats: true,
7220 pauses: Some((&self.pause, &self.interrupt)),
7221 run: &run_id,
7222 node: "review",
7223 prompts: &prompts,
7224 cache: review_cache.as_deref(),
7225 round: Some(round),
7226 };
7227 let results = ask_json_wave::<Review>(
7228 jobs,
7229 Arc::clone(&self.sem),
7230 review_retries,
7231 &self.roles.reviewer_roster,
7232 &ctx,
7233 &mut quota_losses,
7234 &mut self.state,
7235 &|_: &Review| Ok(()),
7236 )
7237 .await;
7238 if self.state.resource_block.is_some() {
7239 return self.stop_for_resource();
7240 }
7241 let round_quota_missing = quota_losses.len();
7245 self.state.quota.extend(quota_losses);
7246
7247 let mut records = Vec::new();
7248 let mut all_findings = Vec::new();
7249 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7250 let agent_id = seat.agent.clone();
7251 self.state.seats.insert(seat.key.clone(), seat);
7252 let mut record = ReviewRecord {
7253 reviewer: r + 1,
7254 agent: agent_id,
7255 summary: String::new(),
7256 findings: Vec::new(),
7257 vote: None,
7258 failed: None,
7259 duration_ms: 0,
7260 attempts,
7266 };
7267 match res {
7268 Ok((review, out)) => {
7269 record.summary =
7277 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7278 record.vote = Some(review.vote);
7279 record.duration_ms = out.duration_ms;
7280 for (n, mut f) in review.findings.into_iter().enumerate() {
7281 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7284 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7285 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7286 f.file = f
7292 .file
7293 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7294 all_findings.push(f.clone());
7295 record.findings.push(f);
7296 }
7297 self.state.event(
7298 "review",
7299 format!(
7300 "round {round}: reviewer {} voted {} with {} finding(s)",
7301 r + 1,
7302 review.vote.label(),
7303 record.findings.len()
7304 ),
7305 );
7306 }
7307 Err(e) => {
7308 record.failed = Some(e.to_string());
7309 self.state.event(
7310 "review",
7311 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7312 );
7313 }
7314 }
7315 records.push(record);
7316 }
7317
7318 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7325 let vote_split =
7326 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7327 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7328 if vote_split {
7329 self.state.event(
7330 "review",
7331 format!(
7332 "round {round}: votes split ({}) — one round of reconsideration",
7333 initial_votes
7334 .iter()
7335 .map(|v| v.label())
7336 .collect::<Vec<_>>()
7337 .join(", ")
7338 ),
7339 );
7340 let panel: Vec<ReviewSeatReport<'_>> = records
7343 .iter()
7344 .filter_map(|r| {
7345 r.vote.map(|vote| ReviewSeatReport {
7346 reviewer: r.reviewer,
7347 vote,
7348 summary: &r.summary,
7349 findings: &r.findings,
7350 })
7351 })
7352 .collect();
7353
7354 let mut jobs = Vec::new();
7355 let mut seats_at = Vec::new();
7356 for (r, spec) in reviewers.iter().cloned().enumerate() {
7357 if records[r].vote.is_none() {
7361 continue;
7362 }
7363 let wt = root.join(format!("review-{}", r + 1));
7364 let seat_key = format!("review-{}", r + 1);
7365 let spec = self.occupant(&seat_key, spec);
7366 let seat = self.seat(&seat_key, &spec.id);
7367 let build = |with_patch: bool| {
7375 prompt::review_reconsider(&ReviewReconsiderCtx {
7376 instruction: &self.state.instruction,
7377 reviewer: r + 1,
7378 lens: Lens::for_seat(r),
7379 panel: &panel,
7380 patch: with_patch.then_some(ReviewPatch {
7381 branch: &winner.branch,
7382 base_short: &base_short,
7383 stat: &stat,
7384 patch: &patch,
7385 }),
7386 round,
7387 rounds: max_rounds,
7388 language: &language,
7389 })
7390 };
7391 let full = build(true);
7392 let prompt = if has_context(&spec, &seat, sessions) {
7393 build(false)
7394 } else {
7395 full.clone()
7396 };
7397 jobs.push(SeatJob {
7398 prompt,
7399 spec,
7400 seat,
7401 cwd: wt,
7402 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7403 allow_write: false,
7404 sessions,
7405 artifacts: artifacts.clone(),
7406 stem: format!("review-{round}-reconsider-{}", r + 1),
7407 handover: Some(full),
7408 });
7409 seats_at.push(r);
7410 }
7411
7412 let mut recon_quota_losses = Vec::new();
7413 let recon_cache = self.state.config.cache_dir();
7414 let recon_ctx = WaveCtx {
7415 carry_seats: true,
7416 pauses: Some((&self.pause, &self.interrupt)),
7417 run: &run_id,
7418 node: "review",
7419 prompts: &prompts,
7420 cache: recon_cache.as_deref(),
7421 round: Some(round),
7422 };
7423 let recon_results = ask_json_wave::<ReviewRevote>(
7424 jobs,
7425 Arc::clone(&self.sem),
7426 review_retries,
7427 &self.roles.reviewer_roster,
7428 &recon_ctx,
7429 &mut recon_quota_losses,
7430 &mut self.state,
7431 &|_: &ReviewRevote| Ok(()),
7432 )
7433 .await;
7434 if self.state.resource_block.is_some() {
7435 return self.stop_for_resource();
7436 }
7437 self.state.quota.extend(recon_quota_losses);
7438
7439 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7440 let agent_id = seat.agent.clone();
7441 self.state.seats.insert(seat.key.clone(), seat);
7442 let mut rec = ReviewRevoteRecord {
7443 reviewer: r + 1,
7444 agent: agent_id,
7445 vote: None,
7446 reason: String::new(),
7447 failed: None,
7448 };
7449 match res {
7450 Ok((rv, _)) => {
7451 rec.vote = Some(rv.vote);
7452 rec.reason =
7453 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7454 self.state.event(
7455 "review",
7456 format!(
7457 "round {round}: reviewer {} revoted {}",
7458 r + 1,
7459 rv.vote.label()
7460 ),
7461 );
7462 }
7463 Err(e) => {
7464 rec.failed = Some(e.to_string());
7465 self.state.event(
7466 "review",
7467 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7468 );
7469 }
7470 }
7471 reconsideration.push(rec);
7472 }
7473 } else if initial_votes.len() > 1 {
7474 self.state.event(
7475 "review",
7476 format!(
7477 "round {round}: votes agreed ({}) — no reconsideration",
7478 initial_votes[0].label()
7479 ),
7480 );
7481 }
7482
7483 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7484 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7485 let defer_e2e =
7496 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7497 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7498 let reason =
7499 format!("{blocking} blocking finding(s) already required a fix this round");
7500 self.state.event(
7501 "verify",
7502 format!(
7503 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7504 {}); it will run once a round has none left",
7505 short(&head)
7506 ),
7507 );
7508 (Vec::new(), false, true, Some(reason))
7509 } else {
7510 let e2e_commands = self.state.config.verify.e2e.clone();
7511 let cache_dir = self.state.config.cache_dir();
7512 let context = format!("round {round}");
7513 let wait = self.repository_wait();
7514 let (e2e, verify_retried) = with_cache_lease(
7515 &mut self.state,
7516 cache_dir.as_deref(),
7517 "e2e",
7518 "e2e",
7519 &winner.worktree,
7520 &head,
7521 verify_timeout,
7522 &context,
7523 &wait,
7524 |state, budget| {
7525 let shell = shell.clone();
7526 let e2e_commands = e2e_commands.clone();
7527 let worktree = winner.worktree.clone();
7528 let context = context.clone();
7529 async move {
7530 run_e2e_with_retry(
7531 state,
7532 &shell,
7533 &e2e_commands,
7534 &worktree,
7535 budget,
7536 &context,
7537 )
7538 .await
7539 }
7540 },
7541 )
7542 .await?;
7543 (e2e, verify_retried, false, None)
7544 };
7545
7546 let expected = records.len();
7547 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7548 let incomplete = answered < expected;
7549 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7550 let policy = self.state.config.graph.incomplete_review;
7551 let clean = round_is_clean(
7552 blocking,
7553 e2e_ok,
7554 answered,
7555 expected,
7556 round_quota_missing,
7557 policy,
7558 );
7559
7560 let mut round_record = ReviewRound {
7561 round,
7562 head: head.clone(),
7563 verified_head: None,
7564 verified_at: None,
7565 reviews: records,
7566 e2e,
7567 verify_retried,
7568 e2e_deferred,
7569 e2e_defer_reason,
7570 fix: None,
7571 blocking,
7572 answered,
7573 expected,
7574 clean,
7575 progressed: false,
7576 vote_split,
7577 reconsideration,
7578 verdict: None,
7579 };
7580 round_record.verdict = ReviewVote::worst(
7584 round_record
7585 .final_votes()
7586 .into_iter()
7587 .map(|(_, _, vote)| vote),
7588 );
7589 if !matches!(
7600 round_record.e2e_status(),
7601 E2eStatus::Deferred | E2eStatus::NotConfigured
7602 ) {
7603 round_record.verified_head = Some(head.clone());
7604 round_record.verified_at = Some(Timestamp::now());
7605 }
7606
7607 if incomplete {
7608 let missing: Vec<String> = round_record
7609 .reviews
7610 .iter()
7611 .filter(|r| r.failed.is_some())
7612 .map(|r| format!("review-{}", r.reviewer))
7613 .collect();
7614 self.state.event(
7615 "review",
7616 format!(
7617 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7618 missing.join(", ")
7619 ),
7620 );
7621 }
7622
7623 if clean {
7624 self.state.event(
7625 "review",
7626 if incomplete && policy == IncompleteReviewPolicy::Warn {
7627 format!(
7628 "round {round}: clean (warn policy, incomplete panel) — no \
7629 blocking findings from the seats that answered, verification green"
7630 )
7631 } else if incomplete {
7632 format!(
7633 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7634 quorum) — no blocking findings from the seats that answered, \
7635 verification green",
7636 expected - answered
7637 )
7638 } else {
7639 format!("round {round}: clean — no blocking findings, verification green")
7640 },
7641 );
7642 self.state.reviews.push(round_record);
7643 self.state.status = RunStatus::Gating;
7644 self.state.save()?;
7645 return Ok(());
7646 }
7647
7648 if incomplete && blocking == 0 && e2e_ok {
7656 self.state.reviews.push(round_record);
7657 self.state.save()?;
7658 if round == max_rounds {
7659 self.state.status = RunStatus::Blocked;
7660 self.state.event(
7661 "review",
7662 format!(
7663 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7664 refusing to call it clean",
7665 expected - answered
7666 ),
7667 );
7668 return Ok(());
7669 }
7670 continue;
7671 }
7672
7673 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7685 self.state.reviews.push(round_record);
7686 return self
7687 .stop_unresolved_review(
7688 "the round's own verification could not run",
7689 &shell,
7690 &winner.worktree,
7691 )
7692 .await;
7693 }
7694
7695 if round == max_rounds {
7696 self.state.reviews.push(round_record);
7697 return self
7698 .stop_reviewing(
7699 &format!(
7700 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7701 ),
7702 &shell,
7703 &winner.worktree,
7704 )
7705 .await;
7706 }
7707
7708 if !self.fix_review_round(&winner, round_record).await? {
7709 return Ok(());
7710 }
7711 }
7712 Ok(())
7713 }
7714
7715 async fn fix_review_round(
7718 &mut self,
7719 winner: &Candidate,
7720 mut round_record: ReviewRound,
7721 ) -> Result<bool> {
7722 let round = round_record.round;
7723 let shell = self.state.config.shell();
7724 let step = self.run_fix_round(winner, &mut round_record).await?;
7725 if matches!(step, FixStep::Resource) {
7726 self.stop_for_resource()?;
7727 return Ok(false);
7728 }
7729 if self.state.reviews.len() < round {
7730 self.state.reviews.push(round_record);
7731 } else {
7732 self.state.reviews[round - 1] = round_record;
7733 }
7734 self.state.save()?;
7735
7736 if matches!(step, FixStep::Recorded { report_lost: true }) {
7749 self.stop_reviewing(
7750 "the fixer's adoption report never came back, even after resuming its \
7751 own seat; refusing to start another round against the same worktree \
7752 while that is unresolved",
7753 &shell,
7754 &winner.worktree,
7755 )
7756 .await?;
7757 return Ok(false);
7758 }
7759
7760 let streak = self
7761 .state
7762 .reviews
7763 .iter()
7764 .rev()
7765 .take_while(|r| !r.progressed)
7766 .count();
7767 if streak >= STAGNANT_LIMIT {
7768 self.stop_reviewing(
7769 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7770 &shell,
7771 &winner.worktree,
7772 )
7773 .await?;
7774 return Ok(false);
7775 }
7776 Ok(true)
7777 }
7778
7779 async fn run_fix_round(
7781 &mut self,
7782 winner: &Candidate,
7783 round_record: &mut ReviewRound,
7784 ) -> Result<FixStep> {
7785 let round = round_record.round;
7786 let max_rounds = self.state.config.graph.review_rounds;
7787 let run_id = self.state.id.clone();
7788 let prompts = self.state.config.prompts.clone();
7789 let language = self.state.config.graph.language.clone();
7790 let sessions = self.state.config.graph.sessions;
7791 let artifacts = self.merge_repair_artifacts();
7792 let base = self.landing_base();
7793 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7794 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7795 let this_round_verification = round_record.verification_summary(&head);
7796 let all_findings: Vec<_> = round_record
7797 .reviews
7798 .iter()
7799 .flat_map(|r| r.findings.iter().cloned())
7800 .collect();
7801 let blocking_findings: Vec<_> = all_findings
7804 .iter()
7805 .filter(|f| f.severity.blocks())
7806 .cloned()
7807 .collect();
7808 let fix_prompt = prompt::fix_in(
7809 &self.state.instruction,
7810 &blocking_findings,
7811 this_round_verification.as_ref(),
7812 round,
7813 max_rounds,
7814 &language,
7815 &self.state.config.graph.github_language,
7816 );
7817 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7818 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7819 let (job, seat, out) = self
7820 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7821 prompt: fix_prompt.clone(),
7822 spec,
7823 seat,
7824 cwd: winner.worktree.clone(),
7825 timeout,
7826 allow_write: true,
7827 sessions,
7828 artifacts: artifacts.clone(),
7829 stem: format!("fix-{round}"),
7830 handover: None,
7831 })
7832 .await;
7833 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7836 return Ok(FixStep::Resource);
7837 }
7838 let agent_id = seat.agent.clone();
7839
7840 let mut fix = FixRecord {
7841 agent: agent_id,
7842 addressed: Vec::new(),
7843 rejected: Vec::new(),
7844 notes: String::new(),
7845 committed: false,
7846 failed: None,
7847 duration_ms: 0,
7848 continuation: None,
7849 };
7850 let mut continuation = ContinuationRecord::not_needed();
7851 let mut final_seat = seat.clone();
7852 match out {
7853 AgentOutcome::Ok(o) => {
7854 fix.duration_ms = o.duration_ms;
7855 let parsed = verdict::extract_json::<FixReport>(&o.text);
7856 let incomplete_reason = match &parsed {
7863 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7864 "the reply parsed, but it reported a command whose own CLI never \
7865 confirmed an exit status"
7866 .to_owned(),
7867 ),
7868 Ok(_) => None,
7869 Err(e) => Some(e.to_string()),
7870 };
7871 match incomplete_reason {
7872 None => {
7873 let report = parsed.expect("checked Ok above");
7874 fix.addressed = report.addressed;
7875 fix.rejected = report.rejected;
7876 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7877 }
7878 Some(reason) => {
7879 let (resumed_seat, resolved, failure, cont) = self
7880 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7881 .await;
7882 fix.duration_ms += cont.cumulative_wait_ms;
7883 continuation = cont;
7884 final_seat = resumed_seat;
7885 match resolved {
7886 Some(report) => {
7887 fix.addressed = report.addressed;
7888 fix.rejected = report.rejected;
7889 fix.notes =
7890 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7891 }
7892 None => fix.failed = failure,
7893 }
7894 }
7895 }
7896 }
7897 AgentOutcome::Dropped(o) => {
7899 fix.duration_ms = o.duration_ms;
7900 let why = o
7901 .dropped
7902 .as_ref()
7903 .map(|d| d.why.as_str())
7904 .unwrap_or("the CLI ended the stream without delivering its answer");
7905 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7906 }
7907 AgentOutcome::Quota(o) => {
7908 self.state.quota.push(QuotaLoss::from_report(
7909 final_seat.key.clone(),
7910 "fix",
7911 o.quota.as_ref(),
7912 ));
7913 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7914 }
7915 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7916 }
7917 if self.state.resource_block.is_some() {
7918 self.state.seats.insert(final_seat.key.clone(), final_seat);
7919 return Ok(FixStep::Resource);
7920 }
7921 fix.continuation = Some(continuation);
7922 self.state.seats.insert(final_seat.key.clone(), final_seat);
7923 let coordination = self.coordinate(&winner.worktree).await?;
7924 if let Ok(r) = git::rescue_commit(
7925 &winner.worktree,
7926 &format!("magi: review round {round} fixes (uncommitted work)"),
7927 )
7928 .await
7929 {
7930 self.state.note_withheld("fix", &r.withheld);
7931 }
7932 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7933 fix.committed = after != before;
7934 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7942 drop(coordination);
7943 let progressed = diff_after != patch;
7944 let commit_note = if fix.committed {
7945 "committed"
7946 } else {
7947 "NO new commit"
7948 };
7949 let tree_note = if progressed {
7950 "changed vs base"
7951 } else {
7952 "unchanged vs base"
7953 };
7954 self.state.event(
7955 "fix",
7956 match &fix.failed {
7957 Some(reason) => {
7963 format!(
7964 "round {round}: fixer's adoption report was lost ({reason}); \
7965 {commit_note}, tree {tree_note}"
7966 )
7967 }
7968 None => format!(
7969 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7970 {tree_note}{}",
7971 fix.addressed.len(),
7972 fix.rejected.len(),
7973 if continuation.outcome == ContinuationOutcome::Resumed {
7974 format!(
7975 " (adoption report recovered after {} continuation(s))",
7976 continuation.attempts
7977 )
7978 } else {
7979 String::new()
7980 },
7981 ),
7982 },
7983 );
7984 round_record.fix = Some(fix);
7985 round_record.progressed = progressed;
7986 Ok(FixStep::Recorded {
7987 report_lost: matches!(
7988 continuation.outcome,
7989 ContinuationOutcome::Exhausted
7990 | ContinuationOutcome::QuotaLost
7991 | ContinuationOutcome::NoSession
7992 ),
7993 })
7994 }
7995
7996 fn record_contested_handoff(&mut self) {
8030 if self.state.contested_handoff.is_some() {
8031 return;
8032 }
8033 let Some(contested) = self
8034 .state
8035 .reviews
8036 .last()
8037 .and_then(ReviewRound::contested_handoff)
8038 else {
8039 return;
8040 };
8041 self.state.event(
8042 "review",
8043 format!(
8044 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8045 wait for the owner's approval",
8046 contested.findings.len(),
8047 contested.rejecters.len()
8048 ),
8049 );
8050 self.state.contested_handoff = Some(contested);
8051 }
8052
8053 async fn stop_unresolved_review(
8054 &mut self,
8055 why: &str,
8056 shell: &[String],
8057 worktree: &Path,
8058 ) -> Result<()> {
8059 self.stop_reviewing_inner(why, shell, worktree, true).await
8060 }
8061
8062 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8063 self.stop_reviewing_inner(why, shell, worktree, false).await
8064 }
8065
8066 async fn stop_reviewing_inner(
8067 &mut self,
8068 why: &str,
8069 shell: &[String],
8070 worktree: &Path,
8071 repairable: bool,
8072 ) -> Result<()> {
8073 let round_idx = self.state.reviews.len() - 1;
8074 let needs_catchup_run = matches!(
8082 self.state.reviews[round_idx].e2e_status(),
8083 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8084 );
8085 if needs_catchup_run {
8086 let round = self.state.reviews[round_idx].round;
8087 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8088 let commands = self.state.config.verify.e2e.clone();
8089 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8090 let cache_dir = self.state.config.cache_dir();
8091 let context = format!(
8092 "round {round}: verification unresolved, catching up before the final decision"
8093 );
8094 let wait = self.repository_wait();
8095 let (outcomes, verify_retried) = with_cache_lease(
8096 &mut self.state,
8097 cache_dir.as_deref(),
8098 "e2e",
8099 "e2e",
8100 worktree,
8101 &attempted_head,
8102 timeout,
8103 &context,
8104 &wait,
8105 |state, budget| {
8106 let shell = shell.to_vec();
8107 let commands = commands.clone();
8108 let context = context.clone();
8109 async move {
8110 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8111 .await
8112 }
8113 },
8114 )
8115 .await?;
8116 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8117 self.state.record_base_progress(&attempted_head);
8118 }
8119 let last = &mut self.state.reviews[round_idx];
8120 last.e2e = outcomes;
8121 last.verify_retried = verify_retried;
8122 last.verified_head = Some(attempted_head);
8129 last.verified_at = Some(Timestamp::now());
8130 if verify_inconclusive(&last.e2e) {
8131 self.state.save()?;
8138 return Ok(());
8139 }
8140 last.e2e_deferred = false;
8141 if last.answered == last.expected {
8146 last.clean = round_is_clean(
8147 last.blocking,
8148 last.e2e.iter().all(CommandOutcome::ok),
8149 last.answered,
8150 last.expected,
8151 0,
8152 self.state.config.graph.incomplete_review,
8153 );
8154 }
8155 }
8156 let last = &self.state.reviews[round_idx];
8157 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8158
8159 match last.e2e_status() {
8160 E2eStatus::Failed => {
8161 let red: Vec<String> = last
8162 .e2e
8163 .iter()
8164 .filter(|o| !o.ok())
8165 .map(|o| {
8166 format!(
8167 "`{}` -> {:?}\n{}",
8168 o.command,
8169 o.code,
8170 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8171 )
8172 })
8173 .collect();
8174 let can_repair = last.blocking == 0 && last.fix.is_none();
8175 self.state.reviews[round_idx].clean = false;
8176 self.state
8177 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8178 self.state.status = RunStatus::Blocked;
8179 if repairable
8180 && can_repair
8181 && self.state.reviews.len() < self.state.config.graph.review_rounds
8182 {
8183 self.state.status = RunStatus::Reviewing;
8184 }
8185 }
8186 E2eStatus::ResourceBlocked => {
8191 self.state.event(
8192 "review",
8193 format!(
8194 "{why}; e2e could not run (resource unavailable); not \
8195 deciding yet"
8196 ),
8197 );
8198 }
8199 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8200 self.state.event(
8201 "review",
8202 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8203 );
8204 self.record_contested_handoff();
8205 self.state.status = RunStatus::Gating;
8206 }
8207 }
8208 self.state.save()?;
8209 Ok(())
8210 }
8211
8212 async fn gate(&mut self) -> Result<()> {
8215 if self.hold_base_sync().await? {
8216 return Ok(());
8217 }
8218 if self.state.status == RunStatus::Failed
8230 || self
8231 .state
8232 .base_sync
8233 .as_ref()
8234 .is_some_and(|s| s.conflict.is_some())
8235 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8236 != Some(RunStatus::Gating)
8237 {
8238 return Ok(());
8239 }
8240 let Some(winner) = self.state.winner().cloned() else {
8241 return Ok(());
8242 };
8243 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8244 self.state.reconcile_verification_head(&head);
8245 self.state.save()?;
8246 let wait = self.repository_wait();
8247 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8248 return Ok(());
8249 }
8250 if self.state.gate_ran {
8251 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8262 self.state.status = RunStatus::Blocked;
8263 self.state.save()?;
8264 }
8265 return Ok(());
8266 }
8267 self.state.status = RunStatus::Gating;
8268 let mut outcomes = self.run_gate(&winner).await?;
8269 let mut attempted_head = self.state.gate_head.clone();
8270 loop {
8271 if verify_inconclusive(&outcomes) {
8282 self.state.save()?;
8283 return Ok(());
8284 }
8285 if outcomes.iter().all(CommandOutcome::ok) {
8286 break;
8287 }
8288 let fixes_before = self.state.gate_fixes.len();
8289 match self.gate_fix_round(&winner, &outcomes).await? {
8290 GateFix::Retry => {
8291 outcomes = self.run_gate(&winner).await?;
8292 attempted_head = self.state.gate_head.clone();
8293 }
8294 GateFix::Stop => {
8295 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8296 if self.state.gate_fixes.len() > fixes_before
8297 && attempted_head.as_deref() != Some(current.as_str())
8298 && verify_current_head(&mut self.state, &winner, &wait).await?
8299 {
8300 outcomes = self.run_gate(&winner).await?;
8303 attempted_head = self.state.gate_head.clone();
8304 } else {
8305 break;
8306 }
8307 }
8308 GateFix::Defer => {
8309 self.state.save()?;
8310 return Ok(());
8311 }
8312 }
8313 }
8314 let passed = outcomes.iter().all(CommandOutcome::ok);
8315 self.state.gate = outcomes;
8316 self.state.gate_ran = true;
8317 self.state.gate_head = attempted_head;
8318 if !self.state.gate.is_empty()
8319 && !verify_inconclusive(&self.state.gate)
8320 && let Some(head) = self.state.gate_head.clone()
8321 {
8322 self.state.record_base_progress(&head);
8323 }
8324 self.state.save()?;
8325 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8326 return Ok(());
8327 }
8328 if !passed {
8329 self.state.status = RunStatus::Blocked;
8330 let spent = self.state.gate_fixes.len();
8331 self.state.event(
8332 "gate",
8333 if spent == 0 {
8334 "gate failed; not merging".to_owned()
8335 } else {
8336 format!("gate failed after {spent} gate-fix round(s); not merging")
8337 },
8338 );
8339 }
8340 self.state.save()?;
8341 Ok(())
8342 }
8343
8344 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8354 let commands = state.config.verify.pre_gate.clone();
8355 if commands.is_empty() {
8356 return;
8357 }
8358 let shell = state.config.shell();
8359 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8360 let (outcomes, _) = run_commands(
8361 state,
8362 "pre_gate",
8363 "pre_gate",
8364 0,
8365 &shell,
8366 &commands,
8367 &winner.worktree,
8368 timeout,
8369 )
8370 .await;
8371 for o in &outcomes {
8372 if !o.ok() {
8373 tracing::warn!(
8374 "pre_gate `{}` failed ({:?}); the gate decides",
8375 o.command,
8376 o.code
8377 );
8378 }
8379 state.event(
8380 "pre_gate",
8381 format!(
8382 "`{}` -> {}",
8383 o.command,
8384 if o.ok() {
8385 "pass".to_owned()
8386 } else {
8387 format!(
8388 "FAIL ({:?})\n{}",
8389 o.code,
8390 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8391 )
8392 }
8393 ),
8394 );
8395 }
8396 state.pre_gate = outcomes;
8397 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8398 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8399 Ok(head) => {
8400 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8401 state.pre_gate_commit = Some(head);
8402 }
8403 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8404 },
8405 Ok(false) => {}
8406 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8407 }
8408 if let Err(e) = state.save() {
8409 tracing::warn!("could not persist the pre_gate record: {e:#}");
8410 }
8411 }
8412
8413 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8416 let wait = self.repository_wait();
8417 run_landing_gate(&mut self.state, winner, &wait).await
8418 }
8419
8420 async fn gate_fix_round(
8433 &mut self,
8434 winner: &Candidate,
8435 outcomes: &[CommandOutcome],
8436 ) -> Result<GateFix> {
8437 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8438 g.phase == crate::repair_budget::Phase::Fixing
8439 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8440 }) {
8441 self.state.gate_fixes.len().saturating_add(1)
8442 } else {
8443 self.state.config.graph.gate_fix_rounds
8444 };
8445 let spent = self.state.gate_fixes.len();
8446 if spent >= cap {
8447 if cap > 0 {
8448 self.state.event(
8449 "gate",
8450 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8451 );
8452 }
8453 return Ok(GateFix::Stop);
8454 }
8455 if !gate_fixable(outcomes) {
8456 self.state.event(
8457 "gate",
8458 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8459 command or similar); not spending a fix round on it",
8460 );
8461 return Ok(GateFix::Stop);
8462 }
8463 let min_free = self.state.config.disk.min_free_bytes;
8464 if min_free > 0 {
8465 match crate::disk::free_bytes(&winner.worktree) {
8466 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8467 Ok(free) => {
8468 self.state.event(
8469 "gate",
8470 format!(
8471 "only {free} bytes free ({min_free} required by `[disk] \
8472 min_free_bytes`); not spending a fix round on a failure the disk \
8473 may explain"
8474 ),
8475 );
8476 return Ok(GateFix::Stop);
8477 }
8478 Err(e) => {
8479 self.state.event(
8480 "gate",
8481 format!("free disk space could not be measured ({e:#}); no fix round"),
8482 );
8483 return Ok(GateFix::Stop);
8484 }
8485 }
8486 }
8487
8488 let attempt = spent + 1;
8489 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8490 let base = self.landing_base();
8491 let fix_prompt = prompt::gate_fix_in(
8492 &self.state.instruction,
8493 &failed,
8494 attempt,
8495 cap,
8496 &self.state.config.graph.language,
8497 &self.state.config.graph.github_language,
8498 );
8499 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8500 let sessions = self.state.config.graph.sessions;
8501 let artifacts = self.merge_repair_artifacts();
8502 self.state.event(
8503 "gate",
8504 format!("gate failed; gate-fix round {attempt} of {cap}"),
8505 );
8506 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8507 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8508 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8511 head: before.clone(),
8512 passed: false,
8513 outcomes: Vec::new(),
8514 });
8515 self.state.save()?;
8516 let (_, seat, out) = self
8517 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8518 prompt: fix_prompt.clone(),
8519 spec,
8520 seat,
8521 cwd: winner.worktree.clone(),
8522 timeout,
8523 allow_write: true,
8524 sessions,
8525 artifacts: artifacts.clone(),
8526 stem: format!("gate-fix-{attempt}"),
8527 handover: None,
8528 })
8529 .await;
8530 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8533 return Ok(GateFix::Defer);
8534 }
8535 let mut record = GateFixRecord {
8536 agent: seat.agent.clone(),
8537 failed,
8538 notes: String::new(),
8539 committed: false,
8540 error: None,
8541 };
8542 match out {
8543 AgentOutcome::Ok(o) => {
8544 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8547 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8548 }
8549 }
8550 AgentOutcome::Dropped(_) => {
8551 record.error = Some("the CLI dropped the stream".to_owned());
8552 }
8553 AgentOutcome::Quota(o) => {
8554 self.state.quota.push(QuotaLoss::from_report(
8555 seat.key.clone(),
8556 "gate-fix",
8557 o.quota.as_ref(),
8558 ));
8559 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8560 }
8561 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8562 }
8563 self.state.seats.insert(seat.key.clone(), seat);
8564 let coordination = self.coordinate(&winner.worktree).await?;
8565 if let Ok(r) = git::rescue_commit(
8566 &winner.worktree,
8567 &format!("magi: gate fix {attempt} (uncommitted work)"),
8568 )
8569 .await
8570 {
8571 self.state.note_withheld("gate-fix", &r.withheld);
8572 }
8573 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8574 record.committed = after != before;
8575 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8576 let note = record.error.clone();
8577 self.state.gate_fixes.push(record);
8578 self.state.save()?;
8579 if !changed {
8580 self.state.event(
8581 "gate",
8582 match note {
8583 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8584 None => format!("gate-fix round {attempt}: the tree did not change"),
8585 },
8586 );
8587 return Ok(GateFix::Stop);
8588 }
8589 self.state.event(
8590 "gate",
8591 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8592 );
8593
8594 drop(coordination);
8595 let wait = self.repository_wait();
8596 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8597 }
8598
8599 async fn guarded_pr_message(
8606 &mut self,
8607 winner: &Candidate,
8608 facts: Option<&BranchFacts>,
8609 posting: bool,
8610 ) -> Result<Option<PrMessage>> {
8611 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8612 let head = match final_head.as_deref() {
8613 Some(h) => crate::pr_outcome::FinalHead::At(h),
8614 None => crate::pr_outcome::FinalHead::Unknown,
8615 };
8616 let summarized = if posting {
8618 Some(self.summarize_change(winner, final_head.as_deref()).await)
8619 } else {
8620 None
8621 };
8622 if self.state.resource_block.is_some() {
8627 self.state.event(
8628 "pr-summary",
8629 "a shared resource was busy; nothing was published, resume to summarize again",
8630 );
8631 return Ok(None);
8632 }
8633 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8634 anyhow::bail!(
8635 "the final commit moved during public description generation; retry before publishing"
8636 );
8637 }
8638 let note = match (&summarized, &self.state.change_summary) {
8639 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8640 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8641 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8642 (None, _) => ChangeNote::NotAsked,
8643 };
8644 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8645 if !posting {
8646 let identity = crate::scrub::Identity::current();
8647 return Ok(Some(PrMessage {
8648 title: crate::scrub::scrub(&pr.title, &identity),
8649 body: crate::scrub::scrub(&pr.body, &identity),
8650 }));
8651 }
8652 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8655 if let Some(known) = self
8656 .state
8657 .github_text
8658 .clone()
8659 .filter(|g| g.fingerprint == raw_fp)
8660 {
8661 return self.settle_withheld(winner, &pr, known).await;
8662 }
8663 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8664 let mut violations =
8665 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8666 let identity = crate::scrub::Identity::current();
8667 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8668 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8669 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8670 {
8671 violations.push(crate::github_text::Violation::SensitiveData);
8672 }
8673 let has_language = violations
8677 .iter()
8678 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8679 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8680 self.state.event(
8681 "github-text",
8682 if has_language {
8683 format!("description rejected: {violations:?}; requesting one rewrite")
8684 } else {
8685 format!("description rejected: {violations:?}; asking the owner")
8686 },
8687 );
8688 let mut rewritten = false;
8689 let mut rejected_rewrite = None;
8690 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8691 {
8692 let output = text;
8693 if let Some(title) = summary_title(&output) {
8694 let raw_body = summary_without_title(&output);
8697 let clean_title = crate::scrub::scrub(&title, &identity);
8698 let mut body = crate::scrub::scrub(&raw_body, &identity);
8699 let redacted = clean_title != title || body != raw_body;
8700 let title = clean_title;
8701 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8702 if !complete {
8703 self.state.event(
8704 "github-text",
8705 "description rewrite omitted required findings",
8706 );
8707 }
8708 if !body.trim().is_empty() {
8709 body.push_str("\n\n");
8710 body.push_str(&public_pr_evidence(&self.state, head));
8711 }
8712 let marker = format!("magi:run/{}", self.state.id);
8713 if !body.trim().is_empty() && !body.contains(&marker) {
8714 body.push_str(&format!("\n\n{marker}"));
8715 }
8716 let rewrite = PrMessage {
8717 title: title.clone(),
8718 body: body.clone(),
8719 };
8720 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8721 let mut failed = crate::github_text::check_for(
8722 &self.state.config.graph,
8723 &title,
8724 &body,
8725 redecision,
8726 );
8727 if (!crate::github_text::shareable(&title)
8728 || !crate::github_text::shareable(&body))
8729 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8730 {
8731 failed.push(crate::github_text::Violation::SensitiveData);
8732 }
8733 if complete
8734 && !title.trim().is_empty()
8735 && !body.trim().is_empty()
8736 && !failed.is_empty()
8737 {
8738 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8739 title: title.clone(),
8740 body: body.clone(),
8741 failed: failed
8742 .iter()
8743 .map(|v| crate::github_text::category(*v).to_owned())
8744 .collect(),
8745 });
8746 }
8747 if complete
8748 && !title.trim().is_empty()
8749 && !raw_body.trim().is_empty()
8750 && failed.is_empty()
8751 {
8752 decision = redecision;
8753 pr = PrMessage { title, body };
8754 rewritten = true;
8755 self.state
8756 .event("github-text", "description rewrite passed");
8757 if redacted {
8758 self.state.event(
8759 "github-text",
8760 "sensitive data removed from description rewrite",
8761 );
8762 }
8763 }
8764 }
8765 }
8766 if !rewritten {
8767 self.state.event(
8768 "github-text",
8769 "description rewrite unavailable or rejected; asking the owner",
8770 );
8771 if let Some(w) =
8772 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8773 {
8774 self.retire_github_text_question("the withheld text changed");
8777 let record = crate::run::GithubTextAsk {
8778 rewrite: rejected_rewrite,
8779 fingerprint: raw_fp,
8780 title: w.title,
8781 body: w.body,
8782 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8783 question: None,
8784 asks: 0,
8785 resolved: false,
8786 chosen_title: None,
8787 choice: None,
8788 };
8789 return self.settle_withheld(winner, &pr, record).await;
8790 }
8791 }
8792 }
8793 let (title, body) =
8794 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8795 Ok(Some(PrMessage { title, body }))
8796 }
8797
8798 async fn summarize_change(
8809 &mut self,
8810 winner: &Candidate,
8811 head: Option<&str>,
8812 ) -> std::result::Result<(), String> {
8813 let Some(head) = head else {
8814 return Err("the final commit could not be read".to_owned());
8815 };
8816 let remote = self.state.config.merge.remote.clone();
8817 let tracking = format!("{remote}/{}", self.state.base_branch);
8818 if !git::rev_exists(&winner.worktree, &tracking).await {
8819 return Err(format!("the remote base `{tracking}` could not be read"));
8820 }
8821 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8822 Ok(b) if !b.is_empty() => b,
8823 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8824 };
8825 let evidence = crate::pr_outcome::summary_evidence(
8826 &self.state,
8827 crate::pr_outcome::FinalHead::At(head),
8828 );
8829 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8830 if self
8831 .state
8832 .change_summary
8833 .as_ref()
8834 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8835 {
8836 return Ok(());
8837 }
8838 if self.state.change_summary.take().is_some() {
8839 self.state.event(
8840 "pr-summary",
8841 "the final diff or evidence changed; the earlier change summary was discarded",
8842 );
8843 }
8844 if let Err(e) = write_artifact(
8846 &self.state,
8847 &format!("public-description-{fingerprint}.md"),
8848 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8849 ) {
8850 tracing::warn!("could not save public description evidence: {e:#}");
8851 }
8852 let chain = match agent::pick_chain(
8853 &self.state.config.agents,
8854 self.state.config.roles.synthesizer.as_ref(),
8855 &agent::installed,
8856 "synthesizer",
8857 ) {
8858 Ok(chain) => chain,
8859 Err(e) => {
8860 tracing::warn!("no agent for the change summary: {e:#}");
8861 self.state.event("pr-summary", "no summary agent available");
8862 return Err("no agent was available to summarize it".to_owned());
8863 }
8864 };
8865 let subjects = git::subjects(&winner.worktree, &base, head)
8866 .await
8867 .unwrap_or_default();
8868 let stat = git::diff_stat(&winner.worktree, &base, head)
8869 .await
8870 .unwrap_or_default();
8871 let stat: String = stat.chars().take(6_000).collect();
8872 let mut prompt = crate::prompt::change_summary_in(
8873 &base,
8874 head,
8875 &subjects,
8876 stat.trim(),
8877 &self.state.config.graph.github_language,
8878 );
8879 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8880 let sessions = self.state.config.graph.sessions;
8881 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8882 let prompts = self.state.config.prompts.clone();
8883 let cache = self.state.config.cache_dir();
8884 let run = self.state.id.clone();
8885 for (i, spec) in chain.into_iter().enumerate() {
8886 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8887 let github_spec = match agent::for_github_text(
8888 &spec,
8889 &self.state.config.graph.github_language,
8890 &winner.worktree,
8891 ) {
8892 Ok(spec) => spec,
8893 Err(e) => {
8894 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8895 continue;
8896 }
8897 };
8898 let job = SeatJob {
8899 spec: github_spec,
8900 seat,
8901 cwd: winner.worktree.clone(),
8902 prompt: prompt.clone(),
8903 timeout,
8904 allow_write: false,
8905 sessions,
8906 artifacts: agent::artifacts_dir(&self.state.dir()),
8907 stem: if i == 0 {
8908 "change-summary".to_owned()
8909 } else {
8910 format!("change-summary-{}", spec.id)
8911 },
8912 handover: None,
8913 };
8914 let ctx = WaveCtx {
8915 run: &run,
8916 node: "pr-summary",
8917 prompts: &prompts,
8918 cache: cache.as_deref(),
8919 round: None,
8920 carry_seats: false,
8921 pauses: Some((&self.pause, &self.interrupt)),
8922 };
8923 let (seat, outcome) =
8924 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8925 self.state.seats.insert(seat.key.clone(), seat);
8926 let text = match outcome {
8927 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8928 AgentOutcome::ResourceBlocked(why) => {
8929 if self.state.resource_block.is_none() {
8932 self.state.resource_block = Some(crate::run::ResourceBlock {
8933 cause: crate::run::ResourceBlockCause::Other,
8934 node: "pr-summary".to_owned(),
8935 kind: "shared resource".to_owned(),
8936 detail: why,
8937 at: Timestamp::now(),
8938 });
8939 }
8940 return Err("a shared resource was busy".to_owned());
8941 }
8942 _ => continue,
8943 };
8944 let blind = &self.state.config.blind;
8945 let Some(title) = summary_title(&text) else {
8946 continue;
8947 };
8948 let body: String = summary_without_title(&text)
8949 .lines()
8950 .filter(|l| !l.contains("magi:run/"))
8951 .collect::<Vec<_>>()
8952 .join("\n");
8953 let body = blind::sanitize_prose(body.trim(), blind);
8954 let title = blind::sanitize_prose(&title, blind);
8955 if body.trim().is_empty() || title.trim().is_empty() {
8956 continue;
8957 }
8958 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8959 continue;
8960 }
8961 if git::rev_parse(&winner.worktree, "HEAD")
8963 .await
8964 .ok()
8965 .as_deref()
8966 != Some(head)
8967 {
8968 return Err("the branch moved while the summary was written".to_owned());
8969 }
8970 if git::merge_base(&winner.worktree, &tracking, head)
8971 .await
8972 .ok()
8973 .as_deref()
8974 != Some(base.as_str())
8975 || crate::pr_outcome::summary_evidence(
8976 &self.state,
8977 crate::pr_outcome::FinalHead::At(head),
8978 ) != evidence
8979 {
8980 return Err("the summary evidence changed while it was written".to_owned());
8981 }
8982 self.state.change_summary = Some(crate::run::ChangeSummary {
8983 head: head.to_owned(),
8984 base,
8985 evidence: fingerprint,
8986 title,
8987 body,
8988 });
8989 self.state.event(
8990 "pr-summary",
8991 format!("change summary written by {} for the final diff", spec.id),
8992 );
8993 if let Err(e) = self.state.save() {
8994 tracing::warn!("could not save the change summary: {e:#}");
8995 }
8996 return Ok(());
8997 }
8998 self.state
8999 .event("pr-summary", "no agent produced a change summary");
9000 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9001 }
9002
9003 async fn rewrite_pr_text(
9013 &mut self,
9014 winner: &Candidate,
9015 pr: &PrMessage,
9016 violations: &[crate::github_text::Violation],
9017 ) -> Option<String> {
9018 let prompt = format!(
9019 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9020 pr.title, pr.body
9021 );
9022 let sessions = self.state.config.graph.sessions;
9023 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9024 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9027 let key = format!("impl-{}", winner.label);
9028 if let Some(spec) = self
9029 .state
9030 .config
9031 .agents
9032 .iter()
9033 .find(|a| a.id == winner.agent)
9034 .cloned()
9035 && let Some(existing) = self.state.seats.get(&key)
9036 && existing.agent == spec.id
9037 && agent::has_session(spec.kind, existing, sessions)
9038 {
9039 attempts.push((spec, Some(existing.clone())));
9040 }
9041 match agent::pick_chain(
9042 &self.state.config.agents,
9043 self.state.config.roles.language_judge.as_ref(),
9044 &agent::installed,
9045 "language_judge",
9046 ) {
9047 Ok(chain) => {
9048 for spec in chain {
9049 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9050 continue;
9051 }
9052 attempts.push((spec, None));
9053 }
9054 }
9055 Err(e) if attempts.is_empty() => {
9056 tracing::warn!("no agent for the description rewrite: {e:#}");
9057 self.state
9058 .event("github-text", "no rewrite agent available");
9059 return None;
9060 }
9061 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9062 }
9063 let prompts = self.state.config.prompts.clone();
9064 let cache = self.state.config.cache_dir();
9065 let run = self.state.id.clone();
9066 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9067 let resumed = resumed_seat.is_some();
9068 let seat = resumed_seat.unwrap_or_else(|| {
9069 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9070 });
9071 let github_spec = match agent::for_github_text(
9072 &spec,
9073 &self.state.config.graph.github_language,
9074 &winner.worktree,
9075 ) {
9076 Ok(spec) => spec,
9077 Err(e) => {
9078 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9079 continue;
9080 }
9081 };
9082 let job = SeatJob {
9083 spec: github_spec,
9084 seat,
9085 cwd: winner.worktree.clone(),
9086 prompt: prompt.clone(),
9087 timeout: retry_budget(timeout_full, resumed),
9088 allow_write: false,
9089 sessions,
9090 artifacts: agent::artifacts_dir(&self.state.dir()),
9091 stem: if i == 0 {
9092 "github-text-rewrite".to_owned()
9093 } else {
9094 format!("github-text-rewrite-{}", spec.id)
9095 },
9096 handover: None,
9097 };
9098 let ctx = WaveCtx {
9099 run: &run,
9100 node: "github-text",
9101 prompts: &prompts,
9102 cache: cache.as_deref(),
9103 round: None,
9104 carry_seats: false,
9105 pauses: Some((&self.pause, &self.interrupt)),
9106 };
9107 let (seat, outcome) =
9108 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9109 self.state.seats.insert(seat.key.clone(), seat);
9110 match outcome {
9111 AgentOutcome::Ok(o)
9112 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9113 {
9114 return Some(o.text);
9115 }
9116 AgentOutcome::ResourceBlocked(_) => return None,
9117 _ => {}
9118 }
9119 }
9120 None
9121 }
9122
9123 fn retire_github_text_question(&mut self, why: &str) {
9125 let Some(id) = self
9126 .state
9127 .github_text
9128 .as_ref()
9129 .and_then(|g| g.question.clone())
9130 else {
9131 return;
9132 };
9133 let _ = ask::Questions::open().update(&id, |q| {
9134 q.abandon(why);
9135 Ok(())
9136 });
9137 }
9138
9139 async fn settle_withheld(
9143 &mut self,
9144 winner: &Candidate,
9145 pr: &PrMessage,
9146 mut g: crate::run::GithubTextAsk,
9147 ) -> Result<Option<PrMessage>> {
9148 use crate::github_text as gt;
9149 let store = ask::Questions::open();
9150 let timeout = self.state.config.graph.answer_timeout;
9151 if !g.resolved {
9152 let mut standing = g
9153 .question
9154 .as_ref()
9155 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9156 if let Some(q) = standing.as_ref()
9157 && q.status.open()
9158 && gt::expired(q, timeout)
9159 {
9160 let why = format!("no answer within {}s of asking", timeout.max(1));
9161 standing = store
9162 .update(&q.id, |q| {
9163 q.abandon(&why);
9164 Ok(())
9165 })
9166 .ok()
9167 .map(|(q, ())| q);
9168 }
9169 let mut ask_again: Option<Option<Vec<String>>> = None;
9170 match standing {
9171 None if g.asks <= 1 => {
9174 g.asks = 0;
9175 ask_again = Some(None);
9176 }
9177 None => {
9178 g.resolved = true;
9179 g.chosen_title = None;
9180 g.choice = Some(GithubTextChoice::Fallback);
9181 }
9182 Some(q) if q.status.open() => {
9183 self.park_on_github_text(&g)?;
9184 return Ok(None);
9185 }
9186 Some(q) => match gt::read_reply(&q) {
9187 gt::Reply::Fallback => {
9188 g.resolved = true;
9189 g.chosen_title = None;
9190 g.choice = Some(GithubTextChoice::Fallback);
9191 }
9192 gt::Reply::Original => {
9193 g.resolved = true;
9194 g.chosen_title = None;
9195 g.choice = Some(GithubTextChoice::Original);
9196 }
9197 gt::Reply::Rewrite => {
9198 g.resolved = true;
9199 g.chosen_title = None;
9200 g.choice = Some(if g.rewrite.is_some() {
9201 GithubTextChoice::Rewrite
9202 } else {
9203 GithubTextChoice::Fallback
9204 });
9205 }
9206 gt::Reply::Title(text) => {
9207 let first = text
9208 .lines()
9209 .map(str::trim)
9210 .find(|l| !l.is_empty())
9211 .unwrap_or("");
9212 let decision = if first.is_empty() {
9213 None
9214 } else {
9215 let candidate = PrMessage {
9216 title: first.to_owned(),
9217 body: String::new(),
9218 };
9219 self.judge_pr_language(&winner.worktree, &candidate).await
9220 };
9221 match gt::vet_title(
9222 &text,
9223 decision,
9224 self.state.config.graph.github_english(),
9225 ) {
9226 Ok(title) => {
9227 g.resolved = true;
9228 g.chosen_title = Some(title);
9229 g.choice = Some(GithubTextChoice::MyTitle);
9230 }
9231 Err(cats) if g.asks < 2 => {
9232 ask_again = Some(Some(
9233 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9234 ))
9235 }
9236 Err(_) => {
9237 g.resolved = true;
9238 g.chosen_title = None;
9239 g.choice = Some(GithubTextChoice::Fallback);
9240 }
9241 }
9242 }
9243 },
9244 }
9245 if let Some(rejected) = ask_again {
9246 let w = gt::Withheld {
9249 title: g.title,
9250 body: g.body,
9251 categories: g.categories.clone(),
9252 };
9253 let artifact = match crate::run::write_artifact(
9256 &self.state,
9257 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9258 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9259 ) {
9260 Ok(p) => Some(p),
9261 Err(e) => {
9262 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9263 None
9264 }
9265 };
9266 let (detail, comparison) = gt::question_content(
9267 &w,
9268 &pr.title,
9269 &pr.body,
9270 &self.state.id,
9271 rejected.as_deref(),
9272 artifact.as_deref(),
9273 g.rewrite.as_ref(),
9274 );
9275 let mut q = ask::Question::new(
9276 self.state.id.clone(),
9277 gt::ASK_NODE.to_owned(),
9278 gt::ASK_SEAT.to_owned(),
9279 gt::question_summary(&self.state.config.graph.language, &w),
9280 detail,
9281 gt::question_choices(&w, g.rewrite.as_ref()),
9282 );
9283 q.posting_comparison = Some(comparison);
9284 q.answer_timeout = timeout;
9285 g.question = Some(q.id.clone());
9288 g.asks += 1;
9289 self.state.github_text = Some(g.clone());
9290 self.state.event(
9291 "github-text",
9292 format!("asking the owner about the withheld text ({})", q.short()),
9293 );
9294 self.state.save()?;
9295 store
9296 .put_executable(&mut q)
9297 .context("file the posting-gate question")?;
9298 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9299 tracing::warn!(
9300 "could not notify about posting-gate question {}: {e:#}",
9301 q.short()
9302 );
9303 }
9304 self.park_on_github_text(&g)?;
9305 return Ok(None);
9306 }
9307 }
9308 let resolved = g.chosen_title.clone();
9309 let original = g.choice == Some(GithubTextChoice::Original);
9310 self.state.event(
9312 "github-text",
9313 if original {
9314 format!(
9315 "posting the original text for: {}",
9316 match (g.title, g.body) {
9317 (true, true) => "title, description",
9318 (true, false) => "title",
9319 _ => "description",
9320 }
9321 )
9322 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9323 "posting the saved rewrite title and description".to_owned()
9324 } else if resolved.is_some() {
9325 "posting the owner's replacement title".to_owned()
9326 } else {
9327 "posting the neutral text for the withheld fields".to_owned()
9328 },
9329 );
9330 self.state.github_text = Some(g.clone());
9331 self.state.save()?;
9332 if g.choice == Some(GithubTextChoice::Rewrite)
9333 && let Some(r) = g.rewrite.as_ref()
9334 {
9335 return Ok(Some(PrMessage {
9336 title: r.title.clone(),
9337 body: r.body.clone(),
9338 }));
9339 }
9340 let w = gt::Withheld {
9341 title: g.title,
9342 body: g.body,
9343 categories: g.categories.clone(),
9344 };
9345 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9346 if let Some(t) = resolved {
9347 title = t;
9348 }
9349 let english = gt::LanguageDecision {
9351 title_english: true,
9352 body_english: true,
9353 body_complete: true,
9354 };
9355 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9356 if original {
9357 if g.title {
9360 title = pr.title.clone();
9361 }
9362 if g.body {
9363 body = pr.body.clone();
9364 }
9365 }
9366 Ok(Some(PrMessage { title, body }))
9367 }
9368
9369 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9370 self.state.github_text = Some(g.clone());
9371 self.state.end_pass_parked(ParkCause::Question);
9372 self.state.parked = true;
9373 self.state.event(
9374 "github-text",
9375 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9376 );
9377 self.state.save()
9378 }
9379
9380 async fn judge_pr_language(
9383 &mut self,
9384 cwd: &Path,
9385 pr: &PrMessage,
9386 ) -> Option<crate::github_text::LanguageDecision> {
9387 self.state.config.roles.language_judge.as_ref()?;
9388 if !self.state.config.graph.language_gate() {
9389 return None;
9390 }
9391 let decision =
9392 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9393 self.state.event(
9394 "github-text",
9395 if decision.is_some() {
9396 "language decided by the language judge"
9397 } else {
9398 "language judge unavailable; using built-in heuristics"
9399 },
9400 );
9401 decision
9402 }
9403
9404 async fn merge(&mut self) -> Result<()> {
9405 if self.resource_halt()? || self.hold_base_sync().await? {
9406 return Ok(());
9407 }
9408 if self
9423 .state
9424 .base_sync
9425 .as_ref()
9426 .is_some_and(|s| s.conflict.is_some())
9427 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9428 != Some(RunStatus::Gating)
9429 || !self.state.gate_allows_progress()
9437 {
9438 return Ok(());
9439 }
9440 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9449 return Ok(());
9450 }
9451 let Some(winner) = self.state.winner().cloned() else {
9452 return Ok(());
9453 };
9454 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9455 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9456 self.state.status = RunStatus::Blocked;
9457 self.state.event(
9458 "gate",
9459 "winner changed after gate; fresh verification required",
9460 );
9461 self.state.save()?;
9462 return Ok(());
9463 }
9464 let wait = self.repository_wait();
9465 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9466 return Ok(());
9467 }
9468 let repo = self.state.repo.clone();
9469 let base = self.state.base_branch.clone();
9470 let mode = self.state.config.merge.mode;
9471 let style = self.state.config.merge.style;
9472 let facts = if is_review_run(&self.state) {
9473 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9474 let start = review_base(
9475 &repo,
9476 &self.state.config.merge.remote,
9477 &base,
9478 &self.state.base_commit,
9479 &winner.branch,
9480 )
9481 .await;
9482 branch_facts(&repo, &start, &winner.branch).await
9483 } else {
9484 None
9485 };
9486 let leaked = if is_review_run(&self.state) {
9489 leaked_subjects(&self.state, &winner.branch).await
9490 } else {
9491 Some(Vec::new())
9492 };
9493 let Some(pr) = self
9497 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9498 .await?
9499 else {
9500 return Ok(());
9501 };
9502 let message = pr.commit_message();
9503
9504 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9505 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9506 return Ok(());
9507 }
9508 let outcome = match mode {
9509 MergeMode::None => MergeOutcome {
9510 mode,
9511 ok: true,
9512 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9513 empty: false,
9514 },
9515 MergeMode::Pr | MergeMode::Local
9516 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9517 {
9518 MergeOutcome {
9519 mode,
9520 ok: false,
9521 detail: empty_candidate_detail(&self.state, &base),
9522 empty: true,
9523 }
9524 }
9525 MergeMode::Local => {
9526 let on = git::current_branch(&repo).await?;
9527 if on.as_deref() != Some(base.as_str()) {
9528 MergeOutcome {
9529 mode,
9530 ok: false,
9531 detail: format!(
9532 "{} has {} checked out, not the base branch {base}",
9533 repo.display(),
9534 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9535 ),
9536 empty: false,
9537 }
9538 } else if !git::is_clean(&repo).await? {
9539 MergeOutcome {
9540 mode,
9541 ok: false,
9542 detail: format!("{} is dirty; refusing to merge", repo.display()),
9543 empty: false,
9544 }
9545 } else {
9546 let out = match style {
9547 MergeStyle::Merge => {
9548 git::merge_no_ff(&repo, &winner.branch, &message).await?
9549 }
9550 MergeStyle::Squash => {
9551 git::merge_squash(&repo, &winner.branch, &message).await?
9552 }
9553 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9554 };
9555 MergeOutcome {
9556 mode,
9557 ok: out.ok(),
9558 detail: if out.ok() { out.stdout } else { out.stderr },
9559 empty: false,
9560 }
9561 }
9562 }
9563 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9564 self.state.pr_publication_pending = true;
9565 self.state.save()?;
9566 let out =
9567 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9568 self.state.pr_publication_pending = out.is_err();
9569 MergeOutcome {
9570 mode,
9571 ok: out.is_ok(),
9572 detail: match out {
9573 Ok(url) => url,
9574 Err(e) => format!("{e:#}"),
9575 },
9576 empty: false,
9577 }
9578 }
9579 MergeMode::Pr => {
9580 let remote = self.state.config.merge.remote.clone();
9581 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9582 let pushed = if self.state.foreign_checkout.is_some() {
9583 let refspec = format!("{delivery_head}:refs/heads/{public}");
9584 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9585 } else {
9586 git::push(&winner.worktree, &remote, &winner.branch).await?
9587 };
9588 if !pushed.ok() {
9589 MergeOutcome {
9590 mode,
9591 ok: false,
9592 detail: pushed.stderr,
9593 empty: false,
9594 }
9595 } else {
9596 if self.state.foreign_checkout.is_some() {
9597 self.state.pr_pushed_head = Some(delivery_head.clone());
9598 }
9599 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9605 let out = match pr_merge_plan(found) {
9606 PrPlan::Create => {
9607 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9608 .await
9609 }
9610 PrPlan::Adopt { url, title } => {
9611 self.state
9612 .event("merge", format!("Pr: adopted open pull request {url}"));
9613 if self.state.github_text.as_ref().is_some_and(|g| {
9614 g.resolved
9615 && g.choice == Some(GithubTextChoice::Rewrite)
9616 && g.rewrite.is_some()
9617 }) {
9618 land::set_pr_text_verbatim(
9621 &winner.worktree,
9622 &url,
9623 &pr.title,
9624 &pr.body,
9625 )
9626 .await
9627 .map(|()| url)
9628 } else {
9629 if title != pr.title
9630 && (!is_review_run(&self.state)
9631 || leaked
9632 .as_deref()
9633 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9634 && let Err(e) =
9635 if self.state.github_text.as_ref().is_some_and(|g| {
9636 g.resolved
9637 && g.title
9638 && g.choice == Some(GithubTextChoice::Original)
9639 }) {
9640 land::set_pr_title_verbatim(
9642 &winner.worktree,
9643 &url,
9644 &pr.title,
9645 )
9646 .await
9647 } else {
9648 land::set_pr_title(
9649 &mut self.state,
9650 &winner.worktree,
9651 &url,
9652 &pr.title,
9653 )
9654 .await
9655 }
9656 {
9657 tracing::warn!("could not refresh title of {url}: {e:#}");
9658 self.state
9659 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9660 }
9661 Ok(url)
9662 }
9663 }
9664 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9665 };
9666 match out {
9667 Ok(url) => MergeOutcome {
9668 mode,
9669 ok: true,
9670 detail: url,
9671 empty: false,
9672 },
9673 Err(e) => MergeOutcome {
9674 mode,
9675 ok: false,
9676 detail: e.to_string(),
9677 empty: false,
9678 },
9679 }
9680 }
9681 }
9682 };
9683
9684 self.state.status = match (mode, outcome.ok) {
9685 (MergeMode::None, _) => RunStatus::Ready,
9686 (_, true) if outcome.empty => RunStatus::Ready,
9687 (MergeMode::Pr, true) => RunStatus::Ready,
9688 (_, true) => RunStatus::Merged,
9689 (_, false) => RunStatus::Blocked,
9690 };
9691 self.state.event(
9692 "merge",
9693 format!(
9694 "{:?}: {}",
9695 mode,
9696 outcome.detail.lines().next().unwrap_or("")
9697 ),
9698 );
9699 self.state.merge = Some(outcome);
9700 self.state.save()?;
9701
9702 if self.state.config.graph.land
9708 && mode == MergeMode::Pr
9709 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9710 {
9711 self.run_land().await?;
9712 }
9713 self.settle_questions();
9718 Ok(())
9719 }
9720
9721 async fn run_land(&mut self) -> Result<()> {
9732 crate::pr_head::ensure_owner(&self.state)?;
9733 let url = self
9734 .state
9735 .pr
9736 .as_ref()
9737 .map(|pr| pr.url.clone())
9738 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9739 .unwrap_or_default();
9740 let url = url.lines().next().unwrap_or("").trim().to_owned();
9741 if !url.starts_with("http") {
9742 return Ok(());
9743 }
9744 loop {
9747 match land::land(&mut self.state, &url).await {
9748 Ok(_)
9749 if self
9750 .state
9751 .merge_repairs
9752 .last()
9753 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9754 {
9755 if self.repair_merge_review().await? {
9756 continue;
9757 }
9758 }
9759 Ok(_) if self.state.resource_block.is_some() => {
9760 return self.stop_for_resource();
9761 }
9762 Ok(pr) if self.state.parked => {
9763 let _ = pr;
9767 }
9768 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9769 self.state.status = RunStatus::Superseded;
9770 self.state.save()?;
9771 }
9772 Ok(pr) => {
9773 self.state.status = match pr.state {
9774 land::PrLifecycle::Merged => RunStatus::Merged,
9775 _ => RunStatus::Blocked,
9776 };
9777 if bump::should_release_bump(self.state.status)
9784 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9785 {
9786 self.state
9792 .event("bump", format!("release bump skipped: {e:#}"));
9793 }
9794 if self.state.status == RunStatus::Merged {
9797 crate::followup::after_merge(&mut self.state, &pr.url).await;
9798 }
9799 self.state.save()?;
9800 }
9801 Err(e) => {
9802 if e.is::<crate::repo_lock::ResourceBlocked>() {
9803 return Err(e);
9804 }
9805 self.state.status = RunStatus::Blocked;
9806 self.state.event("land", format!("gave up: {e}"));
9807 self.state.save()?;
9808 }
9809 }
9810 break;
9811 }
9812 Ok(())
9813 }
9814
9815 async fn repair_merge_review(&mut self) -> Result<bool> {
9818 use crate::run::MergeRepairPhase as Phase;
9819 crate::pr_head::ensure_owner(&self.state)?;
9820 let i = self.state.merge_repairs.len() - 1;
9821 let request = self.state.merge_repairs[i].clone();
9822 if request.phase == Phase::Fixing {
9823 let restored = match self.state.winner().cloned() {
9824 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9825 None => Err(anyhow::anyhow!("repair has no winning branch")),
9826 };
9827 let why = match restored {
9828 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9829 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9830 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9831 };
9832 self.hold_merge_repair(i, why)?;
9833 return Ok(false);
9834 }
9835 if request.phase == Phase::Reviewing && self.state.gate_ran {
9840 let winner = self
9841 .state
9842 .winner()
9843 .context("repair has no winning branch")?;
9844 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9845 if self.state.gate_head.as_deref() != Some(head.as_str())
9846 || !git::is_clean(&winner.worktree).await?
9847 {
9848 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9849 return Ok(false);
9850 }
9851 }
9852 let failure = if i >= self.state.config.graph.land_rounds
9853 || self.state.config.graph.review_rounds == 0
9854 {
9855 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9856 } else {
9857 None
9858 };
9859 if let Some(why) = failure {
9860 self.hold_merge_repair(i, why)?;
9861 return Ok(false);
9862 }
9863 if request.phase == Phase::Requested {
9864 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9865 Ok(pr) => pr,
9866 Err(e) => {
9867 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9868 return Ok(false);
9869 }
9870 };
9871 self.state.merge_repairs[i].phase = Phase::Fixing;
9872 self.state.land_approval = None;
9873 self.state.land_armed_head = None;
9874 self.state.save()?;
9875 let evidence = serde_json::to_string_pretty(&request)?;
9878 let reason = format!(
9879 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9880 );
9881 let budget = self.state.config.graph.land_rounds;
9882 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9883 match result {
9884 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9885 Ok(land::Fixed::Deferred) => {
9886 let worktree = self
9890 .state
9891 .winner()
9892 .context("repair has no winning branch")?
9893 .worktree
9894 .clone();
9895 self.graft_restore(&worktree, &request.head).await?;
9896 self.state.merge_repairs[i].phase = Phase::Requested;
9897 self.stop_for_resource()?;
9898 return Ok(false);
9899 }
9900 Ok(land::Fixed::Failed(why)) => {
9901 self.hold_merge_repair(
9902 i,
9903 format!(
9904 "repair failed: {why}; inspect the branch and run an explicit review"
9905 ),
9906 )?;
9907 return Ok(false);
9908 }
9909 Err(e) => {
9910 self.hold_merge_repair(
9911 i,
9912 format!(
9913 "repair failed: {e:#}; inspect the branch and run an explicit review"
9914 ),
9915 )?;
9916 return Ok(false);
9917 }
9918 }
9919 self.begin_merge_repair_review(i)?;
9923 }
9924 self.review_loop().await?;
9925 self.gate().await?;
9926 if self.resource_halt()? || self.state.parked {
9927 return Ok(false);
9928 }
9929 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9930 != Some(RunStatus::Gating)
9931 || !self.state.gate_allows_progress()
9932 {
9933 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9934 return Ok(false);
9935 }
9936 let winner = self
9939 .state
9940 .winner()
9941 .cloned()
9942 .context("repair has no winning branch")?;
9943 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9944 if self.state.gate_head.as_deref() != Some(after.as_str())
9945 || !git::is_clean(&winner.worktree).await?
9946 {
9947 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9948 return Ok(false);
9949 }
9950 let expected = self
9951 .state
9952 .pr_pushed_head
9953 .as_deref()
9954 .unwrap_or(&request.head)
9955 .to_owned();
9956 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9957 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9960 return Ok(false);
9961 }
9962 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9963 let refspec = format!("{after}:refs/heads/{public}");
9964 let push = git::git_raw(
9965 &winner.worktree,
9966 &["push", &lease, &self.state.config.merge.remote, &refspec],
9967 )
9968 .await;
9969 match push {
9970 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9971 Ok(out) => {
9972 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9973 return Ok(false);
9974 }
9975 Err(e) => {
9976 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9977 return Ok(false);
9978 }
9979 }
9980 self.state.merge_repairs[i].phase = Phase::Complete;
9981 self.state.status = RunStatus::Landing;
9982 self.state.event(
9983 "repair",
9984 "new review and gate completed; fresh owner merge decision required",
9985 );
9986 self.state.save()?;
9987 Ok(true)
9988 }
9989
9990 fn merge_repair_artifacts(&self) -> PathBuf {
9991 let artifacts = agent::artifacts_dir(&self.state.dir());
9992 match self
9993 .state
9994 .merge_repairs
9995 .last()
9996 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9997 {
9998 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9999 None => artifacts,
10000 }
10001 }
10002
10003 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10004 if let Some(e2e) = self.state.head_e2e.take() {
10005 self.state
10006 .verification_history
10007 .push(crate::run::VerificationEvidence {
10008 superseded_by: None,
10009 archived_at: Some(Timestamp::now()),
10010 fixes_spent: Some(self.state.gate_fixes.len()),
10011 gate_head: None,
10012 gate_ran: false,
10013 gate: Vec::new(),
10014 e2e: Some(e2e),
10015 });
10016 }
10017 self.state.reviews.clear();
10018 self.state.gate.clear();
10019 self.state.gate_ran = false;
10020 self.state.gate_head = None;
10021 self.state.gate_fixes.clear();
10022 self.state.gate_repair_e2e = None;
10023 self.state.pre_gate.clear();
10024 self.state.pre_gate_commit = None;
10025 self.state.contested_handoff = None;
10026 self.state.land_reruns = None;
10027 self.state.land_approval = None;
10028 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10029 self.state.status = RunStatus::Reviewing;
10030 self.state.save()
10031 }
10032
10033 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10034 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10035 self.state.merge_repairs[index].failure = Some(why.clone());
10036 self.state.status = RunStatus::Blocked;
10037 self.state.event("repair", why);
10038 self.state.save()
10039 }
10040
10041 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10045 if let Some(existing) = self.state.seats.get(key)
10046 && existing.agent == agent
10047 {
10048 return existing.clone();
10049 }
10050 let fresh = if self.state.seats.contains_key(key) {
10053 handover_seat(key, agent, self.state.next_seat_seed())
10054 } else {
10055 SeatState::new(key, agent, self.state.seed)
10056 };
10057 self.state.seats.insert(key.to_owned(), fresh.clone());
10058 fresh
10059 }
10060
10061 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10067 match self.state.seats.get(key) {
10068 Some(s) if s.agent != spec.id => {
10069 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10070 }
10071 _ => spec,
10072 }
10073 }
10074
10075 fn view(&self, c: &Candidate) -> CandidateView {
10077 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10078 .unwrap_or_default();
10079 let (patch, _) = blind::sanitize_patch(
10080 &format!("candidate {} patch", c.label),
10081 &raw,
10082 &self.state.config.blind,
10083 );
10084 CandidateView {
10085 label: c.label,
10086 branch: c.branch.clone(),
10087 summary: c.summary.clone(),
10088 stat: c.stat.clone(),
10089 patch,
10090 }
10091 }
10092
10093 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10095 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10096 prompt::judge(
10097 "(see above)",
10098 &views,
10099 self.roles.judges.len(),
10100 base_short,
10101 "en",
10102 )
10103 }
10104
10105 fn vote_prompt_full(
10111 &self,
10112 j: usize,
10113 labels: &[char],
10114 language: &str,
10115 candidates: &[Candidate],
10116 base_short: &str,
10117 ) -> String {
10118 let mut text = format!(
10119 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10120 prompt::final_vote(labels, language),
10121 self.state.instruction,
10122 self.candidate_block(candidates, base_short)
10123 );
10124 if let Some(own) = self
10125 .state
10126 .judgements
10127 .get(j)
10128 .filter(|r| !r.ranking.is_empty())
10129 {
10130 let reasons = own
10131 .reasons
10132 .iter()
10133 .map(|(k, v)| format!("- {k}: {v}"))
10134 .collect::<Vec<_>>()
10135 .join("\n");
10136 text.push_str(&format!(
10137 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10138 own.ranking.iter().collect::<String>(),
10139 if reasons.is_empty() {
10140 ""
10141 } else {
10142 ", because:\n"
10143 }
10144 ));
10145 }
10146 if !self.state.deliberation.is_empty() {
10147 text.push_str("\n# What was argued before this vote\n");
10148 for t in self.transcript(&[], j) {
10149 text.push_str(&format!(
10150 "\n## {}{}\n\n{}\n",
10151 t.who,
10152 if t.is_self { " (you)" } else { "" },
10153 t.body.trim()
10154 ));
10155 }
10156 }
10157 text
10158 }
10159
10160 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10167 let mut turns = Vec::new();
10168 for j in &self.state.judgements {
10169 if j.ranking.is_empty() {
10170 continue;
10171 }
10172 let reasons = j
10173 .reasons
10174 .iter()
10175 .map(|(k, v)| format!("- {k}: {v}"))
10176 .collect::<Vec<_>>()
10177 .join("\n");
10178 turns.push(Turn {
10179 who: format!("Judge {} (opening ranking)", j.judge),
10180 is_self: j.judge == self_idx + 1,
10181 body: format!(
10182 "Ranked {}{}{reasons}",
10183 j.ranking.iter().collect::<String>(),
10184 if reasons.is_empty() {
10185 ""
10186 } else {
10187 ", because:\n"
10188 }
10189 ),
10190 });
10191 }
10192 for t in self
10193 .state
10194 .deliberation
10195 .iter()
10196 .flat_map(|r| r.turns.iter())
10197 .chain(current)
10198 {
10199 turns.push(Turn {
10200 who: format!("Judge {}", t.judge),
10201 is_self: t.judge == self_idx + 1,
10202 body: t.body.clone(),
10203 });
10204 }
10205 turns
10206 }
10207}
10208
10209fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10211 agent::has_session(spec.kind, seat, sessions)
10212}
10213
10214fn next_untried_in_roster<'a>(
10235 roster: &'a [AgentSpec],
10236 start: usize,
10237 tried: &BTreeSet<String>,
10238) -> Option<&'a AgentSpec> {
10239 roster
10240 .get(start + 1..)?
10241 .iter()
10242 .find(|s| !tried.contains(&s.id))
10243}
10244
10245fn pick_successor<'a>(
10266 roster: &'a [AgentSpec],
10267 start: usize,
10268 tried: &BTreeSet<String>,
10269 carried: Option<&BTreeSet<String>>,
10270 others: &BTreeSet<String>,
10271) -> Option<&'a AgentSpec> {
10272 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10273 roster
10274 .get(start + 1..)
10275 .and_then(|tail| {
10276 tail.iter()
10277 .filter(free)
10278 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10279 })
10280 .or_else(|| {
10281 carried
10282 .is_some()
10283 .then(|| roster.iter().find(free))
10284 .flatten()
10285 })
10286 .or_else(|| match carried {
10287 Some(c) => next_for_seat(roster, start, tried, c),
10288 None => next_untried_in_roster(roster, start, tried),
10289 })
10290}
10291
10292fn next_for_seat<'a>(
10302 roster: &'a [AgentSpec],
10303 start: usize,
10304 round_tried: &BTreeSet<String>,
10305 carried_failed: &BTreeSet<String>,
10306) -> Option<&'a AgentSpec> {
10307 roster
10308 .get(start + 1..)?
10309 .iter()
10310 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10311 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10312}
10313
10314fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10320 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10321 return spec;
10322 };
10323 let ok = |id: &str| !h.failed.contains(id);
10324 if let Some(last) = h.last_ok.as_deref()
10325 && ok(last)
10326 && let Some(s) = roster.iter().find(|s| s.id == last)
10327 {
10328 return s.clone();
10329 }
10330 if ok(&spec.id) {
10331 return spec;
10332 }
10333 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10334 roster
10335 .get(from + 1..)
10336 .into_iter()
10337 .flatten()
10338 .chain(roster.iter())
10339 .find(|s| ok(&s.id))
10340 .cloned()
10341 .unwrap_or(spec)
10342}
10343
10344pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10348 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10349}
10350
10351const TIMED_OUT: &str = "timed out";
10354
10355impl FailClass {
10356 fn of(out: &AgentOutcome) -> Option<Self> {
10358 match out {
10359 AgentOutcome::Ok(_) => None,
10360 AgentOutcome::Quota(_) => Some(Self::Quota),
10361 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10362 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10363 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10364 AgentOutcome::ResourceBlocked(_) => None,
10367 }
10368 }
10369
10370 fn stem_word(&self) -> &'static str {
10372 match self {
10373 Self::Quota => "quota",
10374 _ => "handover",
10375 }
10376 }
10377}
10378
10379fn failure_signature(msg: &str) -> String {
10383 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10384 if let Some(code) = line
10387 .strip_prefix("process exited with no usable output (exit code ")
10388 .and_then(|s| s.strip_suffix(')'))
10389 .and_then(|s| s.parse::<i32>().ok())
10390 {
10391 line = format!("exited with some({code}) and no usable output");
10392 } else if matches!(
10393 line.as_str(),
10394 "process exited with no usable output (no exit code reported)"
10395 | "process exited with no usable output (exit code unavailable)"
10396 ) {
10397 line = "exited with none and no usable output".to_owned();
10398 }
10399 let mut out = Vec::new();
10400 for word in line.split_whitespace() {
10401 if word.contains('/') || word.contains('\\') {
10402 out.push("<path>".to_owned());
10403 continue;
10404 }
10405 let mut w = String::new();
10406 let mut in_digits = false;
10407 for c in word.chars() {
10408 if c.is_ascii_digit() {
10409 if !in_digits {
10410 w.push('#');
10411 }
10412 in_digits = true;
10413 } else {
10414 in_digits = false;
10415 w.push(c);
10416 }
10417 }
10418 out.push(w);
10419 }
10420 out.join(" ").chars().take(120).collect()
10421}
10422
10423fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10430 match cur {
10431 FailClass::Quota | FailClass::Timeout => true,
10432 FailClass::Other(_) => prev != Some(cur),
10433 }
10434}
10435
10436fn fail_reason(out: &AgentOutcome) -> String {
10438 match out {
10439 AgentOutcome::Ok(_) => String::new(),
10440 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10441 AgentOutcome::Dropped(o) => format!(
10442 "the CLI dropped the stream ({})",
10443 o.dropped
10444 .as_ref()
10445 .map(|d| d.why.as_str())
10446 .unwrap_or("it ended without delivering its answer")
10447 ),
10448 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10449 e.lines().next().unwrap_or("").chars().take(160).collect()
10450 }
10451 }
10452}
10453
10454pub(crate) fn record_handover(
10457 state: &mut RunState,
10458 node: &str,
10459 seat: &str,
10460 from: &str,
10461 to: &str,
10462 class: &FailClass,
10463 reason: &str,
10464) {
10465 let message = if *class == FailClass::Quota {
10466 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10467 } else {
10468 format!("{seat}: handed over {from} -> {to} ({reason})")
10469 };
10470 state.event(node, message);
10471 state.handovers.push(Handover {
10472 at: Timestamp::now(),
10473 node: node.to_owned(),
10474 seat: seat.to_owned(),
10475 from: from.to_owned(),
10476 to: to.to_owned(),
10477 reason: reason.to_owned(),
10478 });
10479}
10480
10481fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10495 commands.iter().any(|c| c.exit_code.is_none())
10496}
10497
10498fn verified_noop_claim(
10511 usable: bool,
10512 commands: &[agent::CommandEvidence],
10513 text: &str,
10514) -> Option<String> {
10515 (usable && !has_unconfirmed_command(commands))
10516 .then(|| verdict::verified_noop(text))
10517 .flatten()
10518}
10519
10520fn short(commit: &str) -> String {
10521 commit.chars().take(7).collect()
10522}
10523
10524fn make_executable(path: &Path) -> Result<()> {
10525 #[cfg(unix)]
10526 {
10527 use std::os::unix::fs::PermissionsExt as _;
10528 let mut perms = std::fs::metadata(path)?.permissions();
10529 perms.set_mode(0o755);
10530 std::fs::set_permissions(path, perms)?;
10531 }
10532 #[cfg(not(unix))]
10533 {
10534 let _ = path;
10535 }
10536 Ok(())
10537}
10538
10539struct WaveCtx<'a> {
10546 run: &'a str,
10549 node: &'a str,
10551 prompts: &'a Prompts,
10552 cache: Option<&'a Path>,
10554 round: Option<usize>,
10557 carry_seats: bool,
10561 pauses: Option<(&'a Pause, &'a Pause)>,
10564}
10565
10566fn repository_wait(
10568 state: &RunState,
10569 pauses: Option<(&Pause, &Pause)>,
10570) -> crate::repo_lock::WaitPolicy {
10571 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10572 let inherited = crate::repo_lock::WaitPolicy::current();
10573 crate::repo_lock::WaitPolicy::new(
10574 Duration::from_secs(state.config.graph.timeout_resource_wait),
10575 move || {
10576 inherited.cancelled()
10577 || pauses
10578 .as_ref()
10579 .is_some_and(|(a, b)| a.parked() || b.parked())
10580 },
10581 )
10582}
10583
10584fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10585 if error.is::<crate::repo_lock::ResourceBlocked>() {
10586 state.resource_block = Some(crate::run::ResourceBlock {
10587 cause: crate::run::ResourceBlockCause::Other,
10588 node: "repository".into(),
10589 kind: if error.to_string().contains("disk pressure") {
10590 "disk pressure"
10591 } else if error.to_string().contains("disk admission") {
10592 "disk admission"
10593 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10594 "incremental cache admission"
10595 } else {
10596 "repository coordination"
10597 }
10598 .into(),
10599 detail: error.to_string(),
10600 at: Timestamp::now(),
10601 });
10602 state.event("resource", error.to_string());
10603 }
10604}
10605
10606async fn coordinate_operation(
10607 state: &mut RunState,
10608 cwd: &Path,
10609 wait: &crate::repo_lock::WaitPolicy,
10610) -> Result<Option<crate::repo_lock::SeatLease>> {
10611 let result =
10612 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10613 .await;
10614 if let Err(e) = &result {
10615 record_repository_block(state, e);
10616 }
10617 result
10618}
10619
10620async fn run_one(
10622 job: SeatJob,
10623 sem: Arc<Semaphore>,
10624 ctx: &WaveCtx<'_>,
10625 state: &mut RunState,
10626 attempt: usize,
10627) -> (SeatState, AgentOutcome) {
10628 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10629 .await
10630 .pop()
10631 .expect("one job in, one result out");
10632 (seat, out)
10633}
10634
10635async fn wave(
10641 jobs: Vec<SeatJob>,
10642 sem: Arc<Semaphore>,
10643 ctx: &WaveCtx<'_>,
10644 state: &mut RunState,
10645 attempt: usize,
10646) -> Vec<(usize, SeatState, AgentOutcome)> {
10647 let WaveCtx {
10648 run,
10649 node,
10650 prompts,
10651 cache,
10652 round,
10653 carry_seats: _,
10654 pauses,
10655 } = *ctx;
10656 for job in &jobs {
10657 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10658 }
10659 if let Err(e) = state.save() {
10660 tracing::warn!("could not persist in-progress seats: {e:#}");
10665 }
10666 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10682 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10686 let cache_guard = if let Some(cache_dir) = cache {
10687 if jobs_had_a_writer {
10688 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10689 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10690 match acquire_cache_lease(
10691 state,
10692 cache_dir,
10693 &owner,
10694 resource_limit,
10695 node,
10696 node,
10697 &cancel,
10698 )
10699 .await
10700 {
10701 Ok(Some(g)) => Some(g),
10702 Ok(None) => {
10705 if let Some(block) = &mut state.resource_block {
10706 block.cause = crate::run::ResourceBlockCause::Cancelled {
10707 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10708 };
10709 }
10710 return block_wave(
10711 jobs,
10712 state,
10713 node,
10714 attempt,
10715 "parked while waiting for the shared build cache",
10716 );
10717 }
10718 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10721 Err(e) => {
10722 let why = format!("{e:#}");
10726 state.event(
10727 "resource",
10728 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10729 );
10730 return block_wave(jobs, state, node, attempt, &why);
10731 }
10732 }
10733 } else {
10734 None
10735 }
10736 } else {
10737 None
10738 };
10739 state.seats_spawning(node, attempt);
10742 if let Err(e) = state.save() {
10743 tracing::warn!("could not persist the start of a wave: {e:#}");
10744 }
10745 let slot_limit = resource_limit
10749 + jobs
10750 .iter()
10751 .map(|j| j.timeout)
10752 .max()
10753 .unwrap_or_default()
10754 .saturating_mul(jobs.len() as u32);
10755 let mut set = tokio::task::JoinSet::new();
10756 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10759 let overlay = prompts.overlay(node);
10760 let repository_wait = repository_wait(state, pauses);
10761 for (i, mut job) in jobs.into_iter().enumerate() {
10762 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10763 if cache.is_some() {
10764 job.prompt.push('\n');
10765 job.prompt
10766 .push_str(&prompt::build_cache_note(node, job.allow_write));
10767 }
10768 let sem = Arc::clone(&sem);
10769 let run = run.to_owned();
10770 let node = node.to_owned();
10771 let attachments = if node == "implement" {
10774 state.attachments.clone()
10775 } else {
10776 Vec::new()
10777 };
10778 let cache = cache
10789 .filter(|_| job.allow_write && cache_guard.is_some())
10790 .map(Path::to_path_buf);
10791 let spawned_tx = spawned_tx.clone();
10792 let repository_wait = repository_wait.clone();
10793 let coordination = state.config.coordination.clone();
10794 let fetched_base = state.fetched_base.clone();
10795 set.spawn(async move {
10796 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10800 Ok(p) => p,
10801 Err(_) => {
10802 return (
10803 i,
10804 job.seat,
10805 AgentOutcome::ResourceBlocked(format!(
10806 "no parallel slot became free within {}s",
10807 slot_limit.as_secs()
10808 )),
10809 false,
10810 );
10811 }
10812 };
10813 spawned_tx.send(job.seat.key.clone()).ok();
10814 let mut seat = job.seat;
10815 let out = repository_wait
10816 .scope(agent::invoke(
10817 &job.spec,
10818 &mut seat,
10819 &Invocation {
10820 coordination: coordination.as_ref(),
10821 fetched_base: fetched_base.as_ref(),
10822 cwd: &job.cwd,
10823 prompt: &job.prompt,
10824 timeout: job.timeout,
10825 allow_write: job.allow_write,
10826 sessions: job.sessions,
10827 artifacts: &job.artifacts,
10828 stem: &job.stem,
10829 run: &run,
10830 node: &node,
10831 cache_dir: cache.as_deref(),
10832 attachments: &attachments,
10833 writable: &[],
10834 },
10835 ))
10836 .await;
10837 let interrupted = out
10838 .as_ref()
10839 .err()
10840 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10841 let out = match out {
10842 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10843 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10844 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10852 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10853 Ok(o) => AgentOutcome::Failed(format!(
10854 "process exited with no usable output ({})",
10855 crate::notices::exit_status(o.exit_code)
10856 )),
10857 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10858 AgentOutcome::ResourceBlocked(e.to_string())
10859 }
10860 Err(e) => AgentOutcome::Failed(e.to_string()),
10861 };
10862 (i, seat, out, interrupted)
10863 });
10864 }
10865 drop(spawned_tx);
10866 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10867 loop {
10868 let joined = tokio::select! {
10869 Some(key) = spawned_rx.recv() => {
10870 state.seat_spawned(&key);
10872 if let Err(e) = state.save() {
10873 tracing::warn!("could not persist a seat's start: {e:#}");
10874 }
10875 continue;
10876 }
10877 joined = set.join_next() => joined,
10878 };
10879 let Some(joined) = joined else {
10880 break;
10881 };
10882 let (i, seat, out, interrupted) = match joined {
10883 Ok(v) => v,
10884 Err(e) => {
10888 tracing::error!("agent task panicked: {e}");
10889 continue;
10890 }
10891 };
10892 state.seat_finished(&seat.key);
10893 if interrupted
10894 && node == "implement"
10895 && let Some(label) = state
10896 .candidates
10897 .iter()
10898 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10899 .map(|candidate| candidate.label)
10900 {
10901 state.candidate_preservation.insert(
10902 label,
10903 "interrupted implementation awaiting persistence".into(),
10904 );
10905 state.interrupted_implementations.insert(label);
10906 }
10907 if let AgentOutcome::ResourceBlocked(why) = &out
10908 && state.resource_block.is_none()
10909 {
10910 state.resource_block = Some(crate::run::ResourceBlock {
10911 cause: crate::run::ResourceBlockCause::Other,
10912 node: node.to_owned(),
10913 kind: if why.contains("disk pressure") {
10914 "disk pressure"
10915 } else if why.contains("disk admission") {
10916 "disk admission"
10917 } else if why.contains(crate::incremental::BLOCKED) {
10918 "incremental cache admission"
10919 } else if why.contains("repository coordination") {
10920 "repository coordination"
10921 } else {
10922 "parallel slot"
10923 }
10924 .to_owned(),
10925 detail: why.clone(),
10926 at: Timestamp::now(),
10927 });
10928 let activity = if interrupted {
10929 "agent execution interrupted"
10930 } else {
10931 "no agent was started"
10932 };
10933 state.event("resource", format!("{node}: {activity} - {why}"));
10934 }
10935 record_jobs(state, node, round, &seat.key, &out);
10936 if let Err(e) = state.save() {
10937 tracing::warn!("could not persist a seat's completion: {e:#}");
10938 }
10939 if collected.len() <= i {
10940 collected.resize_with(i + 1, || None);
10941 }
10942 collected[i] = Some((i, seat, out));
10943 }
10944 if state
10950 .active
10951 .values()
10952 .any(|a| a.node == node && a.attempt == attempt)
10953 {
10954 state
10955 .active
10956 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10957 if let Err(e) = state.save() {
10958 tracing::warn!("could not persist the end of a wave: {e:#}");
10959 }
10960 }
10961 if let Some(cache_dir) = cache
10968 && jobs_had_a_writer
10969 {
10970 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10971 }
10972 if let Some(guard) = cache_guard {
10973 guard.release();
10974 }
10975 collected.into_iter().flatten().collect()
10976}
10977
10978fn record_jobs(
10989 state: &mut RunState,
10990 node: &str,
10991 round: Option<usize>,
10992 seat: &str,
10993 out: &AgentOutcome,
10994) {
10995 let commands: &[agent::CommandEvidence] = match out {
10996 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10997 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10998 };
10999 let checked_at = Timestamp::now();
11000 for c in commands {
11001 state.jobs.push(JobRecord {
11002 node: node.to_owned(),
11003 round,
11004 seat: seat.to_owned(),
11005 id: c.id.clone(),
11006 description: c.description.clone(),
11007 checked_at,
11008 status: match c.exit_code {
11009 Some(0) => JobStatus::Completed,
11010 Some(_) => JobStatus::Failed,
11011 None => JobStatus::Unknown,
11012 },
11013 exit_code: c.exit_code,
11014 result_summary: c.result_summary.clone(),
11015 source: c.source.clone(),
11016 });
11017 }
11018}
11019
11020fn round_is_clean(
11041 blocking: usize,
11042 e2e_ok: bool,
11043 answered: usize,
11044 expected: usize,
11045 quota_missing: usize,
11046 policy: IncompleteReviewPolicy,
11047) -> bool {
11048 if blocking != 0 || !e2e_ok {
11049 return false;
11050 }
11051 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11052 return true;
11053 }
11054 answered > 0 && expected - answered <= quota_missing
11055}
11056
11057pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11081 if max_rounds == 0
11082 || reviews.last().is_some_and(|r| {
11083 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11084 })
11085 {
11086 return Some(RunStatus::Gating);
11087 }
11088 let last = reviews.last()?;
11089 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11090 if reviews.len() < max_rounds && !stagnant {
11091 return None;
11092 }
11093 if last.incomplete() && last.blocking == 0 {
11094 return Some(RunStatus::Blocked);
11095 }
11096 if last.e2e_status() == E2eStatus::ResourceBlocked
11097 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11098 {
11099 return None;
11100 }
11101 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11102 RunStatus::Gating
11103 } else {
11104 RunStatus::Blocked
11105 })
11106}
11107
11108pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11123 if nudged {
11124 (full / 4).max(Duration::from_secs(120)).min(full)
11125 } else {
11126 full
11127 }
11128}
11129
11130#[allow(clippy::too_many_arguments)]
11159async fn ask_json_wave<T>(
11160 jobs: Vec<SeatJob>,
11161 sem: Arc<Semaphore>,
11162 retries: usize,
11163 roster: &[AgentSpec],
11164 ctx: &WaveCtx<'_>,
11165 losses: &mut Vec<QuotaLoss>,
11166 state: &mut RunState,
11167 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11168) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11169where
11170 T: serde::de::DeserializeOwned + Send + 'static,
11171{
11172 ask_wave_with(
11173 jobs,
11174 sem,
11175 retries,
11176 roster,
11177 ctx,
11178 losses,
11179 state,
11180 &|text: &str| {
11181 let v = verdict::extract_json::<T>(text)?;
11182 validate(&v)?;
11183 Ok(v)
11184 },
11185 )
11186 .await
11187}
11188
11189#[allow(clippy::too_many_arguments)]
11198async fn ask_wave_with<T>(
11199 jobs: Vec<SeatJob>,
11200 sem: Arc<Semaphore>,
11201 retries: usize,
11202 roster: &[AgentSpec],
11203 ctx: &WaveCtx<'_>,
11204 losses: &mut Vec<QuotaLoss>,
11205 state: &mut RunState,
11206 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11207) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11208where
11209 T: Send + 'static,
11210{
11211 let n = jobs.len();
11212 let originals: Vec<SeatJob> = jobs;
11213 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11214 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11215 let mut nudges: Vec<usize> = vec![0; n];
11222 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11226 let mut tried: Vec<BTreeSet<String>> = specs
11227 .iter()
11228 .map(|s| BTreeSet::from([s.id.clone()]))
11229 .collect();
11230 let starts: Vec<usize> = specs
11231 .iter()
11232 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11233 .collect();
11234 let carry = ctx.carry_seats && !roster.is_empty();
11235 let carried: Vec<BTreeSet<String>> = originals
11238 .iter()
11239 .map(|j| {
11240 state
11241 .seat_history
11242 .get(&j.seat.key)
11243 .filter(|_| carry)
11244 .map(|h| h.failed.clone())
11245 .unwrap_or_default()
11246 })
11247 .collect();
11248 let mut prev: Vec<Option<FailClass>> = originals
11249 .iter()
11250 .map(|j| {
11251 state
11252 .seat_history
11253 .get(&j.seat.key)
11254 .filter(|_| carry)
11255 .and_then(|h| h.last_fail.clone())
11256 })
11257 .collect();
11258 let next_agent =
11259 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11260 let others: BTreeSet<String> = specs
11261 .iter()
11262 .enumerate()
11263 .filter(|(j, _)| *j != i)
11264 .map(|(_, s)| s.id.clone())
11265 .collect();
11266 pick_successor(
11267 roster,
11268 starts[i],
11269 tried,
11270 carry.then(|| &carried[i]),
11271 &others,
11272 )
11273 .cloned()
11274 };
11275 let mut fresh: Vec<Option<String>> = vec![None; n];
11276 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11277 let mut pending: Vec<usize> = (0..n).collect();
11278
11279 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11284 for round in 0..max_rounds {
11285 if pending.is_empty() {
11286 break;
11287 }
11288 let mut batch = Vec::with_capacity(pending.len());
11289 let mut renudged: Vec<&str> = Vec::new();
11290 for &i in &pending {
11291 let src = &originals[i];
11292 let full: &str = match &src.handover {
11297 Some(h) if specs[i].id != src.spec.id => h,
11298 _ => &src.prompt,
11299 };
11300 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11303 (
11304 full.to_owned(),
11305 src.timeout,
11306 format!("{}-{word}-{}", src.stem, specs[i].id),
11307 )
11308 } else if nudges[i] == 0 {
11309 (full.to_owned(), src.timeout, src.stem.clone())
11310 } else {
11311 renudged.push(src.seat.key.as_str());
11312 let why = done[i]
11313 .as_ref()
11314 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11315 .unwrap_or_else(|| "no parsable answer".to_owned());
11316 let nudge = prompt::nudge(&why);
11317 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11318 let prompt = if nudged {
11319 nudge
11320 } else {
11321 format!("{full}\n\n---\n\n{nudge}")
11322 };
11323 (
11324 prompt,
11325 retry_budget(src.timeout, nudged),
11326 format!("{}-retry{}", src.stem, nudges[i]),
11327 )
11328 };
11329 batch.push(SeatJob {
11330 spec: specs[i].clone(),
11331 seat: seats[i].clone(),
11332 cwd: src.cwd.clone(),
11333 prompt,
11334 timeout,
11335 allow_write: src.allow_write,
11336 sessions: src.sessions,
11337 artifacts: src.artifacts.clone(),
11338 stem,
11339 handover: None,
11340 });
11341 }
11342
11343 if !renudged.is_empty() {
11344 state.event(
11345 ctx.node,
11346 format!("retry {round}: re-asking {}", renudged.join(", ")),
11347 );
11348 }
11349 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11350 let mut still = Vec::new();
11351 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11352 seats[i] = seat;
11353 if let AgentOutcome::ResourceBlocked(why) = &out {
11356 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11357 continue;
11358 }
11359 let class = FailClass::of(&out);
11360 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11363 && nudges[i] < retries
11364 && !roster.is_empty();
11365 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11366 let next = should_hand_over(prev[i].as_ref(), &cur)
11367 .then(|| next_agent(i, &tried[i], &specs))
11368 .flatten();
11369 if carry {
11370 let h = state
11371 .seat_history
11372 .entry(originals[i].seat.key.clone())
11373 .or_default();
11374 h.failed.insert(specs[i].id.clone());
11375 h.last_fail = Some(cur.clone());
11376 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11377 h.last_ok = None;
11378 }
11379 if let Err(e) = state.save() {
11382 tracing::warn!("could not persist a seat's failure history: {e:#}");
11383 }
11384 }
11385 if let Some(next) = next {
11386 record_handover(
11387 state,
11388 ctx.node,
11389 &originals[i].seat.key,
11390 &specs[i].id,
11391 &next.id,
11392 &cur,
11393 &fail_reason(&out),
11394 );
11395 tried[i].insert(next.id.clone());
11396 prev[i] = Some(cur.clone());
11397 seats[i] =
11398 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11399 specs[i] = next;
11400 fresh[i] = Some(cur.stem_word().to_owned());
11401 nudges[i] = 0;
11402 done[i] = Some(Err(anyhow::anyhow!(
11403 "handed over after: {}",
11404 fail_reason(&out)
11405 )));
11406 still.push(i);
11407 continue;
11408 }
11409 }
11410 if carry
11411 && !nudge_first
11412 && let Some(cur) = class.clone()
11413 {
11414 let h = state
11416 .seat_history
11417 .entry(originals[i].seat.key.clone())
11418 .or_default();
11419 h.failed.insert(specs[i].id.clone());
11420 h.last_fail = Some(cur);
11421 }
11422 let parsed = match out {
11423 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11424 AgentOutcome::Quota(o) => {
11425 last_quota[i] = o.quota.clone();
11426 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11427 }
11428 AgentOutcome::Dropped(o) => {
11432 let why = o
11433 .dropped
11434 .as_ref()
11435 .map(|d| d.why.as_str())
11436 .unwrap_or("the CLI ended the stream without delivering its answer");
11437 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11438 }
11439 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11440 Err(anyhow::anyhow!(e))
11441 }
11442 };
11443 let quota = class == Some(FailClass::Quota);
11444 let failed = parsed.is_err();
11445 if !failed && matches!(ctx.node, "judge" | "vote") {
11446 state
11449 .quota
11450 .retain(|loss| loss.seat != originals[i].seat.key);
11451 }
11452 done[i] = Some(parsed);
11453 if carry && !failed {
11454 let h = state
11455 .seat_history
11456 .entry(originals[i].seat.key.clone())
11457 .or_default();
11458 h.failed.remove(&specs[i].id);
11459 h.last_ok = Some(specs[i].id.clone());
11460 h.last_fail = None;
11461 }
11462 let agent_failure = class.is_some()
11470 && !nudge_first
11471 && !roster.is_empty()
11472 && next_agent(i, &tried[i], &specs).is_some();
11473 if failed && !quota && !agent_failure && nudges[i] < retries {
11474 nudges[i] += 1;
11475 still.push(i);
11476 }
11477 }
11478 pending = still;
11479 }
11480
11481 for (i, q) in last_quota.into_iter().enumerate() {
11484 if let Some(quota) = q {
11485 losses.push(QuotaLoss::from_report(
11486 originals[i].seat.key.clone(),
11487 ctx.node,
11488 Some("a),
11489 ));
11490 }
11491 }
11492
11493 if state.status != RunStatus::Stalled
11496 && ctx.node != "deliberate"
11497 && n > 0
11498 && losses.len() == n
11499 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11500 {
11501 block_machine_outage(state, ctx.node, losses);
11502 state.quota.append(losses);
11503 }
11504
11505 seats
11506 .into_iter()
11507 .zip(done)
11508 .zip(nudges)
11509 .map(|((seat, res), attempts)| {
11510 (
11511 seat,
11512 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11513 attempts,
11514 )
11515 })
11516 .collect()
11517}
11518
11519pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11522 let mut auth: Vec<_> = losses
11523 .iter()
11524 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11525 .map(|loss| loss.agent.clone())
11526 .collect();
11527 auth.sort();
11528 auth.dedup();
11529 let (kind, detail) = if auth.is_empty() {
11530 (
11531 "agent usage limit",
11532 "All seats reached their agent usage limits; resume after usage is available"
11533 .to_owned(),
11534 )
11535 } else {
11536 (
11537 "agent authentication",
11538 format!(
11539 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11540 auth.join(", ")
11541 ),
11542 )
11543 };
11544 state.resource_block = Some(crate::run::ResourceBlock {
11545 node: node.to_owned(),
11546 kind: kind.to_owned(),
11547 cause: crate::run::ResourceBlockCause::Other,
11548 detail: detail.clone(),
11549 at: Timestamp::now(),
11550 });
11551 state.event(node, detail);
11552}
11553
11554fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11556 results
11557 .iter()
11558 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11559}
11560
11561fn block_wave(
11565 jobs: Vec<SeatJob>,
11566 state: &mut RunState,
11567 node: &str,
11568 attempt: usize,
11569 why: &str,
11570) -> Vec<(usize, SeatState, AgentOutcome)> {
11571 state.resource_wait = None;
11572 state
11573 .active
11574 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11575 if let Err(e) = state.save() {
11576 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11577 }
11578 jobs.into_iter()
11579 .enumerate()
11580 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11581 .collect()
11582}
11583
11584async fn acquire_cache_lease(
11597 state: &mut RunState,
11598 cache_dir: &Path,
11599 owner: &crate::cache::Owner,
11600 budget: Duration,
11601 node: &str,
11602 context: &str,
11603 cancel: &(dyn Fn() -> bool + Sync),
11604) -> Result<Option<crate::cache::Guard>> {
11605 let home = crate::run::home();
11606 let started = Instant::now();
11607 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11608 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11609 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11610 Err(e) => {
11611 state.event(
11612 "verify",
11613 format!("{context}: could not check the shared build cache: {e:#}"),
11614 );
11615 if let Err(e2) = state.save() {
11616 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11617 }
11618 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11619 }
11620 };
11621 state.event(
11622 "verify",
11623 format!(
11624 "{context}: waiting for the shared build cache at {} ({})",
11625 cache_dir.display(),
11626 busy.describe()
11627 ),
11628 );
11629 state.resource_wait = Some(crate::run::ResourceWait {
11632 node: context.to_owned(),
11633 kind: "shared build cache".to_owned(),
11634 holder: busy.describe(),
11635 since: Timestamp::now(),
11636 limit_secs: budget.as_secs(),
11637 });
11638 if let Err(e) = state.save() {
11639 tracing::warn!("could not persist a cache wait: {e:#}");
11640 }
11641 let remaining = budget.saturating_sub(started.elapsed());
11642 let waited = crate::cache::wait_for_unless(
11643 &home,
11644 cache_dir,
11645 owner,
11646 remaining,
11647 Duration::from_secs(5),
11648 cancel,
11649 )
11650 .await;
11651 state.resource_wait = None;
11652 let cause = match &waited {
11653 Ok(Some(_)) => None,
11654 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11655 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11656 Some(crate::run::ResourceBlockCause::Timeout {
11657 waited_secs: started.elapsed().as_secs(),
11658 limit_secs: budget.as_secs(),
11659 })
11660 }
11661 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11662 };
11663 if let Some(cause) = cause {
11664 let detail = match &waited {
11665 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11666 Err(error) => format!("{error:#}"),
11667 };
11668 state.resource_block = Some(crate::run::ResourceBlock {
11669 cause,
11670 node: node.to_owned(),
11671 kind: "shared build cache".into(),
11672 detail,
11673 at: Timestamp::now(),
11674 });
11675 state.event(
11676 "resource",
11677 state.resource_block.as_ref().unwrap().description(),
11678 );
11679 }
11680 if let Err(error) = state.save() {
11681 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11682 }
11683 waited
11684}
11685
11686#[derive(Debug)]
11691struct CacheCheckError(String);
11692
11693impl std::fmt::Display for CacheCheckError {
11694 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11695 f.write_str(&self.0)
11696 }
11697}
11698
11699impl std::error::Error for CacheCheckError {}
11700
11701fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11703 state
11704 .resource_block
11705 .get_or_insert_with(|| crate::run::ResourceBlock {
11706 cause: crate::run::ResourceBlockCause::Other,
11707 node: node.to_owned(),
11708 kind: "shared build cache".into(),
11709 detail: format!("{error:#}"),
11710 at: Timestamp::now(),
11711 });
11712 if let Err(error) = state.save() {
11713 tracing::warn!("could not persist cache resource block: {error:#}");
11714 }
11715}
11716
11717#[allow(clippy::too_many_arguments)]
11738async fn with_cache_lease<'s, F, Fut>(
11739 state: &'s mut RunState,
11740 cache_dir: Option<&Path>,
11741 node: &str,
11742 seat: &str,
11743 worktree: &Path,
11744 head: &str,
11745 budget: Duration,
11746 context: &str,
11747 wait: &crate::repo_lock::WaitPolicy,
11748 body: F,
11749) -> Result<(Vec<CommandOutcome>, bool)>
11750where
11751 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11752 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11753{
11754 let started = Instant::now();
11757 let home = crate::run::home();
11758 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11759 let guard = if let Some(cache_dir) = cache_dir {
11760 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11761 match acquire_cache_lease(
11762 state,
11763 cache_dir,
11764 &owner,
11765 resource_limit,
11766 node,
11767 context,
11768 &|| wait.cancelled(),
11769 )
11770 .await
11771 .and_then(|g| {
11772 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11773 }) {
11774 Ok(g) => Some(g),
11775 Err(e) => {
11776 record_cache_error(state, node, &e);
11777 return Ok((
11778 vec![CommandOutcome {
11779 command: "(waiting for the shared build cache)".into(),
11780 code: None,
11781 output_tail: e.to_string(),
11782 duration_ms: started.elapsed().as_millis() as u64,
11783 resource_blocked: true,
11784 }],
11785 false,
11786 ));
11787 }
11788 }
11789 } else {
11790 None
11791 };
11792 let _coordination = coordinate_operation(state, worktree, wait).await?;
11793 if let Some(cache_dir) = cache_dir {
11794 let identity = crate::cache::Identity::new(worktree, head);
11795 if let Err(e) = crate::cache::ensure_fresh(
11796 &home,
11797 cache_dir,
11798 &identity,
11799 &state.config.verify.cargo_manifests(),
11800 ) {
11801 record_cache_error(state, node, &e);
11802 state.event(
11803 "verify",
11804 format!(
11805 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11806 worktree.display(),
11807 short(head)
11808 ),
11809 );
11810 if let Some(guard) = guard {
11811 guard.release();
11812 }
11813 return Ok((
11814 vec![CommandOutcome {
11815 command: "(confirming the shared build cache is fresh)".into(),
11816 code: None,
11817 output_tail: e.to_string(),
11818 duration_ms: started.elapsed().as_millis() as u64,
11819 resource_blocked: true,
11820 }],
11821 false,
11822 ));
11823 }
11824 }
11825 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11826 if !timed_out_pids.is_empty() {
11827 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11828 }
11829 drop(_coordination);
11832 if let Some(guard) = guard {
11833 guard.release();
11834 }
11835 Ok((outcomes, retried))
11836}
11837
11838async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11850 wait_for_pids_with(
11851 pids,
11852 crate::proc::pid_alive,
11853 LEASE_RELEASE_POLL,
11854 LEASE_RELEASE_MAX_WAIT,
11855 )
11856 .await;
11857}
11858
11859async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11865 pids: &[u32],
11866 alive: F,
11867 poll: Duration,
11868 max_wait: Duration,
11869) {
11870 let deadline = Instant::now() + max_wait;
11871 loop {
11872 if pids.iter().all(|&pid| !alive(pid)) {
11873 return;
11874 }
11875 if Instant::now() >= deadline {
11876 return;
11877 }
11878 tokio::time::sleep(poll).await;
11879 }
11880}
11881
11882fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11889 outcomes.iter().any(|o| o.resource_blocked)
11890}
11891
11892pub(crate) enum GateFix {
11894 Retry,
11896 Stop,
11899 Defer,
11902}
11903
11904pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11912 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11913 red.peek().is_some()
11914 && red.all(|o| {
11915 !o.resource_blocked
11916 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11917 && !o.output_tail.trim().is_empty()
11918 })
11919}
11920
11921fn e2e_outcome_label(o: &CommandOutcome) -> String {
11925 if o.ok() {
11926 return "pass".to_owned();
11927 }
11928 let reason = if o.build_failed() {
11929 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11930 } else {
11931 format!("FAIL ({:?})", o.code)
11932 };
11933 format!(
11934 "{reason}\n{}",
11935 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11936 )
11937}
11938
11939async fn run_e2e_with_retry(
11947 state: &mut RunState,
11948 shell: &[String],
11949 commands: &[String],
11950 worktree: &Path,
11951 timeout: Duration,
11952 context: &str,
11953) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11954 let (mut e2e, mut timed_out_pids) = run_commands(
11955 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11956 )
11957 .await;
11958 for o in &e2e {
11959 state.event(
11960 "verify",
11961 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11962 );
11963 }
11964 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11968 if verify_retried {
11969 state.event(
11970 "verify",
11971 format!(
11972 "{context}: verify could not build/link, not a test result — retrying once \
11973 before concluding"
11974 ),
11975 );
11976 let retried = run_commands(
11977 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11978 )
11979 .await;
11980 e2e = retried.0;
11981 timed_out_pids.extend(retried.1);
11984 for o in &e2e {
11985 state.event(
11986 "verify",
11987 format!(
11988 "{context}: retry `{}` -> {}",
11989 o.command,
11990 e2e_outcome_label(o)
11991 ),
11992 );
11993 }
11994 }
11995 (e2e, verify_retried, timed_out_pids)
11996}
11997
11998#[allow(clippy::too_many_arguments)]
12014async fn run_commands(
12015 state: &mut RunState,
12016 node: &str,
12017 task: &str,
12018 attempt: usize,
12019 shell: &[String],
12020 commands: &[String],
12021 cwd: &Path,
12022 timeout: Duration,
12023) -> (Vec<CommandOutcome>, Vec<u32>) {
12024 if commands.is_empty() {
12025 return (Vec::new(), Vec::new());
12030 }
12031 let mut out = Vec::new();
12032 let mut timed_out_pids = Vec::new();
12033 let total = commands.len();
12034 for (idx, command) in commands.iter().enumerate() {
12035 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12036 if let Err(e) = state.save() {
12037 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12038 }
12039 let started = Instant::now();
12040 let mut cmd = tokio::process::Command::new(&shell[0]);
12041 cmd.quiet();
12042 for key in crate::worker::INVOCATION_ENV {
12046 cmd.env_remove(key);
12047 }
12048 cmd.args(&shell[1..])
12049 .arg(command)
12050 .current_dir(cwd)
12051 .stdin(std::process::Stdio::null())
12052 .stdout(std::process::Stdio::piped())
12053 .stderr(std::process::Stdio::piped())
12054 .kill_on_drop(true);
12055 let mut outputs = Vec::new();
12056 let protected = (|| -> Result<()> {
12057 crate::incremental::configure(&mut cmd, cwd)?;
12058 let mut paths = vec![cwd.to_owned()];
12059 if let Some(cache) = state.config.cache_dir() {
12060 paths.push(cache);
12061 }
12062 outputs.extend(crate::retention::OutputLease::admit(
12063 &crate::run::home(),
12064 &paths,
12065 state.config.disk.min_free_bytes,
12066 )?);
12067 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12068 outputs.push(guard);
12069 }
12070 if let Some(cache) = state.config.cache_dir() {
12071 outputs.push(crate::retention::OutputLease::cache(
12072 &crate::run::home(),
12073 &cache,
12074 false,
12075 )?);
12076 }
12077 Ok(())
12078 })();
12079 if let Err(e) = protected {
12080 let detail = format!("execution admission unavailable: {e:#}");
12081 state.resource_block = Some(crate::run::ResourceBlock {
12082 cause: crate::run::ResourceBlockCause::Other,
12083 node: node.to_owned(),
12084 kind: if detail.contains("disk admission") {
12085 "disk admission"
12086 } else {
12087 "output ownership"
12088 }
12089 .to_owned(),
12090 detail: detail.clone(),
12091 at: Timestamp::now(),
12092 });
12093 out.push(CommandOutcome {
12094 command: command.clone(),
12095 code: None,
12096 output_tail: detail,
12097 duration_ms: 0,
12098 resource_blocked: true,
12099 });
12100 break;
12101 }
12102 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12103 let (code, body) = match spawned {
12104 Ok(mut running) => {
12105 let pid = running.child().id();
12106 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12107 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12108 let mut paths = vec![cwd.to_owned()];
12109 if let Some(cache) = state.config.cache_dir() {
12110 paths.push(cache);
12111 }
12112 let mut pressure = None;
12113 let status = tokio::select! {
12114 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12115 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12116 pressure = Some(reason); None
12117 }
12118 };
12119 let (code, failure) = match status {
12120 Some(Ok(Ok(status))) => (
12121 status.code(),
12122 status
12123 .code()
12124 .is_none()
12125 .then(|| format!("command terminated: {status}")),
12126 ),
12127 Some(Ok(Err(error))) => {
12128 (None, Some(format!("failed waiting for command: {error}")))
12129 }
12130 Some(Err(_)) => {
12131 if !running.coordinated()
12135 && let Some(pid) = pid
12136 {
12137 timed_out_pids.push(pid);
12138 }
12139 (
12140 None,
12141 Some(format!("timed out after {}s", timeout.as_secs())),
12142 )
12143 }
12144 None => (None, None),
12145 };
12146 running.finish().await;
12147 if let Some(reason) = pressure {
12148 state.resource_block = Some(crate::run::ResourceBlock {
12149 cause: crate::run::ResourceBlockCause::Other,
12150 node: node.to_owned(),
12151 kind: "disk pressure".to_owned(),
12152 detail: reason.clone(),
12153 at: Timestamp::now(),
12154 });
12155 out.push(CommandOutcome {
12156 command: command.clone(),
12157 code: None,
12158 output_tail: reason,
12159 duration_ms: started.elapsed().as_millis() as u64,
12160 resource_blocked: true,
12161 });
12162 break;
12163 }
12164 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12165 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12166 if let Some(failure) = failure {
12167 body.push('\n');
12168 body.push_str(&failure);
12169 }
12170 (code, body)
12171 }
12172 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12173 };
12174 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12175 if !body.is_empty() {
12176 match crate::run::write_verification_artifact(state, task, &body) {
12180 Ok(name) if summary != body => {
12181 summary = format!("[full output: artifacts/{name}]\n{summary}");
12182 }
12183 Ok(_) => {}
12184 Err(e) => {
12185 let warning = format!("could not save the full {task} output: {e:#}");
12186 tracing::warn!("{warning}");
12187 state.event(node, warning);
12188 if summary != body {
12189 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12190 }
12191 }
12192 }
12193 }
12194 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12195 out.push(CommandOutcome {
12196 command: command.clone(),
12197 code,
12198 output_tail: summary,
12199 duration_ms: started.elapsed().as_millis() as u64,
12200 resource_blocked,
12201 });
12202 if resource_blocked {
12203 state.resource_block = Some(crate::run::ResourceBlock {
12204 cause: crate::run::ResourceBlockCause::Other,
12205 node: node.to_owned(),
12206 kind: "incremental cache admission".into(),
12207 detail: body,
12208 at: Timestamp::now(),
12209 });
12210 break;
12211 }
12212 }
12213 state.task_finished(task);
12214 if let Err(e) = state.save() {
12215 tracing::warn!("could not persist the end of {task}: {e:#}");
12216 }
12217 (out, timed_out_pids)
12218}
12219
12220fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12232 let repo = repo.display();
12233 match style {
12234 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12235 MergeStyle::Squash => {
12236 let subject = message
12239 .lines()
12240 .next()
12241 .unwrap_or(branch)
12242 .replace(['\\', '"', '$', '`'], "");
12243 format!(
12244 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12245 )
12246 }
12247 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12248 }
12249}
12250
12251const PR_TITLE_MAX: usize = 240;
12262
12263fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12274 let line = queue::first_line(text)?;
12275 let chars: Vec<char> = line.chars().collect();
12276 let (kept, mark) = if chars.len() <= max {
12277 (line.to_owned(), "")
12278 } else if let Some(end) = sentence_end(&chars, max) {
12279 (chars[..end].iter().collect::<String>(), "")
12280 } else {
12281 let room = max.saturating_sub(3);
12282 let cut = if chars[room].is_whitespace() {
12285 room
12286 } else {
12287 chars[..room]
12288 .iter()
12289 .rposition(|c| c.is_whitespace())
12290 .unwrap_or(room)
12291 };
12292 let head: String = chars[..cut].iter().collect();
12293 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12294 (head.to_owned(), "...")
12295 };
12296 if kept.is_empty() {
12297 return None;
12298 }
12299 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12300 return None;
12301 }
12302 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12303 return None;
12304 }
12305 Some(format!("{kept}{mark}"))
12306}
12307
12308fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12312 const MIN: usize = 20;
12313 for i in MIN..max.min(chars.len()) {
12314 if !matches!(chars[i], '.' | '!' | '?') {
12315 continue;
12316 }
12317 let Some(&next) = chars.get(i + 1) else {
12318 continue;
12319 };
12320 if !next.is_whitespace() {
12321 continue;
12322 }
12323 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12324 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12325 continue;
12326 }
12327 let word: String = chars[..i]
12328 .iter()
12329 .rev()
12330 .take_while(|c| !c.is_whitespace())
12331 .collect::<Vec<_>>()
12332 .into_iter()
12333 .rev()
12334 .collect();
12335 let word = word.to_ascii_lowercase();
12336 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12337 continue;
12338 }
12339 let end = chars[..i]
12340 .iter()
12341 .rposition(|c| !c.is_whitespace())
12342 .map_or(i, |p| p + 1);
12343 return Some(end);
12344 }
12345 None
12346}
12347
12348struct PrMessage {
12353 title: String,
12354 body: String,
12355}
12356
12357impl PrMessage {
12358 fn commit_message(&self) -> String {
12362 format!("{}\n\n{}", self.title, self.body)
12363 }
12364}
12365
12366fn title_marker(line: &str) -> Option<&str> {
12368 let line = line.trim();
12369 let head = line.get(..6)?;
12370 head.eq_ignore_ascii_case("title:")
12371 .then(|| line[6..].trim())
12372}
12373
12374fn summary_title(summary: &str) -> Option<String> {
12379 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12380 let raw = title_marker(first)?;
12381 if raw.is_empty() {
12382 return None;
12383 }
12384 let title = queue::title_from(raw, PR_TITLE_MAX);
12385 let lower = title.to_ascii_lowercase();
12386 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12387 return None;
12388 }
12389 Some(title)
12390}
12391
12392const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12394
12395pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12397
12398pub(crate) fn is_review_run(state: &RunState) -> bool {
12402 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12403}
12404
12405fn review_title(state: &RunState) -> Option<String> {
12410 english_subject(
12411 state.reviewed_commits.as_ref()?.first()?,
12412 state.config.graph.github_english(),
12413 )
12414}
12415
12416fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12419 let raw = raw.trim();
12420 if raw.is_empty() {
12421 return None;
12422 }
12423 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12424 return None;
12425 }
12426 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12427 return None;
12428 }
12429 let title = queue::title_from(raw, PR_TITLE_MAX);
12430 let lower = title.to_ascii_lowercase();
12431 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12432 return None;
12433 }
12434 Some(title)
12435}
12436
12437#[derive(Debug, Clone, PartialEq, Eq)]
12440struct BranchFacts {
12441 commits: Vec<(String, String)>,
12443 stat: String,
12445}
12446
12447const STAT_FILE_LINES: usize = 25;
12449const COMMITS_MAX_CHARS: usize = 20_000;
12451
12452async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12455 let commits = git::commit_log(repo, base, branch).await.ok()?;
12456 if commits.is_empty() {
12457 return None;
12458 }
12459 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12460 let lines: Vec<&str> = stat.lines().collect();
12461 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12462 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12463 let more = format!(" ... {omitted} more file(s)");
12464 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12465 kept.push(&more);
12466 kept.push(lines[lines.len() - 1]);
12467 kept.join("\n")
12468 } else {
12469 lines.join("\n")
12470 };
12471 Some(BranchFacts { commits, stat })
12472}
12473
12474fn markdown_safe(text: &str) -> String {
12477 text.replace("</details>", "</details>")
12478 .replace("\x60\x60\x60", "~~~")
12479}
12480
12481fn neutral_title(state: &RunState, winner: char) -> String {
12482 format!(
12483 "chore: land candidate {} of run {}",
12484 winner.to_ascii_uppercase(),
12485 state.id
12486 )
12487}
12488
12489pub(crate) async fn repair_landing_gate(
12492 state: &mut RunState,
12493 winner: &Candidate,
12494 outcomes: &[CommandOutcome],
12495) -> Result<GateFix> {
12496 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12497 return Ok(GateFix::Stop);
12498 }
12499 let mut runner = Runner {
12500 roles: state.config.resolve_roles()?,
12501 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12502 state: state.clone(),
12503 pause: Pause::new(),
12504 interrupt: Pause::new(),
12505 };
12506 let result = runner.gate_fix_round(winner, outcomes).await;
12507 *state = runner.state;
12508 result
12509}
12510
12511pub(crate) async fn verify_current_head(
12514 state: &mut RunState,
12515 winner: &Candidate,
12516 wait: &crate::repo_lock::WaitPolicy,
12517) -> Result<bool> {
12518 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12519 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12521 state
12522 .verification_history
12523 .push(crate::run::VerificationEvidence {
12524 superseded_by: Some(head.clone()),
12525 archived_at: Some(Timestamp::now()),
12526 fixes_spent: Some(state.gate_fixes.len()),
12527 gate_head: None,
12528 gate_ran: false,
12529 gate: Vec::new(),
12530 e2e: state.head_e2e.take(),
12531 });
12532 }
12533 if let Some(e) = &state.head_e2e
12534 && e.head == head
12535 {
12536 if e.passed {
12537 return Ok(true);
12538 }
12539 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12540 state.status = RunStatus::Blocked;
12541 state.save()?;
12542 return Ok(false);
12543 }
12544 } else if state.reviews.last().is_some_and(|r| {
12545 r.verified_head.as_deref() == Some(head.as_str())
12546 && (r.e2e_status() == E2eStatus::Passed
12547 || (r.e2e_status() == E2eStatus::NotConfigured
12548 && state.config.verify.e2e.is_empty()))
12549 }) || state
12550 .gate_repair_e2e
12551 .as_ref()
12552 .is_some_and(|e| e.head == head && e.passed)
12553 {
12554 return Ok(true);
12555 }
12556 if let Some(e) = &state.gate_repair_e2e
12557 && e.head == head
12558 && !e.passed
12559 && !e.outcomes.is_empty()
12560 && !verify_inconclusive(&e.outcomes)
12561 {
12562 state.status = RunStatus::Blocked;
12563 state.save()?;
12564 return Ok(false);
12565 }
12566 state.head_e2e = Some(crate::run::GateRepairE2e {
12567 head: head.clone(),
12568 passed: false,
12569 outcomes: Vec::new(),
12570 });
12571 state.save()?;
12572 let commands = state.config.verify.e2e.clone();
12573 let shell = state.config.shell();
12574 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12575 let cache_dir = state.config.cache_dir();
12576 let outcomes = if commands.is_empty() {
12577 Vec::new()
12578 } else {
12579 let (outcomes, _) = with_cache_lease(
12580 state,
12581 cache_dir.as_deref(),
12582 "e2e",
12583 "e2e",
12584 &winner.worktree,
12585 &head,
12586 timeout,
12587 "current head e2e",
12588 wait,
12589 |state, budget| async move {
12590 run_e2e_with_retry(
12591 state,
12592 &shell,
12593 &commands,
12594 &winner.worktree,
12595 budget,
12596 "current head e2e",
12597 )
12598 .await
12599 },
12600 )
12601 .await?;
12602 outcomes
12603 };
12604 let unresolved = verify_inconclusive(&outcomes);
12605 let passed = !unresolved
12606 && outcomes.iter().all(CommandOutcome::ok)
12607 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12608 if !outcomes.is_empty() && !unresolved {
12609 state.record_base_progress(&head);
12610 }
12611 state.head_e2e = Some(crate::run::GateRepairE2e {
12612 head: head.clone(),
12613 passed,
12614 outcomes,
12615 });
12616 if !passed && !unresolved {
12617 state.status = RunStatus::Blocked;
12618 state.event(
12619 "e2e",
12620 format!("verify.e2e failed on head {head}; nothing published"),
12621 );
12622 }
12623 state.save()?;
12624 Ok(passed)
12625}
12626
12627pub(crate) async fn check_gate_repair_e2e(
12630 state: &mut RunState,
12631 winner: &Candidate,
12632 wait: &crate::repo_lock::WaitPolicy,
12633) -> Result<GateFix> {
12634 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12635 return Ok(GateFix::Retry);
12636 }
12637 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12638 if state
12639 .gate_repair_e2e
12640 .as_ref()
12641 .is_some_and(|r| r.head == head && r.passed)
12642 {
12643 return Ok(GateFix::Retry);
12644 }
12645 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12646 head: head.clone(),
12647 passed: false,
12648 outcomes: Vec::new(),
12649 });
12650 state.save()?;
12651 let commands = state.config.verify.e2e.clone();
12652 let outcomes = if commands.is_empty() {
12653 Vec::new()
12654 } else {
12655 let shell = state.config.shell();
12656 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12657 let cache_dir = state.config.cache_dir();
12658 let (outcomes, _) = with_cache_lease(
12659 state,
12660 cache_dir.as_deref(),
12661 "e2e",
12662 "e2e",
12663 &winner.worktree,
12664 &head,
12665 timeout,
12666 "gate repair e2e",
12667 wait,
12668 |state, budget| async move {
12669 run_e2e_with_retry(
12670 state,
12671 &shell,
12672 &commands,
12673 &winner.worktree,
12674 budget,
12675 "gate repair e2e",
12676 )
12677 .await
12678 },
12679 )
12680 .await?;
12681 outcomes
12682 };
12683 let deferred = verify_inconclusive(&outcomes);
12684 let passed = outcomes.iter().all(CommandOutcome::ok)
12685 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12686 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12687 head,
12688 passed,
12689 outcomes,
12690 });
12691 state.save()?;
12692 if deferred {
12693 Ok(GateFix::Defer)
12694 } else if !passed {
12695 state.event(
12696 "gate",
12697 "verify.e2e failed after gate repair; nothing published",
12698 );
12699 Ok(GateFix::Stop)
12700 } else {
12701 Ok(GateFix::Retry)
12702 }
12703}
12704
12705pub(crate) async fn run_landing_gate(
12707 state: &mut RunState,
12708 winner: &Candidate,
12709 wait: &crate::repo_lock::WaitPolicy,
12710) -> Result<Vec<CommandOutcome>> {
12711 let shell = state.config.shell();
12712 let gate_commands = state.config.verify.gate.clone();
12713 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12714 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12715 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12716 Vec::new()
12717 } else {
12718 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12719 let cache_dir = state.config.cache_dir();
12720 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12721 let (outcomes, _) = with_cache_lease(
12722 state,
12723 cache_dir.as_deref(),
12724 "gate",
12725 "gate",
12726 &winner.worktree,
12727 &head,
12728 timeout,
12729 "final gate",
12730 wait,
12731 |state, budget| {
12732 let shell = shell.clone();
12733 let gate_commands = gate_commands.clone();
12734 let winner = winner.clone();
12735 async move {
12736 Runner::run_pre_gate(state, &winner).await;
12737 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12739 let (outcomes, timed_out_pids) = run_commands(
12740 state,
12741 "gate",
12742 "gate",
12743 0,
12744 &shell,
12745 &gate_commands,
12746 &winner.worktree,
12747 budget,
12748 )
12749 .await;
12750 (outcomes, false, timed_out_pids)
12751 }
12752 },
12753 )
12754 .await?;
12755 outcomes
12756 };
12757 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12758 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12759 let mut outcomes = outcomes;
12760 outcomes.push(CommandOutcome {
12761 command: "gate head stability".into(), code: None,
12762 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12763 duration_ms: 0, resource_blocked: false,
12764 });
12765 return Ok(outcomes);
12766 }
12767 if outcomes.is_empty() {
12768 state.event("gate", "no gate commands configured; not tested");
12773 }
12774 for o in &outcomes {
12775 state.event(
12776 "gate",
12777 format!(
12778 "`{}` -> {}",
12779 o.command,
12780 if o.ok() {
12781 "pass".to_owned()
12782 } else {
12783 format!(
12784 "FAIL ({:?})\n{}",
12785 o.code,
12786 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12787 )
12788 }
12789 ),
12790 );
12791 }
12792 Ok(outcomes)
12793}
12794
12795pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12801 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12802 ""
12803 } else {
12804 pr_title
12805 }
12806}
12807
12808pub fn landing_subject_source(state: &RunState) -> String {
12812 if is_review_run(state) {
12813 let winner = state.candidates.first().map_or('A', |c| c.label);
12814 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12815 }
12816 state.instruction.clone()
12817}
12818
12819fn summary_without_title(summary: &str) -> String {
12822 let mut lines = summary.trim().lines().peekable();
12823 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12824 lines.next();
12825 }
12826 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12827}
12828
12829#[cfg(test)]
12841fn pr_message(state: &RunState, winner: char) -> PrMessage {
12842 pr_message_with(state, winner, None)
12843}
12844
12845#[cfg(test)]
12848fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12849 let raw = pr_message_raw(
12850 state,
12851 winner,
12852 facts,
12853 crate::pr_outcome::FinalHead::Unreported,
12854 ChangeNote::NotAsked,
12855 );
12856 let id = crate::scrub::Identity::current();
12857 PrMessage {
12858 title: crate::scrub::scrub(&raw.title, &id),
12859 body: crate::scrub::scrub(&raw.body, &id),
12860 }
12861}
12862
12863#[derive(Debug, Clone, Copy)]
12865enum ChangeNote<'a> {
12866 NotAsked,
12868 Summary(&'a crate::run::ChangeSummary),
12870 Unavailable(&'a str),
12872}
12873
12874fn pr_message_raw(
12875 state: &RunState,
12876 winner: char,
12877 facts: Option<&BranchFacts>,
12878 head: crate::pr_outcome::FinalHead<'_>,
12879 change: ChangeNote<'_>,
12880) -> PrMessage {
12881 if !matches!(change, ChangeNote::NotAsked) {
12882 let (title, mut body) = match change {
12883 ChangeNote::Summary(c) => (
12884 c.title.clone(),
12885 format!("## Summary\n\n{}\n\n", c.body.trim()),
12886 ),
12887 ChangeNote::Unavailable(why) => (
12888 neutral_title(state, winner),
12889 format!(
12890 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12891 ),
12892 ),
12893 ChangeNote::NotAsked => unreachable!(),
12894 };
12895 body.push_str(&public_pr_evidence(state, head));
12896 body.push_str(&format!(
12897 "\n---\nmagi:run/{} magi:candidate-{}\n",
12898 state.id,
12899 winner.to_ascii_lowercase()
12900 ));
12901 return PrMessage { title, body };
12902 }
12903 pr_preview(state, winner, facts, head)
12904}
12905
12906fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12909 let mut body = crate::pr_outcome::review_rounds(state);
12910 body.push_str(&crate::pr_outcome::final_verification(state, head));
12911 let open = state.last_round_findings();
12912 if !open.is_empty() {
12913 body.push_str("## Open review findings\n\n");
12914 for f in open {
12915 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12916 }
12917 body.push('\n');
12918 }
12919 if let Some(c) = state.change_summary.as_ref()
12920 && head == crate::pr_outcome::FinalHead::At(&c.head)
12921 {
12922 body.push_str(&format!(
12923 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12924 c.base, c.head
12925 ));
12926 }
12927 body
12928}
12929
12930fn pr_preview(
12932 state: &RunState,
12933 winner: char,
12934 facts: Option<&BranchFacts>,
12935 head: crate::pr_outcome::FinalHead<'_>,
12936) -> PrMessage {
12937 let summary = state
12938 .candidates
12939 .iter()
12940 .find(|c| c.label == winner)
12941 .map(|c| c.summary.as_str())
12942 .unwrap_or_default();
12943 let review = is_review_run(state);
12946 let title = if review {
12947 facts
12948 .and_then(|f| {
12949 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12950 })
12951 .or_else(|| review_title(state))
12952 .or_else(|| {
12953 state
12954 .candidates
12955 .iter()
12956 .find(|c| c.label == winner)
12957 .filter(|c| !c.branch.starts_with("magi/"))
12958 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12959 })
12960 .unwrap_or_else(|| neutral_title(state, winner))
12961 } else {
12962 summary_title(summary).unwrap_or_else(|| {
12963 english_title(
12964 &state.instruction,
12965 PR_TITLE_MAX,
12966 state.config.graph.github_english(),
12967 )
12968 .unwrap_or_else(|| neutral_title(state, winner))
12969 })
12970 };
12971
12972 let mut body = String::new();
12973 let what = summary_without_title(summary);
12974 if !what.is_empty() {
12975 body.push_str("## Summary\n\n");
12976 body.push_str(&what);
12977 body.push_str("\n\n");
12978 }
12979
12980 body.push_str(&crate::pr_outcome::review_rounds(state));
12982 body.push_str(&crate::pr_outcome::final_verification(state, head));
12983 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12984
12985 let open = state.open_findings();
12986 if !open.is_empty() {
12987 body.push_str("## Open review findings\n\n");
12988 for f in &open {
12989 body.push_str(&format!(
12990 "- `{}` [{:?}] {}\n",
12991 f.id, f.severity, "Context retained in local review records."
12992 ));
12993 }
12994 body.push('\n');
12995 }
12996
12997 if let Some(fix) = fix
12998 && !fix.rejected.is_empty()
12999 {
13000 body.push_str("## Declined by the fixer\n\n");
13001 for r in &fix.rejected {
13002 body.push_str(&format!(
13003 "- `{}`: {}\n",
13004 r.id, "Reason retained in local review records."
13005 ));
13006 }
13007 body.push('\n');
13008 }
13009
13010 if review {
13011 body.push_str("## Commits under review\n\n");
13013 if let Some(facts) = facts {
13014 let mut left = COMMITS_MAX_CHARS;
13015 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13016 let mut entry = format!("- {}\n", markdown_safe(subject));
13017 for l in markdown_safe(text).lines() {
13018 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13019 }
13020 if left == 0 {
13021 body.push_str(&format!(
13022 "- ... {} more commit(s)\n",
13023 facts.commits.len() - i
13024 ));
13025 break;
13026 }
13027 if entry.len() > left {
13028 let mut end = left;
13031 while !entry.is_char_boundary(end) {
13032 end -= 1;
13033 }
13034 entry.truncate(end);
13035 entry.push_str("\n ... (truncated)\n");
13036 left = 0;
13037 } else {
13038 left -= entry.len();
13039 }
13040 body.push_str(&entry);
13041 }
13042 if !facts.stat.trim().is_empty() {
13043 body.push_str(&format!(
13044 "\n## Diff stat\n\n```\n{}\n```\n",
13045 markdown_safe(facts.stat.trim())
13046 ));
13047 }
13048 } else {
13049 match &state.reviewed_commits {
13050 Some(subjects) => {
13051 for s in subjects {
13052 body.push_str(&format!("- {}\n", s.trim()));
13053 }
13054 }
13055 None => {
13056 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13059 body.push_str(rest.trim());
13060 body.push('\n');
13061 }
13062 }
13063 }
13064 } else {
13065 let task = state.instruction.trim();
13066 let task = if task.is_empty() {
13067 "(empty task)"
13068 } else {
13069 task
13070 };
13071 body.push_str(&format!(
13072 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13073 task.replace("</details>", "</details>")
13074 ));
13075 }
13076
13077 body.push_str(&format!(
13078 "\n---\nmagi:run/{} magi:candidate-{}\n",
13079 state.id,
13080 winner.to_ascii_lowercase()
13081 ));
13082
13083 PrMessage { title, body }
13084}
13085
13086fn seeded_instruction(state: &RunState) -> String {
13089 let mut inputs = Vec::new();
13090 if let Some(start) = &state.sources.start_from {
13091 inputs.push(format!(
13092 "Start from `{}` at `{}` (including ancestry).",
13093 start.token, start.sha
13094 ));
13095 }
13096 for pick in &state.sources.pick_commits {
13097 inputs.push(format!(
13098 "Pick `{}` at `{}` (single commit).",
13099 pick.token, pick.sha
13100 ));
13101 }
13102 if inputs.is_empty() {
13103 state.instruction.clone()
13104 } else {
13105 format!(
13106 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13107 state.instruction,
13108 inputs.join("\n")
13109 )
13110 }
13111}
13112
13113async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13117 let base = &state.base_branch;
13118 let mut against = base.clone();
13119 if mode == MergeMode::Pr {
13120 let remote = &state.config.merge.remote;
13124 let tracking = format!("{remote}/{base}");
13125 let fetched = git::fetch(repo, remote, base).await;
13126 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13127 against = tracking;
13128 } else {
13129 return false;
13130 }
13131 }
13132 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13133}
13134
13135fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13138 let mut detail = format!(
13139 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13140 nothing to open a pull request for"
13141 );
13142 detail.push_str(
13143 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13144 );
13145 detail
13146}
13147
13148#[derive(Debug, PartialEq, Eq)]
13151enum PrPlan {
13152 Create,
13153 Adopt { url: String, title: String },
13154 Stop(String),
13155}
13156
13157fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13160 match found {
13161 Ok(land::OpenPr::None) => PrPlan::Create,
13162 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13163 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13164 "several open pull requests exist for this branch, not picking one: {}",
13165 urls.join(" ")
13166 )),
13167 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13168 }
13169}
13170
13171pub(crate) async fn gh_pr_create(
13173 cwd: &Path,
13174 base: &str,
13175 head: &str,
13176 title: &str,
13177 body: &str,
13178) -> Result<String> {
13179 let out = tokio::process::Command::new("gh")
13180 .args([
13181 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13182 ])
13183 .current_dir(cwd)
13184 .quiet()
13185 .stdin(std::process::Stdio::null())
13186 .output()
13187 .await
13188 .context("spawn gh")?;
13189 if out.status.success() {
13190 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13191 } else {
13192 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13193 }
13194}
13195
13196async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13198 match std::fs::symlink_metadata(path) {
13199 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13200 git::worktree_remove_clean(repo, path).await?;
13203 return Ok(());
13204 }
13205 Err(e) => return Err(e.into()),
13206 Ok(_) => {}
13207 }
13208 if !git::is_clean(path).await? {
13209 bail!(
13210 "worktree has uncommitted work; refusing removal: {}",
13211 path.display()
13212 );
13213 }
13214 if !git::worktree_remove_clean(repo, path).await? {
13215 bail!("worktree removal refused: {}", path.display());
13216 }
13217 Ok(())
13218}
13219
13220pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13229 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13230 bail!("refusing fold: {reason}");
13231 }
13232 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13233 bail!("candidate preservation incomplete; refusing cleanup");
13234 }
13235 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13239 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13240 let _driver = crate::driver::maintenance(home, &state.id)?;
13241 let mut output_guards = Vec::new();
13242 for entry in std::fs::read_dir(state.worktree_root())
13243 .into_iter()
13244 .flatten()
13245 .flatten()
13246 {
13247 if entry.path().join(".git").is_file() {
13248 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13249 }
13250 }
13251 let repo = state.repo.clone();
13252 let wait = repository_wait(state, None);
13255 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13256 Ok(lease) => lease,
13257 Err(e) => {
13258 state.save_under(home)?;
13259 return Err(e);
13260 }
13261 };
13262 let _source_outputs = if state.status == RunStatus::Merged
13265 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13266 {
13267 verify_merged_fold_sources(state, drop_winner, home).await?
13268 } else {
13269 Vec::new()
13270 };
13271 let root = state.worktree_root();
13272 let winner = state.tally.as_ref().map(|t| t.winner);
13273 let mut removed = Vec::new();
13274
13275 for i in 0..state.candidates.len() {
13276 let c = state.candidates[i].clone();
13277 let is_winner = Some(c.label) == winner;
13278 if is_winner && !drop_winner {
13279 continue;
13280 }
13281 let existed = c.worktree.exists();
13282 remove_preserved_worktree(&repo, &c.worktree).await?;
13285 if existed {
13286 removed.push(c.worktree.to_string_lossy().into_owned());
13287 }
13288 let handed_over = state.released_branches.contains(&c.branch);
13291 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13292 if !git::branch_delete(&repo, &c.branch).await? {
13293 bail!("candidate branch removal refused");
13294 }
13295 removed.push(c.branch.clone());
13296 }
13297 state.candidates[i].folded = true;
13298 }
13299
13300 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13301 let path = name.path();
13302 let keep = !drop_winner
13303 && winner.is_some_and(|w| {
13304 path.file_name()
13305 .is_some_and(|n| n == format!("cand-{w}").as_str())
13306 });
13307 if keep {
13308 continue;
13309 }
13310 remove_preserved_worktree(&repo, &path).await?;
13311 removed.push(path.to_string_lossy().into_owned());
13312 }
13313
13314 remove_if_empty(&root);
13323
13324 if state.enabled_worktree_config && drop_winner {
13325 git::release_worktree_config(&repo).await.ok();
13329 state.enabled_worktree_config = false;
13330 }
13331 state.save_under(home)?;
13332 Ok(removed)
13333}
13334
13335async fn verify_merged_fold_sources(
13336 state: &RunState,
13337 drop_winner: bool,
13338 home: &Path,
13339) -> Result<Vec<std::fs::File>> {
13340 let winner = state.tally.as_ref().map(|t| t.winner);
13341 let mut heads = BTreeSet::new();
13342 for candidate in &state.candidates {
13343 if Some(candidate.label) == winner && !drop_winner {
13344 continue;
13345 }
13346 anyhow::ensure!(
13347 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13348 "candidate recovery source is outside the inventoried run root: {}",
13349 candidate.worktree.display()
13350 );
13351 match std::fs::symlink_metadata(&candidate.worktree) {
13352 Ok(_) => {}
13353 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13354 Err(error) => return Err(error.into()),
13355 }
13356 if !state.released_branches.contains(&candidate.branch)
13357 && git::branch_exists(&state.repo, &candidate.branch).await?
13358 {
13359 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13360 }
13361 }
13362 let mut cargo_guards = Vec::new();
13363 let entries = match std::fs::read_dir(state.worktree_root()) {
13364 Ok(entries) => Some(entries),
13365 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13366 Err(error) => return Err(error.into()),
13367 };
13368 for entry in entries.into_iter().flatten() {
13369 let path = entry?.path();
13370 let metadata = std::fs::symlink_metadata(&path)?;
13371 anyhow::ensure!(
13372 metadata.is_dir() && !metadata.file_type().is_symlink(),
13373 "non-directory or symlink recovery source retained: {}",
13374 path.display()
13375 );
13376 if !drop_winner
13377 && winner.is_some_and(|w| {
13378 path.file_name()
13379 .is_some_and(|name| name == format!("cand-{w}").as_str())
13380 })
13381 {
13382 continue;
13383 }
13384 anyhow::ensure!(
13385 path.join(".git").is_file(),
13386 "unregistered recovery source retained: {}",
13387 path.display()
13388 );
13389 anyhow::ensure!(
13390 git::git(
13391 &path,
13392 &[
13393 "status",
13394 "--porcelain",
13395 "--untracked-files=all",
13396 "--ignored",
13397 "--",
13398 ".",
13399 ":(exclude)target"
13400 ]
13401 )
13402 .await?
13403 .is_empty(),
13404 "source changes or ignored recovery files retained: {}",
13405 path.display()
13406 );
13407 match std::fs::symlink_metadata(path.join("target")) {
13408 Ok(_) => {
13409 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13410 }
13411 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13412 Err(error) => return Err(error.into()),
13413 }
13414 for checkpoint in [
13415 "rebase-merge",
13416 "rebase-apply",
13417 "sequencer",
13418 "CHERRY_PICK_HEAD",
13419 "MERGE_HEAD",
13420 "REBASE_HEAD",
13421 "REVERT_HEAD",
13422 "BISECT_LOG",
13423 "AUTO_MERGE",
13424 "MERGE_MSG",
13425 ] {
13426 let location = git::git(
13427 &path,
13428 &[
13429 "rev-parse",
13430 "--path-format=absolute",
13431 "--git-path",
13432 checkpoint,
13433 ],
13434 )
13435 .await?;
13436 anyhow::ensure!(
13437 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13438 if e.kind() == std::io::ErrorKind::NotFound),
13439 "Git recovery checkpoint retained: {checkpoint}"
13440 );
13441 }
13442 heads.insert(git::rev_parse(&path, "HEAD").await?);
13443 }
13444 crate::fold_evidence::preserve(state, home, &heads).await?;
13445
13446 Ok(cargo_guards)
13447}
13448
13449fn remove_if_empty(dir: &Path) {
13460 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13461 std::fs::remove_dir(dir).ok();
13462 }
13463}
13464
13465pub fn worst_open(state: &RunState) -> Option<Severity> {
13467 state
13468 .reviews
13469 .last()?
13470 .reviews
13471 .iter()
13472 .flat_map(|r| r.findings.iter())
13473 .map(|f| f.severity)
13474 .max()
13475}
13476
13477async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13481 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13482 git::acquire_worktree_config(repo).await?;
13483 state.enabled_worktree_config = true;
13484 }
13485 Ok(())
13486}
13487
13488#[cfg(test)]
13489mod tests {
13490 #[tokio::test]
13491 async fn launcher_base_fetch_requires_remote_success() {
13492 let tmp = tempfile::tempdir().unwrap();
13493 let repo = tmp.path().join("repo");
13494 let remote = tmp.path().join("remote");
13495 std::fs::create_dir(&repo).unwrap();
13496 std::fs::create_dir(&remote).unwrap();
13497 git::git(&remote, &["init", "--bare", "-b", "main"])
13498 .await
13499 .unwrap();
13500 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13501 git::git(
13502 &repo,
13503 &[
13504 "-c",
13505 "user.name=Fixture",
13506 "-c",
13507 "user.email=fixture@example.invalid",
13508 "commit",
13509 "--allow-empty",
13510 "-m",
13511 "fixture",
13512 ],
13513 )
13514 .await
13515 .unwrap();
13516 git::git(
13517 &repo,
13518 &["remote", "add", "origin", remote.to_str().unwrap()],
13519 )
13520 .await
13521 .unwrap();
13522 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13523 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13524 assert_eq!(
13525 resolve_base(&repo, "main", "origin", None)
13526 .await
13527 .unwrap()
13528 .commit,
13529 head
13530 );
13531 #[cfg(unix)]
13532 {
13533 let policy: crate::config::Coordination = toml::from_str(
13534 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13535 ).unwrap();
13536 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13537 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13538 let held = crate::repo_lock::SeatLease::operation_with(
13539 &repo,
13540 Some(&policy),
13541 &crate::repo_lock::WaitPolicy::default(),
13542 )
13543 .await
13544 .unwrap();
13545 assert!(held.is_some());
13546 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13547 tokio::pin!(fetch);
13548 assert!(
13549 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13550 .await
13551 .is_err()
13552 );
13553 assert!(
13554 !repo.join(".git/FETCH_HEAD").exists(),
13555 "no fetch before admission"
13556 );
13557 drop(held);
13558 let fetched = fetch.await.unwrap();
13559 assert_eq!(fetched.commit, head);
13560 assert_eq!(fetched.remote, "origin");
13561 assert_eq!(fetched.branch, "main");
13562 }
13563 git::git(
13565 &repo,
13566 &[
13567 "remote",
13568 "set-url",
13569 "origin",
13570 tmp.path().join("missing").to_str().unwrap(),
13571 ],
13572 )
13573 .await
13574 .unwrap();
13575 let error = resolve_base(&repo, "main", "origin", None)
13576 .await
13577 .unwrap_err()
13578 .to_string();
13579 assert!(error.contains("cannot read origin/main"));
13580 assert!(error.contains("refusing to branch off"));
13581 }
13582
13583 #[test]
13584 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13585 let leaked = vec!["chore(deps): update a crate".to_owned()];
13586 let own = "fix(daemon): apply a chosen action";
13587 assert!(should_retitle("", own, &leaked));
13588 assert!(should_retitle(
13589 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13590 own,
13591 &leaked
13592 ));
13593 assert!(should_retitle(
13594 "chore: land candidate A of run 1",
13595 own,
13596 &leaked
13597 ));
13598 assert!(should_retitle(
13599 "magi: candidate A (uncommitted work)",
13600 own,
13601 &leaked
13602 ));
13603 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13604 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13605 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13606 }
13607
13608 #[test]
13609 fn pr_merge_plan_creates_adopts_or_stops() {
13610 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13611 assert_eq!(
13612 pr_merge_plan(Ok(land::OpenPr::One {
13613 url: "u".into(),
13614 title: "t".into()
13615 })),
13616 PrPlan::Adopt {
13617 url: "u".into(),
13618 title: "t".into()
13619 }
13620 );
13621 let PrPlan::Stop(many) =
13622 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13623 else {
13624 panic!("many must stop");
13625 };
13626 assert!(many.contains('a') && many.contains('b'));
13627 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13628 panic!("a failed lookup must stop");
13629 };
13630 assert!(err.contains("bad token"));
13631 }
13632
13633 use super::*;
13634 use crate::run::GateStatus;
13635 use std::collections::BTreeMap;
13636 use std::time::Duration;
13637
13638 fn conductor() -> AgentSpec {
13639 AgentSpec {
13640 id: "conductor".to_owned(),
13641 kind: crate::config::AgentKind::Command,
13642 model: None,
13643 command: vec!["true".to_owned()],
13644 extra_args: Vec::new(),
13645 env: BTreeMap::new(),
13646 prompt_delivery: None,
13647 max_turn_secs: None,
13648 }
13649 }
13650
13651 fn spec(id: &str) -> AgentSpec {
13652 AgentSpec {
13653 id: id.to_owned(),
13654 kind: crate::config::AgentKind::Command,
13655 model: None,
13656 command: vec!["true".to_owned()],
13657 extra_args: Vec::new(),
13658 env: BTreeMap::new(),
13659 prompt_delivery: None,
13660 max_turn_secs: None,
13661 }
13662 }
13663
13664 fn ids(xs: &[&str]) -> BTreeSet<String> {
13665 xs.iter().map(|s| (*s).to_owned()).collect()
13666 }
13667
13668 #[test]
13669 fn pick_successor_skips_an_agent_another_seat_holds() {
13670 let roster = [spec("a"), spec("b"), spec("c")];
13672 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13673 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13674 }
13675
13676 #[test]
13677 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13678 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13680 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13681 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13682 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13683 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13684 }
13685
13686 #[test]
13687 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13688 let roster = [spec("a"), spec("b")];
13689 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13690 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13691 let carried = ids(&["b"]);
13692 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13693 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13694 }
13695
13696 #[test]
13697 fn pick_successor_returns_none_without_any_untried_successor() {
13698 let roster = [spec("a"), spec("b")];
13699 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13700 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13701 let carried = ids(&["a"]);
13702 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13703 }
13704
13705 #[test]
13706 fn pick_successor_rescue_avoids_another_seats_occupant() {
13707 let roster = [spec("a"), spec("b"), spec("c")];
13709 let carried = ids(&["b", "c"]);
13710 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13711 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13712 }
13713
13714 #[test]
13715 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13716 let roster = [spec("a"), spec("b"), spec("c")];
13717 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13718 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13719 }
13720
13721 #[test]
13722 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13723 let roster = [spec("a"), spec("b"), spec("c")];
13724 let failed = ids(&["a", "b", "c"]);
13725 let mut tried = ids(&["b"]);
13727 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13728 assert_eq!(first.id, "a");
13729 tried.insert(first.id.clone());
13730 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13731 assert_eq!(second.id, "c");
13732 tried.insert(second.id.clone());
13733 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13734 }
13735
13736 #[test]
13737 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13738 let roster = [spec("a"), spec("b")];
13739 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13740 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13741 }
13742
13743 #[test]
13744 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13745 let roster = [spec("a"), spec("b"), spec("c")];
13746 let h = SeatHistory {
13747 failed: ids(&["a"]),
13748 last_ok: Some("b".to_owned()),
13749 last_fail: None,
13750 };
13751 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13752 let gone = SeatHistory {
13754 last_ok: Some("zzz".to_owned()),
13755 ..h.clone()
13756 };
13757 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13758 let failed = SeatHistory {
13759 failed: ids(&["a", "b"]),
13760 last_ok: Some("b".to_owned()),
13761 last_fail: None,
13762 };
13763 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13764 }
13765
13766 #[test]
13767 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13768 let first = SeatState::new("review-1", "alpha", 7);
13769 let next = handover_seat("review-1", "gamma", 7);
13770 assert_ne!(first.claude_session, next.claude_session);
13771 }
13772
13773 #[test]
13774 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13775 let mut state = state_with_summary("x", "y");
13776 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13777 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13778 assert_ne!(a.claude_session, b.claude_session);
13779 }
13780
13781 #[test]
13782 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13783 let roster = [spec("a"), spec("b")];
13784 let h = SeatHistory {
13785 failed: ids(&["a", "b"]),
13786 ..SeatHistory::default()
13787 };
13788 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13789 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13790 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13791 }
13792
13793 #[test]
13800 fn failure_signature_ignores_numbers_and_paths() {
13801 assert_eq!(
13802 failure_signature("exited with Some(2) and no usable output"),
13803 failure_signature("exited with Some(137) and no usable output")
13804 );
13805 assert_eq!(
13806 failure_signature("process exited with no usable output (exit code 2)"),
13807 failure_signature("process exited with no usable output (exit code 137)")
13808 );
13809 for (old, new) in [
13810 (
13811 "exited with Some(2) and no usable output",
13812 "process exited with no usable output (exit code 2)",
13813 ),
13814 (
13815 "exited with None and no usable output",
13816 "process exited with no usable output (exit code unavailable)",
13817 ),
13818 ] {
13819 let previous = FailClass::Other(failure_signature(old));
13820 let current = FailClass::Other(failure_signature(new));
13821 assert_eq!(previous, current, "resumption retains the failure class");
13822 assert!(!should_hand_over(Some(&previous), ¤t));
13823 }
13824 assert_eq!(
13825 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13826 failure_signature("cannot open /var/x.txt: denied")
13827 );
13828 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13829 }
13830
13831 #[test]
13832 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13833 let other = FailClass::Other("x".into());
13834 assert!(should_hand_over(None, &FailClass::Quota));
13835 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13836 assert!(should_hand_over(
13837 Some(&FailClass::Timeout),
13838 &FailClass::Timeout
13839 ));
13840 assert!(should_hand_over(None, &other));
13841 assert!(!should_hand_over(Some(&other), &other));
13842 assert!(should_hand_over(
13843 Some(&other),
13844 &FailClass::Other("y".into())
13845 ));
13846 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13848 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13849 }
13850
13851 #[test]
13852 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13853 let a = SeatState::new("judge-1", "alpha", 7);
13854 let b = handover_seat("judge-1", "beta", 7);
13855 assert_ne!(a.claude_session, b.claude_session);
13856 assert_eq!(b.turns, 0);
13857 }
13858
13859 #[test]
13860 fn a_timeout_is_classified_apart_from_other_failures() {
13861 assert_eq!(
13862 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13863 Some(FailClass::Timeout)
13864 );
13865 assert!(matches!(
13866 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13867 Some(FailClass::Other(_))
13868 ));
13869 }
13870
13871 #[test]
13872 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13873 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13874 let tried = BTreeSet::from(["beta".to_owned()]);
13875 let next = next_untried_in_roster(&roster, 1, &tried);
13878 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13879 }
13880
13881 #[test]
13882 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13883 let roster = vec![spec("alpha"), spec("beta")];
13884 let tried = BTreeSet::from(["beta".to_owned()]);
13885 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13889 }
13890
13891 #[test]
13892 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13893 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13894 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13895 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13899 }
13900
13901 #[test]
13902 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13903 let roster = vec![spec("a"), spec("a"), spec("b")];
13904 let tried = BTreeSet::from(["a".to_owned()]);
13905 let next = next_untried_in_roster(&roster, 0, &tried);
13906 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13907 }
13908
13909 #[test]
13910 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13911 let roster = vec![spec("a"), spec("b")];
13912 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13913 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13914 }
13915
13916 #[test]
13917 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13918 let dir = tempfile::tempdir().unwrap();
13919 let bay = dir.path().join("ffff");
13920
13921 remove_if_empty(&bay);
13923 assert!(!bay.exists());
13924
13925 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13928 remove_if_empty(&bay);
13929 assert!(bay.exists(), "non-empty directory must survive");
13930
13931 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13933 remove_if_empty(&bay);
13934 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13935 }
13936
13937 #[test]
13946 fn a_full_panel_that_found_nothing_is_clean() {
13947 assert!(round_is_clean(
13948 0,
13949 true,
13950 2,
13951 2,
13952 0,
13953 IncompleteReviewPolicy::Block
13954 ));
13955 }
13956
13957 #[test]
13958 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13959 assert!(!round_is_clean(
13960 0,
13961 true,
13962 1,
13963 2,
13964 0,
13965 IncompleteReviewPolicy::Block
13966 ));
13967 }
13968
13969 #[test]
13970 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13971 assert!(!round_is_clean(
13972 1,
13973 true,
13974 1,
13975 2,
13976 0,
13977 IncompleteReviewPolicy::Warn
13978 ));
13979 }
13980
13981 #[test]
13982 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13983 assert!(round_is_clean(
13984 0,
13985 true,
13986 1,
13987 2,
13988 0,
13989 IncompleteReviewPolicy::Warn
13990 ));
13991 }
13992
13993 #[test]
13994 fn a_full_panel_with_an_open_finding_is_not_clean() {
13995 assert!(!round_is_clean(
13996 1,
13997 true,
13998 2,
13999 2,
14000 0,
14001 IncompleteReviewPolicy::Block
14002 ));
14003 }
14004
14005 #[test]
14006 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14007 assert!(!round_is_clean(
14008 0,
14009 false,
14010 2,
14011 2,
14012 0,
14013 IncompleteReviewPolicy::Block
14014 ));
14015 }
14016
14017 #[test]
14024 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14025 assert!(round_is_clean(
14028 0,
14029 true,
14030 1,
14031 2,
14032 1,
14033 IncompleteReviewPolicy::Block
14034 ));
14035 }
14036
14037 #[test]
14038 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14039 assert!(!round_is_clean(
14042 0,
14043 true,
14044 1,
14045 2,
14046 0,
14047 IncompleteReviewPolicy::Block
14048 ));
14049 }
14050
14051 #[test]
14052 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14053 assert!(!round_is_clean(
14054 1,
14055 true,
14056 1,
14057 2,
14058 1,
14059 IncompleteReviewPolicy::Block
14060 ));
14061 assert!(!round_is_clean(
14062 0,
14063 false,
14064 1,
14065 2,
14066 1,
14067 IncompleteReviewPolicy::Block
14068 ));
14069 }
14070
14071 #[test]
14072 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14073 assert!(!round_is_clean(
14077 0,
14078 true,
14079 0,
14080 2,
14081 2,
14082 IncompleteReviewPolicy::Block
14083 ));
14084 }
14085
14086 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14087 CommandOutcome {
14088 command: "test".to_owned(),
14089 code,
14090 output_tail: String::new(),
14091 duration_ms: 0,
14092 resource_blocked,
14093 }
14094 }
14095
14096 #[cfg(unix)]
14097 #[tokio::test]
14098 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14099 ask_test_home();
14100 let temp = tempfile::tempdir().unwrap();
14101 let mut runner = runner_at(RunStatus::Gating);
14102 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14103 let commands = vec![
14104 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14105 "touch must-not-run".to_owned(),
14106 ];
14107 let (outcomes, retried, pids) = run_e2e_with_retry(
14108 &mut runner.state,
14109 &shell,
14110 &commands,
14111 temp.path(),
14112 Duration::from_secs(5),
14113 "fixture",
14114 )
14115 .await;
14116 assert_eq!(outcomes.len(), 1);
14117 assert!(outcomes[0].resource_blocked);
14118 assert!(!outcomes[0].build_failed());
14119 assert!(!retried);
14120 assert!(pids.is_empty());
14121 assert!(!temp.path().join("must-not-run").exists());
14122 assert_eq!(
14123 runner.state.resource_block.as_ref().unwrap().kind,
14124 "incremental cache admission"
14125 );
14126 }
14127
14128 #[test]
14129 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14130 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14131 assert!(
14132 !verify_inconclusive(&[outcome(Some(1), false)]),
14133 "an ordinary failure is still evidence about the patch"
14134 );
14135 assert!(verify_inconclusive(&[outcome(None, true)]));
14136 assert!(
14137 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14138 "one inconclusive outcome taints the whole batch"
14139 );
14140 assert!(!verify_inconclusive(&[]));
14141 }
14142
14143 #[tokio::test]
14144 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14145 let calls = std::sync::atomic::AtomicUsize::new(0);
14149 let started = Instant::now();
14150 wait_for_pids_with(
14151 &[123],
14152 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14153 Duration::from_millis(5),
14154 Duration::from_secs(5),
14155 )
14156 .await;
14157 assert!(
14158 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14159 "must keep checking rather than deciding on the first answer"
14160 );
14161 assert!(
14162 started.elapsed() < Duration::from_secs(1),
14163 "must return the moment it is confirmed dead, not wait out the ceiling"
14164 );
14165 }
14166
14167 #[tokio::test]
14168 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14169 let started = Instant::now();
14170 wait_for_pids_with(
14171 &[123],
14172 |_| true, Duration::from_millis(5),
14174 Duration::from_millis(30),
14175 )
14176 .await;
14177 let elapsed = started.elapsed();
14178 assert!(
14179 elapsed >= Duration::from_millis(30),
14180 "must not give up before its own ceiling: {elapsed:?}"
14181 );
14182 assert!(
14183 elapsed < Duration::from_secs(1),
14184 "must not wait past its own ceiling either: {elapsed:?}"
14185 );
14186 }
14187
14188 #[tokio::test]
14189 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14190 let started = Instant::now();
14191 wait_for_pids_with(
14192 &[],
14193 |_| true,
14194 Duration::from_secs(5),
14195 Duration::from_secs(5),
14196 )
14197 .await;
14198 assert!(
14199 started.elapsed() < Duration::from_millis(200),
14200 "an empty pid list has nothing to confirm"
14201 );
14202 }
14203
14204 fn review_round(
14210 clean: bool,
14211 blocking: usize,
14212 answered: usize,
14213 expected: usize,
14214 progressed: bool,
14215 e2e_ok: bool,
14216 ) -> ReviewRound {
14217 ReviewRound {
14218 round: 1,
14219 head: "h".to_owned(),
14220 verified_head: None,
14221 verified_at: None,
14222 reviews: Vec::new(),
14223 e2e: vec![CommandOutcome {
14224 command: "test".to_owned(),
14225 code: Some(if e2e_ok { 0 } else { 1 }),
14226 output_tail: String::new(),
14227 duration_ms: 0,
14228 resource_blocked: false,
14229 }],
14230 verify_retried: false,
14231 e2e_deferred: false,
14232 e2e_defer_reason: None,
14233 fix: None,
14234 blocking,
14235 answered,
14236 expected,
14237 clean,
14238 progressed,
14239 vote_split: false,
14240 reconsideration: Vec::new(),
14241 verdict: None,
14242 }
14243 }
14244
14245 #[test]
14246 fn review_conclusion_is_none_when_nothing_has_run() {
14247 assert_eq!(review_conclusion(&[], 3), None);
14248 }
14249
14250 #[test]
14251 fn review_conclusion_is_none_while_rounds_remain() {
14252 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14253 assert_eq!(review_conclusion(&rounds, 3), None);
14254 }
14255
14256 #[test]
14257 fn review_conclusion_is_gating_once_a_round_is_clean() {
14258 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14259 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14260 }
14261
14262 #[test]
14263 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14264 let rounds = vec![
14265 review_round(false, 1, 2, 2, true, true),
14266 review_round(false, 1, 2, 2, true, true),
14267 ];
14268 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14269 }
14270
14271 #[test]
14272 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14273 let rounds = vec![
14274 review_round(false, 1, 2, 2, true, true),
14275 review_round(false, 1, 2, 2, true, false),
14276 ];
14277 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14278 }
14279
14280 #[test]
14281 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14282 let mut blocked = review_round(false, 1, 2, 2, true, false);
14289 blocked.e2e[0].resource_blocked = true;
14290 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14291 assert_eq!(review_conclusion(&rounds, 2), None);
14292 }
14293
14294 #[test]
14295 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14296 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14298 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14299 }
14300
14301 #[test]
14302 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14303 let rounds = vec![
14304 review_round(false, 1, 2, 2, false, true),
14305 review_round(false, 1, 2, 2, false, true),
14306 ];
14307 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14308 }
14309
14310 fn secs(n: u64) -> Duration {
14311 Duration::from_secs(n)
14312 }
14313
14314 #[tokio::test]
14317 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14318 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14319 if std::env::var_os(CHILD).is_none() {
14320 let output = std::process::Command::new(std::env::current_exe().unwrap())
14324 .args([
14325 "--exact",
14326 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14327 "--nocapture",
14328 ])
14329 .env(CHILD, "1")
14330 .env_remove("MAGI_WORKER_HOME")
14331 .env_remove("MAGI_WORKER_ID")
14332 .output()
14333 .unwrap();
14334 assert!(
14335 output.status.success()
14336 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14337 "{}\n{}",
14338 String::from_utf8_lossy(&output.stdout),
14339 String::from_utf8_lossy(&output.stderr)
14340 );
14341 return;
14342 }
14343 ask_test_home();
14344 let tmp = tempfile::tempdir().unwrap();
14345 let repo = tmp.path().join("repo");
14346 std::fs::create_dir_all(&repo).unwrap();
14347 init_repo(&repo);
14348 let g = |args: &[&str]| {
14349 let out = std::process::Command::new("git")
14350 .args(args)
14351 .current_dir(&repo)
14352 .quiet()
14353 .output()
14354 .unwrap();
14355 assert!(
14356 out.status.success(),
14357 "{}",
14358 String::from_utf8_lossy(&out.stderr)
14359 );
14360 };
14361 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14362 g(&["checkout", "-b", "historical"]);
14363 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14364 g(&["add", "."]);
14365 g(&["commit", "-m", "old alternative"]);
14366 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14367 g(&["checkout", "-b", "magi/task-selected", &base]);
14368 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14369 g(&["add", "."]);
14370 g(&["commit", "-m", "measurement infrastructure"]);
14371 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14372 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14373 g(&["add", "."]);
14374 g(&["commit", "-m", "dependent implementation"]);
14375 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14376 g(&["checkout", "main"]);
14377 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14378 g(&["add", "."]);
14379 g(&["commit", "-m", "unrelated base change"]);
14380 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14381 let origin = tmp.path().join("origin.git");
14382 g(&[
14383 "clone",
14384 "--bare",
14385 repo.to_str().unwrap(),
14386 origin.to_str().unwrap(),
14387 ]);
14388 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14389 g(&["fetch", "origin"]);
14390 let selection = crate::pr_recovery::Selection {
14391 url: "https://example.test/project/pull/1".into(),
14392 branch: "magi/task-selected".into(),
14393 head: head.clone(),
14394 source_head: None,
14395 source_run: None,
14396 source_proof: None,
14397 disposition: Some(crate::pr_recovery::Disposition::Review),
14398 instruction: "preserve measured behavior".into(),
14399 approved_at: jiff::Timestamp::now(),
14400 };
14401 let mut template = runner_at(RunStatus::Reviewing);
14402 template.state.repo = repo.clone();
14403 template.state.base_commit = current.clone();
14404 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14405 template.state.origin = Some(crate::run::Origin::queue("selected"));
14406 template.state.pr_head = Some(selection.branch.clone());
14407 template.state.pr_recovery = Some(selection.clone());
14408 let mut runner =
14409 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14410 .await
14411 .unwrap();
14412 let wt = runner.state.winner().unwrap().worktree.clone();
14413 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14414 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14415 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14416 assert!(runner.state.seeds.is_empty());
14417 assert_eq!(runner.state.instruction, "preserve measured behavior");
14418 runner.sync_to_base().await.unwrap();
14419 assert_eq!(
14420 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14421 "prerequisite"
14422 );
14423 assert_eq!(
14424 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14425 "measured implementation"
14426 );
14427 assert_eq!(
14428 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14429 "current main change"
14430 );
14431 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14432 assert_eq!(
14433 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14434 head
14435 );
14436
14437 let mut partial = runner_at(RunStatus::Reviewing);
14439 partial.state.repo = repo.clone();
14440 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14441 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14442 run: Some("exit 1".into()),
14443 ..Default::default()
14444 }];
14445 let selection = runner.state.pr_recovery.clone().unwrap();
14446 partial.state.pr_recovery = Some(selection.clone());
14447 partial.state.pr_head = Some(selection.branch.clone());
14448 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14449 let id = partial.state.id.clone();
14450 let error = match Runner::open_selected(
14451 &repo,
14452 partial.state,
14453 partial.roles,
14454 1,
14455 runner.state.base_commit.clone(),
14456 selection,
14457 )
14458 .await
14459 {
14460 Ok(_) => panic!("setup must fail"),
14461 Err(e) => e,
14462 };
14463 assert_eq!(
14464 error
14465 .downcast_ref::<crate::run::StartupFailure>()
14466 .unwrap()
14467 .run,
14468 id
14469 );
14470 let failed = RunState::load(&id).unwrap();
14471 assert_eq!(failed.status, RunStatus::Failed);
14472 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14473 assert!(Runner::resume(&id).is_err());
14474 runner.state.status = RunStatus::Ready;
14475 runner.state.driver_exited = true;
14476 runner.state.save().unwrap();
14477 let mut retry = runner_at(RunStatus::Reviewing);
14478 retry.state.repo = repo.clone();
14479 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14480 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14481 reselected.approved_at = jiff::Timestamp::now();
14482 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14483 retry.state.pr_head = Some(reselected.branch.clone());
14484 retry.state.pr_recovery = Some(reselected.clone());
14485 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14486 .await
14487 .unwrap();
14488 let restarted = Runner::open_selected(
14489 &repo,
14490 retry.state,
14491 retry.roles,
14492 1,
14493 runner.state.base_commit.clone(),
14494 reselected,
14495 )
14496 .await
14497 .unwrap();
14498 assert_eq!(
14499 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14500 .await
14501 .unwrap(),
14502 head
14503 );
14504 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14505 assert_eq!(
14506 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14507 head
14508 );
14509 }
14510
14511 #[tokio::test]
14512 #[cfg(unix)]
14513 async fn fallback_and_interrupted_restoration_require_coordination() {
14514 use std::os::fd::AsRawFd as _;
14515 ask_test_home();
14516 let tmp = tempfile::tempdir().unwrap();
14517 let mut runner = registered_runner(tmp.path()).await;
14518 runner.state.config.graph.timeout_resource_wait = 1;
14519 let winner = runner.state.candidates[0].clone();
14520 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14521 let leftover = winner.worktree.join("leftover");
14522 std::fs::write(&leftover, "untrusted edits").unwrap();
14523 let next = tmp.path().join("next-fixer");
14524 let mut failed = spec("first");
14525 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14526 let mut successor = spec("second");
14527 successor.command = vec![
14528 "sh".into(),
14529 "-c".into(),
14530 format!("touch '{}'; echo done", next.display()),
14531 ];
14532 let attempts = vec![
14533 (failed.clone(), "first".into()),
14534 (successor.clone(), "second".into()),
14535 ];
14536 let lock_path = runner.state.repo.join(".git/updater.lock");
14537 let external = std::fs::File::create(&lock_path).unwrap();
14538 assert_eq!(
14539 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14540 0
14541 );
14542 let artifacts = tmp.path().join("artifacts");
14543 let build = |spec, seat| SeatJob {
14544 spec,
14545 seat,
14546 cwd: winner.worktree.clone(),
14547 prompt: "fixture".into(),
14548 timeout: Duration::from_secs(10),
14549 allow_write: true,
14550 sessions: false,
14551 artifacts: artifacts.clone(),
14552 stem: "fix".into(),
14553 handover: None,
14554 };
14555 let (_, _, out) = runner
14556 .ask_fixer_among(
14557 attempts.clone(),
14558 "graft",
14559 None,
14560 Some((&winner.worktree, &head)),
14561 build,
14562 )
14563 .await;
14564 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14565 assert!(leftover.exists(), "contention must prevent clean");
14566 assert!(!next.exists());
14567 assert!(runner.state.handovers.is_empty());
14568 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14570 from_run: "earlier".into(),
14571 generation: 1,
14572 mode: crate::recovery::Mode::VerifyRepair,
14573 reasons: Vec::new(),
14574 reviewed_head: None,
14575 rounds: Vec::new(),
14576 open_findings: Vec::new(),
14577 evidence: None,
14578 criteria_inherited: true,
14579 repair: crate::recovery::Repair::Started {
14580 before: head.clone(),
14581 },
14582 });
14583 assert!(
14584 runner
14585 .recovery_repair(&winner)
14586 .await
14587 .unwrap_err()
14588 .is::<crate::repo_lock::ResourceBlocked>()
14589 );
14590 assert!(matches!(
14591 runner.state.recovery.as_ref().unwrap().repair,
14592 crate::recovery::Repair::Started { .. }
14593 ));
14594 assert!(leftover.exists());
14595 drop(external);
14596 runner.recovery_repair(&winner).await.unwrap();
14597 assert!(!leftover.exists());
14598 assert!(matches!(
14599 runner.state.recovery.as_ref().unwrap().repair,
14600 crate::recovery::Repair::Done { .. }
14601 ));
14602 std::fs::remove_file(&lock_path).unwrap();
14604 std::fs::create_dir(&lock_path).unwrap();
14605 std::fs::write(&leftover, "untrusted edits").unwrap();
14606 let (_, _, out) = runner
14607 .ask_fixer_among(
14608 attempts,
14609 "graft",
14610 None,
14611 Some((&winner.worktree, &head)),
14612 build,
14613 )
14614 .await;
14615 assert!(matches!(out, AgentOutcome::Failed(_)));
14616 assert!(leftover.exists());
14617 assert!(!next.exists());
14618 assert_eq!(
14619 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14620 head
14621 );
14622 }
14623
14624 #[cfg(unix)]
14625 async fn registered_runner(root: &Path) -> Runner {
14626 repository_runner(root, crate::config::test_coordination()).await
14627 }
14628
14629 async fn repository_runner(
14630 root: &Path,
14631 coordination: Option<crate::config::Coordination>,
14632 ) -> Runner {
14633 ask_test_home();
14634 let repo = root.join("repo");
14635 std::fs::create_dir_all(&repo).unwrap();
14636 init_repo(&repo);
14637 if coordination.is_some() {
14638 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14639 }
14640 let mut runner = runner_at(RunStatus::Prep);
14641 runner.state.repo = repo.clone();
14642 runner.state.config.coordination = coordination;
14643 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14644 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14645 runner.state.config.graph.advise = false;
14646 runner.roles.implementers = vec![spec("implement")];
14647 runner.roles.judges = vec![spec("judge")];
14648 runner.prep().await.unwrap();
14649 runner
14650 }
14651
14652 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14653
14654 #[tokio::test]
14655 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14656 let tmp = tempfile::tempdir().unwrap();
14657 let mut runner = repository_runner(tmp.path(), None).await;
14658 let prompts = runner.state.config.prompts.clone();
14659 let run = runner.state.id.clone();
14660 let ctx = WaveCtx {
14661 carry_seats: false,
14662 pauses: None,
14663 run: &run,
14664 node: "advise",
14665 prompts: &prompts,
14666 cache: None,
14667 round: None,
14668 };
14669 let jobs: Vec<_> = ["alpha", "beta"]
14670 .into_iter()
14671 .map(|id| SeatJob {
14672 spec: scripted(id, AUTH_OUTAGE),
14673 seat: SeatState::new(id, id, 7),
14674 prompt: "propose".into(),
14675 cwd: runner.state.candidates[0].worktree.clone(),
14676 timeout: Duration::from_secs(10),
14677 allow_write: false,
14678 sessions: false,
14679 artifacts: agent::artifacts_dir(&runner.state.dir()),
14680 stem: id.into(),
14681 handover: None,
14682 })
14683 .collect();
14684 let mut losses = Vec::new();
14685 let results = ask_wave_with(
14686 jobs.clone(),
14687 runner.sem.clone(),
14688 2,
14689 &[],
14690 &ctx,
14691 &mut losses,
14692 &mut runner.state,
14693 &|_: &str| Ok(()),
14694 )
14695 .await;
14696 assert!(
14697 results
14698 .iter()
14699 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14700 );
14701 assert_eq!(runner.state.quota.len(), 2);
14702 assert!(
14703 runner
14704 .state
14705 .quota
14706 .iter()
14707 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14708 );
14709 assert!(!runner.state.advise_attempted);
14710 runner.stop_for_resource().unwrap();
14711 assert_eq!(runner.state.status, RunStatus::Blocked);
14712 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14713 assert_eq!(
14714 runner.state.resource_block.as_ref().unwrap().kind,
14715 "agent authentication"
14716 );
14717 runner.state.resource_block = None;
14720 runner.state.quota.clear();
14721 let mut mixed = jobs;
14722 mixed[1].spec.command = vec!["false".into()];
14723 let results = ask_wave_with(
14724 mixed,
14725 runner.sem.clone(),
14726 0,
14727 &[],
14728 &ctx,
14729 &mut losses,
14730 &mut runner.state,
14731 &|_: &str| Ok(()),
14732 )
14733 .await;
14734 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14735 assert!(runner.state.resource_block.is_none());
14736 assert_eq!(losses.len(), 1);
14737 }
14738
14739 #[tokio::test]
14740 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14741 let tmp = tempfile::tempdir().unwrap();
14742 let mut runner = repository_runner(tmp.path(), None).await;
14743 let implementer = scripted("implement", AUTH_OUTAGE);
14744 runner.roles.implementers = vec![implementer.clone()];
14745 runner.state.config.agents = vec![implementer];
14746 runner.implement().await.unwrap();
14747 assert_eq!(runner.state.status, RunStatus::Blocked);
14748 assert_eq!(runner.state.quota.len(), 1);
14749 assert_eq!(
14750 runner.state.quota[0].cause,
14751 agent::QuotaCause::Authentication
14752 );
14753 let candidate = &runner.state.candidates[0];
14754 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14755 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14756 }
14757
14758 #[tokio::test]
14759 #[cfg(unix)]
14760 async fn parked_implementer_persists_completed_work() {
14761 for registered in [false, true] {
14762 let tmp = tempfile::tempdir().unwrap();
14763 let mut runner = registered_runner(tmp.path()).await;
14764 if !registered {
14765 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14766 }
14767 let marker = tmp.path().join("started");
14768 let release = tmp.path().join("release");
14769 let mut implementer = spec("implement");
14770 implementer.command = vec![
14771 "sh".into(),
14772 "-c".into(),
14773 format!(
14774 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14775 marker.display(),
14776 release.display()
14777 ),
14778 ];
14779 runner.roles.implementers = vec![implementer.clone()];
14780 runner.state.config.agents = vec![implementer];
14781 let pause = runner.pause.clone();
14782 let (result, ()) = tokio::join!(runner.implement(), async {
14783 tokio::time::timeout(Duration::from_secs(10), async {
14784 while !marker.exists() {
14785 tokio::time::sleep(Duration::from_millis(10)).await;
14786 }
14787 })
14788 .await
14789 .unwrap();
14790 pause.park();
14791 std::fs::write(&release, "go").unwrap();
14792 });
14793 result.unwrap();
14794 let candidate = &runner.state.candidates[0];
14795 assert!(candidate.commits > 0, "{candidate:?}");
14796 assert!(candidate.summary.contains("saved-result"));
14797 assert_eq!(candidate.files, 1);
14798 let before = candidate.commits;
14799 runner.implement().await.unwrap();
14800 assert_eq!(runner.state.candidates[0].commits, before);
14801 }
14802 }
14803
14804 #[tokio::test]
14805 #[cfg(unix)]
14806 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14807 let tmp = tempfile::tempdir().unwrap();
14808 let mut runner = registered_runner(tmp.path()).await;
14809 let winner = runner.state.candidates[0].clone();
14810 let cache = tmp.path().join("cache");
14811 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14812 runner.state.config.graph.timeout_resource_wait = 0;
14813 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14814 let crate::cache::AcquireOutcome::Acquired(held) =
14815 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14816 else {
14817 panic!("cache must be free")
14818 };
14819 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14820 from_run: "earlier".into(),
14821 generation: 1,
14822 mode: crate::recovery::Mode::VerifyRepair,
14823 reasons: Vec::new(),
14824 reviewed_head: None,
14825 rounds: Vec::new(),
14826 open_findings: Vec::new(),
14827 evidence: None,
14828 criteria_inherited: true,
14829 repair: crate::recovery::Repair::NotStarted,
14830 });
14831 runner.recovery_repair(&winner).await.unwrap();
14832 assert!(matches!(
14833 runner.state.recovery.as_ref().unwrap().repair,
14834 crate::recovery::Repair::WaitingForRestore { .. }
14835 ));
14836 drop(held);
14837 runner.state.resource_block = None;
14838 runner.state.config.graph.timeout_resource_wait = 1;
14839 runner.recovery_repair(&winner).await.unwrap();
14840 assert!(matches!(
14841 runner.state.recovery.as_ref().unwrap().repair,
14842 crate::recovery::Repair::Done { .. }
14843 ));
14844 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14845 }
14846
14847 #[tokio::test]
14848 #[cfg(unix)]
14849 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14850 use std::os::fd::AsRawFd as _;
14851 for graft in [false, true] {
14852 let tmp = tempfile::tempdir().unwrap();
14853 let mut runner = registered_runner(tmp.path()).await;
14854 runner.state.config.graph.timeout_resource_wait = 1;
14855 let winner = runner.state.candidates[0].clone();
14856 let marker = tmp.path().join("fixer-started");
14857 let mut fixer = spec("fixer");
14858 fixer.command = vec![
14859 "sh".into(),
14860 "-c".into(),
14861 format!("echo started >> '{}'; echo done", marker.display()),
14862 ];
14863 runner.roles.fixer = Some(vec![fixer]);
14864 if graft {
14865 let mut loser = winner.clone();
14866 loser.label = 'B';
14867 loser.index = 1;
14868 runner.state.candidates.push(loser);
14869 runner.state.config.graph.graft_runner_up = true;
14870 runner.state.tally = Some(Tally {
14871 first_choice: BTreeMap::new(),
14872 borda: BTreeMap::new(),
14873 winner: winner.label,
14874 rankings: 1,
14875 unanimous_initial: true,
14876 deliberated: false,
14877 changed_votes: 0,
14878 unanimous_final: true,
14879 tie_break: None,
14880 judges: 1,
14881 present: 1,
14882 quorum: 1,
14883 met_quorum: true,
14884 uncontested: None,
14885 });
14886 runner.state.judgements.push(crate::run::Judgement {
14887 judge: 1,
14888 seat: "judge-1".into(),
14889 agent: "judge".into(),
14890 ranking: vec![winner.label, 'B'],
14891 reasons: BTreeMap::new(),
14892 confidence: None,
14893 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14894 order: vec![0, 1],
14895 failed: None,
14896 duration_ms: 0,
14897 });
14898 } else {
14899 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14900 from_run: "earlier".into(),
14901 generation: 1,
14902 mode: crate::recovery::Mode::VerifyRepair,
14903 reasons: Vec::new(),
14904 reviewed_head: None,
14905 rounds: Vec::new(),
14906 open_findings: Vec::new(),
14907 evidence: None,
14908 criteria_inherited: true,
14909 repair: crate::recovery::Repair::NotStarted,
14910 });
14911 }
14912 let reader = crate::repo_lock::SeatLease::for_checkout(
14913 &winner.worktree,
14914 false,
14915 crate::config::test_coordination().as_ref(),
14916 )
14917 .await
14918 .unwrap();
14919 let pause = runner.pause.clone();
14920 let (result, ()) = tokio::join!(
14921 async {
14922 if graft {
14923 runner.graft().await
14924 } else {
14925 runner.recovery_repair(&winner).await
14926 }
14927 },
14928 async {
14929 tokio::time::sleep(Duration::from_millis(100)).await;
14930 pause.park();
14931 }
14932 );
14933 result.unwrap();
14934 assert!(!marker.exists(), "a blocked seat must not spawn");
14935 runner.state = RunState::load(&runner.state.id).unwrap();
14937 if graft {
14938 assert_eq!(
14939 runner.state.graft.as_ref().unwrap().outcome,
14940 GraftOutcome::WaitingForRestore
14941 );
14942 } else {
14943 assert!(matches!(
14944 runner.state.recovery.as_ref().unwrap().repair,
14945 crate::recovery::Repair::WaitingForRestore { .. }
14946 ));
14947 }
14948 drop(reader);
14949 runner.pause = Pause::new();
14950 runner.state.resource_block = None;
14951 let external =
14953 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14954 assert_eq!(
14955 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14956 0
14957 );
14958 let leftover = winner.worktree.join("leftover");
14959 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14960 let result = if graft {
14961 runner.graft().await
14962 } else {
14963 runner.recovery_repair(&winner).await
14964 };
14965 assert!(
14966 result
14967 .unwrap_err()
14968 .is::<crate::repo_lock::ResourceBlocked>()
14969 );
14970 assert!(leftover.exists());
14971 assert!(!marker.exists());
14972 drop(external);
14973 runner.state.resource_block = None;
14974 let result = if graft {
14975 runner.graft().await
14976 } else {
14977 runner.recovery_repair(&winner).await
14978 };
14979 result.unwrap();
14980 assert!(!leftover.exists(), "restore must precede retry");
14981 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14982 if graft {
14983 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14984 } else {
14985 assert!(matches!(
14986 runner.state.recovery.as_ref().unwrap().repair,
14987 crate::recovery::Repair::Done { .. }
14988 ));
14989 }
14990 }
14991 }
14992
14993 #[tokio::test]
14994 #[cfg(unix)]
14995 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14996 use std::os::fd::AsRawFd as _;
14997 let tmp = tempfile::tempdir().unwrap();
14998 let mut runner = registered_runner(tmp.path()).await;
14999 runner.state.config.graph.timeout_resource_wait = 1;
15000 let winner = runner.state.candidates[0].clone();
15001 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15002 runner.state.tally = Some(Tally {
15003 first_choice: BTreeMap::new(),
15004 borda: BTreeMap::new(),
15005 winner: winner.label,
15006 rankings: 0,
15007 unanimous_initial: false,
15008 deliberated: false,
15009 changed_votes: 0,
15010 unanimous_final: false,
15011 tie_break: None,
15012 judges: 0,
15013 present: 0,
15014 quorum: 0,
15015 met_quorum: true,
15016 uncontested: Some("fixture".into()),
15017 });
15018 runner.state.graft = Some(GraftRecord {
15019 pre: pre.clone(),
15020 post: None,
15021 lines: None,
15022 limit: None,
15023 agent: None,
15024 outcome: GraftOutcome::Running,
15025 requested: Vec::new(),
15026 adoption: None,
15027 adoption_error: None,
15028 });
15029 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15030 git::commit_all(&winner.worktree, "unfinished graft")
15031 .await
15032 .unwrap();
15033 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15034 let leftover = winner.worktree.join("leftover");
15035 std::fs::write(&leftover, "untrusted edits").unwrap();
15036 let path = runner.state.repo.join(".git/updater.lock");
15037 let external = std::fs::File::create(&path).unwrap();
15038 assert_eq!(
15039 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15040 0
15041 );
15042 assert!(
15043 runner
15044 .graft()
15045 .await
15046 .unwrap_err()
15047 .is::<crate::repo_lock::ResourceBlocked>()
15048 );
15049 assert!(matches!(
15050 runner.state.graft.as_ref().unwrap().outcome,
15051 GraftOutcome::Running
15052 ));
15053 assert_eq!(
15054 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15055 unfinished
15056 );
15057 assert!(leftover.exists());
15058 drop(external);
15059 std::fs::remove_file(&path).unwrap();
15060 std::fs::create_dir(&path).unwrap();
15061 assert!(runner.graft().await.is_err());
15062 assert!(matches!(
15063 runner.state.graft.as_ref().unwrap().outcome,
15064 GraftOutcome::Running
15065 ));
15066 assert_eq!(
15067 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15068 unfinished
15069 );
15070 assert!(leftover.exists());
15071 std::fs::remove_dir(&path).unwrap();
15072 runner.graft().await.unwrap();
15073 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15074 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15075 assert!(!leftover.exists());
15076 }
15077
15078 #[tokio::test]
15079 #[cfg(unix)]
15080 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15081 use std::os::fd::AsRawFd as _;
15082 let tmp = tempfile::tempdir().unwrap();
15083 let mut runner = registered_runner(tmp.path()).await;
15084 runner.state.prep_complete = Some(false);
15085 runner.state.config.graph.timeout_resource_wait = 1;
15086 runner.state.save().unwrap();
15087 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15088 assert_eq!(
15089 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15090 0
15091 );
15092 runner.execute().await.unwrap();
15093 let disk = RunState::load(&runner.state.id).unwrap();
15094 assert!(disk.driver_exited);
15095 assert_eq!(disk.driver_pid, None);
15096 assert_eq!(runner.state.status, RunStatus::Blocked);
15097 assert_eq!(
15098 runner.state.resource_block.as_ref().unwrap().kind,
15099 "repository coordination"
15100 );
15101 assert!(runner.state.prep_failure.is_none());
15102 assert!(runner.state.seats.is_empty());
15103 runner.pause.park();
15104 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15107 .await
15108 .unwrap()
15109 .unwrap();
15110 assert!(runner.state.parked);
15111 assert!(runner.state.prep_failure.is_none());
15112 let disk = RunState::load(&runner.state.id).unwrap();
15113 assert!(disk.driver_exited);
15114 assert_eq!(disk.driver_pid, None);
15115 }
15116
15117 #[tokio::test]
15118 #[cfg(unix)]
15119 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15120 use std::os::fd::AsRawFd as _;
15121 let tmp = tempfile::tempdir().unwrap();
15122 let mut runner = registered_runner(tmp.path()).await;
15123 runner.state.prep_complete = Some(false);
15124 runner.state.config.graph.timeout_resource_wait = 30;
15125 runner.state.save().unwrap();
15126 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15127 assert_eq!(
15128 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15129 0
15130 );
15131 assert!(
15132 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15133 .await
15134 .is_err()
15135 );
15136 let disk = RunState::load(&runner.state.id).unwrap();
15137 assert!(disk.driver_exited);
15138 assert_eq!(disk.driver_pid, None);
15139 assert!(disk.status.resumable());
15140 assert!(runner.state.driver_exited);
15141 assert_eq!(runner.state.driver_pid, None);
15142 runner.state.save().unwrap();
15143 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15144 }
15145
15146 #[tokio::test]
15147 #[cfg(unix)]
15148 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15149 let tmp = tempfile::tempdir().unwrap();
15150 let mut runner = registered_runner(tmp.path()).await;
15151 let winner = runner.state.candidates[0].clone();
15152 let cache = tmp.path().join("cache");
15153 let started = tmp.path().join("pre-gate-started");
15154 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15155 runner.state.config.verify.gate =
15156 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15157 let home = crate::run::home();
15158 let owner =
15159 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15160 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15161 crate::cache::AcquireOutcome::Acquired(g) => g,
15162 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15163 };
15164 let mut gate = Box::pin(runner.run_gate(&winner));
15165 assert!(
15166 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15167 .await
15168 .is_err()
15169 );
15170 assert!(!started.exists(), "pre-gate must wait for the cache too");
15171 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15175 let editor = crate::repo_lock::SeatLease::checkout_with(
15176 &winner.worktree,
15177 true,
15178 crate::config::test_coordination().as_ref(),
15179 &wait,
15180 )
15181 .await
15182 .unwrap();
15183 drop(editor);
15184 guard.release();
15185 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15186 .await
15187 .unwrap()
15188 .unwrap();
15189 drop(gate);
15190 assert!(outcomes.iter().all(CommandOutcome::ok));
15191 assert!(started.exists());
15192 }
15193
15194 #[tokio::test]
15195 #[cfg(unix)]
15196 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15197 use std::os::fd::AsRawFd as _;
15198
15199 ask_test_home();
15200 let tmp = tempfile::tempdir().unwrap();
15201 let repo = tmp.path().join("repo");
15202 std::fs::create_dir_all(&repo).unwrap();
15203 init_repo(&repo);
15204 let common = repo.join(".git");
15205 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15206 let mut runner = runner_at(RunStatus::Prep);
15207 runner.state.repo = repo.clone();
15208 runner.state.config.coordination = crate::config::test_coordination();
15209 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15210 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15211 runner.state.config.graph.advise = false;
15212 runner.roles.implementers = vec![spec("implement")];
15213 runner.roles.judges = vec![spec("judge")];
15214 runner.prep().await.unwrap();
15215 let winner = runner.state.candidates[0].clone();
15216 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15219 let completed = tmp.path().join("seat-completed");
15220 runner.roles.implementers[0].command = vec![
15221 "sh".into(),
15222 "-c".into(),
15223 format!("touch '{}'; echo completed", completed.display()),
15224 ];
15225 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15226 assert_eq!(
15227 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15228 0
15229 );
15230 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15231 let mut implement = Box::pin(runner.implement());
15232 tokio::select! {
15233 result = &mut implement => panic!("rescue must wait: {result:?}"),
15234 result = tokio::time::timeout(Duration::from_secs(10), async {
15235 while !completed.exists() {
15236 tokio::time::sleep(Duration::from_millis(20)).await;
15237 }
15238 }) => result.unwrap(),
15239 }
15240 assert!(
15241 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15242 .await
15243 .is_err()
15244 );
15245 assert_eq!(
15246 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15247 before
15248 );
15249 assert_eq!(
15250 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15251 0
15252 );
15253 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15254 .await
15255 .unwrap()
15256 .unwrap();
15257 drop(implement);
15258 assert_ne!(
15259 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15260 before
15261 );
15262 assert!(git::is_clean(&winner.worktree).await.unwrap());
15263
15264 let started = tmp.path().join("pre-gate-started");
15265 let proceed = tmp.path().join("pre-gate-proceed");
15266 let checked = tmp.path().join("gate-ran");
15267 runner.state.config.verify.pre_gate = vec![format!(
15268 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15269 started.display(),
15270 proceed.display()
15271 )];
15272 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15273 assert_eq!(
15274 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15275 0
15276 );
15277 let mut gate = Box::pin(runner.run_gate(&winner));
15278 assert!(
15279 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15280 .await
15281 .is_err()
15282 );
15283 assert!(!started.exists());
15284 assert!(!checked.exists());
15285 assert!(git::is_clean(&winner.worktree).await.unwrap());
15286 assert_eq!(
15287 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15288 0
15289 );
15290 tokio::select! {
15291 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15292 result = tokio::time::timeout(Duration::from_secs(10), async {
15293 while !started.exists() {
15294 tokio::time::sleep(Duration::from_millis(20)).await;
15295 }
15296 }) => result.unwrap(),
15297 }
15298 assert_eq!(
15299 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15300 -1,
15301 "mechanical edits must hold the real lock"
15302 );
15303 std::fs::write(proceed, "ready").unwrap();
15304 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15305 .await
15306 .unwrap()
15307 .unwrap();
15308 drop(gate);
15309 assert!(outcomes.iter().all(CommandOutcome::ok));
15310 assert!(checked.exists());
15311 assert!(runner.state.pre_gate_commit.is_some());
15312 assert!(git::is_clean(&winner.worktree).await.unwrap());
15313 assert_eq!(
15314 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15315 0,
15316 "gate completion must release coordination"
15317 );
15318 drop(outsider);
15319 std::fs::remove_file(common.join("updater.lock")).unwrap();
15320 std::fs::create_dir(common.join("updater.lock")).unwrap();
15321 std::fs::remove_file(&started).unwrap();
15322 std::fs::remove_file(&checked).unwrap();
15323 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15324 let error = runner.run_gate(&winner).await.unwrap_err();
15325 assert!(
15326 error
15327 .to_string()
15328 .contains("acquire mandatory repository lock")
15329 );
15330 assert!(
15331 !started.exists(),
15332 "failed acquisition must prevent pre-gate"
15333 );
15334 assert!(!checked.exists(), "failed acquisition must prevent gate");
15335 assert!(git::is_clean(&winner.worktree).await.unwrap());
15336 assert_eq!(
15337 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15338 head
15339 );
15340 }
15341
15342 #[tokio::test]
15343 #[cfg(unix)]
15344 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15345 use std::os::fd::AsRawFd as _;
15346 use std::time::Duration;
15347
15348 ask_test_home();
15349 let tmp = tempfile::tempdir().unwrap();
15350 let repo = tmp.path().join("repo");
15351 std::fs::create_dir_all(&repo).unwrap();
15352 init_repo(&repo);
15353 let common = repo.join(".git");
15354 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15355 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15356 assert_eq!(
15357 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15358 0
15359 );
15360 let mut runner = runner_at(RunStatus::Prep);
15361 runner.state.repo = repo.clone();
15362 runner.state.config.coordination = crate::config::test_coordination();
15363 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15364 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15365 runner.state.config.graph.advise = true;
15366 runner.state.config.graph.advisors = 1;
15367 runner.roles.implementers = vec![spec("implement")];
15368 runner.roles.judges = vec![spec("judge")];
15369 let root = runner.state.worktree_root();
15370 let started = tmp.path().join("setup-started");
15371 let proceed = tmp.path().join("setup-proceed");
15372 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15373 run: Some(format!(
15374 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15375 started.display(),
15376 proceed.display()
15377 )),
15378 ..Default::default()
15379 }];
15380 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15381 let config = git::git(&repo, &["config", "--local", "--list"])
15382 .await
15383 .unwrap();
15384 let mut prep = Box::pin(runner.prep());
15385 assert!(
15386 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15387 .await
15388 .is_err()
15389 );
15390 assert!(
15391 !root.exists(),
15392 "contention must prevent even checkout creation"
15393 );
15394 assert!(!started.exists(), "contention must prevent setup");
15395 assert_eq!(
15396 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15397 branches
15398 );
15399 assert_eq!(
15400 git::git(&repo, &["config", "--local", "--list"])
15401 .await
15402 .unwrap(),
15403 config
15404 );
15405 drop(lock);
15406 tokio::select! {
15407 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15408 result = tokio::time::timeout(Duration::from_secs(10), async {
15409 while !started.exists() {
15410 tokio::time::sleep(Duration::from_millis(20)).await;
15411 }
15412 }) => result.unwrap(),
15413 }
15414 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15415 assert_eq!(
15416 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15417 -1
15418 );
15419 assert!(
15420 tokio::time::timeout(
15421 Duration::from_millis(100),
15422 crate::repo_lock::SeatLease::for_checkout(
15423 &repo,
15424 true,
15425 crate::config::test_coordination().as_ref()
15426 )
15427 )
15428 .await
15429 .is_err(),
15430 "preparation must also exclude local editors"
15431 );
15432 std::fs::write(proceed, "ready").unwrap();
15433 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15434 .await
15435 .unwrap()
15436 .unwrap();
15437 drop(prep);
15438 assert_eq!(runner.state.prep_complete, Some(true));
15439 assert!(root.join("judge-1").exists());
15440 assert!(root.join("advisor-1").exists());
15441 assert_eq!(
15442 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15443 0
15444 );
15445 }
15446
15447 #[tokio::test]
15448 #[cfg(unix)]
15449 async fn preparation_lock_failure_prevents_repository_changes() {
15450 ask_test_home();
15451 let tmp = tempfile::tempdir().unwrap();
15452 let repo = tmp.path().join("repo");
15453 std::fs::create_dir_all(&repo).unwrap();
15454 init_repo(&repo);
15455 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15456 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15459 let mut runner = runner_at(RunStatus::Prep);
15460 runner.state.repo = repo.clone();
15461 runner.state.config.coordination = crate::config::test_coordination();
15462 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15463 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15464 runner.roles.implementers = vec![spec("implement")];
15465 let error = runner.prep().await.unwrap_err();
15466 assert!(
15467 error
15468 .to_string()
15469 .contains("acquire mandatory repository lock")
15470 );
15471 assert!(!runner.state.worktree_root().exists());
15472 assert!(runner.state.candidates.is_empty());
15473 assert!(!repo.join(".git/config.worktree").exists());
15474 assert_eq!(
15475 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15476 "* main"
15477 );
15478 }
15479
15480 #[tokio::test]
15481 async fn preparation_records_creation_intent_before_git_can_fail() {
15482 ask_test_home();
15483 let tmp = tempfile::tempdir().unwrap();
15484 let repo = tmp.path().join("repo");
15485 std::fs::create_dir_all(&repo).unwrap();
15486 init_repo(&repo);
15487 let mut runner = runner_at(RunStatus::Prep);
15488 runner.state.repo = repo.clone();
15489 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15490 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15491 runner.state.config.blind.commit_msg_hook = false;
15492 runner.roles.implementers = vec![spec("one")];
15493 let label = blind::assign_labels(1, runner.state.seed)[0];
15494 let branch = runner.state.branch_for(label);
15495 git::git(&repo, &["branch", &branch]).await.unwrap();
15498 assert!(runner.prep().await.is_err());
15499 let saved = RunState::load(&runner.state.id).unwrap();
15500 assert!(saved.candidates.is_empty());
15501 let event = saved
15502 .events
15503 .iter()
15504 .find(|e| e.node == "prep-intent")
15505 .unwrap();
15506 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15507 assert_eq!(intent["branch"], branch);
15508 assert_eq!(intent["start"], runner.state.base_commit);
15509 assert_eq!(
15510 intent["worktree"],
15511 runner
15512 .state
15513 .worktree_root()
15514 .join(format!("cand-{label}"))
15515 .to_string_lossy()
15516 .as_ref()
15517 );
15518 assert_eq!(intent["picks"], serde_json::json!([]));
15519 }
15520
15521 #[tokio::test]
15522 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15523 ask_test_home();
15524 let tmp = tempfile::tempdir().unwrap();
15525 let repo = tmp.path().join("repo");
15526 std::fs::create_dir_all(&repo).unwrap();
15527 init_repo(&repo);
15528 let mut runner = runner_at(RunStatus::Prep);
15529 runner.state.repo = repo.clone();
15530 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15531 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15532 runner.state.sources.pick_commits.push(refs::Input {
15533 token: "missing".into(),
15534 sha: "0".repeat(40),
15535 provenance: refs::Provenance::ExplicitCli,
15536 });
15537 assert!(runner.prep().await.is_err());
15538 assert!(runner.state.candidates.is_empty());
15539 assert!(!runner.state.worktree_root().exists());
15540 }
15541
15542 #[tokio::test]
15543 async fn prose_and_legacy_inference_never_import_into_candidates() {
15544 ask_test_home();
15545 let tmp = tempfile::tempdir().unwrap();
15546 let repo = tmp.path().join("repo");
15547 std::fs::create_dir_all(&repo).unwrap();
15548 init_repo(&repo);
15549 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15550 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15551 .await
15552 .unwrap();
15553 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15554 git::git(&repo, &["add", "."]).await.unwrap();
15555 git::git(&repo, &["commit", "-m", "historical implementation"])
15556 .await
15557 .unwrap();
15558 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15559 git::git(&repo, &["checkout", "main"]).await.unwrap();
15560 for instruction in [
15561 format!("Example: magi/history/A at {source}"),
15562 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15563 format!("Never import magi/history/A or {source}"),
15564 format!("Conversation answer: I mentioned {source} only as background"),
15565 ] {
15566 let mut runner = runner_at(RunStatus::Prep);
15567 runner.state.repo = repo.clone();
15568 runner.state.base_commit = base.clone();
15569 runner.state.instruction = instruction.clone();
15570 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15571 runner.state.config.blind.commit_msg_hook = false;
15572 runner.state.seeds = vec![refs::Seed {
15573 token: "magi/history/A".into(),
15574 sha: source.clone(),
15575 kind: refs::SeedKind::Unmerged,
15576 branch: true,
15577 detail: String::new(),
15578 }];
15579 runner.roles.implementers = vec![spec("one")];
15580 assert_eq!(seeded_instruction(&runner.state), instruction);
15581 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15583 legacy.as_object_mut().unwrap().remove("sources");
15584 runner.state = serde_json::from_value(legacy).unwrap();
15585 assert_eq!(runner.state.sources, refs::Sources::default());
15586 runner.state.save().unwrap();
15587 runner.state = RunState::load(&runner.state.id).unwrap();
15588 runner.prep().await.unwrap();
15589 for candidate in &runner.state.candidates {
15590 assert_eq!(
15591 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15592 base
15593 );
15594 assert!(!candidate.worktree.join("unwanted").exists());
15595 }
15596 }
15597 }
15598
15599 #[tokio::test]
15600 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15601 ask_test_home();
15602 let tmp = tempfile::tempdir().unwrap();
15603 let repo = tmp.path().join("repo");
15604 std::fs::create_dir_all(&repo).unwrap();
15605 init_repo(&repo);
15606 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15607 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15608 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15609 git::git(&repo, &["add", "."]).await.unwrap();
15610 git::git(&repo, &["commit", "-m", "required seed"])
15611 .await
15612 .unwrap();
15613 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15614 git::git(&repo, &["checkout", "main"]).await.unwrap();
15615 let mut runner = runner_at(RunStatus::Prep);
15616 runner.state.repo = repo.clone();
15617 runner.state.base_commit = base.clone();
15618 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15619 runner.state.config.graph.advise = false;
15620 runner.state.config.blind.commit_msg_hook = false;
15621 runner.state.sources =
15622 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15623 .await
15624 .unwrap();
15625 runner.roles.implementers = vec![spec("one"), spec("two")];
15626 runner.roles.judges = vec![spec("judge")];
15627 let label = blind::assign_labels(2, runner.state.seed)[0];
15628 let branch = runner.state.branch_for(label);
15629 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15630 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15631 .await
15632 .unwrap();
15633 runner.state.candidates.push(Candidate {
15634 index: 0,
15635 label,
15636 agent: "one".into(),
15637 branch,
15638 worktree,
15639 summary: String::new(),
15640 stat: String::new(),
15641 files: 0,
15642 commits: 0,
15643 empty: false,
15644 failed: None,
15645 verified_noop: None,
15646 duration_ms: 0,
15647 folded: false,
15648 });
15649 runner.state.save().unwrap();
15650 runner.state = RunState::load(&runner.state.id).unwrap();
15652 assert_eq!(runner.state.prep_complete, Some(false));
15653 runner.prep().await.unwrap();
15654 assert_eq!(runner.state.prep_complete, Some(true));
15655 assert_eq!(runner.state.candidates.len(), 2);
15656 for candidate in &runner.state.candidates {
15657 assert_eq!(
15658 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15659 git::tree_of(&repo, &seed).await.unwrap()
15660 );
15661 assert_eq!(
15662 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15663 "required commit"
15664 );
15665 }
15666 assert!(runner.state.worktree_root().join("judge-1").exists());
15667 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15668 runner.prep().await.unwrap();
15669 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15670
15671 runner.state.prep_complete = Some(false);
15672 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15673 run: Some(format!(
15674 "test -f '{}'",
15675 tmp.path().join("dependency-ready").display()
15676 )),
15677 ..Default::default()
15678 }];
15679 assert!(runner.prep().await.is_err());
15680 assert!(runner.state.prep_failure.is_some());
15681 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15682 runner.state = RunState::load(&runner.state.id).unwrap();
15683 assert!(
15684 runner.prep().await.is_err(),
15685 "correcting the cause alone must not auto-retry"
15686 );
15687 let failed_at = runner.state.events.last().unwrap().at;
15688 assert!(
15689 runner.retry_preparation(failed_at).is_err(),
15690 "old approvals cannot unlock the hold"
15691 );
15692 let mut task = crate::queue::Task::new(
15693 "retry".into(),
15694 "retry setup".into(),
15695 repo.clone(),
15696 crate::queue::Source::Human,
15697 );
15698 task.start(runner.state.id.clone());
15699 task.hold_machine(Some("setup failed".into()));
15700 task.release_operator("cli", None);
15701 runner
15702 .retry_preparation(task.resumption.as_ref().unwrap().at)
15703 .unwrap();
15704 runner.prep().await.unwrap();
15705 assert!(runner.state.prep_failure.is_none());
15706 assert_eq!(runner.state.prep_complete, Some(true));
15707 assert_eq!(runner.state.candidates.len(), 2);
15708 assert!(runner.state.worktree_root().join("judge-1").exists());
15709 for candidate in &runner.state.candidates {
15710 assert_eq!(
15711 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15712 git::tree_of(&repo, &seed).await.unwrap()
15713 );
15714 }
15715 }
15716
15717 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15718 let mut heads = Vec::new();
15719 for candidate in candidates {
15720 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15721 }
15722 heads
15723 }
15724
15725 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15726 ask_test_home();
15727 let tmp = tempfile::tempdir().unwrap();
15728 let repo = tmp.path().join("repo");
15729 std::fs::create_dir(&repo).unwrap();
15730 init_repo(&repo);
15731 let mut runner = runner_at(RunStatus::Reviewing);
15732 runner.state.repo = repo.clone();
15733 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15734 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15735 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15736 let branch = format!("candidate-{label}");
15737 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15738 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15739 .await
15740 .unwrap();
15741 if label == 'B' {
15742 std::fs::write(worktree.join("change"), "winner").unwrap();
15743 git::rescue_commit(&worktree, "candidate change")
15744 .await
15745 .unwrap();
15746 }
15747 runner.state.candidates.push(Candidate {
15748 index,
15749 label,
15750 agent: "alpha".into(),
15751 branch,
15752 worktree,
15753 summary: "completed reply".into(),
15754 stat: String::new(),
15755 files: usize::from(label == 'B'),
15756 commits: usize::from(label == 'B'),
15757 empty: label == 'A',
15758 failed: None,
15759 verified_noop: None,
15760 duration_ms: 1,
15761 folded: false,
15762 });
15763 }
15764 runner.state.judge_skipped = true;
15765 runner.state.tally = Some(Tally {
15766 first_choice: BTreeMap::new(),
15767 borda: BTreeMap::new(),
15768 winner: 'B',
15769 rankings: 0,
15770 unanimous_initial: false,
15771 deliberated: false,
15772 changed_votes: 0,
15773 unanimous_final: false,
15774 tie_break: None,
15775 judges: 0,
15776 present: 0,
15777 quorum: 0,
15778 met_quorum: true,
15779 uncontested: Some("only B produced a change".into()),
15780 });
15781 runner.state.save().unwrap();
15782 (tmp, runner)
15783 }
15784
15785 #[tokio::test]
15786 #[cfg(unix)]
15787 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15788 for (solo, failed_checkpoint) in
15789 [(true, false), (false, false), (true, true), (false, true)]
15790 {
15791 let (_tmp, mut runner) = preservation_resume_fixture().await;
15792 if solo {
15793 runner.state.candidates.retain(|c| c.label == 'A');
15794 }
15795 runner.state.candidates[0].empty = false;
15796 runner.state.candidates[0].summary.clear();
15797 runner.state.tally = None;
15798 runner.state.judge_skipped = false;
15799 runner.state.status = RunStatus::Implementing;
15800 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15801 let mut implementer = spec("alpha");
15802 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15803 runner.roles.implementers = vec![implementer];
15804 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15805 if failed_checkpoint {
15806 std::fs::create_dir_all(&patch).unwrap();
15807 }
15808 runner.state.save().unwrap();
15809 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15810 drop(pressure);
15811 runner.state = RunState::load(&runner.state.id).unwrap();
15812 assert_eq!(runner.state.status, RunStatus::Blocked);
15813 assert!(runner.state.interrupted_implementations.contains(&'A'));
15814 assert_eq!(runner.state.candidates[0].commits, 0);
15815 assert!(!runner.state.candidates[0].empty);
15816 assert!(runner.state.candidates[0].failed.is_none());
15817 if failed_checkpoint {
15818 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15819 std::fs::remove_dir(patch).unwrap();
15820 }
15821 runner.state.resource_block = None;
15822 runner.state.config.disk.min_free_bytes = 0;
15823 runner.roles.implementers[0].command = vec![
15824 "sh".into(),
15825 "-c".into(),
15826 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15827 ];
15828 runner.implement().await.unwrap();
15829 assert!(runner.state.interrupted_implementations.is_empty());
15830 assert!(runner.state.candidate_preservation.is_empty());
15831 assert_eq!(runner.state.candidates[0].commits, 1);
15832 assert!(!runner.state.candidates[0].empty);
15833 assert!(
15834 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15835 .unwrap()
15836 .contains("retried empty interrupted seat")
15837 );
15838 if !solo {
15839 assert_eq!(runner.state.candidates[1].commits, 1);
15840 }
15841 }
15842 }
15843
15844 #[tokio::test]
15845 #[cfg(unix)]
15846 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15847 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15848 let (_tmp, mut runner) = preservation_resume_fixture().await;
15849 runner.state.candidates.retain(|c| c.label == 'A');
15850 runner.state.candidates[0].empty = false;
15851 runner.state.candidates[0].summary.clear();
15852 runner.state.tally = None;
15853 runner.state.judge_skipped = false;
15854 runner.state.status = RunStatus::Implementing;
15855 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15858 let wt = runner.state.candidates[0].worktree.clone();
15859 std::fs::write(
15860 wt.join("interrupted-source"),
15861 "retain implementation work\n",
15862 )
15863 .unwrap();
15864 if committed {
15865 git::rescue_commit(&wt, "implementation checkpoint")
15866 .await
15867 .unwrap();
15868 }
15869 let lock = git::git(
15870 &wt,
15871 &[
15872 "rev-parse",
15873 "--path-format=absolute",
15874 "--git-path",
15875 "index.lock",
15876 ],
15877 )
15878 .await
15879 .unwrap();
15880 if block_rescue {
15881 std::fs::write(&lock, "fixture-owned lock").unwrap();
15882 }
15883 let mut implementer = spec("alpha");
15884 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15885 runner.roles.implementers = vec![implementer];
15886 runner.state.save().unwrap();
15887 let result = runner.implement().await;
15888 drop(pressure);
15889 assert_eq!(result.is_err(), block_rescue);
15890 assert_eq!(
15891 runner.state.resource_block.as_ref().unwrap().kind,
15892 "disk pressure"
15893 );
15894 assert!(
15895 runner
15896 .state
15897 .events
15898 .iter()
15899 .any(|e| e.message.contains("agent execution interrupted"))
15900 );
15901 assert!(
15902 runner
15903 .state
15904 .dir()
15905 .join("artifacts/candidate-A-recovery/staged.patch")
15906 .exists()
15907 );
15908 if block_rescue {
15909 let saved = RunState::load(&runner.state.id).unwrap();
15910 assert!(saved.candidate_preservation.contains_key(&'A'));
15911 assert!(!saved.candidates[0].empty);
15912 assert_eq!(
15913 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15914 "retain implementation work\n"
15915 );
15916 std::fs::remove_file(lock).unwrap();
15917 runner.state = saved;
15918 } else {
15919 assert_eq!(runner.state.status, RunStatus::Blocked);
15920 assert_eq!(runner.state.candidates[0].commits, 1);
15921 }
15922 runner.roles.implementers.clear();
15925 runner.state.config.disk.min_free_bytes = 0;
15926 runner.state.resource_block = None;
15927 runner.implement().await.unwrap();
15928 assert!(runner.state.candidate_preservation.is_empty());
15929 assert_eq!(runner.state.candidates[0].commits, 1);
15930 assert!(!runner.state.candidates[0].empty);
15931 assert!(git::is_clean(&wt).await.unwrap());
15932 assert!(
15933 crate::run::read_artifact(&runner.state, "cand-A.patch")
15934 .unwrap()
15935 .contains("retain implementation work")
15936 );
15937 }
15938 }
15939
15940 #[tokio::test]
15941 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15942 let (_tmp, mut runner) = preservation_resume_fixture().await;
15943 runner.implement().await.unwrap();
15945 assert!(runner.state.candidate_preservation.is_empty());
15946 assert!(
15947 !runner
15948 .state
15949 .dir()
15950 .join("artifacts/candidate-A-recovery")
15951 .exists()
15952 );
15953 runner.fold_losers().await.unwrap();
15954 assert!(!runner.state.candidates[0].worktree.exists());
15955 for schema in [14, crate::run::SCHEMA] {
15956 runner.state.schema = schema;
15957 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15958 runner.state.save().unwrap();
15959 runner.state = RunState::load(&runner.state.id).unwrap();
15960 runner.implement().await.unwrap();
15961 assert!(runner.state.candidate_preservation.is_empty());
15962 assert!(runner.state.candidates[0].folded);
15963 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15964 }
15965 }
15966
15967 #[tokio::test]
15968 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15969 let (_tmp, mut runner) = preservation_resume_fixture().await;
15970 runner.state.candidates.retain(|c| c.label == 'A');
15971 runner.state.tally = None;
15972 runner.state.judge_skipped = false;
15973 let candidate = &mut runner.state.candidates[0];
15974 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15975 candidate.failed = Some("deployment validation unavailable".into());
15976 std::fs::write(
15977 candidate.worktree.join("repair"),
15978 "future launcher policy\n",
15979 )
15980 .unwrap();
15981 runner
15982 .state
15983 .candidate_preservation
15984 .insert('A', "awaiting persistence".into());
15985 runner.persist_candidates().await.unwrap();
15986 let candidate = &runner.state.candidates[0];
15987 assert!(!candidate.empty);
15988 assert_eq!(candidate.commits, 1);
15989 assert_eq!(candidate.files, 1);
15990 assert!(candidate.failed.is_none());
15991 assert!(candidate.summary.contains("EPERM"));
15992 assert!(
15993 crate::run::read_artifact(&runner.state, "cand-A.patch")
15994 .unwrap()
15995 .contains("future launcher policy")
15996 );
15997 assert!(runner.state.candidate_preservation.is_empty());
15998 }
15999
16000 #[tokio::test]
16001 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16002 let (_tmp, mut runner) = preservation_resume_fixture().await;
16003 let wt = runner.state.candidates[0].worktree.clone();
16004 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16005 runner.state.schema = 14;
16006 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16007 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16008 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16009 for _ in 0..2 {
16010 assert!(runner.implement().await.is_err());
16011 assert_eq!(runner.state.status, RunStatus::Blocked);
16012 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16013 assert!(
16014 runner.state.candidates[0].empty,
16015 "old verdict must remain unusable"
16016 );
16017 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16018 assert!(runner.state.judge_skipped);
16019 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16020 assert!(
16021 crate::run::read_artifact(&runner.state, "cand-A.patch")
16022 .unwrap()
16023 .contains("owned edits")
16024 );
16025 assert!(wt.exists());
16026 assert_eq!(
16027 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16028 .await
16029 .unwrap(),
16030 1
16031 );
16032 runner.state = RunState::load(&runner.state.id).unwrap();
16033 }
16034 }
16035
16036 #[tokio::test]
16037 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16038 let (_tmp, mut runner) = preservation_resume_fixture().await;
16039 let wt = runner.state.candidates[0].worktree.clone();
16040 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16041 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16042 let index_lock = PathBuf::from(
16043 git::git(
16044 &wt,
16045 &[
16046 "rev-parse",
16047 "--path-format=absolute",
16048 "--git-path",
16049 "index.lock",
16050 ],
16051 )
16052 .await
16053 .unwrap(),
16054 );
16055 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16056 runner.state.schema = 14;
16057 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16058 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16059 assert!(runner.implement().await.is_err());
16060 assert!(runner.after_implement().is_err());
16061 assert!(runner.fold_losers().await.is_err());
16062 assert_eq!(
16063 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16064 "failed seat edits"
16065 );
16066 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16067 runner.state = RunState::load(&runner.state.id).unwrap();
16068 std::fs::remove_file(index_lock).unwrap();
16069 assert!(runner.implement().await.is_err());
16071 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16072 assert!(runner.state.candidates[0].empty);
16073 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16074 assert!(
16075 crate::run::read_artifact(&runner.state, "cand-A.patch")
16076 .unwrap()
16077 .contains("failed seat edits")
16078 );
16079 assert!(wt.exists());
16080 }
16081
16082 #[tokio::test]
16083 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16084 let (_tmp, mut runner) = preservation_resume_fixture().await;
16085 let repo = runner.state.repo.clone();
16086 let common = git::git(
16087 &repo,
16088 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16089 )
16090 .await
16091 .unwrap();
16092 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16093 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16094 assert!(runner.fold_losers().await.is_err());
16095 assert!(!runner.state.candidates[0].worktree.exists());
16096 assert!(!runner.state.candidates[0].folded);
16097 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16098 runner.state = RunState::load(&runner.state.id).unwrap();
16099 std::fs::remove_file(branch_lock).unwrap();
16100 let judge = runner.state.worktree_root().join("judge-1");
16101 git::worktree_add_detached(&repo, &judge, "main")
16102 .await
16103 .unwrap();
16104 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16105 runner.roles.judges = vec![spec("judge")];
16106 assert!(runner.fold_losers().await.is_err());
16107 runner.state = RunState::load(&runner.state.id).unwrap();
16108 assert!(
16109 runner.state.candidates[0].folded,
16110 "candidate progress saved before judge failure"
16111 );
16112 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16113 assert!(judge.join("uncommitted").exists());
16114 std::fs::remove_file(judge.join("uncommitted")).unwrap();
16115 runner.fold_losers().await.unwrap();
16116 assert!(!judge.exists());
16117 runner.fold_losers().await.unwrap();
16118 }
16119
16120 #[tokio::test]
16121 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16122 let (_tmp, mut runner) = preservation_resume_fixture().await;
16123 runner.state.candidates.retain(|c| c.label == 'B');
16124 runner.state.tally = None;
16125 runner.state.judge_skipped = false;
16126 runner.state.prep_complete = Some(true);
16127 runner.state.config.graph.advise = false;
16128 runner.state.config.graph.selected_implementer = Some("alpha".into());
16129 let wt = runner.state.candidates[0].worktree.clone();
16130 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16131 runner
16132 .state
16133 .candidate_preservation
16134 .insert('B', "awaiting persistence".into());
16135 let index_lock = git::git(
16136 &wt,
16137 &[
16138 "rev-parse",
16139 "--path-format=absolute",
16140 "--git-path",
16141 "index.lock",
16142 ],
16143 )
16144 .await
16145 .unwrap();
16146 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16147 runner.state.save().unwrap();
16148 assert!(runner.execute_graph().await.is_err());
16149 assert_eq!(runner.state.status, RunStatus::Blocked);
16150 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16151 std::fs::remove_file(index_lock).unwrap();
16152 for _ in 0..2 {
16155 runner.state = RunState::load(&runner.state.id).unwrap();
16156 runner.pause.park();
16157 runner.execute_graph().await.unwrap();
16158 assert_eq!(runner.state.status, RunStatus::Implementing);
16159 assert!(runner.state.candidate_preservation.is_empty());
16160 assert_eq!(runner.state.candidates[0].commits, 2);
16161 assert!(runner.state.tally.is_none());
16162 }
16163 runner.implement().await.unwrap();
16164 assert_eq!(runner.state.status, RunStatus::Judging);
16165 runner.state.status = RunStatus::Blocked;
16167 runner.state.candidates[0].failed = Some("implementation failed".into());
16168 runner.state.save().unwrap();
16169 let error = runner.execute_graph().await.unwrap_err();
16170 assert!(
16171 error
16172 .to_string()
16173 .contains("selected implementation is held")
16174 );
16175 }
16176
16177 #[tokio::test]
16178 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16179 let (_tmp, mut runner) = preservation_resume_fixture().await;
16180 let repo = runner.state.repo.clone();
16181 let candidate = runner.state.candidates[0].clone();
16182 let registered = git::worktree_holding(&repo, &candidate.branch)
16185 .await
16186 .unwrap()
16187 .unwrap();
16188 assert_eq!(
16189 registered.canonicalize().unwrap(),
16190 candidate.worktree.canonicalize().unwrap()
16191 );
16192 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16193 assert_eq!(
16194 git::worktree_holding(&repo, &candidate.branch)
16195 .await
16196 .unwrap(),
16197 Some(registered)
16198 );
16199 assert!(
16200 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16201 "the stale registration initially keeps the branch checked out"
16202 );
16203 runner.fold_losers().await.unwrap();
16204 assert!(runner.state.candidates[0].folded);
16205 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16206 assert_eq!(
16207 git::worktree_holding(&repo, &candidate.branch)
16208 .await
16209 .unwrap(),
16210 None
16211 );
16212 runner.state = RunState::load(&runner.state.id).unwrap();
16213 runner.fold_losers().await.unwrap();
16214 }
16215
16216 #[tokio::test]
16217 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16218 let (_tmp, mut runner) = preservation_resume_fixture().await;
16219 let repo = runner.state.repo.clone();
16220 let candidate = runner.state.candidates[0].clone();
16221 let winner = runner.state.candidates[1].clone();
16222 let registered = git::worktree_holding(&repo, &candidate.branch)
16223 .await
16224 .unwrap()
16225 .unwrap();
16226 assert_eq!(
16227 registered.canonicalize().unwrap(),
16228 candidate.worktree.canonicalize().unwrap()
16229 );
16230 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16231 assert_eq!(
16232 git::worktree_holding(&repo, &candidate.branch)
16233 .await
16234 .unwrap(),
16235 Some(registered)
16236 );
16237 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16238
16239 for _ in 0..2 {
16240 fold_run(&mut runner.state, false, &crate::run::home())
16241 .await
16242 .unwrap();
16243 runner.state = RunState::load(&runner.state.id).unwrap();
16244 assert!(runner.state.candidates[0].folded);
16245 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16246 assert_eq!(
16247 git::worktree_holding(&repo, &candidate.branch)
16248 .await
16249 .unwrap(),
16250 None
16251 );
16252 assert!(!runner.state.candidates[1].folded);
16253 assert!(winner.worktree.exists());
16254 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16255 }
16256 }
16257
16258 #[tokio::test]
16259 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16260 ask_test_home();
16261 let tmp = tempfile::tempdir().unwrap();
16262 let repo = tmp.path().join("repo");
16263 std::fs::create_dir(&repo).unwrap();
16264 init_repo(&repo);
16265 let mut runner = runner_at(RunStatus::Implementing);
16266 runner.state.repo = repo.clone();
16267 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16268 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16269 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16270 let branch = format!("candidate-{label}");
16271 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16272 git::worktree_add_branch(&repo, &wt, &branch, "main")
16273 .await
16274 .unwrap();
16275 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16276 git::git(&wt, &["add", "README.md"]).await.unwrap();
16277 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16278 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16279 runner.state.candidates.push(Candidate {
16280 index,
16281 label,
16282 agent: "alpha".to_owned(),
16283 branch,
16284 worktree: wt,
16285 summary: "completed implementation".to_owned(),
16286 stat: String::new(),
16287 files: 0,
16288 commits: 0,
16289 empty: false,
16290 failed: None,
16291 verified_noop: None,
16292 duration_ms: 42,
16293 folded: false,
16294 });
16295 runner
16296 .state
16297 .candidate_preservation
16298 .insert(label, "awaiting persistence".to_owned());
16299 }
16300 let wt = runner.state.candidates[0].worktree.clone();
16301 let index_lock = git::git(
16302 &wt,
16303 &[
16304 "rev-parse",
16305 "--path-format=absolute",
16306 "--git-path",
16307 "index.lock",
16308 ],
16309 )
16310 .await
16311 .unwrap();
16312 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16313 runner.state.save().unwrap();
16314 assert!(runner.persist_candidates().await.is_err());
16315 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16316 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16317 assert_eq!(
16318 runner.state.candidates[1].commits, 1,
16319 "other results are still collected"
16320 );
16321 assert!(
16322 !runner.state.candidates[0].empty,
16323 "failed rescue is not no change"
16324 );
16325 assert_eq!(
16326 runner.state.candidates[0].summary,
16327 "completed implementation"
16328 );
16329 assert!(runner.after_implement().is_err());
16330 let home = tmp.path().join("fold-home");
16331 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16332 assert!(wt.join("new-file").exists());
16333 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16334 assert!(recovery.join("staged.patch").exists());
16335 assert!(recovery.join("unstaged.patch").exists());
16336 assert_eq!(
16337 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16338 "untracked edits\n"
16339 );
16340 runner.state.begin_pass();
16341 runner.state.end_pass_parked(ParkCause::Upgrade {
16342 from: "1.0".into(),
16343 to: Some("1.1".into()),
16344 });
16345 runner.state.save().unwrap();
16346 runner.state = RunState::load(&runner.state.id).unwrap();
16347 assert!(matches!(
16348 runner.state.passes.last().unwrap().end,
16349 Some(crate::run::PassEnd::Parked {
16350 cause: ParkCause::Upgrade { .. }
16351 })
16352 ));
16353 std::fs::remove_file(index_lock).unwrap();
16354 runner.implement().await.unwrap();
16356 assert!(runner.state.candidate_preservation.is_empty());
16357 assert!(
16358 runner
16359 .state
16360 .candidates
16361 .iter()
16362 .all(|c| c.commits == 1 && !c.empty)
16363 );
16364 assert!(git::is_clean(&wt).await.unwrap());
16365 assert!(
16366 crate::run::read_artifact(&runner.state, "cand-A.patch")
16367 .unwrap()
16368 .contains("later tracked edits")
16369 );
16370 }
16371
16372 #[tokio::test]
16373 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16374 ask_test_home();
16375 let tmp = tempfile::tempdir().unwrap();
16376 let repo = tmp.path().join("repo");
16377 std::fs::create_dir(&repo).unwrap();
16378 init_repo(&repo);
16379 let wt = tmp.path().join("candidate");
16380 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16381 .await
16382 .unwrap();
16383 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16384 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16385 assert!(wt.join("uncommitted").exists());
16386 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16387 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16388 let mut runner = runner_at(RunStatus::Implementing);
16389 runner.state.repo = repo.clone();
16390 runner.state.base_commit = "missing-base".to_owned();
16391 runner.state.candidates.push(Candidate {
16392 index: 0,
16393 label: 'A',
16394 agent: "alpha".into(),
16395 branch: "candidate".into(),
16396 worktree: wt.clone(),
16397 summary: "completed".into(),
16398 stat: String::new(),
16399 files: 0,
16400 commits: 0,
16401 empty: false,
16402 failed: None,
16403 verified_noop: None,
16404 duration_ms: 1,
16405 folded: false,
16406 });
16407 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16408 assert!(
16409 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16410 .await
16411 .is_err()
16412 );
16413 assert!(!runner.state.candidates[0].folded);
16414 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16415 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16416 runner
16417 .state
16418 .candidate_preservation
16419 .insert('A', "pending".into());
16420 assert!(runner.persist_candidates().await.is_err());
16421 assert!(!runner.state.candidates[0].empty);
16422 assert!(!runner.state.candidates[0].folded);
16423 assert!(wt.exists());
16424 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16425 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16426 std::fs::create_dir_all(&patch_path).unwrap();
16427 assert!(
16428 runner.persist_candidates().await.is_err(),
16429 "artifact failure stays pending"
16430 );
16431 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16432 assert!(!runner.state.candidates[0].empty);
16433 std::fs::remove_dir(patch_path).unwrap();
16434 runner.persist_candidates().await.unwrap();
16435 assert!(
16436 runner.state.candidates[0].empty,
16437 "only successful inspection confirms no change"
16438 );
16439 }
16440
16441 fn init_repo(dir: &Path) {
16442 let run = |args: &[&str]| {
16443 let out = std::process::Command::new("git")
16444 .args(args)
16445 .current_dir(dir)
16446 .quiet()
16447 .output()
16448 .expect("spawn git");
16449 assert!(
16450 out.status.success(),
16451 "git {args:?} failed: {}",
16452 String::from_utf8_lossy(&out.stderr)
16453 );
16454 };
16455 run(&["init", "-b", "main"]);
16456 run(&["config", "user.name", "magi test"]);
16457 run(&["config", "user.email", "magi@example.com"]);
16458 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16459 run(&["add", "-A"]);
16460 run(&["commit", "-m", "init"]);
16461 }
16462
16463 fn ask_test_home() {
16471 crate::run::pin_test_home();
16472 }
16473
16474 #[test]
16475 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16476 ask_test_home();
16477 let mut runner = runner_at(RunStatus::Prep);
16478 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16479 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16482 let error = guard
16483 .finish(
16484 &mut runner.state,
16485 Err(
16486 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16487 .into(),
16488 ),
16489 )
16490 .unwrap_err();
16491 assert_eq!(
16492 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16493 std::io::ErrorKind::PermissionDenied
16494 );
16495 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16496 let message = format!("{error:#}");
16497 assert!(message.contains("original refusal"));
16498 assert!(!message.contains("could not save terminal"));
16499 assert_eq!(
16500 error.chain().count(),
16501 2,
16502 "only startup attribution wraps the original error"
16503 );
16504 let saved = RunState::load(&runner.state.id).unwrap();
16505 assert_eq!(saved.status, RunStatus::Blocked);
16506 assert!(!saved.startup.unwrap().complete);
16507 }
16508
16509 #[test]
16510 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16511 ask_test_home();
16512 let mut runner = runner_at(RunStatus::Prep);
16513 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16514 let id = runner.state.id.clone();
16515 drop(guard);
16516 let saved = RunState::load(&id).unwrap();
16517 assert_eq!(saved.status, RunStatus::Blocked);
16518 assert!(!saved.startup.as_ref().unwrap().complete);
16519 assert!(
16520 saved
16521 .startup
16522 .unwrap()
16523 .failure
16524 .unwrap()
16525 .contains("interrupted")
16526 );
16527 assert!(Runner::resume(&id).is_err());
16528 }
16529
16530 #[test]
16531 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16532 ask_test_home();
16533 let mut runner = runner_at(RunStatus::Prep);
16534 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16535 let error = guard
16536 .finish(
16537 &mut runner.state,
16538 Err(
16539 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16540 .into(),
16541 ),
16542 )
16543 .unwrap_err();
16544 assert_eq!(
16545 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16546 std::io::ErrorKind::PermissionDenied
16547 );
16548 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16549 let mut task = crate::queue::Task::new(
16550 "review".into(),
16551 "criteria".into(),
16552 runner.state.repo.clone(),
16553 crate::queue::Source::Human,
16554 );
16555 task.record_startup_failure(failure.clone());
16556 task.record_startup_failure(failure.clone());
16557 assert_eq!(task.startup_failures.len(), 1);
16558 assert!(task.runs.is_empty());
16559 assert_eq!(task.attempts, 0);
16560 }
16561
16562 #[cfg(unix)]
16563 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16564 ask_test_home();
16565 let tmp = tempfile::tempdir().unwrap();
16566 let repo = tmp.path().join("repo");
16567 std::fs::create_dir(&repo).unwrap();
16568 init_repo(&repo);
16569 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16570 let origin = tmp.path().join("origin.git");
16571 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16572 .await
16573 .unwrap();
16574 git::git(
16575 &repo,
16576 &["remote", "add", "origin", origin.to_str().unwrap()],
16577 )
16578 .await
16579 .unwrap();
16580 git::git(&repo, &["checkout", "-b", "fix/retained"])
16581 .await
16582 .unwrap();
16583 std::fs::write(repo.join("retained"), "retained change").unwrap();
16584 git::git(&repo, &["add", "."]).await.unwrap();
16585 git::git(&repo, &["commit", "-m", "retained implementation"])
16586 .await
16587 .unwrap();
16588 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16589 git::git(&repo, &["checkout", "main"]).await.unwrap();
16590 std::fs::write(repo.join("base-change"), "new base").unwrap();
16591 git::git(&repo, &["add", "."]).await.unwrap();
16592 git::git(&repo, &["commit", "-m", "base moved"])
16593 .await
16594 .unwrap();
16595 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16596 git::git(&repo, &["checkout", "fix/retained"])
16597 .await
16598 .unwrap();
16599 let mut runner = runner_at(RunStatus::Voting);
16600 runner.state.repo = repo.clone();
16601 runner.state.base_commit = base;
16602 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16603 runner.state.config.disk.min_free_bytes = 0;
16604 runner.state.config.graph.review_rounds = 1;
16605 runner.state.config.verify.e2e = vec!["true".into()];
16606 runner.state.config.verify.gate = vec!["true".into()];
16607 runner.state.candidates = vec![Candidate {
16608 index: 0,
16609 label: 'A',
16610 agent: "fixture".into(),
16611 branch: "fix/retained".into(),
16612 worktree: repo.clone(),
16613 summary: String::new(),
16614 stat: String::new(),
16615 files: 1,
16616 commits: 1,
16617 empty: false,
16618 failed: None,
16619 verified_noop: None,
16620 duration_ms: 0,
16621 folded: false,
16622 }];
16623 runner.state.tally = Some(Tally {
16624 first_choice: BTreeMap::new(),
16625 borda: BTreeMap::new(),
16626 winner: 'A',
16627 rankings: 0,
16628 unanimous_initial: true,
16629 deliberated: false,
16630 changed_votes: 0,
16631 unanimous_final: true,
16632 tie_break: None,
16633 judges: 0,
16634 present: 0,
16635 quorum: 0,
16636 met_quorum: true,
16637 uncontested: Some("review-only".into()),
16638 });
16639 let mut task = crate::queue::Task::new(
16640 "fixture".into(),
16641 "fixture".into(),
16642 repo,
16643 crate::queue::Source::Human,
16644 );
16645 task.link_run(&runner.state.id);
16646 crate::queue::Queue::open().put(&mut task).unwrap();
16647 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16648 runner.state.base_sync = Some(BaseSync {
16649 tip: String::new(),
16650 behind: 1,
16651 attempts: BASE_SYNC_ROUNDS,
16652 conflict: None,
16653 already_in: None,
16654 });
16655 runner.sync_to_base().await.unwrap();
16656 assert_eq!(runner.state.status, RunStatus::Blocked);
16657 (tmp, runner, head, task.id)
16658 }
16659
16660 #[cfg(unix)]
16661 #[tokio::test]
16662 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16663 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16664 if std::env::var_os(CHILD).is_none() {
16665 let home = tempfile::tempdir().unwrap();
16666 let output = std::process::Command::new(std::env::current_exe().unwrap())
16667 .args([
16668 "--exact",
16669 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16670 "--nocapture",
16671 ])
16672 .env(CHILD, "1")
16673 .env("MAGI_HOME", home.path())
16674 .env("MAGI_WORKER_HOME", home.path())
16675 .env("MAGI_WORKER_ID", "abcdef")
16676 .output()
16677 .unwrap();
16678 assert!(
16679 output.status.success()
16680 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16681 "{}\n{}",
16682 String::from_utf8_lossy(&output.stdout),
16683 String::from_utf8_lossy(&output.stderr)
16684 );
16685 return;
16686 }
16687 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16688 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16689 let home = crate::run::home();
16690 let queue = crate::queue::Queue::open();
16691 let mut owner = queue.get(&task).unwrap();
16692 owner.release_operator("test-release", None);
16693 owner.start(runner.state.id.clone());
16694 queue.put(&mut owner).unwrap();
16695 runner.state.driver_exited = true;
16696 runner.state.task_instruction = Some(owner.instruction.clone());
16697 runner.state.save().unwrap();
16698 let _claim = queue.claim(&task).unwrap();
16699 let reserve = |id: &str| {
16700 let directory = home.join("workers/jobs").join(id);
16701 std::fs::create_dir_all(&directory).unwrap();
16702 let job = crate::worker::Job {
16703 id: id.into(),
16704 sequence: 1,
16705 kind: "task".into(),
16706 key: format!("task:{task}"),
16707 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16708 generation: "fixture-generation".into(),
16709 version: None,
16710 status: crate::worker::Status::Running,
16711 result: None,
16712 pid: Some(std::process::id()),
16713 created_at: 0,
16714 finished_at: None,
16715 };
16716 std::fs::write(
16717 directory.join("job.json"),
16718 serde_json::to_vec(&job).unwrap(),
16719 )
16720 .unwrap();
16721 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16722 };
16723 let _own = reserve("abcdef");
16724 let other = reserve("fedcba");
16725 assert!(
16726 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16727 .await
16728 .is_err()
16729 );
16730 assert!(
16731 RunState::load(&runner.state.id)
16732 .unwrap()
16733 .base_sync_recovery
16734 .authorizations
16735 .is_empty()
16736 );
16737 drop(other);
16738 assert!(crate::daemon::is_working_on(
16739 &home,
16740 &runner.state.id,
16741 Timestamp::now()
16742 ));
16743 assert!(
16744 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16745 );
16746 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16747 .await
16748 .unwrap();
16749 assert!(
16750 RunState::load(&runner.state.id)
16751 .unwrap()
16752 .base_recovery_pending()
16753 );
16754 }
16755
16756 #[cfg(unix)]
16757 #[tokio::test]
16758 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16759 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16760 let home = crate::run::home();
16761 runner.state.driver_pid = Some(std::process::id());
16762 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16763 runner.state.driver_exited = false;
16764 runner.state.begin_pass();
16765 runner.state.save().unwrap();
16766 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16767 driver.record(&runner.state).unwrap();
16768 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16769 drop(driver);
16771 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16772 assert_eq!(
16773 runner.state.liveness_under(&home, false),
16774 crate::run::Liveness::Dead
16775 );
16776 let queue = crate::queue::Queue::open();
16777 let _claim = queue.claim(&task).unwrap();
16778 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16779 .await
16780 .unwrap();
16781 let disk = RunState::load(&runner.state.id).unwrap();
16782 assert!(!disk.driver_exited);
16783 assert!(disk.base_recovery_pending());
16784 }
16785
16786 #[cfg(unix)]
16787 #[tokio::test]
16788 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16789 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16790 if std::env::var_os(CHILD).is_none() {
16791 use std::os::unix::fs::PermissionsExt;
16792 let tmp = tempfile::tempdir().unwrap();
16793 let gh = tmp.path().join("gh");
16794 std::fs::write(&gh, r#"#!/bin/sh
16795case "$1 $2" in
16796 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16797 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16798 *) exit 2 ;;
16799esac
16800"#).unwrap();
16801 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16802 let mut paths = vec![tmp.path().to_path_buf()];
16803 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16804 let output = std::process::Command::new(std::env::current_exe().unwrap())
16805 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16806 .env(CHILD, "1")
16807 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16808 .env("PATH", std::env::join_paths(paths).unwrap())
16809 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16810 .output().unwrap();
16811 assert!(
16812 output.status.success()
16813 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16814 "{}\n{}",
16815 String::from_utf8_lossy(&output.stdout),
16816 String::from_utf8_lossy(&output.stderr)
16817 );
16818 return;
16819 }
16820 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16821 let repo = runner.state.repo.clone();
16822 git::git(&repo, &["checkout", "main"]).await.unwrap();
16823 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16824 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16825 git::git(&repo, &["checkout", "fix/retained"])
16826 .await
16827 .unwrap();
16828 runner.state.pr = Some(crate::run::PrRecord {
16829 url: "https://example.test/project/pull/1".into(),
16830 number: 1,
16831 state: "open".into(),
16832 checks: "unknown".into(),
16833 round: 0,
16834 rounds: 1,
16835 red_at_merge: Vec::new(),
16836 });
16837 runner.sync_to_base().await.unwrap();
16838 assert_eq!(runner.state.status, RunStatus::Blocked);
16839 assert_eq!(
16840 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16841 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16842 );
16843 assert!(runner.state.base_sync_closure_pending());
16844 assert!(crate::handover::can_resume(&runner.state));
16845 let queue = crate::queue::Queue::open();
16846 assert!(!crate::base_sync_recovery::held_task(
16847 &queue,
16848 &queue.get(&task).unwrap()
16849 ));
16850 assert!(runner.state.authorize_base_sync(&task).is_err());
16851 runner.state.base_sync_recovery.stop = None;
16853 runner.state.status = RunStatus::Voting;
16854 runner.state.gate_ran = true;
16855 runner.state.config.graph.review_rounds = 0;
16856 runner.state.save().unwrap();
16857 assert!(crate::handover::can_resume(&runner.state));
16858 runner.state = RunState::load(&runner.state.id).unwrap();
16859 runner.execute_graph().await.unwrap();
16860 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16861 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16862 assert_eq!(
16863 runner.state.base_sync.as_ref().unwrap().attempts,
16864 BASE_SYNC_ROUNDS
16865 );
16866 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16867 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16868 assert!(runner.state.reviews.is_empty());
16869 assert!(runner.state.gate.is_empty());
16870 }
16871
16872 #[cfg(unix)]
16873 #[tokio::test]
16874 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16875 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16876 runner.state.driver_exited = true;
16877 runner.state.save().unwrap();
16878 let home = crate::run::home();
16879 let queue = crate::queue::Queue::open();
16880 let _claim = queue.claim(&task).unwrap();
16881 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16882 assert!(
16883 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16884 .await
16885 .is_err()
16886 );
16887 drop(lease);
16888 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16889 assert!(
16890 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16891 .await
16892 .is_err()
16893 );
16894 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16895 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16896 .await
16897 .unwrap();
16898 let disk = RunState::load(&runner.state.id).unwrap();
16899 assert!(disk.base_recovery_pending());
16900 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16901 assert!(
16902 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16903 .await
16904 .is_err()
16905 );
16906 assert_eq!(
16907 RunState::load(&runner.state.id)
16908 .unwrap()
16909 .base_sync_recovery
16910 .authorizations
16911 .len(),
16912 1
16913 );
16914 let mut owner = queue.get(&task).unwrap();
16915 owner.runs.push("20260101-000000-ffff".into());
16916 queue.put(&mut owner).unwrap();
16917 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16918 .await
16919 .unwrap_err();
16920 assert!(error.to_string().contains("run task ownership changed"));
16921 }
16922
16923 #[cfg(unix)]
16924 #[tokio::test]
16925 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16926 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16927 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16928 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16929 legacy["status"] = serde_json::json!("voting");
16930 runner.state = serde_json::from_value(legacy).unwrap();
16931 runner.state.save().unwrap();
16932 for _ in 0..3 {
16933 runner.execute_graph().await.unwrap();
16934 assert_eq!(runner.state.status, RunStatus::Blocked);
16935 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16936 assert!(!crate::handover::can_resume(&runner.state));
16937 assert_eq!(
16938 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16939 head
16940 );
16941 assert!(runner.state.reviews.is_empty());
16942 assert!(!runner.state.gate_ran);
16943 }
16944 let notice = crate::notices::run_ended(&runner.state).unwrap();
16945 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16946 runner.state.begin_pass();
16947 runner.state.status = RunStatus::Voting;
16948 runner.deliberate().await.unwrap();
16949 runner.vote().await.unwrap();
16950 runner.finish_after_tally().await.unwrap();
16951 assert_eq!(runner.state.status, RunStatus::Blocked);
16952 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16953 assert_eq!(
16954 crate::notices::run_ended(&runner.state).unwrap().since,
16955 notice.since
16956 );
16957 }
16958
16959 #[cfg(unix)]
16960 #[tokio::test]
16961 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16962 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16963 for detail in [
16964 "fix/retained conflicts with origin/main and did not rebase: conflict",
16965 "rebased fix/retained locally but origin refused the push",
16966 "unknown failure",
16967 ] {
16968 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16969 runner.state.record_base_sync_stop(&head);
16970 assert!(runner.state.authorize_base_sync(&task).is_err());
16971 runner.state.status = RunStatus::Voting;
16972 runner.execute_graph().await.unwrap();
16973 assert_eq!(runner.state.status, RunStatus::Blocked);
16974 assert!(!crate::handover::can_resume(&runner.state));
16975 assert_eq!(
16976 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16977 head
16978 );
16979 }
16980 }
16981
16982 #[cfg(unix)]
16983 #[tokio::test]
16984 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16985 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16986 runner.state.authorize_base_sync(&task).unwrap();
16987 assert!(runner.state.consume_base_recovery(&head));
16988 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
16989 runner.state.reserve_base_operation(&head, &tip);
16990 runner.state.save().unwrap();
16991 runner.state = RunState::load(&runner.state.id).unwrap();
16992 assert!(!runner.state.consume_base_recovery(&head));
16993 runner.sync_to_base().await.unwrap();
16994 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16995 assert_eq!(runner.state.base_episode_used(), 1);
16996 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16997 assert!(
16998 runner
16999 .state
17000 .base_sync_recovery
17001 .operations
17002 .last()
17003 .unwrap()
17004 .finished
17005 );
17006 assert!(!runner.state.base_sync_blocked());
17007 runner.sync_to_base().await.unwrap();
17008 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17009 }
17010
17011 #[cfg(unix)]
17012 #[tokio::test]
17013 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17014 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17015 let original = runner
17017 .state
17018 .base_sync
17019 .as_ref()
17020 .unwrap()
17021 .conflict
17022 .clone()
17023 .unwrap();
17024 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17025 original
17026 .replace("synchronization(s)", "rebase(s)")
17027 .replace("synchronizing", "rebasing"),
17028 );
17029 runner.state.base_sync_recovery = Default::default();
17030 runner.state.record_base_sync_stop(&head);
17031 assert_eq!(
17032 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17033 crate::base_sync_recovery::StopKind::Exhausted
17034 );
17035 runner.state.authorize_base_sync(&task).unwrap();
17036 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17037 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17038 .as_object_mut()
17039 .unwrap();
17040 auth.remove("interval");
17041 auth.remove("scope");
17042 runner.state = serde_json::from_value(legacy).unwrap();
17043 assert!(runner.state.consume_base_recovery(&head));
17044 assert_eq!(runner.state.base_episode_used(), 3);
17045 runner.sync_to_base().await.unwrap();
17046 assert_eq!(runner.state.base_episode_used(), 4);
17047 runner.state.resource_block = Some(crate::run::ResourceBlock {
17048 node: "gate".into(),
17049 kind: "cache".into(),
17050 detail: "legacy wait".into(),
17051 at: Timestamp::now(),
17052 cause: crate::run::ResourceBlockCause::Unrecorded,
17053 });
17054 runner.state.save().unwrap();
17055 assert!(
17056 runner
17057 .state
17058 .base_sync_recovery
17059 .episodes
17060 .last()
17061 .unwrap()
17062 .boundary
17063 .is_none()
17064 );
17065 runner.state.resource_block = None;
17066 advance_base_fixture(&runner).await;
17067 runner.sync_to_base().await.unwrap();
17068 assert!(runner.state.base_sync_blocked());
17069 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17070 }
17071
17072 #[cfg(unix)]
17073 async fn advance_base_fixture(runner: &Runner) {
17074 let repo = &runner.state.repo;
17075 for args in [
17076 vec!["checkout", "main"],
17077 vec!["commit", "--allow-empty", "-m", "base advanced"],
17078 vec!["push", "origin", "main"],
17079 vec!["checkout", "fix/retained"],
17080 ] {
17081 git::git(repo, &args).await.unwrap();
17082 }
17083 }
17084
17085 #[cfg(unix)]
17086 fn fresh_base_interval_fixture(runner: &mut Runner) {
17087 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17088 runner.state.base_sync.as_mut().unwrap().conflict = None;
17089 runner.state.base_sync_recovery = Default::default();
17090 runner.state.status = RunStatus::Voting;
17091 }
17092
17093 #[cfg(unix)]
17094 #[tokio::test]
17095 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17096 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17097 fresh_base_interval_fixture(&mut runner);
17098 runner.state.begin_pass();
17099 runner.sync_to_base().await.unwrap();
17100 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17101 runner.state.resource_block = Some(crate::run::ResourceBlock {
17102 node: "gate".into(),
17103 kind: "build cache".into(),
17104 detail: "busy".into(),
17105 at: Timestamp::now(),
17106 cause: crate::run::ResourceBlockCause::Timeout {
17107 waited_secs: 1,
17108 limit_secs: 1,
17109 },
17110 });
17111 runner.state.save().unwrap();
17112 let queue = crate::queue::Queue::open();
17113 let mut owner = queue.get(&task).unwrap();
17114 runner.state.task_instruction = Some(owner.instruction.clone());
17115 owner.release_operator("fixture-owner", None);
17116 queue.put(&mut owner).unwrap();
17117 let approval = owner.resumption.clone().unwrap();
17118 runner.state.resource_block = None;
17119 runner.state.save().unwrap();
17120 runner.state = RunState::load(&runner.state.id).unwrap();
17121 advance_base_fixture(&runner).await;
17122 runner.sync_to_base().await.unwrap();
17123 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17124 assert_eq!(runner.state.base_episode_used(), 1);
17125 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17126 assert!(
17127 runner.state.base_sync_recovery.episodes[0]
17128 .boundary
17129 .as_ref()
17130 .unwrap()
17131 .consumed
17132 );
17133 assert!(
17134 !runner
17135 .state
17136 .renew_base_episode(&task, &approval, &head, "base")
17137 );
17138 for _ in 0..3 {
17139 owner.release_operator("fixture-owner", None);
17140 queue.put(&mut owner).unwrap();
17141 runner.state = RunState::load(&runner.state.id).unwrap();
17142 runner.state.begin_pass();
17143 runner.sync_to_base().await.unwrap();
17144 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17145 assert_eq!(runner.state.base_episode_used(), 1);
17146 }
17147 runner.state.resource_block = Some(crate::run::ResourceBlock {
17148 node: "gate".into(),
17149 kind: "disk pressure".into(),
17150 detail: "free space required".into(),
17151 at: Timestamp::now(),
17152 cause: crate::run::ResourceBlockCause::Other,
17153 });
17154 runner.state.end_pass_parked(ParkCause::Interrupt {
17155 reason: "disk pressure".into(),
17156 });
17157 runner.state.save().unwrap();
17158 owner.release_operator("fixture-owner-after-disk", None);
17159 queue.put(&mut owner).unwrap();
17160 runner.state.resource_block = None;
17161 advance_base_fixture(&runner).await;
17162 runner.sync_to_base().await.unwrap();
17163 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17164 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17165 assert_eq!(runner.state.base_episode_used(), 1);
17166 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17167 }
17168
17169 #[cfg(unix)]
17170 #[tokio::test]
17171 async fn base_recovery_continuous_advancement_stops_at_four() {
17172 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17173 fresh_base_interval_fixture(&mut runner);
17174 for spent in 1..=BASE_SYNC_ROUNDS {
17175 runner.sync_to_base().await.unwrap();
17176 assert!(!runner.state.base_sync_blocked());
17177 assert_eq!(runner.state.base_episode_used(), spent);
17178 advance_base_fixture(&runner).await;
17179 }
17180 runner.sync_to_base().await.unwrap();
17181 assert_eq!(
17182 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17183 crate::base_sync_recovery::StopKind::Exhausted
17184 );
17185 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17186 assert!(stop.unverified_sync);
17187 let action = runner.state.base_sync_next_action();
17188 assert!(action.contains("never reached its gate"));
17189 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17190 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17191 runner.state = RunState::load(&runner.state.id).unwrap();
17192 runner.sync_to_base().await.unwrap();
17193 assert_eq!(runner.state.base_episode_used(), 4);
17194 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17195 }
17196
17197 #[cfg(unix)]
17198 #[tokio::test]
17199 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17200 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17201 runner.state.ensure_base_episode();
17202 runner.state.record_base_progress(&head);
17204 runner.state.base_sync_recovery.stop = None;
17205 runner.state.record_base_sync_stop(&head);
17206 assert!(
17207 !runner
17208 .state
17209 .base_sync_recovery
17210 .stop
17211 .as_ref()
17212 .unwrap()
17213 .unverified_sync
17214 );
17215
17216 runner.state.finish_base_operation(Some(&head));
17217 runner.state.gate_ran = true;
17219 runner.state.gate_head = Some("older-head".into());
17220 runner.state.base_sync_recovery.stop = None;
17221 runner.state.record_base_sync_stop(&head);
17222 runner.state.save().unwrap();
17223 runner.state = RunState::load(&runner.state.id).unwrap();
17224 assert!(
17225 runner
17226 .state
17227 .base_sync_recovery
17228 .stop
17229 .as_ref()
17230 .unwrap()
17231 .unverified_sync
17232 );
17233
17234 runner.state.gate_head = Some(head.clone());
17235 runner.state.base_sync_recovery.stop = None;
17236 runner.state.record_base_sync_stop(&head);
17237 assert!(
17238 !runner
17239 .state
17240 .base_sync_recovery
17241 .stop
17242 .as_ref()
17243 .unwrap()
17244 .unverified_sync
17245 );
17246 assert!(
17247 !runner
17248 .state
17249 .base_sync_next_action()
17250 .contains("never reached its gate")
17251 );
17252
17253 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17255 legacy["base_sync_recovery"]["stop"]
17256 .as_object_mut()
17257 .unwrap()
17258 .remove("unverified_sync");
17259 legacy["base_sync_recovery"]["episodes"][0]
17260 .as_object_mut()
17261 .unwrap()
17262 .remove("last_sync_head");
17263 runner.state = serde_json::from_value(legacy).unwrap();
17264 assert!(
17265 !runner
17266 .state
17267 .base_sync_recovery
17268 .stop
17269 .as_ref()
17270 .unwrap()
17271 .unverified_sync
17272 );
17273 assert!(
17274 runner.state.base_sync_recovery.episodes[0]
17275 .last_sync_head
17276 .is_none()
17277 );
17278 }
17279
17280 #[cfg(unix)]
17281 #[tokio::test]
17282 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17283 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17284 runner.state.authorize_base_sync(&task).unwrap();
17285 assert!(
17286 runner
17287 .state
17288 .base_sync_next_action()
17289 .contains("Approval recorded")
17290 );
17291 assert!(
17292 runner
17293 .state
17294 .base_sync_next_action()
17295 .contains("4/4 in episode 0, 4 lifetime")
17296 );
17297 assert!(runner.state.consume_base_recovery(&head));
17298 assert!(
17299 runner
17300 .state
17301 .base_sync_next_action()
17302 .contains("0/4 in episode 1, 4 lifetime")
17303 );
17304 }
17305
17306 #[cfg(unix)]
17307 #[tokio::test]
17308 async fn base_recovery_changed_head_invalidates_without_spending() {
17309 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17310 runner.state.authorize_base_sync(&task).unwrap();
17311 git::git(
17312 &runner.state.repo,
17313 &["commit", "--allow-empty", "-m", "external head change"],
17314 )
17315 .await
17316 .unwrap();
17317 runner.sync_to_base().await.unwrap();
17318 let a = runner
17319 .state
17320 .base_sync_recovery
17321 .authorizations
17322 .last()
17323 .unwrap();
17324 assert!(a.consumed);
17325 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17326 runner.state = RunState::load(&runner.state.id).unwrap();
17327 assert!(
17328 runner
17329 .state
17330 .base_sync_next_action()
17331 .contains("Approval invalidated")
17332 );
17333 assert!(!runner.state.base_recovery_pending());
17334 assert!(runner.state.base_sync_blocked());
17335 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17336 }
17337
17338 #[cfg(unix)]
17339 #[tokio::test]
17340 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17341 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17342 runner.state.driver_exited = true;
17343 runner.state.save().unwrap();
17344 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17345 .await
17346 .unwrap();
17347 runner.state = RunState::load(&runner.state.id).unwrap();
17348 advance_base_fixture(&runner).await;
17349 runner.sync_to_base().await.unwrap();
17350 let a = runner
17351 .state
17352 .base_sync_recovery
17353 .authorizations
17354 .last()
17355 .unwrap();
17356 assert!(
17357 a.consumed
17358 && a.invalidated
17359 .as_deref()
17360 .unwrap()
17361 .contains("fetched base changed")
17362 );
17363 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17364 assert_eq!(
17365 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17366 head
17367 );
17368 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17369 .await
17370 .unwrap();
17371 runner.state = RunState::load(&runner.state.id).unwrap();
17372 runner.sync_to_base().await.unwrap();
17373 assert!(!runner.state.base_sync_blocked());
17374 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17375 }
17376
17377 #[cfg(unix)]
17378 #[tokio::test]
17379 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17380 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17381 runner.state.authorize_base_sync(&task).unwrap();
17382 assert!(runner.state.consume_base_recovery(&head));
17383 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17384 runner.state.reserve_base_operation(&head, &tip);
17385 runner.state.save().unwrap();
17386 let scratch = runner.state.dir().join("base-sync");
17387 assert_eq!(
17388 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17389 .await
17390 .unwrap(),
17391 crate::rebase::Rebased::Applied
17392 );
17393 runner.state = RunState::load(&runner.state.id).unwrap();
17394 runner.sync_to_base().await.unwrap();
17395 assert!(
17396 runner
17397 .state
17398 .base_sync_recovery
17399 .operations
17400 .last()
17401 .unwrap()
17402 .finished
17403 );
17404 assert_eq!(runner.state.base_episode_used(), 1);
17405 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17406 assert!(!runner.state.base_sync_blocked());
17407 }
17408
17409 #[cfg(unix)]
17410 #[tokio::test]
17411 async fn base_recovery_crash_preserves_untracked_files() {
17412 for ignored in [false, true] {
17413 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17414 runner.state.authorize_base_sync(&task).unwrap();
17415 assert!(runner.state.consume_base_recovery(&head));
17416 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17417 runner.state.reserve_base_operation(&head, &tip);
17418 runner.state.save().unwrap();
17419 let scratch = runner.state.dir().join("base-sync");
17420 assert_eq!(
17421 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17422 .await
17423 .unwrap(),
17424 crate::rebase::Rebased::Applied
17425 );
17426 let repo = &runner.state.repo;
17427 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17428 assert_ne!(result, head);
17429 if ignored {
17430 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17431 .await
17432 .unwrap();
17433 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17434 }
17435 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17436 assert!(
17437 git::git_raw(repo, &["diff", "--quiet", &head])
17438 .await
17439 .unwrap()
17440 .ok()
17441 );
17442 runner.state = RunState::load(&runner.state.id).unwrap();
17444 runner.sync_to_base().await.unwrap();
17445 assert!(runner.state.base_sync_blocked());
17446 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17447 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17448 assert_eq!(
17449 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17450 result
17451 );
17452 assert_eq!(
17453 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17454 "retained work\n"
17455 );
17456 assert!(
17457 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17458 .await
17459 .unwrap()
17460 .ok()
17461 );
17462 }
17463 }
17464
17465 #[cfg(unix)]
17466 #[tokio::test]
17467 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17468 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17469 runner.state.authorize_base_sync(&task).unwrap();
17470 assert!(runner.state.consume_base_recovery(&head));
17471 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17472 runner.state.reserve_base_operation(&head, &tip);
17473 runner.state.save().unwrap();
17474 let scratch = runner.state.dir().join("base-sync");
17475 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17476 .await
17477 .unwrap();
17478 git::sync_to_head(&runner.state.repo).await.unwrap();
17479 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17480 git::git(&runner.state.repo, &["add", "foreign"])
17481 .await
17482 .unwrap();
17483 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17484 .await
17485 .unwrap();
17486 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17487 runner.state = RunState::load(&runner.state.id).unwrap();
17488 runner.sync_to_base().await.unwrap();
17489 assert!(runner.state.base_sync_blocked());
17490 assert!(
17491 !runner
17492 .state
17493 .base_sync_recovery
17494 .operations
17495 .last()
17496 .unwrap()
17497 .finished
17498 );
17499 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17500 assert_eq!(
17501 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17502 foreign_head
17503 );
17504 }
17505
17506 #[cfg(unix)]
17507 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17508 {
17509 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17510 fresh_base_interval_fixture(&mut runner);
17511 let repo = &runner.state.repo;
17512 git::git(repo, &["checkout", "main"]).await.unwrap();
17513 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17514 git::git(repo, &["add", "retained"]).await.unwrap();
17515 git::git(repo, &["commit", "-m", "upstream retained change"])
17516 .await
17517 .unwrap();
17518 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17519 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17520 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17521 let scratch = runner.state.dir().join("base-sync");
17522 assert!(matches!(
17523 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17524 .await
17525 .unwrap(),
17526 git::RebaseStart::Conflicted(_)
17527 ));
17528 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17529 agent: "fixture".into(),
17530 paths: vec!["retained".into()],
17531 from: Some(head.clone()),
17532 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17533 index_outside: None,
17534 finished: false,
17535 error: None,
17536 launcher: None,
17537 infra: None,
17538 outage: None,
17539 });
17540 (tmp, runner, head, tip, scratch)
17541 }
17542
17543 #[cfg(unix)]
17544 #[tokio::test]
17545 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17546 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17547 std::fs::write(
17550 scratch.join("retained"),
17551 "upstream and retained resolution\n",
17552 )
17553 .unwrap();
17554 runner.state.resource_block = Some(crate::run::ResourceBlock {
17555 node: "rebase".into(),
17556 kind: "repository coordination".into(),
17557 detail: "busy".into(),
17558 at: Timestamp::now(),
17559 cause: crate::run::ResourceBlockCause::Other,
17560 });
17561 runner.state.save().unwrap();
17562 let newer = git::git(
17565 &runner.state.repo,
17566 &[
17567 "commit-tree",
17568 "main^{tree}",
17569 "-p",
17570 "main",
17571 "-m",
17572 "base advanced during wait",
17573 ],
17574 )
17575 .await
17576 .unwrap();
17577 git::git(
17578 &runner.state.repo,
17579 &[
17580 "push",
17581 "origin",
17582 &format!("{}:refs/heads/main", newer.trim()),
17583 ],
17584 )
17585 .await
17586 .unwrap();
17587 runner.state = RunState::load(&runner.state.id).unwrap();
17588 runner.state.resource_block = None;
17589 runner.sync_to_base().await.unwrap();
17590 assert!(!runner.state.base_sync_blocked());
17591 let op = &runner.state.base_sync_recovery.operations[0];
17592 assert_eq!(op.head, head);
17593 assert_eq!(op.tip, tip);
17594 assert!(op.finished);
17595 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17596 assert_eq!(runner.state.rebase_fixes.len(), 1);
17597 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17598 .await
17599 .unwrap();
17600 assert_ne!(latest, tip);
17601 runner.state = RunState::load(&runner.state.id).unwrap();
17602 runner.sync_to_base().await.unwrap();
17603 assert!(!runner.state.base_sync_blocked());
17604 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17605 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17606 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17607 }
17608
17609 #[cfg(unix)]
17610 #[tokio::test]
17611 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17612 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17613 runner.state.reserve_base_operation(&head, &tip);
17614 runner.state.save().unwrap();
17615 std::fs::write(
17616 scratch.join("retained"),
17617 "upstream and retained resolution\n",
17618 )
17619 .unwrap();
17620 git::git(&scratch, &["add", "retained"]).await.unwrap();
17621 git::git(
17622 &scratch,
17623 &["-c", "core.editor=true", "rebase", "--continue"],
17624 )
17625 .await
17626 .unwrap();
17627 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17628 assert!(
17629 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17630 .await
17631 .unwrap()
17632 .is_empty()
17633 );
17634 assert_eq!(
17635 crate::rebase::finish(
17636 &mut runner.state,
17637 &scratch,
17638 "fix/retained",
17639 &head,
17640 &tip,
17641 &["retained".into()],
17642 ""
17643 )
17644 .await
17645 .unwrap(),
17646 crate::rebase::Rebased::Applied
17647 );
17648 assert!(!scratch.exists());
17649 runner.state = RunState::load(&runner.state.id).unwrap();
17651 let op = &runner.state.base_sync_recovery.operations[0];
17652 assert!(!op.finished);
17653 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17654 runner.sync_to_base().await.unwrap();
17655 assert!(!runner.state.base_sync_blocked());
17656 assert!(runner.state.base_sync_recovery.operations[0].finished);
17657 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17658 assert_eq!(runner.state.rebase_fixes.len(), 1);
17659 let winner = runner.state.winner().unwrap().clone();
17660 let wait = runner.repository_wait();
17661 assert!(
17662 verify_current_head(&mut runner.state, &winner, &wait)
17663 .await
17664 .unwrap()
17665 );
17666 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17667 }
17668
17669 #[cfg(unix)]
17670 #[tokio::test]
17671 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17672 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17673 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17674 runner.state.reserve_base_operation(&head, &tip);
17675 runner.state.save().unwrap();
17676 std::fs::write(
17677 scratch.join("retained"),
17678 "upstream and retained resolution\n",
17679 )
17680 .unwrap();
17681 git::git(&scratch, &["add", "retained"]).await.unwrap();
17682 git::git(
17683 &scratch,
17684 &["-c", "core.editor=true", "rebase", "--continue"],
17685 )
17686 .await
17687 .unwrap();
17688 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17689 assert!(!git::rebase_in_progress(&scratch).await);
17690 assert!(
17691 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17692 .await
17693 .unwrap()
17694 .is_empty()
17695 );
17696 match damage {
17697 Some("record") => {
17698 let path =
17699 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17700 .await
17701 .unwrap();
17702 std::fs::write(scratch.join(path), "").unwrap();
17703 }
17704 Some("foreign") => {
17705 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17706 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17707 git::git(&scratch, &["commit", "-m", "foreign work"])
17708 .await
17709 .unwrap();
17710 }
17711 Some("dirty") => {
17712 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17713 }
17714 _ => {}
17715 }
17716 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17717 runner.state = RunState::load(&runner.state.id).unwrap();
17719 assert!(
17720 runner.state.base_sync_recovery.operations[0]
17721 .validated_head
17722 .is_none()
17723 );
17724 runner.sync_to_base().await.unwrap();
17725 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17726 assert_eq!(runner.state.base_episode_used(), 1);
17727 assert_eq!(runner.state.rebase_fixes.len(), 1);
17728 assert_eq!(
17729 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17730 retained_head
17731 );
17732 if damage.is_some() {
17733 assert!(runner.state.base_sync_blocked());
17734 assert!(scratch.exists());
17735 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17736 assert!(
17737 runner.state.base_sync_recovery.operations[0]
17738 .validated_head
17739 .is_none()
17740 );
17741 } else {
17742 assert!(!runner.state.base_sync_blocked());
17743 assert!(!scratch.exists());
17744 assert!(runner.state.base_sync_recovery.operations[0].finished);
17745 assert_eq!(
17746 runner.state.base_sync_recovery.operations[0]
17747 .validated_head
17748 .as_deref(),
17749 Some(result.as_str())
17750 );
17751 let winner = runner.state.winner().unwrap().clone();
17752 let wait = runner.repository_wait();
17753 assert!(
17754 verify_current_head(&mut runner.state, &winner, &wait)
17755 .await
17756 .unwrap()
17757 );
17758 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17759 runner.state = RunState::load(&runner.state.id).unwrap();
17760 runner.sync_to_base().await.unwrap();
17761 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17762 }
17763 }
17764 }
17765
17766 #[cfg(unix)]
17767 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17768 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17769 runner.state.prep_complete = Some(true);
17771 runner.state.base_sync = None;
17772 runner.state.base_sync_recovery = Default::default();
17773 runner.state.config.graph.gate_fix_rounds = 1;
17774 let mut review = review_round(true, 0, 1, 1, false, true);
17775 review.head = head.clone();
17776 review.verified_head = Some(head.clone());
17777 runner.state.reviews = vec![review];
17778 runner.state.gate = vec![CommandOutcome {
17779 command: "old gate".into(),
17780 code: Some(1),
17781 output_tail: "old failure".into(),
17782 duration_ms: 0,
17783 resource_blocked: false,
17784 }];
17785 runner.state.gate_head = Some(head.clone());
17786 runner.state.gate_ran = true;
17787 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17788 agent: "fixture".into(),
17789 failed: runner.state.gate.clone(),
17790 notes: "spent".into(),
17791 committed: false,
17792 error: None,
17793 }];
17794 runner.state.land_approval = Some(crate::run::LandApproval {
17795 question: "old approval".into(),
17796 head: head.clone(),
17797 });
17798 runner.state.status = RunStatus::Blocked;
17799 (tmp, runner, head)
17800 }
17801
17802 #[cfg(unix)]
17803 #[tokio::test]
17804 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17805 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17806 let (tmp, mut runner, old) = resumed_gate_fixture().await;
17807 let phases = tmp.path().join("phases");
17808 runner.state.config.verify.e2e = vec![format!(
17809 "echo e2e >> '{}'; {}",
17810 phases.display(),
17811 if e2e_ok {
17812 "true"
17813 } else {
17814 "echo new-e2e-failure; false"
17815 }
17816 )];
17817 runner.state.config.verify.gate = vec![format!(
17818 "echo gate >> '{}'; {}",
17819 phases.display(),
17820 if gate_ok {
17821 "true"
17822 } else {
17823 "echo new-gate-failure; false"
17824 }
17825 )];
17826 runner.state.save().unwrap();
17827 runner.state = RunState::load(&runner.state.id).unwrap();
17828 runner.sync_to_base().await.unwrap();
17829 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17830 assert_ne!(head, old);
17831 runner.state = RunState::load(&runner.state.id).unwrap();
17833 runner.execute_graph().await.unwrap();
17834 assert_eq!(runner.state.gate_fixes.len(), 1);
17835 assert_eq!(runner.state.reviews.len(), 1);
17836 assert_eq!(runner.state.reviews[0].head, old);
17837 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17838 assert_eq!(runner.state.verification_history.len(), 1);
17839 let history = &runner.state.verification_history[0];
17840 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17841 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17842 assert!(history.archived_at.is_some());
17843 assert_eq!(history.fixes_spent, Some(1));
17844 assert_eq!(history.gate[0].output_tail, "old failure");
17845 let archive_events: Vec<_> = runner
17846 .state
17847 .events
17848 .iter()
17849 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17850 .collect();
17851 assert_eq!(archive_events.len(), 1);
17852 assert!(archive_events[0].message.contains(&short(&old)));
17853 assert!(archive_events[0].message.contains(&short(&head)));
17854 assert!(
17855 archive_events[0]
17856 .message
17857 .contains("1 gate-fix round(s) already spent stay spent")
17858 );
17859 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17860 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17861 if e2e_ok {
17862 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17863 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17864 } else {
17865 assert!(!runner.state.gate_ran);
17866 }
17867 assert_eq!(
17868 runner.state.status,
17869 if e2e_ok && gate_ok {
17870 RunStatus::Ready
17871 } else {
17872 RunStatus::Blocked
17873 }
17874 );
17875 if !e2e_ok || !gate_ok {
17876 let cause = runner.state.verification_blocking_cause().unwrap();
17877 assert!(cause.contains(&head));
17878 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17879 runner.state.event("land", "rebased successfully");
17881 let notice = crate::notices::run_ended(&runner.state).unwrap();
17882 assert!(!format!("{notice:?}").contains("rebased successfully"));
17883 assert!(runner.state.merge.is_none());
17884 }
17885 let before = std::fs::read_to_string(&phases).unwrap();
17886 runner.state.save().unwrap();
17887 runner.state = RunState::load(&runner.state.id).unwrap();
17888 runner.review_loop().await.unwrap();
17889 runner.gate().await.unwrap();
17890 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17891 assert_eq!(runner.state.verification_history.len(), 1);
17892 let history = &runner.state.verification_history[0];
17893 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17894 assert!(history.archived_at.is_some());
17895 assert_eq!(history.fixes_spent, Some(1));
17896 }
17897 }
17898
17899 #[cfg(unix)]
17900 #[tokio::test]
17901 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17902 let (tmp, mut runner, head) = resumed_gate_fixture().await;
17903 let marker = tmp.path().join("gate-ran");
17904 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17905 runner.review_loop().await.unwrap();
17906 runner.gate().await.unwrap();
17907 assert_eq!(runner.state.status, RunStatus::Blocked);
17908 assert!(!marker.exists());
17909 assert!(runner.state.verification_history.is_empty());
17910 runner.state.gate_head = None;
17911 runner.gate().await.unwrap();
17912 assert!(marker.exists());
17913 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17914 assert_eq!(runner.state.verification_history.len(), 1);
17915 assert_eq!(runner.state.gate_fixes.len(), 1);
17916 }
17917
17918 #[cfg(unix)]
17919 #[tokio::test]
17920 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17921 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17922 runner.sync_to_base().await.unwrap();
17923 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17924 runner.state.reconcile_verification_head(&head);
17925 for outcomes in [
17927 Vec::new(),
17928 vec![CommandOutcome {
17929 command: "true".into(),
17930 code: None,
17931 output_tail: "cache unavailable".into(),
17932 duration_ms: 0,
17933 resource_blocked: true,
17934 }],
17935 ] {
17936 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17937 head: head.clone(),
17938 passed: false,
17939 outcomes,
17940 });
17941 runner.state.gate_ran = false;
17942 runner.state.gate.clear();
17943 runner.state.save().unwrap();
17944 runner.state = RunState::load(&runner.state.id).unwrap();
17945 runner.merge().await.unwrap();
17946 assert!(runner.state.merge.is_none());
17947 runner.review_loop().await.unwrap();
17948 runner.gate().await.unwrap();
17949 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17950 assert!(runner.state.gate_allows_progress());
17951 assert_eq!(runner.state.gate_fixes.len(), 1);
17952 assert!(runner.state.merge.is_none());
17953 }
17954 git::git(
17956 &runner.state.repo,
17957 &["commit", "--allow-empty", "-m", "external update"],
17958 )
17959 .await
17960 .unwrap();
17961 runner.merge().await.unwrap();
17962 assert!(runner.state.merge.is_none());
17963 assert_eq!(runner.state.status, RunStatus::Blocked);
17964 }
17965
17966 #[cfg(unix)]
17967 #[tokio::test]
17968 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17969 for waiting_on_gate in [false, true] {
17970 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17971 runner.sync_to_base().await.unwrap();
17972 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17973 runner.state.reconcile_verification_head(&head);
17974 if waiting_on_gate {
17975 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17976 head: head.clone(),
17977 passed: true,
17978 outcomes: Vec::new(),
17979 });
17980 }
17981 let cache = tmp.path().join("cache");
17982 runner.state.config.verify.e2e =
17983 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17984 runner.state.config.verify.gate = vec!["true".into()];
17985 runner.state.config.graph.timeout_resource_wait = 1;
17986 let owner =
17987 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
17988 let guard =
17989 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
17990 crate::cache::AcquireOutcome::Acquired(g) => g,
17991 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
17992 };
17993 runner.review_loop().await.unwrap();
17994 runner.gate().await.unwrap();
17995 assert!(!runner.state.gate_ran);
17996 assert_eq!(runner.state.gate_fixes.len(), 1);
17997 assert!(runner.state.resource_block.is_some());
17998 runner.merge().await.unwrap();
17999 assert!(runner.state.merge.is_none());
18000 drop(guard);
18001 runner.state = RunState::load(&runner.state.id).unwrap();
18002 runner.state.resource_block = None;
18004 runner.review_loop().await.unwrap();
18005 runner.gate().await.unwrap();
18006 assert!(runner.state.gate_allows_progress());
18007 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18008 assert_eq!(runner.state.gate_fixes.len(), 1);
18009 }
18010 }
18011
18012 #[cfg(unix)]
18013 #[tokio::test]
18014 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18015 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18016 runner.sync_to_base().await.unwrap();
18017 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18018 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18019 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18021 runner.review_loop().await.unwrap();
18022 runner.gate().await.unwrap();
18023 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18024 assert_ne!(before, after);
18025 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18026 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18027 assert!(!runner.state.gate_allows_progress());
18028 runner.merge().await.unwrap();
18029 assert!(runner.state.merge.is_none());
18030 assert!(
18031 runner
18032 .state
18033 .verification_history
18034 .iter()
18035 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18036 );
18037
18038 runner.state.reviews[0].clean = false;
18040 runner.state.reviews[0].answered = 0;
18041 runner.state.config.verify.e2e = vec!["true".into()];
18042 runner.review_loop().await.unwrap();
18043 runner.gate().await.unwrap();
18044 runner.merge().await.unwrap();
18045 assert_eq!(runner.state.status, RunStatus::Blocked);
18046 assert!(runner.state.merge.is_none());
18047 assert!(
18048 runner
18049 .state
18050 .verification_blocking_cause()
18051 .unwrap()
18052 .contains("review blocked")
18053 );
18054 }
18055
18056 #[cfg(unix)]
18057 #[tokio::test]
18058 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18059 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18060 runner.state.gate_head = None;
18061 runner.state.config.verify.gate =
18062 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18063 runner.review_loop().await.unwrap();
18064 runner.gate().await.unwrap();
18065 assert_ne!(
18066 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18067 old
18068 );
18069 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18070 assert!(!runner.state.gate_allows_progress());
18071 assert_eq!(runner.state.status, RunStatus::Blocked);
18072 runner.merge().await.unwrap();
18073 assert!(runner.state.merge.is_none());
18074 }
18075
18076 #[cfg(unix)]
18077 #[tokio::test]
18078 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18079 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18080 let mut old_review = review_round(true, 0, 1, 1, false, true);
18081 old_review.head = head.clone();
18082 old_review.verified_head = Some(head.clone());
18083 runner.state.reviews = vec![old_review];
18084 runner.state.gate = vec![CommandOutcome {
18085 command: "old gate".into(),
18086 code: None,
18087 output_tail: "timed out after 2400s".into(),
18088 duration_ms: 2_400_000,
18089 resource_blocked: false,
18090 }];
18091 runner.state.gate_ran = true;
18092 runner.state.gate_head = Some(head.clone());
18093 runner.state.land_approval = Some(crate::run::LandApproval {
18094 question: "old-head-approval".into(),
18095 head: head.clone(),
18096 });
18097 runner.state.authorize_base_sync(&task).unwrap();
18098 runner.sync_to_base().await.unwrap();
18099 assert!(!runner.state.base_sync_blocked());
18100 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18101 assert_ne!(head, fresh);
18102 assert_eq!(runner.state.reviews.len(), 1);
18103 assert!(!runner.state.gate_ran);
18104 let saved = &runner.state.base_sync_recovery.authorizations[0];
18105 assert_eq!(saved.reviews[0].head, head);
18106 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18107 assert!(saved.gate_ran);
18108 assert!(!saved.gate[0].ok());
18109 runner.review_loop().await.unwrap();
18110 runner.gate().await.unwrap();
18111 assert_eq!(runner.state.reviews[0].head, head);
18112 assert_eq!(
18113 runner.state.reviews[0].verified_head.as_deref(),
18114 Some(head.as_str())
18115 );
18116 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18117 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18118 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18119 assert!(runner.state.gate_ran);
18120 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18121 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18122 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18123 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18124 assert_eq!(
18125 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18126 "retained change"
18127 );
18128 }
18129
18130 #[cfg(unix)]
18131 #[tokio::test]
18132 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18133 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18134 let queue = crate::queue::Queue::open();
18135 let mut owner = queue.get(&task).unwrap();
18136 owner.release_operator("test-resume", None);
18137 queue.put(&mut owner).unwrap();
18138 let generation = owner.resumption.as_ref().unwrap().generation;
18139 runner.state.task_instruction = Some(owner.instruction.clone());
18140 runner.state.driver_exited = true;
18141 runner.state.save().unwrap();
18142 let home = crate::run::home();
18143 let _claim = queue.claim(&task).unwrap();
18144 owner.resumption.as_mut().unwrap().at =
18145 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18146 queue.put(&mut owner).unwrap();
18147 assert!(
18148 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18149 .await
18150 .is_err(),
18151 "an approval from before the stop grants nothing"
18152 );
18153 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18154 queue.put(&mut owner).unwrap();
18155 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18156 .await
18157 .unwrap();
18158 assert!(
18159 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18160 .await
18161 .is_err()
18162 );
18163 runner.state = RunState::load(&runner.state.id).unwrap();
18164 assert_eq!(
18165 runner.state.base_sync_recovery.authorizations[0].generation,
18166 Some(generation)
18167 );
18168 runner.sync_to_base().await.unwrap();
18169 let repo = runner.state.repo.clone();
18170 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18171 git::git(&repo, &["checkout", "main"]).await.unwrap();
18172 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18173 git::git(&repo, &["add", "."]).await.unwrap();
18174 git::git(&repo, &["commit", "-m", "move base again"])
18175 .await
18176 .unwrap();
18177 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18178 git::git(&repo, &["checkout", "fix/retained"])
18179 .await
18180 .unwrap();
18181 runner.sync_to_base().await.unwrap();
18182 assert!(!runner.state.base_sync_blocked());
18183 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18184 assert_eq!(runner.state.base_episode_used(), 2);
18185 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18186 assert_ne!(fresh, head);
18187 assert!(!runner.state.base_recovery_pending());
18188 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18190 queue.put(&mut owner).unwrap();
18191 assert!(
18192 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18193 .await
18194 .is_err()
18195 );
18196 runner.state = RunState::load(&runner.state.id).unwrap();
18197 runner.sync_to_base().await.unwrap();
18198 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18199 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18200 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18201 }
18202
18203 #[cfg(unix)]
18204 #[tokio::test]
18205 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18206 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18207 runner.state.base_sync = None;
18208 runner.state.status = RunStatus::Blocked;
18209 let mut second = runner.state.candidates[0].clone();
18210 second.label = 'B';
18211 runner.state.candidates.push(second);
18212 assert!(runner.state.votes.is_empty());
18213 assert!(runner.state.deliberation.is_empty());
18214 runner.deliberate().await.unwrap();
18215 runner.vote().await.unwrap();
18216 assert_eq!(runner.state.status, RunStatus::Blocked);
18217 assert!(runner.state.votes.is_empty());
18218 assert!(runner.state.deliberation.is_empty());
18219 }
18220
18221 fn runner_at(status: RunStatus) -> Runner {
18224 let mut state = RunState::new(
18225 PathBuf::from("/nonexistent/repo"),
18226 "main".to_owned(),
18227 "deadbeef".to_owned(),
18228 "task".to_owned(),
18229 Config::default(),
18230 );
18231 state.config.disk.min_free_bytes = 0;
18233 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18236 state.id = format!(
18237 "20000101-000000-{:04x}",
18238 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18239 );
18240 state.status = status;
18241 Runner {
18242 state,
18243 roles: ResolvedRoles {
18244 implementers: Vec::new(),
18245 judges: Vec::new(),
18246 reviewers: Vec::new(),
18247 fixer: None,
18248 conductor: conductor(),
18249 implementer_roster: Vec::new(),
18250 judge_roster: Vec::new(),
18251 reviewer_roster: Vec::new(),
18252 },
18253 sem: Arc::new(Semaphore::new(1)),
18254 pause: Pause::new(),
18255 interrupt: Pause::new(),
18256 }
18257 }
18258
18259 #[tokio::test]
18260 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18261 crate::run::pin_test_home();
18262 let temp = tempfile::tempdir().unwrap();
18263 let root = temp.path().canonicalize().unwrap();
18264 let repo = root.join("repo");
18265 std::fs::create_dir(&repo).unwrap();
18266 init_repo(&repo);
18267 git::git(&repo, &["config", "core.autocrlf", "false"])
18268 .await
18269 .unwrap();
18270 let mut runner = runner_at(RunStatus::Blocked);
18271 runner.state.repo = repo.clone();
18272 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18273 runner.state.config.agents = vec![conductor()];
18274 runner.state.config.graph.selected_implementer = Some("conductor".into());
18275 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18276 let (shell, command, setup_output) = if cfg!(windows) {
18277 (
18278 vec!["cmd".into(), "/C".into()],
18279 "echo ready>seat-ready",
18280 "ready\r\n",
18281 )
18282 } else {
18283 (
18284 vec!["sh".into(), "-c".into()],
18285 "printf ready > seat-ready",
18286 "ready",
18287 )
18288 };
18289 runner.state.config.verify.shell = Some(shell);
18290 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18291 run: Some(command.into()),
18292 ..Default::default()
18293 }];
18294 runner.state.save().unwrap();
18295 let path = runner.state.worktree_root().join("judge-1");
18296 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18297 .await
18298 .unwrap();
18299 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18300 git::git(&path, &["add", "."]).await.unwrap();
18301 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18302 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18303 .await
18304 .unwrap();
18305 git::git(&path, &["checkout", "--detach", &head])
18306 .await
18307 .unwrap();
18308 let evidence = crate::fold_evidence::snapshot(
18309 &runner.state,
18310 &crate::run::home().canonicalize().unwrap(),
18311 &head,
18312 64 * 1024 * 1024,
18313 )
18314 .await
18315 .unwrap();
18316 let mut contents = BTreeMap::new();
18317 contents.insert(
18318 repo.join("logs/HEAD"),
18319 format!("{head} {head} fixture\n").into_bytes(),
18320 );
18321 let record = serde_json::json!({
18322 "version":1, "run":runner.state.id, "source":path, "head":head,
18323 "evidence":evidence, "admin":{}, "admin_contents":contents,
18324 "history":{head.clone():evidence}, "phase":"evicted"
18325 });
18326 std::fs::write(
18327 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18328 serde_json::to_vec(&record).unwrap(),
18329 )
18330 .unwrap();
18331 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18332 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18333 .await
18334 .unwrap();
18335 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18336 assert!(
18337 !git::git_raw(&repo, &["cat-file", "-e", &head])
18338 .await
18339 .unwrap()
18340 .ok()
18341 );
18342 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18343 let error = resumed.execute().await.unwrap_err();
18344 assert!(
18345 error
18346 .to_string()
18347 .contains("selected implementation is held"),
18348 "{error:#}"
18349 );
18350 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18351 assert_eq!(
18352 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18353 setup_output
18354 );
18355 assert_eq!(
18356 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18357 "unrelated seat"
18358 );
18359 let error = resumed.execute().await.unwrap_err();
18360 assert!(
18361 error
18362 .to_string()
18363 .contains("selected implementation is held"),
18364 "{error:#}"
18365 );
18366 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18367 git::git(&path, &["add", "."]).await.unwrap();
18368 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18369 let next = git::git(
18370 &path,
18371 &[
18372 "commit-tree",
18373 &next_tree,
18374 "-p",
18375 &head,
18376 "-m",
18377 "next review checkpoint",
18378 ],
18379 )
18380 .await
18381 .unwrap();
18382 git::git(&path, &["checkout", "--detach", &next])
18383 .await
18384 .unwrap();
18385 resumed.state.reviews.push(
18386 serde_json::from_value(serde_json::json!({
18387 "round":1, "head":next, "reviews":[]
18388 }))
18389 .unwrap(),
18390 );
18391 resumed.state.save().unwrap();
18392 let error = resumed.execute().await.unwrap_err();
18393 assert!(
18394 error
18395 .to_string()
18396 .contains("selected implementation is held"),
18397 "{error:#}"
18398 );
18399 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18400 }
18401
18402 #[tokio::test]
18403 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18404 crate::run::pin_test_home();
18405 let mut runner = runner_at(RunStatus::Gating);
18406 runner.state.save().unwrap();
18407 let id = runner.state.id.clone();
18408 let passes = runner.state.passes.len();
18409 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18410 let mut claiming = Box::pin(runner.claim_driver());
18411 std::future::poll_fn(|cx| {
18412 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18413 std::task::Poll::Ready(())
18414 })
18415 .await;
18416 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18417 drop(owner);
18418 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18419 .await
18420 .unwrap()
18421 .unwrap();
18422 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18423 assert!(execution.runner.state.resource_block.is_none());
18424 assert_eq!(
18425 execution.runner.state.liveness(false),
18426 crate::run::Liveness::Live
18427 );
18428 drop(execution);
18429 assert!(RunState::load(&id).unwrap().driver_exited);
18430 }
18431
18432 #[tokio::test]
18433 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18434 crate::run::pin_test_home();
18435 let mut runner = runner_at(RunStatus::Gating);
18436 runner.state.save().unwrap();
18437 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18440 assert!(runner.claim_driver().await.is_err());
18441 assert!(runner.state.driver_exited);
18442 assert_eq!(runner.state.driver_pid, None);
18443 let disk = RunState::load(&runner.state.id).unwrap();
18444 assert!(disk.driver_exited);
18445 assert_eq!(disk.driver_pid, None);
18446 assert_eq!(disk.status, RunStatus::Gating);
18447 }
18448
18449 #[tokio::test]
18450 async fn an_early_graph_error_releases_its_driver() {
18451 crate::run::pin_test_home();
18452 let mut runner = runner_at(RunStatus::Blocked);
18453 runner.state.config.graph.selected_implementer = Some("selected".into());
18454 runner.state.save().unwrap();
18455 assert!(runner.execute().await.is_err());
18456 let disk = RunState::load(&runner.state.id).unwrap();
18457 assert!(disk.driver_exited);
18458 assert_eq!(disk.driver_pid, None);
18459 assert_eq!(disk.status, RunStatus::Blocked);
18460 }
18461
18462 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18463 crate::run::MergeRepair {
18464 question: "repair-question".into(),
18465 head: "old-head".into(),
18466 pr: "https://github.com/o/r/pull/42".into(),
18467 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18468 gate: vec![outcome(Some(0), false)],
18469 gate_head: Some("old-head".into()),
18470 gate_fixes: Vec::new(),
18471 pre_gate: Vec::new(),
18472 phase,
18473 failure: None,
18474 }
18475 }
18476
18477 #[tokio::test]
18478 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18479 crate::run::pin_test_home();
18480 let mut runner = runner_at(RunStatus::Landing);
18481 runner.state.merge_repairs.push(merge_repair_request(
18482 crate::run::MergeRepairPhase::Requested,
18483 ));
18484 runner.state.save().unwrap();
18485 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18486 assert!(
18487 runner
18488 .execute()
18489 .await
18490 .unwrap_err()
18491 .to_string()
18492 .contains("already running")
18493 );
18494 assert_eq!(
18495 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18496 crate::run::MergeRepairPhase::Requested
18497 );
18498 drop(claim);
18499 let mut disk = runner.state.clone();
18500 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18501 disk.save().unwrap();
18502 assert!(
18503 runner
18504 .execute()
18505 .await
18506 .unwrap_err()
18507 .to_string()
18508 .contains("decision changed")
18509 );
18510 assert_eq!(
18511 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18512 crate::run::MergeRepairPhase::Complete
18513 );
18514 }
18515
18516 #[test]
18517 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18518 crate::run::pin_test_home();
18519 let mut runner = runner_at(RunStatus::Landing);
18520 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18521 runner.state.reviews = request.reviews.clone();
18522 runner.state.gate = request.gate.clone();
18523 runner.state.gate_ran = true;
18524 runner.state.land_approval = Some(crate::run::LandApproval {
18525 question: "old".into(),
18526 head: "old-head".into(),
18527 });
18528 runner.state.merge_repairs.push(request);
18529 runner.begin_merge_repair_review(0).unwrap();
18530 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18531 assert!(!runner.state.gate_status().ok());
18532 assert!(runner.state.land_approval.is_none());
18533 let disk = RunState::load(&runner.state.id).unwrap();
18534 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18535 assert!(disk.merge_repairs[0].gate[0].ok());
18536 assert_eq!(
18537 disk.merge_repairs[0].phase,
18538 crate::run::MergeRepairPhase::Reviewing
18539 );
18540 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18541 assert!(disk.reviews.is_empty());
18542 }
18543
18544 #[tokio::test]
18545 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18546 crate::run::pin_test_home();
18547 for phase in [
18548 crate::run::MergeRepairPhase::Fixing,
18549 crate::run::MergeRepairPhase::Requested,
18550 ] {
18551 let mut runner = runner_at(RunStatus::Landing);
18552 runner.state.config.graph.land_rounds = 0;
18553 runner.state.merge_repairs.push(merge_repair_request(phase));
18554 runner.state.save().unwrap();
18555 assert!(!runner.repair_merge_review().await.unwrap());
18556 assert_eq!(runner.state.status, RunStatus::Blocked);
18557 assert_eq!(
18558 runner.state.merge_repairs[0].phase,
18559 crate::run::MergeRepairPhase::Failed
18560 );
18561 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18562 assert!(
18563 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18564 "interrupted"
18565 } else {
18566 "budget exhausted"
18567 })
18568 );
18569 assert!(runner.state.merge.is_none());
18570 }
18571 }
18572
18573 #[test]
18577 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18578 crate::run::pin_test_home();
18579 let mut runner = runner_at(RunStatus::Implementing);
18580 let interrupt = Pause::new();
18581 runner.watch_interrupt(interrupt.clone());
18582
18583 interrupt.park_because("task a1b2 asked to run first");
18584
18585 assert!(runner.park_here().expect("park_here"));
18586 assert!(runner.state.parked);
18587 let last = runner.state.events.last().expect("a park event");
18588 assert_eq!(last.node, "park");
18589 assert!(
18590 last.message.contains("task a1b2 asked to run first"),
18591 "expected the interrupt reason in {:?}",
18592 last.message
18593 );
18594 }
18595
18596 #[test]
18599 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18600 crate::run::pin_test_home();
18601 let mut runner = runner_at(RunStatus::Implementing);
18602 runner.state.begin_pass();
18603 let shutdown = Pause::new();
18604 runner.on_pause(shutdown.clone());
18605 let upgrade = ParkCause::Upgrade {
18606 from: "0.121.0".to_owned(),
18607 to: Some("0.121.1".to_owned()),
18608 };
18609 shutdown.park_for(upgrade.clone());
18610 shutdown.park_for(ParkCause::Operator);
18611
18612 assert!(runner.park_here().expect("park_here"));
18613 assert_eq!(
18614 runner.state.passes[0].end,
18615 Some(crate::run::PassEnd::Parked { cause: upgrade })
18616 );
18617 let last = runner.state.events.last().expect("a park event");
18618 assert!(last.message.contains("0.121.0"), "{}", last.message);
18619 }
18620
18621 #[tokio::test]
18622 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18623 crate::run::pin_test_home();
18624 let mut runner = runner_at(RunStatus::Reviewing);
18625 runner.state.save().unwrap();
18626 let mut execution = runner.claim_driver().await.expect("first execution");
18627 assert_eq!(execution.runner.state.passes.len(), 1);
18628 execution
18629 .driver
18630 .finish(&mut execution.runner.state)
18631 .await
18632 .unwrap();
18633 }
18634
18635 #[tokio::test]
18636 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18637 crate::run::pin_test_home();
18638 let mut runner = runner_at(RunStatus::Reviewing);
18639 runner.state.config.agents = vec![conductor()];
18640 runner.state.config.graph.timeout_resource_wait = 0;
18641 runner.state.save().unwrap();
18642 let id = runner.state.id.clone();
18643 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18644 assert!(runner.claim_driver().await.is_err());
18645 drop(owner);
18646 drop(runner);
18647 let mut resumed = Runner::resume(&id).unwrap();
18648 assert_eq!(resumed.state.driver_claimed, Some(false));
18649 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18650 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18651 let mut execution = resumed
18652 .claim_driver()
18653 .await
18654 .expect("retry initial claim after reload");
18655 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18656 assert_eq!(execution.runner.state.passes.len(), 1);
18657 execution
18658 .driver
18659 .finish(&mut execution.runner.state)
18660 .await
18661 .unwrap();
18662 }
18663
18664 #[tokio::test]
18665 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18666 crate::run::pin_test_home();
18667 let mut runner = runner_at(RunStatus::Reviewing);
18668 runner.state.config.agents = vec![conductor()];
18669 runner.state.save().unwrap();
18670 let id = runner.state.id.clone();
18671 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18672 let mut resumed = Runner::resume(&id).unwrap();
18673 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18674 assert!(resumed.claim_driver().await.is_err());
18675 assert!(RunState::load(&id).unwrap().passes.is_empty());
18676 drop(lease);
18677 let mut execution = resumed.claim_driver().await.unwrap();
18678 execution
18679 .driver
18680 .finish(&mut execution.runner.state)
18681 .await
18682 .unwrap();
18683 }
18684
18685 #[tokio::test]
18686 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18687 crate::run::pin_test_home();
18688 let mut runner = runner_at(RunStatus::Reviewing);
18689 let state = &mut runner.state;
18690 state.config.agents = vec![conductor()];
18691 state.driver_claimed = None;
18692 state.driver_pid = Some(std::process::id());
18693 state.driver_started_at = None;
18694 state.begin_pass();
18695 state.save().unwrap();
18696 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18697 std::fs::write(
18698 state.dir().join("driver.json"),
18699 serde_json::to_vec(&serde_json::json!({
18700 "pid": state.driver_pid, "started_at": null,
18701 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18702 }))
18703 .unwrap(),
18704 )
18705 .unwrap();
18706 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18707 drop(lease);
18708 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18709 let mut resumed = Runner::resume(&state.id).unwrap();
18710 assert!(!resumed.state.driver_exited);
18711 let mut execution = resumed
18712 .claim_driver()
18713 .await
18714 .expect("lease proves previous execution ended");
18715 assert_eq!(execution.runner.state.passes.len(), 2);
18716 execution
18717 .driver
18718 .finish(&mut execution.runner.state)
18719 .await
18720 .unwrap();
18721 }
18722
18723 #[tokio::test]
18724 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18725 crate::run::pin_test_home();
18726 let mut runner = runner_at(RunStatus::Reviewing);
18727 let mut json = serde_json::to_value(&runner.state).unwrap();
18728 json.as_object_mut().unwrap().remove("driver_claimed");
18729 runner.state = serde_json::from_value(json).unwrap();
18730 assert_eq!(runner.state.driver_claimed, None);
18731 runner.state.save().unwrap();
18732 let mut execution = runner
18733 .claim_driver()
18734 .await
18735 .expect("the execution lease, not a missing marker, excludes a second driver");
18736 assert_eq!(execution.runner.state.passes.len(), 1);
18737 execution
18738 .driver
18739 .finish(&mut execution.runner.state)
18740 .await
18741 .unwrap();
18742 }
18743
18744 #[tokio::test]
18745 async fn driver_admission_refuses_a_proven_live_driver() {
18746 crate::run::pin_test_home();
18747 let mut runner = runner_at(RunStatus::Reviewing);
18748 let pid = std::process::id();
18749 runner.state.driver_pid = Some(pid);
18750 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18751 assert!(runner.state.driver_started_at.is_some());
18752 runner.state.save().unwrap();
18753 let error = runner
18754 .claim_driver()
18755 .await
18756 .err()
18757 .expect("a live driver must refuse");
18758 assert!(error.to_string().contains("live driver"), "{error:#}");
18759 let saved = RunState::load(&runner.state.id).unwrap();
18760 assert_eq!(saved.driver_pid, Some(pid));
18761 assert_eq!(saved.status, RunStatus::Reviewing);
18762 assert!(saved.passes.is_empty());
18763 }
18764
18765 #[test]
18766 fn parking_resource_halt_persists_the_pass_once() {
18767 crate::run::pin_test_home();
18768 let mut runner = runner_at(RunStatus::Reviewing);
18769 runner.state.begin_pass();
18770 runner.state.resource_block = Some(crate::run::ResourceBlock {
18771 node: "fix".into(),
18772 kind: "shared build cache".into(),
18773 detail: "cancelled".into(),
18774 at: Timestamp::now(),
18775 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18776 });
18777 let cause = ParkCause::Upgrade {
18778 from: "old".into(),
18779 to: Some("new".into()),
18780 };
18781 runner.pause.park_for(cause.clone());
18782 runner.stop_for_resource().unwrap();
18785 assert!(runner.resource_halt().unwrap());
18786 assert!(runner.park_here().unwrap());
18787 let saved = RunState::load(&runner.state.id).unwrap();
18788 assert!(saved.parked);
18789 assert_eq!(saved.status, RunStatus::Reviewing);
18790 assert!(saved.passes[0].ended_at.is_some());
18791 assert_eq!(
18792 saved.passes[0].end,
18793 Some(crate::run::PassEnd::Parked {
18794 cause: cause.clone()
18795 })
18796 );
18797 assert_eq!(
18798 saved.resource_block.unwrap().cause,
18799 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18800 );
18801 assert_eq!(
18802 saved
18803 .events
18804 .iter()
18805 .filter(|event| event.node == "park")
18806 .count(),
18807 1
18808 );
18809 }
18810
18811 #[test]
18819 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18820 crate::run::pin_test_home();
18821 let mut runner = runner_at(RunStatus::Implementing);
18822 let shutdown = Pause::new();
18823 runner.on_pause(shutdown.clone());
18824 let interrupt = Pause::new();
18825 runner.watch_interrupt(interrupt.clone());
18826
18827 assert!(!runner.park_here().expect("park_here"));
18829 assert!(!runner.state.parked);
18830
18831 interrupt.park_because("test");
18833 assert!(!shutdown.parked());
18834 assert!(runner.park_here().expect("park_here"));
18835 }
18836
18837 #[tokio::test]
18851 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18852 crate::run::pin_test_home();
18853 let mut runner = runner_at(RunStatus::Implementing);
18854 let interrupt = Pause::new();
18855 runner.watch_interrupt(interrupt.clone());
18856
18857 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18858 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18859
18860 let node = async move {
18864 started_tx.send(()).expect("send started");
18865 finish_rx.await.expect("recv finish");
18866 "node finished"
18867 };
18868
18869 let interrupter = async move {
18870 started_rx.await.expect("recv started");
18871 interrupt.park_because("higher-priority task waiting");
18873 tokio::task::yield_now().await;
18877 finish_tx.send(()).expect("send finish");
18878 };
18879
18880 let (node_result, ()) = tokio::join!(node, interrupter);
18881 assert_eq!(
18882 node_result, "node finished",
18883 "the in-flight call ran to completion"
18884 );
18885
18886 assert!(runner.park_here().expect("park_here"));
18889 assert!(runner.state.parked);
18890 }
18891
18892 #[test]
18898 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18899 crate::run::pin_test_home();
18900 let mut runner = runner_at(RunStatus::Judging);
18901 runner.state.config.agents = vec![conductor()];
18905 runner.state.candidates = vec![Candidate {
18906 index: 0,
18907 label: 'A',
18908 agent: "alpha".to_owned(),
18909 branch: "magi/x/A".to_owned(),
18910 worktree: PathBuf::from("/nonexistent/worktree"),
18911 summary: "did the thing".to_owned(),
18912 stat: "1 file changed".to_owned(),
18913 files: 1,
18914 commits: 1,
18915 empty: false,
18916 failed: None,
18917 verified_noop: None,
18918 duration_ms: 1234,
18919 folded: false,
18920 }];
18921 let run_id = runner.state.id.clone();
18922
18923 let interrupt = Pause::new();
18924 runner.watch_interrupt(interrupt.clone());
18925 interrupt.park_because("task c3d4 asked to run first");
18926 assert!(runner.park_here().expect("park_here"));
18927
18928 let resumed = Runner::resume(&run_id).expect("resume");
18929 assert_eq!(resumed.state.candidates.len(), 1);
18930 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18931 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18932 assert_eq!(resumed.state.status, runner.state.status);
18933 assert!(
18934 resumed.state.parked,
18935 "still parked until `execute` actually walks the graph again"
18936 );
18937 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18938 }
18939
18940 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18942 let mut q = ask::Question::new(
18943 run.to_owned(),
18944 "implement".to_owned(),
18945 "impl-A".to_owned(),
18946 "Which storage backend should the cache use?".to_owned(),
18947 String::new(),
18948 vec!["SQLite".to_owned(), "Redis".to_owned()],
18949 );
18950 store.put(&mut q).unwrap();
18951 q
18952 }
18953
18954 #[test]
18955 fn a_failed_runs_open_question_is_abandoned() {
18956 ask_test_home();
18957 let store = ask::Questions::open();
18958 let mut runner = runner_at(RunStatus::Failed);
18959 let run = runner.state.id.clone();
18960 let q = ask_open_question(&store, &run);
18961
18962 runner.settle_questions();
18963
18964 let back = store.get(&q.id).unwrap();
18965 assert!(
18966 !back.status.open(),
18967 "the seat that asked died with the run; nobody is left to read an answer"
18968 );
18969 assert!(
18970 back.detail.contains(&run) && back.detail.contains("failed"),
18971 "the reason names what the run became, not just that it is gone: {}",
18972 back.detail
18973 );
18974 }
18975
18976 #[test]
18977 fn a_merged_runs_open_question_is_abandoned_too() {
18978 ask_test_home();
18979 let store = ask::Questions::open();
18980 for status in [RunStatus::Merged, RunStatus::Ready] {
18983 let mut runner = runner_at(status);
18984 let run = runner.state.id.clone();
18985 let q = ask_open_question(&store, &run);
18986
18987 runner.settle_questions();
18988
18989 let back = store.get(&q.id).unwrap();
18990 assert!(
18991 !back.status.open(),
18992 "{status:?} run's question must not outlive the run"
18993 );
18994 }
18995 }
18996
18997 #[test]
18998 fn a_still_resumable_runs_open_question_is_left_alone() {
18999 ask_test_home();
19000 let store = ask::Questions::open();
19001 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19007 let mut runner = runner_at(status);
19008 let run = runner.state.id.clone();
19009 let q = ask_open_question(&store, &run);
19010
19011 runner.settle_questions();
19012
19013 let back = store.get(&q.id).unwrap();
19014 assert!(
19015 back.status.open(),
19016 "{status:?} is still alive; the question must still be waiting"
19017 );
19018 }
19019 }
19020
19021 #[test]
19022 fn settle_questions_never_touches_an_already_answered_question() {
19023 ask_test_home();
19024 let store = ask::Questions::open();
19025 let mut runner = runner_at(RunStatus::Failed);
19026 let run = runner.state.id.clone();
19027 let mut q = ask_open_question(&store, &run);
19028 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19029 .unwrap();
19030 store.put(&mut q).unwrap();
19031
19032 runner.settle_questions();
19037 runner.settle_questions();
19038
19039 let back = store.get(&q.id).unwrap();
19040 assert_eq!(
19041 back.status,
19042 ask::QuestionStatus::Answered,
19043 "a real answer is a decision on record, never overwritten by a sweep"
19044 );
19045 }
19046
19047 #[tokio::test]
19058 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19059 crate::run::pin_test_home();
19060 let tmp = tempfile::tempdir().expect("tempdir");
19061 let repo = tmp.path().join("repo");
19062 std::fs::create_dir_all(&repo).unwrap();
19063 init_repo(&repo);
19064
19065 let mut config = Config::default();
19066 config.graph.worktree_root = Some(tmp.path().join("wt"));
19067
19068 let mut state = RunState::new(
19069 repo.clone(),
19070 "main".to_owned(),
19071 "deadbeef".to_owned(),
19072 "task".to_owned(),
19073 config,
19074 );
19075 let root = state.worktree_root();
19076 let wt_a = root.join("cand-A");
19077 let wt_b = root.join("cand-B");
19078 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19079 .await
19080 .expect("worktree A");
19081 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19082 .await
19083 .expect("worktree B");
19084
19085 state.candidates = vec![
19086 Candidate {
19087 index: 0,
19088 label: 'A',
19089 agent: "alpha".to_owned(),
19090 branch: "magi/x/A".to_owned(),
19091 worktree: wt_a.clone(),
19092 summary: String::new(),
19093 stat: String::new(),
19094 files: 0,
19095 commits: 0,
19096 empty: false,
19097 failed: None,
19098 verified_noop: None,
19099 duration_ms: 0,
19100 folded: false,
19101 },
19102 Candidate {
19103 index: 1,
19104 label: 'B',
19105 agent: "beta".to_owned(),
19106 branch: "magi/x/B".to_owned(),
19107 worktree: wt_b.clone(),
19108 summary: String::new(),
19109 stat: String::new(),
19110 files: 0,
19111 commits: 0,
19112 empty: false,
19113 failed: None,
19114 verified_noop: None,
19115 duration_ms: 0,
19116 folded: false,
19117 },
19118 ];
19119 state.tally = Some(Tally {
19120 first_choice: BTreeMap::from([('A', 1)]),
19121 borda: BTreeMap::new(),
19122 winner: 'A',
19123 rankings: 1,
19124 unanimous_initial: true,
19125 deliberated: false,
19126 changed_votes: 0,
19127 unanimous_final: true,
19128 tie_break: None,
19129 judges: 1,
19130 present: 1,
19131 quorum: 1,
19132 met_quorum: true,
19133 uncontested: None,
19134 });
19135 state.status = RunStatus::Ready;
19136
19137 fold_run(&mut state, false, &crate::run::home())
19138 .await
19139 .expect("fold_run");
19140
19141 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19142 assert!(
19143 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19144 "the unmerged winner's branch survives"
19145 );
19146 assert!(
19147 !state.candidates[0].folded,
19148 "the winner is not marked folded"
19149 );
19150
19151 assert!(!wt_b.exists(), "the loser's worktree is removed");
19152 assert!(
19153 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19154 "the loser's branch is removed"
19155 );
19156 assert!(state.candidates[1].folded, "the loser is marked folded");
19157 }
19158
19159 #[tokio::test]
19160 #[cfg(unix)]
19161 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19162 use std::os::fd::AsRawFd as _;
19163 use std::time::Duration;
19164
19165 let tmp = tempfile::tempdir().unwrap();
19166 let repo = tmp.path().join("repo");
19167 std::fs::create_dir_all(&repo).unwrap();
19168 init_repo(&repo);
19169 let common = repo.join(".git");
19170 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19171 let home = tmp.path().join("home");
19172 let mut config = Config::default();
19173 config.coordination = crate::config::test_coordination();
19174 config.graph.worktree_root = Some(tmp.path().join("wt"));
19175 config.graph.timeout_resource_wait = 1;
19176 let mut state = RunState::new(
19177 repo.clone(),
19178 "main".into(),
19179 "deadbeef".into(),
19180 "task".into(),
19181 config,
19182 );
19183 let root = state.worktree_root();
19184 let candidate = root.join("cand-A");
19185 let reviewer = root.join("review-1");
19186 let branch = state.branch_for('A');
19187 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19188 .await
19189 .unwrap();
19190 git::worktree_add_detached(&repo, &reviewer, "main")
19191 .await
19192 .unwrap();
19193 git::acquire_worktree_config(&repo).await.unwrap();
19194 state.enabled_worktree_config = true;
19195 state.status = RunStatus::Ready;
19196 state.candidates.push(Candidate {
19197 index: 0,
19198 label: 'A',
19199 agent: "implement".into(),
19200 branch: branch.clone(),
19201 worktree: candidate.clone(),
19202 summary: String::new(),
19203 stat: String::new(),
19204 files: 0,
19205 commits: 0,
19206 empty: false,
19207 failed: None,
19208 verified_noop: None,
19209 duration_ms: 0,
19210 folded: false,
19211 });
19212 state.save_under(&home).unwrap();
19213 let config_before = std::fs::read(common.join("config")).unwrap();
19214 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19215 .await
19216 .unwrap();
19217 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19220 assert_eq!(
19221 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19222 0
19223 );
19224 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19225 assert!(
19226 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19227 .await
19228 .is_err()
19229 );
19230 assert!(candidate.exists());
19231 assert!(reviewer.exists());
19232 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19233 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19234 assert_eq!(
19235 git::git(&repo, &["worktree", "list", "--porcelain"])
19236 .await
19237 .unwrap(),
19238 worktrees_before
19239 );
19240 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19241 .await
19242 .unwrap()
19243 .unwrap_err();
19244 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19245 drop(fold);
19246 assert!(candidate.exists() && reviewer.exists());
19247 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19248 assert_eq!(
19249 state.resource_block.as_ref().unwrap().kind,
19250 "repository coordination"
19251 );
19252 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19253 drop(external);
19254 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19255 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19256 .await
19257 .unwrap()
19258 .unwrap();
19259 drop(fold);
19260 assert!(removed.contains(&branch));
19261 assert!(!root.exists());
19262 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19263 assert!(state.candidates[0].folded);
19264 assert!(!state.enabled_worktree_config);
19265 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19266 let enabled = git::git_raw(
19267 &repo,
19268 &["config", "--local", "--get", "extensions.worktreeConfig"],
19269 )
19270 .await
19271 .unwrap();
19272 assert!(!enabled.ok(), "the last config reference was released");
19273 }
19274
19275 #[tokio::test]
19278 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19279 let tmp = tempfile::tempdir().expect("tempdir");
19280 let repo = tmp.path().join("repo");
19281 std::fs::create_dir_all(&repo).unwrap();
19282 init_repo(&repo);
19283 let home = tmp.path().join("home");
19284
19285 let mut config = Config::default();
19286 config.graph.worktree_root = Some(tmp.path().join("wt"));
19287 let mut state = RunState::new(
19288 repo.clone(),
19289 "main".to_owned(),
19290 "deadbeef".to_owned(),
19291 "task".to_owned(),
19292 config,
19293 );
19294 git::git(&repo, &["branch", "magi/x/A", "main"])
19296 .await
19297 .expect("branch");
19298 state.candidates = vec![Candidate {
19299 index: 0,
19300 label: 'A',
19301 agent: "alpha".to_owned(),
19302 branch: "magi/x/A".to_owned(),
19303 worktree: state.worktree_root().join("cand-A"),
19304 summary: String::new(),
19305 stat: String::new(),
19306 files: 0,
19307 commits: 0,
19308 empty: false,
19309 failed: None,
19310 verified_noop: None,
19311 duration_ms: 0,
19312 folded: true,
19313 }];
19314 state.released_to = Some("20260901-000000-new1".to_owned());
19315 state.released_branches = vec!["magi/x/A".to_owned()];
19316
19317 fold_run(&mut state, true, &home).await.expect("fold_run");
19318 state.status = RunStatus::Merged;
19319 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19320 state.merge = Some(crate::run::MergeOutcome {
19321 mode: MergeMode::Pr,
19322 ok: true,
19323 detail: "confirmed merged".into(),
19324 empty: false,
19325 });
19326 fold_run(&mut state, true, &home)
19327 .await
19328 .expect("merged fold");
19329
19330 assert!(
19331 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19332 "the handed-over branch survives a fold"
19333 );
19334 }
19335
19336 #[tokio::test]
19340 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19341 let tmp = tempfile::tempdir().expect("tempdir");
19342 let repo = tmp.path().join("repo");
19343 std::fs::create_dir_all(&repo).unwrap();
19344 init_repo(&repo);
19345 let run = |args: &[&str]| {
19346 let out = std::process::Command::new("git")
19347 .quiet()
19348 .args(args)
19349 .current_dir(&repo)
19350 .output()
19351 .expect("spawn git");
19352 assert!(out.status.success(), "git {args:?}");
19353 };
19354 run(&["branch", "magi/x/A"]);
19355 run(&["checkout", "-q", "-b", "magi/x/B"]);
19356 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19357 run(&["add", "-A"]);
19358 run(&["commit", "-q", "-m", "work"]);
19359 run(&["checkout", "-q", "main"]);
19360
19361 let probe = RunState::new(
19365 repo.clone(),
19366 "main".to_owned(),
19367 "deadbeef".to_owned(),
19368 "task".to_owned(),
19369 Config::default(),
19370 );
19371 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19372 let bare = tmp.path().join("origin.git");
19373 let out = std::process::Command::new("git")
19374 .quiet()
19375 .args(["init", "-q", "--bare"])
19376 .arg(&bare)
19377 .output()
19378 .expect("spawn git");
19379 assert!(out.status.success(), "git init --bare");
19380 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19381 run(&["push", "-q", "origin", "main"]);
19382
19383 let mut state = RunState::new(
19384 repo.clone(),
19385 "main".to_owned(),
19386 "deadbeef".to_owned(),
19387 "task".to_owned(),
19388 Config::default(),
19389 );
19390 state.seeds = vec![refs::Seed {
19391 token: "magi/27b2/A".to_owned(),
19392 kind: refs::SeedKind::Unresolved,
19393 sha: String::new(),
19394 branch: true,
19395 detail: "no branch or commit named magi/27b2/A".to_owned(),
19396 }];
19397
19398 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19399 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19400 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19401 let detail = empty_candidate_detail(&state, "main");
19402 assert!(detail.starts_with("empty candidate"), "{detail}");
19403 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19404 assert!(detail.contains("--start-from"), "{detail}");
19405 }
19406
19407 #[tokio::test]
19416 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19417 let tmp = tempfile::tempdir().expect("tempdir");
19418 let repo = tmp.path().join("repo");
19419 std::fs::create_dir_all(&repo).unwrap();
19420 init_repo(&repo);
19421
19422 let mut config = Config::default();
19423 config.merge.mode = MergeMode::Local;
19424
19425 let mut state = RunState::new(
19426 repo.clone(),
19427 "main".to_owned(),
19428 "deadbeef".to_owned(),
19429 "task".to_owned(),
19430 config,
19431 );
19432 state.candidates = vec![Candidate {
19433 index: 0,
19434 label: 'A',
19435 agent: "alpha".to_owned(),
19436 branch: "does-not-exist".to_owned(),
19437 worktree: repo.clone(),
19438 summary: String::new(),
19439 stat: String::new(),
19440 files: 0,
19441 commits: 0,
19442 empty: false,
19443 failed: None,
19444 verified_noop: None,
19445 duration_ms: 0,
19446 folded: false,
19447 }];
19448 state.tally = Some(Tally {
19449 first_choice: BTreeMap::from([('A', 1)]),
19450 borda: BTreeMap::new(),
19451 winner: 'A',
19452 rankings: 1,
19453 unanimous_initial: true,
19454 deliberated: false,
19455 changed_votes: 0,
19456 unanimous_final: true,
19457 tie_break: None,
19458 judges: 0,
19459 present: 0,
19460 quorum: 0,
19461 met_quorum: true,
19462 uncontested: Some("only candidate A produced a change".to_owned()),
19463 });
19464 state.reviews = vec![ReviewRound {
19465 round: 1,
19466 head: "deadbeef".to_owned(),
19467 verified_head: None,
19468 verified_at: None,
19469 reviews: Vec::new(),
19470 e2e: Vec::new(),
19471 fix: None,
19472 blocking: 0,
19473 answered: 0,
19474 expected: 0,
19475 clean: true,
19476 verify_retried: false,
19477 e2e_deferred: false,
19478 e2e_defer_reason: None,
19479 progressed: false,
19480 vote_split: false,
19481 reconsideration: Vec::new(),
19482 verdict: None,
19483 }];
19484 state.gate = vec![CommandOutcome {
19485 command: "test".to_owned(),
19486 code: Some(0),
19487 output_tail: String::new(),
19488 duration_ms: 0,
19489 resource_blocked: false,
19490 }];
19491 state.gate_ran = true;
19492 state.status = RunStatus::Ready;
19497 state.merge = Some(MergeOutcome {
19498 mode: MergeMode::Local,
19499 ok: false,
19500 detail: "already concluded".to_owned(),
19501 empty: false,
19502 });
19503
19504 let mut runner = Runner {
19505 state,
19506 roles: ResolvedRoles {
19507 implementers: Vec::new(),
19508 judges: Vec::new(),
19509 reviewers: Vec::new(),
19510 fixer: None,
19511 conductor: conductor(),
19512 implementer_roster: Vec::new(),
19513 judge_roster: Vec::new(),
19514 reviewer_roster: Vec::new(),
19515 },
19516 sem: Arc::new(Semaphore::new(1)),
19517 pause: Pause::new(),
19518 interrupt: Pause::new(),
19519 };
19520
19521 runner.merge().await.expect("merge");
19522
19523 assert_eq!(
19524 runner.state.status,
19525 RunStatus::Ready,
19526 "a concluded run's status must not change on reentry"
19527 );
19528 assert_eq!(
19529 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19530 Some("already concluded"),
19531 "merge must not run again once the node already recorded an outcome"
19532 );
19533 }
19534
19535 #[tokio::test]
19544 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19545 let tmp = tempfile::tempdir().expect("tempdir");
19546 let repo = tmp.path().join("repo");
19547 std::fs::create_dir_all(&repo).unwrap();
19548 init_repo(&repo);
19549
19550 let mut config = Config::default();
19551 config.merge.mode = MergeMode::Local;
19552
19553 let mut state = RunState::new(
19554 repo.clone(),
19555 "main".to_owned(),
19556 "deadbeef".to_owned(),
19557 "task".to_owned(),
19558 config,
19559 );
19560 state.candidates = vec![Candidate {
19561 index: 0,
19562 label: 'A',
19563 agent: "alpha".to_owned(),
19564 branch: "does-not-exist".to_owned(),
19565 worktree: repo.clone(),
19566 summary: String::new(),
19567 stat: String::new(),
19568 files: 0,
19569 commits: 0,
19570 empty: false,
19571 failed: None,
19572 verified_noop: None,
19573 duration_ms: 0,
19574 folded: false,
19575 }];
19576 state.tally = Some(Tally {
19577 first_choice: BTreeMap::from([('A', 1)]),
19578 borda: BTreeMap::new(),
19579 winner: 'A',
19580 rankings: 1,
19581 unanimous_initial: true,
19582 deliberated: false,
19583 changed_votes: 0,
19584 unanimous_final: true,
19585 tie_break: None,
19586 judges: 0,
19587 present: 0,
19588 quorum: 0,
19589 met_quorum: true,
19590 uncontested: Some("only candidate A produced a change".to_owned()),
19591 });
19592 state.reviews = vec![ReviewRound {
19593 round: 1,
19594 head: "deadbeef".to_owned(),
19595 verified_head: None,
19596 verified_at: None,
19597 reviews: Vec::new(),
19598 e2e: Vec::new(),
19599 fix: None,
19600 blocking: 0,
19601 answered: 0,
19602 expected: 0,
19603 clean: true,
19604 verify_retried: false,
19605 e2e_deferred: false,
19606 e2e_defer_reason: None,
19607 progressed: false,
19608 vote_split: false,
19609 reconsideration: Vec::new(),
19610 verdict: None,
19611 }];
19612 state.gate = Vec::new();
19614 state.gate_ran = false;
19615 state.status = RunStatus::Gating;
19616
19617 let mut runner = Runner {
19618 state,
19619 roles: ResolvedRoles {
19620 implementers: Vec::new(),
19621 judges: Vec::new(),
19622 reviewers: Vec::new(),
19623 fixer: None,
19624 conductor: conductor(),
19625 implementer_roster: Vec::new(),
19626 judge_roster: Vec::new(),
19627 reviewer_roster: Vec::new(),
19628 },
19629 sem: Arc::new(Semaphore::new(1)),
19630 pause: Pause::new(),
19631 interrupt: Pause::new(),
19632 };
19633
19634 runner.merge().await.expect("merge");
19635
19636 assert!(
19637 runner.state.merge.is_none(),
19638 "an empty gate must never be read as a passing one: {:?}",
19639 runner.state.merge
19640 );
19641 }
19642
19643 #[tokio::test]
19650 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19651 let tmp = tempfile::tempdir().expect("tempdir");
19652 let repo = tmp.path().join("repo");
19653 std::fs::create_dir_all(&repo).unwrap();
19654 init_repo(&repo);
19655
19656 let config = Config::default();
19658
19659 let mut state = RunState::new(
19660 repo.clone(),
19661 "main".to_owned(),
19662 "deadbeef".to_owned(),
19663 "task".to_owned(),
19664 config,
19665 );
19666 state.candidates = vec![Candidate {
19667 index: 0,
19668 label: 'A',
19669 agent: "alpha".to_owned(),
19670 branch: "does-not-exist".to_owned(),
19671 worktree: repo.clone(),
19672 summary: String::new(),
19673 stat: String::new(),
19674 files: 0,
19675 commits: 0,
19676 empty: false,
19677 failed: None,
19678 verified_noop: None,
19679 duration_ms: 0,
19680 folded: false,
19681 }];
19682 state.tally = Some(Tally {
19683 first_choice: BTreeMap::from([('A', 1)]),
19684 borda: BTreeMap::new(),
19685 winner: 'A',
19686 rankings: 1,
19687 unanimous_initial: true,
19688 deliberated: false,
19689 changed_votes: 0,
19690 unanimous_final: true,
19691 tie_break: None,
19692 judges: 0,
19693 present: 0,
19694 quorum: 0,
19695 met_quorum: true,
19696 uncontested: Some("only candidate A produced a change".to_owned()),
19697 });
19698 state.reviews = vec![ReviewRound {
19699 round: 1,
19700 head: "deadbeef".to_owned(),
19701 verified_head: None,
19702 verified_at: None,
19703 reviews: Vec::new(),
19704 e2e: Vec::new(),
19705 fix: None,
19706 blocking: 0,
19707 answered: 0,
19708 expected: 0,
19709 clean: true,
19710 verify_retried: false,
19711 e2e_deferred: false,
19712 e2e_defer_reason: None,
19713 progressed: false,
19714 vote_split: false,
19715 reconsideration: Vec::new(),
19716 verdict: None,
19717 }];
19718
19719 let mut runner = Runner {
19720 state,
19721 roles: ResolvedRoles {
19722 implementers: Vec::new(),
19723 judges: Vec::new(),
19724 reviewers: Vec::new(),
19725 fixer: None,
19726 conductor: conductor(),
19727 implementer_roster: Vec::new(),
19728 judge_roster: Vec::new(),
19729 reviewer_roster: Vec::new(),
19730 },
19731 sem: Arc::new(Semaphore::new(1)),
19732 pause: Pause::new(),
19733 interrupt: Pause::new(),
19734 };
19735
19736 runner.gate().await.expect("gate");
19737 assert!(
19738 runner.state.gate_ran,
19739 "zero configured commands is still a real attempt, not an unrun gate"
19740 );
19741 assert!(runner.state.gate.is_empty());
19742 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19743 assert_ne!(
19744 runner.state.status,
19745 RunStatus::Blocked,
19746 "a gate with nothing to check must not read as failed"
19747 );
19748
19749 runner.merge().await.expect("merge");
19750 assert_eq!(
19751 runner.state.status,
19752 RunStatus::Ready,
19753 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19754 );
19755 }
19756
19757 #[tokio::test]
19768 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19769 crate::run::pin_test_home();
19770 let home = crate::run::home();
19771
19772 let tmp = tempfile::tempdir().expect("tempdir");
19773 let repo = tmp.path().join("repo");
19774 std::fs::create_dir_all(&repo).unwrap();
19775 init_repo(&repo);
19776 let cache_dir = tmp.path().join("target");
19779
19780 let mut config = Config::default();
19781 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19782 config.graph.timeout_verify = Some(2);
19786 config.graph.timeout_resource_wait = 0;
19787
19788 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19789 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19790 .expect("no io error acquiring directly")
19791 {
19792 crate::cache::AcquireOutcome::Acquired(g) => g,
19793 crate::cache::AcquireOutcome::Busy(b) => {
19794 panic!("expected the direct acquire to win the lease first: {b:?}")
19795 }
19796 };
19797
19798 let mut state = RunState::new(
19799 repo.clone(),
19800 "main".to_owned(),
19801 "deadbeef".to_owned(),
19802 "task".to_owned(),
19803 config,
19804 );
19805 state.candidates = vec![Candidate {
19806 index: 0,
19807 label: 'A',
19808 agent: "alpha".to_owned(),
19809 branch: "does-not-exist".to_owned(),
19810 worktree: repo.clone(),
19811 summary: String::new(),
19812 stat: String::new(),
19813 files: 0,
19814 commits: 0,
19815 empty: false,
19816 failed: None,
19817 verified_noop: None,
19818 duration_ms: 0,
19819 folded: false,
19820 }];
19821 state.tally = Some(Tally {
19822 first_choice: BTreeMap::from([('A', 1)]),
19823 borda: BTreeMap::new(),
19824 winner: 'A',
19825 rankings: 1,
19826 unanimous_initial: true,
19827 deliberated: false,
19828 changed_votes: 0,
19829 unanimous_final: true,
19830 tie_break: None,
19831 judges: 0,
19832 present: 0,
19833 quorum: 0,
19834 met_quorum: true,
19835 uncontested: Some("only candidate A produced a change".to_owned()),
19836 });
19837 state.reviews = vec![ReviewRound {
19838 round: 1,
19839 head: "deadbeef".to_owned(),
19840 verified_head: None,
19841 verified_at: None,
19842 reviews: Vec::new(),
19843 e2e: Vec::new(),
19844 fix: None,
19845 blocking: 0,
19846 answered: 0,
19847 expected: 0,
19848 clean: true,
19849 verify_retried: false,
19850 e2e_deferred: false,
19851 e2e_defer_reason: None,
19852 progressed: false,
19853 vote_split: false,
19854 reconsideration: Vec::new(),
19855 verdict: None,
19856 }];
19857
19858 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19859 state.reviews[0].e2e = vec![CommandOutcome {
19860 command: state.config.verify.e2e[0].clone(),
19861 code: Some(0),
19862 output_tail: String::new(),
19863 duration_ms: 0,
19864 resource_blocked: false,
19865 }];
19866 let mut runner = Runner {
19867 state,
19868 roles: ResolvedRoles {
19869 implementers: Vec::new(),
19870 judges: Vec::new(),
19871 reviewers: Vec::new(),
19872 fixer: None,
19873 conductor: conductor(),
19874 implementer_roster: Vec::new(),
19875 judge_roster: Vec::new(),
19876 reviewer_roster: Vec::new(),
19877 },
19878 sem: Arc::new(Semaphore::new(1)),
19879 pause: Pause::new(),
19880 interrupt: Pause::new(),
19881 };
19882
19883 runner.gate().await.expect("gate");
19884 assert!(runner.state.resource_block.is_none());
19885 assert!(
19886 !runner
19887 .state
19888 .events
19889 .iter()
19890 .any(|event| { event.message.contains("waiting for the shared build cache") }),
19891 "a gate with nothing to run must never request the held cache lease"
19892 );
19893 assert!(
19894 runner.state.gate_ran,
19895 "zero commands is still a real, immediate attempt"
19896 );
19897 assert!(runner.state.gate.is_empty());
19898 assert_ne!(
19899 runner.state.status,
19900 RunStatus::Blocked,
19901 "must not read as resource-blocked on a lease it never asked for"
19902 );
19903 }
19904
19905 #[cfg(unix)]
19908 #[tokio::test]
19909 async fn verification_is_standalone_inside_a_live_worker() {
19910 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19911 if std::env::var_os(CHILD).is_none() {
19912 let home = tempfile::tempdir().unwrap();
19913 let generation = crate::worker::pin_current(home.path()).unwrap();
19914 let output = std::process::Command::new(std::env::current_exe().unwrap())
19915 .args([
19916 "--exact",
19917 "graph::tests::verification_is_standalone_inside_a_live_worker",
19918 "--nocapture",
19919 ])
19920 .env(CHILD, "1")
19921 .env("MAGI_HOME", home.path())
19922 .env("MAGI_WORKER_HOME", home.path())
19923 .env("MAGI_WORKER_ID", "abcdef")
19924 .env("MAGI_WORKER_GENERATION", &generation.id)
19925 .env("MAGI_WORKER_EXE", &generation.executable)
19926 .env("MAGI_RUN", "verification-parent")
19927 .env("MAGI_NODE", "gate")
19928 .env("MAGI_SEAT", "worker-fixture")
19929 .env("MAGI_PROCESS_OWNER", "fixture-owner")
19930 .env("MAGI_NO_AUTOUPDATE", "0")
19931 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19932 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19933 .env(
19934 "VERIFY_EXPECT_TOOLCHAIN",
19935 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19936 )
19937 .env(
19938 "VERIFY_EXPECT_CACHE",
19939 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19940 )
19941 .output()
19942 .unwrap();
19943 assert!(
19944 output.status.success()
19945 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19946 "{}\n{}",
19947 String::from_utf8_lossy(&output.stdout),
19948 String::from_utf8_lossy(&output.stderr)
19949 );
19950 return;
19951 }
19952 let home = crate::run::home();
19953 let directory = home.join("workers/jobs/abcdef");
19954 std::fs::create_dir_all(&directory).unwrap();
19955 let job = crate::worker::Job {
19956 id: "abcdef".into(),
19957 sequence: 1,
19958 kind: "task".into(),
19959 key: "task:verification".into(),
19960 payload: serde_json::json!({}),
19961 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19962 version: Some(env!("CARGO_PKG_VERSION").into()),
19963 status: crate::worker::Status::Running,
19964 result: None,
19965 pid: Some(std::process::id()),
19966 created_at: 0,
19967 finished_at: None,
19968 };
19969 std::fs::write(
19970 directory.join("job.json"),
19971 serde_json::to_vec(&job).unwrap(),
19972 )
19973 .unwrap();
19974 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19975 let assert_worker = || {
19976 assert_eq!(
19977 crate::worker::current_job(&home).unwrap().unwrap().id,
19978 job.id
19979 );
19980 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19981 assert_eq!(
19982 crate::worker::helper_executable(&home).unwrap(),
19983 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19984 );
19985 assert!(
19986 crate::updater::Checker::new(&crate::config::Update {
19987 mode: crate::config::UpdateMode::Notify,
19988 interval: None,
19989 auto_restart: false,
19990 })
19991 .is_none(),
19992 "the driver remains pinned, even while tests are standalone"
19993 );
19994 };
19995 assert_worker();
19996 let repo = home.join("repo");
19997 std::fs::create_dir_all(&repo).unwrap();
19998 init_repo(&repo);
19999 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20002 test -z "${MAGI_WORKER_GENERATION+x}" &&
20003 test -z "${MAGI_WORKER_HOME+x}" &&
20004 test -z "${MAGI_WORKER_EXE+x}" &&
20005 test "$MAGI_HOME/repo" -ef . &&
20006 test "$MAGI_RUN" = verification-parent &&
20007 test "$MAGI_NODE" = gate &&
20008 test "$MAGI_SEAT" = worker-fixture &&
20009 test -n "$MAGI_PROCESS_OWNER" &&
20010 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20011 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20012 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20013 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20014 assert!(
20016 !std::process::Command::new("sh")
20017 .args(["-c", probe])
20018 .current_dir(&repo)
20019 .status()
20020 .unwrap()
20021 .success()
20022 );
20023 let unrelated_home = tempfile::tempdir().unwrap();
20024 assert!(
20025 crate::worker::current_job(unrelated_home.path()).is_err(),
20026 "an inherited worker id cannot be resolved in a unit fixture's home"
20027 );
20028 let mut runner = runner_at(RunStatus::Gating);
20029 runner.state.repo = repo.clone();
20030 runner.state.config.disk.min_free_bytes = 0;
20031 runner.state.config.coordination = crate::config::test_coordination();
20032 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20033 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20034 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20035 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20036 let winner = Candidate {
20037 index: 0,
20038 label: 'A',
20039 agent: "fixture".into(),
20040 branch: "main".into(),
20041 worktree: repo.clone(),
20042 summary: String::new(),
20043 stat: String::new(),
20044 files: 0,
20045 commits: 0,
20046 empty: false,
20047 failed: None,
20048 verified_noop: None,
20049 duration_ms: 0,
20050 folded: false,
20051 };
20052 let (e2e, retried, pids) = run_e2e_with_retry(
20053 &mut runner.state,
20054 &["sh".into(), "-c".into()],
20055 &[format!("{probe} && printf initial")],
20056 &repo,
20057 Duration::from_secs(10),
20058 "initial",
20059 )
20060 .await;
20061 assert!(!retried && pids.is_empty());
20062 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20063 assert_eq!(e2e[0].output_tail, "initial");
20064 let gate = runner.run_gate(&winner).await.unwrap();
20065 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20066 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20069 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20070 passed: false,
20071 outcomes: Vec::new(),
20072 });
20073 runner.state.save().unwrap();
20074 let mut resumed = RunState::load(&runner.state.id).unwrap();
20075 let wait = crate::repo_lock::WaitPolicy::default();
20076 assert!(matches!(
20077 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20078 .await
20079 .unwrap(),
20080 GateFix::Retry
20081 ));
20082 let saved = RunState::load(&resumed.id).unwrap();
20083 let repair = saved.gate_repair_e2e.unwrap();
20084 assert!(repair.passed);
20085 assert_eq!(repair.outcomes.len(), 1);
20086 assert!(repair.outcomes[0].ok());
20087 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20088 .await
20089 .unwrap();
20090 assert_eq!(landing.len(), 1);
20091 assert!(landing[0].ok(), "{landing:?}");
20092 assert_eq!(
20093 std::fs::read_to_string(repo.join("phases")).unwrap(),
20094 "pre\ngate\ne2e\npre\ngate\n"
20095 );
20096 assert_worker();
20097 }
20098
20099 #[cfg(unix)]
20100 #[tokio::test]
20101 async fn verification_keeps_the_panic_and_the_full_output() {
20102 crate::run::pin_test_home();
20103 let tmp = tempfile::tempdir().unwrap();
20104 let mut config = Config::default();
20105 config.disk.min_free_bytes = 0;
20106 let mut state = RunState::new(
20107 tmp.path().to_owned(),
20108 "main".into(),
20109 "base".into(),
20110 "verification evidence".into(),
20111 config,
20112 );
20113 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20114 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20115 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20116 let mut names = Vec::new();
20117 for _ in 0..2 {
20118 let (outcomes, _) = run_commands(
20119 &mut state,
20120 "gate",
20121 "gate",
20122 0,
20123 &["sh".into(), "-c".into()],
20124 &[script.into()],
20125 tmp.path(),
20126 Duration::from_secs(60),
20127 )
20128 .await;
20129 let tail = &outcomes[0].output_tail;
20130 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20131 assert!(tail.contains("boom assertion"));
20132 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20133 let line = tail.lines().next().unwrap();
20134 names.push(line.to_owned());
20135 }
20136 assert_ne!(names[0], names[1], "the second run must not overwrite");
20137 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20138 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20139 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20140
20141 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20143 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20144 let (outcomes, _) = run_commands(
20145 &mut state,
20146 "gate",
20147 "gate",
20148 0,
20149 &["sh".into(), "-c".into()],
20150 &["echo failure-output; exit 7".into()],
20151 tmp.path(),
20152 Duration::from_secs(10),
20153 )
20154 .await;
20155 assert_eq!(outcomes[0].code, Some(7));
20156 assert!(state.events.iter().any(|e| {
20157 e.node == "gate" && e.message.contains("could not save the full gate output:")
20158 }));
20159 }
20160
20161 #[cfg(unix)]
20162 #[tokio::test]
20163 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20164 crate::run::pin_test_home();
20165 let tmp = tempfile::tempdir().unwrap();
20166 let mut config = Config::default();
20167 config.disk.min_free_bytes = 0;
20168 let mut state = RunState::new(
20169 tmp.path().to_owned(),
20170 "main".into(),
20171 "base".into(),
20172 "verification evidence".into(),
20173 config,
20174 );
20175 let (outcomes, pids) = run_commands(
20176 &mut state,
20177 "gate",
20178 "gate",
20179 1,
20180 &["sh".into(), "-c".into()],
20181 &[
20182 "printf before-timeout; exec sleep 30".into(),
20183 "kill -TERM $$".into(),
20184 "exit 7".into(),
20185 "printf success".into(),
20186 ],
20187 tmp.path(),
20188 Duration::from_secs(1),
20189 )
20190 .await;
20191 wait_for_timed_out_children_to_die(&pids).await;
20192 assert_eq!(
20193 pids.len(),
20194 1,
20195 "unleased timeout retains its death-check PID"
20196 );
20197 assert_eq!(outcomes.len(), 4);
20198 assert_eq!(outcomes[0].code, None);
20199 assert!(outcomes[0].output_tail.contains("before-timeout"));
20200 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20201 assert_eq!(outcomes[1].code, None);
20202 assert!(outcomes[1].output_tail.contains("command terminated:"));
20203 assert!(!outcomes[1].output_tail.contains("timed out"));
20204 assert_eq!(outcomes[2].code, Some(7));
20205 assert!(!outcomes[2].output_tail.contains("timed out"));
20206 assert_eq!(outcomes[3].code, Some(0));
20207 assert_eq!(outcomes[3].output_tail, "success");
20208 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20209 }
20210
20211 #[tokio::test]
20221 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20222 crate::run::pin_test_home();
20223
20224 let tmp = tempfile::tempdir().expect("tempdir");
20225 let repo = tmp.path().join("repo");
20226 std::fs::create_dir_all(&repo).unwrap();
20227 init_repo(&repo);
20228
20229 let mut config = Config::default();
20230 config.disk.min_free_bytes = 0;
20231 config.verify.gate = vec![
20232 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20233 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20234 .to_owned(),
20235 ];
20236
20237 let mut state = RunState::new(
20238 repo.clone(),
20239 "main".to_owned(),
20240 "deadbeef".to_owned(),
20241 "task".to_owned(),
20242 config,
20243 );
20244 let run_id = state.id.clone();
20245 state.candidates = vec![Candidate {
20246 index: 0,
20247 label: 'A',
20248 agent: "alpha".to_owned(),
20249 branch: "does-not-exist".to_owned(),
20250 worktree: repo.clone(),
20251 summary: String::new(),
20252 stat: String::new(),
20253 files: 0,
20254 commits: 0,
20255 empty: false,
20256 failed: None,
20257 verified_noop: None,
20258 duration_ms: 0,
20259 folded: false,
20260 }];
20261 state.tally = Some(Tally {
20262 first_choice: BTreeMap::from([('A', 1)]),
20263 borda: BTreeMap::new(),
20264 winner: 'A',
20265 rankings: 1,
20266 unanimous_initial: true,
20267 deliberated: false,
20268 changed_votes: 0,
20269 unanimous_final: true,
20270 tie_break: None,
20271 judges: 0,
20272 present: 0,
20273 quorum: 0,
20274 met_quorum: true,
20275 uncontested: Some("only candidate A produced a change".to_owned()),
20276 });
20277 state.reviews = vec![ReviewRound {
20278 round: 1,
20279 head: "deadbeef".to_owned(),
20280 verified_head: None,
20281 verified_at: None,
20282 reviews: Vec::new(),
20283 e2e: Vec::new(),
20284 fix: None,
20285 blocking: 0,
20286 answered: 0,
20287 expected: 0,
20288 clean: true,
20289 verify_retried: false,
20290 e2e_deferred: false,
20291 e2e_defer_reason: None,
20292 progressed: false,
20293 vote_split: false,
20294 reconsideration: Vec::new(),
20295 verdict: None,
20296 }];
20297
20298 let mut runner = Runner {
20299 state,
20300 roles: ResolvedRoles {
20301 implementers: Vec::new(),
20302 judges: Vec::new(),
20303 reviewers: Vec::new(),
20304 fixer: None,
20305 conductor: conductor(),
20306 implementer_roster: Vec::new(),
20307 judge_roster: Vec::new(),
20308 reviewer_roster: Vec::new(),
20309 },
20310 sem: Arc::new(Semaphore::new(1)),
20311 pause: Pause::new(),
20312 interrupt: Pause::new(),
20313 };
20314
20315 let started_marker = repo.join("started.marker");
20316 let release_marker = repo.join("release.marker");
20317 let poller = tokio::spawn(async move {
20318 for _ in 0..100 {
20323 if started_marker.exists()
20324 && let Ok(s) = crate::run::RunState::load(&run_id)
20325 && let Some(a) = s.active.get("gate")
20326 {
20327 std::fs::write(&release_marker, b"go").expect("release marker");
20328 return Some(a.clone());
20329 }
20330 tokio::time::sleep(Duration::from_millis(50)).await;
20331 }
20332 None
20333 });
20334
20335 runner.gate().await.expect("gate");
20336 let captured = poller.await.expect("poller task");
20337 let captured = captured.expect(
20338 "the poller never saw a `gate` task entry in run.json while the command was \
20339 still blocked on its own release marker",
20340 );
20341
20342 assert_eq!(captured.task.as_deref(), Some("gate"));
20343 assert_eq!(captured.node, "gate");
20344 assert_eq!(captured.index, Some(1));
20345 assert_eq!(captured.total, Some(1));
20346 assert!(
20347 captured
20348 .command
20349 .as_deref()
20350 .is_some_and(|c| c.contains("started.marker")),
20351 "{captured:?}"
20352 );
20353
20354 assert!(
20355 runner.state.active.is_empty(),
20356 "the entry must be cleared once the command actually finished: {:?}",
20357 runner.state.active
20358 );
20359 assert!(runner.state.gate_ran);
20360 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20361 }
20362
20363 #[tokio::test]
20366 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20367 use crate::verdict::{Finding, ReviewVote, Severity};
20368 crate::run::pin_test_home();
20369 let tmp = tempfile::tempdir().expect("tempdir");
20370 let repo = tmp.path().join("repo");
20371 std::fs::create_dir_all(&repo).unwrap();
20372 init_repo(&repo);
20373
20374 for (severity, vote, expect) in [
20375 (Severity::Major, ReviewVote::Reject, true),
20376 (Severity::Minor, ReviewVote::Reject, false),
20377 (Severity::Major, ReviewVote::Approve, false),
20378 ] {
20379 let mut round = review_round(false, 1, 1, 1, false, true);
20380 round.reviews = vec![ReviewRecord {
20381 reviewer: 1,
20382 agent: "alpha".to_owned(),
20383 summary: String::new(),
20384 findings: vec![Finding {
20385 id: "R1-1-1".to_owned(),
20386 severity,
20387 file: None,
20388 line: None,
20389 title: "t".to_owned(),
20390 detail: String::new(),
20391 }],
20392 vote: Some(vote),
20393 failed: None,
20394 duration_ms: 0,
20395 attempts: 0,
20396 }];
20397 let mut state = RunState::new(
20398 repo.clone(),
20399 "main".to_owned(),
20400 "deadbeef".to_owned(),
20401 "task".to_owned(),
20402 Config::default(),
20403 );
20404 state.reviews = vec![round];
20405 let mut runner = Runner {
20406 state,
20407 roles: ResolvedRoles {
20408 implementers: Vec::new(),
20409 judges: Vec::new(),
20410 reviewers: Vec::new(),
20411 fixer: None,
20412 conductor: conductor(),
20413 implementer_roster: Vec::new(),
20414 judge_roster: Vec::new(),
20415 reviewer_roster: Vec::new(),
20416 },
20417 sem: Arc::new(Semaphore::new(1)),
20418 pause: Pause::new(),
20419 interrupt: Pause::new(),
20420 };
20421 let shell = runner.state.config.shell();
20422 runner
20423 .stop_reviewing("round budget spent", &shell, &repo)
20424 .await
20425 .expect("stop_reviewing");
20426 assert_eq!(runner.state.status, RunStatus::Gating);
20427 assert_eq!(
20428 runner.state.contested_handoff.is_some(),
20429 expect,
20430 "{severity:?} + {vote:?}"
20431 );
20432 }
20433 }
20434
20435 #[tokio::test]
20448 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20449 crate::run::pin_test_home();
20450 let home = crate::run::home();
20451
20452 let tmp = tempfile::tempdir().expect("tempdir");
20453 let repo = tmp.path().join("repo");
20454 std::fs::create_dir_all(&repo).unwrap();
20455 init_repo(&repo);
20456 let head = crate::git::rev_parse(&repo, "HEAD")
20457 .await
20458 .expect("rev-parse");
20459 let cache_dir = tmp.path().join("target");
20462
20463 let mut config = Config::default();
20464 config.verify.e2e = vec![format!(
20465 "CARGO_TARGET_DIR='{}' test -f README.md",
20466 cache_dir.display()
20467 )];
20468 config.graph.review_rounds = 1;
20469 config.graph.timeout_verify = Some(2);
20472 config.graph.timeout_resource_wait = 2;
20473
20474 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20475 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20476 .expect("no io error acquiring directly")
20477 {
20478 crate::cache::AcquireOutcome::Acquired(g) => g,
20479 crate::cache::AcquireOutcome::Busy(b) => {
20480 panic!("expected the direct acquire to win the lease first: {b:?}")
20481 }
20482 };
20483
20484 let mut state = RunState::new(
20485 repo.clone(),
20486 "main".to_owned(),
20487 head.clone(),
20488 "task".to_owned(),
20489 config,
20490 );
20491 state.candidates = vec![Candidate {
20492 index: 0,
20493 label: 'A',
20494 agent: "alpha".to_owned(),
20495 branch: "does-not-exist".to_owned(),
20496 worktree: repo.clone(),
20497 summary: String::new(),
20498 stat: String::new(),
20499 files: 0,
20500 commits: 0,
20501 empty: false,
20502 failed: None,
20503 verified_noop: None,
20504 duration_ms: 0,
20505 folded: false,
20506 }];
20507 state.tally = Some(Tally {
20508 first_choice: BTreeMap::from([('A', 1)]),
20509 borda: BTreeMap::new(),
20510 winner: 'A',
20511 rankings: 1,
20512 unanimous_initial: true,
20513 deliberated: false,
20514 changed_votes: 0,
20515 unanimous_final: true,
20516 tie_break: None,
20517 judges: 0,
20518 present: 0,
20519 quorum: 0,
20520 met_quorum: true,
20521 uncontested: Some("only candidate A produced a change".to_owned()),
20522 });
20523 state.reviews = vec![ReviewRound {
20527 round: 1,
20528 head: head.clone(),
20529 verified_head: None,
20530 verified_at: None,
20531 reviews: Vec::new(),
20532 e2e: Vec::new(),
20533 fix: None,
20534 blocking: 1,
20535 answered: 1,
20536 expected: 1,
20537 clean: false,
20538 verify_retried: false,
20539 e2e_deferred: true,
20540 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20541 progressed: false,
20542 vote_split: false,
20543 reconsideration: Vec::new(),
20544 verdict: None,
20545 }];
20546
20547 let mut runner = Runner {
20548 state,
20549 roles: ResolvedRoles {
20550 implementers: Vec::new(),
20551 judges: Vec::new(),
20552 reviewers: Vec::new(),
20553 fixer: None,
20554 conductor: conductor(),
20555 implementer_roster: Vec::new(),
20556 judge_roster: Vec::new(),
20557 reviewer_roster: Vec::new(),
20558 },
20559 sem: Arc::new(Semaphore::new(1)),
20560 pause: Pause::new(),
20561 interrupt: Pause::new(),
20562 };
20563
20564 let shell = runner.state.config.shell();
20565 runner
20566 .stop_reviewing("round budget spent", &shell, &repo)
20567 .await
20568 .expect("stop_reviewing");
20569
20570 let last = runner.state.reviews.last().expect("round record");
20571 assert_eq!(
20572 last.e2e_status(),
20573 E2eStatus::ResourceBlocked,
20574 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20575 failed: {last:?}"
20576 );
20577 assert_eq!(
20578 last.verified_head.as_deref(),
20579 Some(head.as_str()),
20580 "which commit this attempt targeted is known even though nothing finished checking \
20581 it"
20582 );
20583 let first_attempt_at = last
20584 .verified_at
20585 .expect("when this attempt ran is known too");
20586 assert_ne!(
20587 runner.state.status,
20588 RunStatus::Blocked,
20589 "contention is evidence about the machine, not the patch — it must not settle the \
20590 run as blocked: {:?}",
20591 runner.state.status
20592 );
20593 assert!(
20594 !runner
20595 .state
20596 .events
20597 .iter()
20598 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20599 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20600 runner.state.events
20601 );
20602
20603 runner
20608 .stop_reviewing("round budget spent", &shell, &repo)
20609 .await
20610 .expect("stop_reviewing retry");
20611 assert_eq!(
20612 runner.state.reviews.len(),
20613 1,
20614 "no new round was started: {:?}",
20615 runner.state.reviews
20616 );
20617 let last = runner.state.reviews.last().expect("round record");
20618 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20619 assert!(
20620 last.verified_at.expect("still known") > first_attempt_at,
20621 "a second reentry must be a fresh attempt, not a stale copy of the first"
20622 );
20623 assert_ne!(runner.state.status, RunStatus::Blocked);
20624
20625 held.release();
20626 }
20627
20628 #[cfg(unix)]
20629 #[tokio::test]
20630 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20631 ask_test_home();
20632 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20633 let tmp = tempfile::tempdir().unwrap();
20634 let repo = tmp.path().join("repo");
20635 std::fs::create_dir(&repo).unwrap();
20636 init_repo(&repo);
20637 let run_git = |args: &[&str]| {
20638 let out = std::process::Command::new("git")
20639 .args(args)
20640 .current_dir(&repo)
20641 .output()
20642 .unwrap();
20643 assert!(
20644 out.status.success(),
20645 "{args:?}: {}",
20646 String::from_utf8_lossy(&out.stderr)
20647 );
20648 };
20649 let origin = tmp.path().join("origin.git");
20650 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20651 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20652 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20653 run_git(&["add", "spec.rs"]);
20654 run_git(&["commit", "-m", "initial spec"]);
20655 run_git(&["push", "origin", "main"]);
20656 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20657 run_git(&["checkout", "-b", "fix/review"]);
20658 std::fs::write(
20659 repo.join("check.rs"),
20660 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20661 )
20662 .unwrap();
20663 run_git(&["add", "check.rs"]);
20664 run_git(&["commit", "-m", "use spec"]);
20665 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20666 run_git(&["checkout", "main"]);
20667 std::fs::write(
20668 repo.join("spec.rs"),
20669 "struct AgentSpec { max_turn_secs: u64 }\n",
20670 )
20671 .unwrap();
20672 run_git(&["add", "spec.rs"]);
20673 run_git(&["commit", "-m", "extend spec"]);
20674 run_git(&["push", "origin", "main"]);
20675 run_git(&["checkout", "fix/review"]);
20676
20677 let mut runner = runner_at(RunStatus::Reviewing);
20678 runner.state.repo = repo.clone();
20679 runner.state.base_commit = base;
20680 runner.state.pr_head = Some("fix/review".into());
20681 runner.state.config.graph.review_rounds = max_rounds;
20682 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20683 runner.state.config.graph.sessions = false;
20684 runner.state.config.disk.min_free_bytes = 0;
20685 runner.state.config.verify.e2e =
20686 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20687 runner.state.candidates = vec![Candidate {
20688 index: 0,
20689 label: 'A',
20690 agent: "fixture".into(),
20691 branch: "fix/review".into(),
20692 worktree: repo.clone(),
20693 summary: String::new(),
20694 stat: String::new(),
20695 files: 1,
20696 commits: 1,
20697 empty: false,
20698 failed: None,
20699 verified_noop: None,
20700 duration_ms: 0,
20701 folded: false,
20702 }];
20703 runner.state.tally = Some(Tally {
20704 first_choice: BTreeMap::new(),
20705 borda: BTreeMap::new(),
20706 winner: 'A',
20707 rankings: 0,
20708 unanimous_initial: true,
20709 deliberated: false,
20710 changed_votes: 0,
20711 unanimous_final: true,
20712 tie_break: None,
20713 judges: 0,
20714 present: 0,
20715 quorum: 0,
20716 met_quorum: true,
20717 uncontested: Some("review-only".into()),
20718 });
20719 let mut last = review_round(false, 0, 1, 1, false, false);
20720 last.head = reviewed.clone();
20721 last.e2e_deferred = true;
20722 last.e2e[0].resource_blocked = true;
20723 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20724 if max_rounds == 1 {
20725 last.e2e.clear();
20726 }
20727 runner.state.reviews = vec![last];
20728 let calls = tmp.path().join("fix-calls");
20729 let mut fixer = spec("repair");
20730 fixer
20731 .env
20732 .insert("FIX_CALLS".into(), calls.display().to_string());
20733 fixer.command = vec!["sh".into(), "-c".into(), r#"
20734 echo fix >> "$FIX_CALLS"
20735 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20736 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20737 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20738 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20739 "#.into()];
20740 let mut reviewer = spec("reviewer");
20741 reviewer.command = vec![
20742 "sh".into(),
20743 "-c".into(),
20744 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20745 .into(),
20746 ];
20747 runner.roles.fixer = Some(vec![fixer]);
20748 runner.roles.reviewers = vec![reviewer];
20749
20750 runner.sync_to_base().await.unwrap();
20751 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20752 assert_ne!(rebased, reviewed);
20753 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20754 if reentry {
20755 let shell = runner.state.config.shell();
20758 runner
20759 .stop_unresolved_review("deferred verification", &shell, &repo)
20760 .await
20761 .unwrap();
20762 assert!(runner.catchup_repairable());
20763 assert!(!calls.exists());
20764 runner.state = RunState::load(&runner.state.id).unwrap();
20765 assert!(runner.catchup_repairable());
20766 }
20767 runner.review_loop().await.unwrap();
20768 let first = &runner.state.reviews[0];
20769 assert_eq!(first.head, reviewed);
20770 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20771 assert!(
20772 first.verify_retried,
20773 "a reproduced build failure gets exactly one retry: {:?}",
20774 first.e2e
20775 );
20776 assert!(first.e2e[0].output_tail.contains("E0063"));
20777 assert!(!runner.catchup_repairable());
20778 if max_rounds == 2 {
20779 assert_eq!(
20780 runner.state.status,
20781 RunStatus::Gating,
20782 "{}",
20783 crate::report::run(&runner.state)
20784 );
20785 assert_eq!(runner.state.reviews.len(), 2);
20786 assert!(first.fix.as_ref().unwrap().committed);
20787 assert!(runner.state.reviews[1].clean);
20788 assert_eq!(
20789 runner
20790 .state
20791 .reviews
20792 .iter()
20793 .filter(|r| r.fix.is_some())
20794 .count(),
20795 1
20796 );
20797 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20798 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20799 } else {
20800 assert_eq!(runner.state.status, RunStatus::Blocked);
20801 assert!(first.fix.is_none());
20802 assert!(!calls.exists());
20803 let recovery = crate::recovery::assess(
20804 "prior",
20805 Some(&runner.state),
20806 &runner.landing_base(),
20807 crate::recovery::Relation::Diverged,
20808 );
20809 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20810 let reasons = recovery.reasons.join("\n");
20811 assert!(
20812 reasons.contains(&reviewed) && reasons.contains(&rebased),
20813 "{reasons}"
20814 );
20815 assert!(
20816 reasons.contains("base sync to")
20817 && reasons.contains("additional review is required"),
20818 "{reasons}"
20819 );
20820 }
20821 }
20822 }
20823
20824 #[tokio::test]
20836 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20837 crate::run::pin_test_home();
20838 let home = crate::run::home();
20839
20840 let tmp = tempfile::tempdir().expect("tempdir");
20841 let repo = tmp.path().join("repo");
20842 std::fs::create_dir_all(&repo).unwrap();
20843 init_repo(&repo);
20844 let head = crate::git::rev_parse(&repo, "HEAD")
20845 .await
20846 .expect("rev-parse");
20847 let cache_dir = tmp.path().join("target");
20848
20849 let mut config = Config::default();
20850 config.verify.e2e = vec![format!(
20851 "CARGO_TARGET_DIR='{}' test -f README.md",
20852 cache_dir.display()
20853 )];
20854 config.graph.review_rounds = 1;
20855 config.graph.timeout_verify = Some(2);
20856 config.graph.timeout_resource_wait = 2;
20857
20858 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20859 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20860 .expect("no io error acquiring directly")
20861 {
20862 crate::cache::AcquireOutcome::Acquired(g) => g,
20863 crate::cache::AcquireOutcome::Busy(b) => {
20864 panic!("expected the direct acquire to win the lease first: {b:?}")
20865 }
20866 };
20867
20868 let mut state = RunState::new(
20869 repo.clone(),
20870 "main".to_owned(),
20871 head.clone(),
20872 "task".to_owned(),
20873 config,
20874 );
20875 state.candidates = vec![Candidate {
20876 index: 0,
20877 label: 'A',
20878 agent: "alpha".to_owned(),
20879 branch: "does-not-exist".to_owned(),
20880 worktree: repo.clone(),
20881 summary: String::new(),
20882 stat: String::new(),
20883 files: 0,
20884 commits: 0,
20885 empty: false,
20886 failed: None,
20887 verified_noop: None,
20888 duration_ms: 0,
20889 folded: false,
20890 }];
20891 state.tally = Some(Tally {
20892 first_choice: BTreeMap::from([('A', 1)]),
20893 borda: BTreeMap::new(),
20894 winner: 'A',
20895 rankings: 1,
20896 unanimous_initial: true,
20897 deliberated: false,
20898 changed_votes: 0,
20899 unanimous_final: true,
20900 tie_break: None,
20901 judges: 0,
20902 present: 0,
20903 quorum: 0,
20904 met_quorum: true,
20905 uncontested: Some("only candidate A produced a change".to_owned()),
20906 });
20907 state.reviews = vec![ReviewRound {
20911 round: 1,
20912 head: head.clone(),
20913 verified_head: Some(head.clone()),
20914 verified_at: Some(jiff::Timestamp::now()),
20915 reviews: Vec::new(),
20916 e2e: vec![CommandOutcome {
20917 command: format!(
20918 "CARGO_TARGET_DIR='{}' test -f README.md",
20919 cache_dir.display()
20920 ),
20921 code: None,
20922 output_tail: "waiting for the shared build cache".to_owned(),
20923 duration_ms: 0,
20924 resource_blocked: true,
20925 }],
20926 fix: None,
20927 blocking: 1,
20928 answered: 1,
20929 expected: 1,
20930 clean: false,
20931 verify_retried: false,
20932 e2e_deferred: false,
20933 e2e_defer_reason: None,
20934 progressed: false,
20935 vote_split: false,
20936 reconsideration: Vec::new(),
20937 verdict: None,
20938 }];
20939
20940 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20941 let mut runner = Runner {
20942 state,
20943 roles: ResolvedRoles {
20944 implementers: Vec::new(),
20945 judges: Vec::new(),
20946 reviewers: Vec::new(),
20947 fixer: None,
20948 conductor: conductor(),
20949 implementer_roster: Vec::new(),
20950 judge_roster: Vec::new(),
20951 reviewer_roster: Vec::new(),
20952 },
20953 sem: Arc::new(Semaphore::new(1)),
20954 pause: Pause::new(),
20955 interrupt: Pause::new(),
20956 };
20957
20958 runner.review_loop().await.expect("review_loop");
20963
20964 assert_eq!(
20965 runner.state.reviews.len(),
20966 1,
20967 "no new round was started on top of the unresolved one: {:?}",
20968 runner.state.reviews
20969 );
20970 let last = &runner.state.reviews[0];
20971 assert_eq!(
20972 last.e2e_status(),
20973 E2eStatus::ResourceBlocked,
20974 "still contended: {last:?}"
20975 );
20976 assert!(
20977 last.verified_at.expect("still known") > first_attempt_at,
20978 "review_loop must have actually retried the check, not left it exactly as found"
20979 );
20980 assert_ne!(
20981 runner.state.status,
20982 RunStatus::Blocked,
20983 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20984 runner.state.status
20985 );
20986
20987 held.release();
20988 }
20989
20990 #[tokio::test]
20991 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
20992 crate::run::pin_test_home();
20993 for status in [RunStatus::Blocked, RunStatus::Gating] {
20994 let mut runner = runner_at(status);
20995 runner.state.config.graph.land = true;
20996 runner.state.gate_ran = true;
20997 runner.state.merge = Some(MergeOutcome {
20998 mode: MergeMode::Pr,
20999 ok: false,
21000 detail: "the fixer produced no commit".into(),
21001 empty: false,
21002 });
21003 let url = "https://example.invalid/o/r/pull/1";
21004 runner.state.pr = Some(crate::run::PrRecord {
21005 url: url.into(),
21006 number: 1,
21007 state: "open".into(),
21008 checks: "red".into(),
21009 round: 1,
21010 rounds: 1,
21011 red_at_merge: Vec::new(),
21012 });
21013 runner.state.save().unwrap();
21014 runner.execute_graph().await.unwrap();
21018 assert_eq!(runner.state.status, RunStatus::Blocked);
21019 assert!(runner.state.land_sync_pending);
21020 assert!(runner.state.events.iter().any(|event|
21021 event.node == "land" && event.message == format!("watching {url}")));
21022 assert_eq!(
21023 runner.state.merge.as_ref().unwrap().detail,
21024 "the fixer produced no commit"
21025 );
21026 }
21027 }
21028
21029 #[tokio::test]
21030 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21031 crate::run::pin_test_home();
21032 let tmp = tempfile::tempdir().expect("tempdir");
21033 let repo = tmp.path().join("repo");
21034 std::fs::create_dir_all(&repo).unwrap();
21035 init_repo(&repo);
21036
21037 let mut config = Config::default();
21038 config.merge.mode = MergeMode::Pr;
21039 config.graph.land = true;
21040 config.graph.land_approval = false;
21041
21042 let mut state = RunState::new(
21043 repo.clone(),
21044 "main".to_owned(),
21045 "deadbeef".to_owned(),
21046 "task".to_owned(),
21047 config,
21048 );
21049 state.candidates = vec![Candidate {
21050 index: 0,
21051 label: 'A',
21052 agent: "alpha".to_owned(),
21053 branch: "does-not-exist".to_owned(),
21054 worktree: repo.clone(),
21055 summary: String::new(),
21056 stat: String::new(),
21057 files: 0,
21058 commits: 0,
21059 empty: false,
21060 failed: None,
21061 verified_noop: None,
21062 duration_ms: 0,
21063 folded: false,
21064 }];
21065 state.tally = Some(Tally {
21066 first_choice: BTreeMap::from([('A', 1)]),
21067 borda: BTreeMap::new(),
21068 winner: 'A',
21069 rankings: 1,
21070 unanimous_initial: true,
21071 deliberated: false,
21072 changed_votes: 0,
21073 unanimous_final: true,
21074 tie_break: None,
21075 judges: 0,
21076 present: 0,
21077 quorum: 0,
21078 met_quorum: true,
21079 uncontested: Some("only candidate A produced a change".to_owned()),
21080 });
21081 state.reviews = vec![ReviewRound {
21082 round: 1,
21083 head: "deadbeef".to_owned(),
21084 verified_head: None,
21085 verified_at: None,
21086 reviews: Vec::new(),
21087 e2e: Vec::new(),
21088 fix: None,
21089 blocking: 0,
21090 answered: 0,
21091 expected: 0,
21092 clean: true,
21093 verify_retried: false,
21094 e2e_deferred: false,
21095 e2e_defer_reason: None,
21096 progressed: false,
21097 vote_split: false,
21098 reconsideration: Vec::new(),
21099 verdict: None,
21100 }];
21101 state.gate = vec![CommandOutcome {
21102 command: "test".to_owned(),
21103 code: Some(0),
21104 output_tail: String::new(),
21105 duration_ms: 0,
21106 resource_blocked: false,
21107 }];
21108 state.gate_ran = true;
21109 state.status = RunStatus::Landing;
21113 state.merge = Some(MergeOutcome {
21114 mode: MergeMode::Pr,
21115 ok: true,
21116 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21117 empty: false,
21118 });
21119
21120 ask_test_home();
21124 state.save().expect("save landing run");
21127 let store = ask::Questions::open();
21128 let q = ask_open_question(&store, &state.id);
21129
21130 let mut runner = Runner {
21131 state,
21132 roles: ResolvedRoles {
21133 implementers: Vec::new(),
21134 judges: Vec::new(),
21135 reviewers: Vec::new(),
21136 fixer: None,
21137 conductor: conductor(),
21138 implementer_roster: Vec::new(),
21139 judge_roster: Vec::new(),
21140 reviewer_roster: Vec::new(),
21141 },
21142 sem: Arc::new(Semaphore::new(1)),
21143 pause: Pause::new(),
21144 interrupt: Pause::new(),
21145 };
21146
21147 runner.execute().await.expect("execute");
21152
21153 assert_eq!(
21154 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21155 Some("https://example.invalid/x/y/pull/1"),
21156 "reentry must not push again or open a second pull request over the \
21157 one `land` is already watching"
21158 );
21159 assert_ne!(
21160 runner.state.status,
21161 RunStatus::Landing,
21162 "land could not actually reach the fake pull request, so it must \
21163 have given up rather than left the run silently parked forever"
21164 );
21165 assert_eq!(runner.state.status, RunStatus::Blocked);
21169 assert!(
21170 store.get(&q.id).unwrap().status.open(),
21171 "Blocked is still alive; settle_questions must have been a no-op here"
21172 );
21173 }
21174
21175 fn state_with_round(round: ReviewRound) -> RunState {
21176 let mut s = RunState::new(
21177 PathBuf::from("/repo"),
21178 "main".to_owned(),
21179 "abc1234".to_owned(),
21180 "add retries".to_owned(),
21181 Config::default(),
21182 );
21183 s.reviews = vec![round];
21184 s
21185 }
21186
21187 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21188 crate::verdict::Finding {
21189 id: id.to_owned(),
21190 severity,
21191 file: None,
21192 line: None,
21193 title: title.to_owned(),
21194 detail: String::new(),
21195 }
21196 }
21197
21198 #[test]
21199 fn pr_body_names_open_findings_and_declined_ones() {
21200 let round = ReviewRound {
21201 round: 2,
21202 head: "deadbee".to_owned(),
21203 verified_head: None,
21204 verified_at: None,
21205 reviews: vec![ReviewRecord {
21206 attempts: 0,
21207 reviewer: 1,
21208 agent: "alpha".to_owned(),
21209 summary: String::new(),
21210 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21211 vote: None,
21212 failed: None,
21213 duration_ms: 0,
21214 }],
21215 e2e: vec![CommandOutcome {
21216 command: "cargo test".to_owned(),
21217 code: Some(0),
21218 output_tail: String::new(),
21219 duration_ms: 0,
21220 resource_blocked: false,
21221 }],
21222 verify_retried: false,
21223 e2e_deferred: false,
21224 e2e_defer_reason: None,
21225 fix: Some(FixRecord {
21226 agent: "alpha".to_owned(),
21227 addressed: Vec::new(),
21228 rejected: vec![crate::verdict::Rejection {
21229 id: "R1-1-1".to_owned(),
21230 why: "not reachable from any caller".to_owned(),
21231 }],
21232 notes: String::new(),
21233 committed: true,
21234 failed: None,
21235 duration_ms: 0,
21236 continuation: None,
21237 }),
21238 blocking: 0,
21239 answered: 1,
21240 expected: 1,
21241 clean: false,
21242 progressed: true,
21243 vote_split: false,
21244 reconsideration: Vec::new(),
21245 verdict: None,
21246 };
21247 let state = state_with_round(round);
21248 let body = pr_message(&state, 'A').body;
21249
21250 assert!(body.contains("add retries"), "the task must still be there");
21251 assert!(body.contains("R2-1-1"), "{body}");
21252 assert!(!body.contains("unused import"), "{body}");
21253 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21254 assert!(
21255 !body.contains("not reachable from any caller"),
21256 "the reason it was declined: {body}"
21257 );
21258 }
21259
21260 #[test]
21261 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21262 let round = ReviewRound {
21263 round: 1,
21264 head: "deadbee".to_owned(),
21265 verified_head: None,
21266 verified_at: None,
21267 reviews: vec![ReviewRecord {
21268 attempts: 0,
21269 reviewer: 1,
21270 agent: "alpha".to_owned(),
21271 summary: String::new(),
21272 findings: Vec::new(),
21273 vote: None,
21274 failed: None,
21275 duration_ms: 0,
21276 }],
21277 e2e: Vec::new(),
21278 verify_retried: false,
21279 e2e_deferred: false,
21280 e2e_defer_reason: None,
21281 fix: None,
21282 blocking: 0,
21283 answered: 1,
21284 expected: 1,
21285 clean: true,
21286 progressed: false,
21287 vote_split: false,
21288 reconsideration: Vec::new(),
21289 verdict: None,
21290 };
21291 let state = state_with_round(round);
21292 let body = pr_message(&state, 'A').body;
21293 assert!(!body.contains("Open review findings"), "{body}");
21294 assert!(!body.contains("Declined"), "{body}");
21295 }
21296
21297 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21298 let mut state = RunState::new(
21299 PathBuf::from("/repo"),
21300 "main".to_owned(),
21301 "abc1234".to_owned(),
21302 instruction.to_owned(),
21303 Config::default(),
21304 );
21305 state.config.disk.min_free_bytes = 0;
21307 state.candidates.push(Candidate {
21308 index: 0,
21309 label: 'A',
21310 agent: "alpha".to_owned(),
21311 branch: "magi/x/A".to_owned(),
21312 worktree: PathBuf::from("/wt"),
21313 summary: summary.to_owned(),
21314 stat: String::new(),
21315 files: 1,
21316 commits: 1,
21317 empty: false,
21318 failed: None,
21319 verified_noop: None,
21320 folded: false,
21321 duration_ms: 0,
21322 });
21323 state
21324 }
21325
21326 fn review_state(subjects: &[&str]) -> RunState {
21327 let mut state = state_with_summary(
21328 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21329 "",
21330 );
21331 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21332 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21333 state
21334 }
21335
21336 #[tokio::test]
21339 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21340 ask_test_home();
21341 let tmp = tempfile::tempdir().unwrap();
21342 let repo = tmp.path().join("repo");
21343 std::fs::create_dir_all(&repo).unwrap();
21344 init_repo(&repo);
21345 let origin = tmp.path().join("origin.git");
21346 let g = |dir: &Path, args: &[&str]| {
21347 let out = std::process::Command::new("git")
21348 .args(args)
21349 .current_dir(dir)
21350 .quiet()
21351 .output()
21352 .expect("spawn git");
21353 assert!(
21354 out.status.success(),
21355 "git {args:?}: {}",
21356 String::from_utf8_lossy(&out.stderr)
21357 );
21358 };
21359 g(
21360 tmp.path(),
21361 &[
21362 "clone",
21363 "--bare",
21364 "-q",
21365 repo.to_str().unwrap(),
21366 origin.to_str().unwrap(),
21367 ],
21368 );
21369 g(
21370 &repo,
21371 &["remote", "add", "origin", origin.to_str().unwrap()],
21372 );
21373 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21374
21375 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21377 g(&repo, &["add", "-A"]);
21378 g(
21379 &repo,
21380 &["commit", "-q", "-m", "chore(deps): update a crate"],
21381 );
21382 g(&repo, &["push", "-q", "origin", "main"]);
21383 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21384 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21385 g(&repo, &["add", "-A"]);
21386 g(
21387 &repo,
21388 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21389 );
21390 g(&repo, &["checkout", "-q", "main"]);
21391
21392 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21393 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21394 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21396 assert_eq!(fallback, c1);
21397
21398 let mut state = review_state(&[
21399 "chore(deps): update a crate",
21400 "fix(daemon): apply a chosen action",
21401 ]);
21402 state.repo = repo.clone();
21403 state.base_branch = "main".to_owned();
21404 state.base_commit = c1;
21405 refresh_reviewed_commits(&mut state, "feat/own").await;
21406 assert_eq!(
21407 state.reviewed_commits,
21408 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21409 );
21410 assert_eq!(
21411 review_title(&state).as_deref(),
21412 Some("fix(daemon): apply a chosen action")
21413 );
21414 assert_eq!(
21415 leaked_subjects(&state, "feat/own").await,
21416 Some(vec!["chore(deps): update a crate".to_owned()])
21417 );
21418 state.config.merge.remote = "nowhere".to_owned();
21421 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21422 }
21423
21424 struct SummaryFixture {
21428 _tmp: tempfile::TempDir,
21429 repo: PathBuf,
21430 reply: PathBuf,
21431 runner: Runner,
21432 }
21433
21434 fn summary_fixture() -> SummaryFixture {
21435 ask_test_home();
21436 let tmp = tempfile::tempdir().unwrap();
21437 let repo = tmp.path().join("repo");
21438 std::fs::create_dir_all(&repo).unwrap();
21439 init_repo(&repo);
21440 let origin = tmp.path().join("origin.git");
21441 let g = |args: &[&str]| {
21442 let out = std::process::Command::new("git")
21443 .args(args)
21444 .current_dir(&repo)
21445 .quiet()
21446 .output()
21447 .expect("spawn git");
21448 assert!(
21449 out.status.success(),
21450 "git {args:?}: {}",
21451 String::from_utf8_lossy(&out.stderr)
21452 );
21453 };
21454 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21455 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21456 g(&["push", "-q", "origin", "main"]);
21457 g(&["checkout", "-q", "-b", "feat/own"]);
21458 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21459 g(&["add", "-A"]);
21460 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21461 let reply = tmp.path().join("reply.txt");
21462 std::fs::write(
21463 &reply,
21464 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21465 )
21466 .unwrap();
21467 let mut runner = runner_at(RunStatus::Gating);
21468 let mut state = review_state(&["feat(own): add the own file"]);
21469 state.repo = repo.clone();
21470 state.base_branch = "main".to_owned();
21471 state.candidates[0].branch = "feat/own".to_owned();
21472 state.candidates[0].worktree = repo.clone();
21473 let mut author = spec("alpha");
21474 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21475 author
21476 .env
21477 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21478 state.config.agents = vec![author];
21479 runner.state = state;
21480 SummaryFixture {
21481 _tmp: tmp,
21482 repo,
21483 reply,
21484 runner,
21485 }
21486 }
21487
21488 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21489 let winner = f.runner.state.candidates[0].clone();
21490 f.runner
21491 .guarded_pr_message(&winner, None, true)
21492 .await
21493 .unwrap()
21494 .expect("an English description never parks")
21495 }
21496
21497 fn clean_round_with_minor_findings() -> ReviewRound {
21498 let mut round = passing_round(2, None);
21499 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21500 round.reviews = vec![ReviewRecord {
21501 attempts: 0,
21502 reviewer: 1,
21503 agent: "alpha".into(),
21504 summary: "内部レビュー".into(),
21505 findings: vec![
21506 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21507 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21508 ],
21509 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21510 failed: None,
21511 duration_ms: 0,
21512 }];
21513 round
21514 }
21515
21516 #[tokio::test]
21517 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21518 for review_only in [false, true] {
21519 let mut f = summary_fixture();
21520 if !review_only {
21521 f.runner.state.candidates[0].agent = "alpha".into();
21522 }
21523 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21524 assert!(f.runner.state.reviews[0].clean);
21525 assert!(f.runner.state.reviews[0].fix.is_none());
21526 assert!(f.runner.state.open_findings().is_empty());
21527 assert_eq!(
21528 crate::pr_outcome::required_finding_ids(&f.runner.state),
21529 ["R2-1-1", "R2-1-2"]
21530 );
21531 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21532 let evidence = crate::pr_outcome::summary_evidence(
21533 &f.runner.state,
21534 crate::pr_outcome::FinalHead::At(&head),
21535 );
21536 assert!(
21537 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21538 );
21539 let omitted = posted(&mut f).await;
21541 assert!(omitted.body.contains("No description"));
21542 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21543 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21544 assert!(f.runner.state.change_summary.is_none());
21545 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21546 let described = posted(&mut f).await;
21547 assert!(described.body.contains("improve documentation for callers"));
21548 assert!(described.body.contains("remaining edge case"));
21549 assert!(described.body.contains("## Open review findings"));
21550 assert!(!described.body.contains("未修正"));
21551 }
21552 }
21553
21554 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21555 let mut f = summary_fixture();
21556 if !review_only {
21557 f.runner.state.candidates[0].agent = "alpha".into();
21558 }
21559 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21560 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21561 fixed
21562 .fix
21563 .as_mut()
21564 .unwrap()
21565 .rejected
21566 .push(crate::verdict::Rejection {
21567 id: "R1-1-1".into(),
21568 why: "呼び出されないコードのため辞退".into(),
21569 });
21570 let mut clean = clean_round_with_minor_findings();
21571 clean.verified_head = Some(head.clone());
21572 f.runner.state.reviews = vec![fixed, clean];
21573 ok_gate(&mut f.runner.state, Some(&head));
21574 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21575 f.runner.state.config.agents.push(scripted("beta", reply));
21576 f.runner.state.config.roles.language_judge =
21577 Some(crate::config::AgentChoice::One("beta".into()));
21578 f
21579 }
21580
21581 #[tokio::test]
21582 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21583 for review_only in [false, true] {
21584 let mut f = rewrite_fixture_with_findings(review_only,
21585 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21586 let m = posted(&mut f).await;
21587 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21588 let expected =
21589 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21590 assert!(m.body.contains(&expected));
21591 assert!(m.body.contains("## Review rounds"));
21592 assert!(
21593 m.body
21594 .contains("Gate (`verify.gate`): passed on the final commit")
21595 );
21596 assert!(
21597 m.body
21598 .contains("Tests (`verify.e2e`): passed on the final commit")
21599 );
21600 let c = f.runner.state.change_summary.as_ref().unwrap();
21601 assert!(
21602 m.body
21603 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21604 );
21605 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21606 assert!(crate::pr_outcome::covers_required_findings(
21607 &f.runner.state,
21608 &m.body
21609 ));
21610 assert!(m.body.contains("documentation remains incomplete"));
21611 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21612 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21613 assert!(f.runner.state.github_text.is_none());
21614 }
21615 }
21616
21617 #[tokio::test]
21618 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21619 for review_only in [false, true] {
21620 for reply in [
21621 "TITLE: fix: public description\nA short English description.",
21622 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21623 ] {
21624 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21625 let winner = f.runner.state.candidates[0].clone();
21626 let posted = f
21627 .runner
21628 .guarded_pr_message(&winner, None, true)
21629 .await
21630 .unwrap();
21631 assert!(posted.is_none());
21632 assert!(f.runner.state.parked);
21633 let withheld = f.runner.state.github_text.as_ref().unwrap();
21634 assert!(
21635 withheld.rewrite.is_none(),
21636 "an incomplete rewrite must not be selectable"
21637 );
21638 let q = ask::Questions::open()
21639 .get(withheld.question.as_ref().unwrap())
21640 .unwrap();
21641 assert!(
21642 !q.choices
21643 .contains(&crate::github_text::USE_REWRITE.to_owned())
21644 );
21645 assert!(
21646 q.detail.contains("R1-1-1")
21647 && q.detail.contains("R2-1-1")
21648 && q.detail.contains("R2-1-2")
21649 );
21650 }
21651 }
21652 }
21653
21654 #[tokio::test]
21655 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21656 for review_only in [false, true] {
21657 let mut f = summary_fixture();
21658 if !review_only {
21659 f.runner.state.candidates[0].agent = "alpha".into();
21660 }
21661 f.runner.state.instruction = "内部タスクの説明".into();
21662 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21663 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21664 f.runner.state.reviews = vec![
21665 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21666 round_with_notes(2, Some("二回目の内部修正報告です。")),
21667 passing_round(3, Some(&head)),
21668 ];
21669 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21670 ok_gate(&mut f.runner.state, Some(&head));
21671 let captured = f.repo.join("public-prompt.txt");
21672 let author = &mut f.runner.state.config.agents[0];
21673 author.command = vec![
21674 "sh".into(),
21675 "-c".into(),
21676 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21677 ];
21678 author.env.insert(
21679 "PROMPT_FILE".into(),
21680 captured.to_string_lossy().into_owned(),
21681 );
21682 let m = posted(&mut f).await;
21683 assert!(m.body.contains("Adds `own.txt`"));
21684 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21685 assert!(m.body.contains("R1-1-1"));
21686 assert!(
21687 m.body
21688 .contains("Gate (`verify.gate`): passed on the final commit")
21689 );
21690 assert!(
21691 m.body
21692 .contains("Tests (`verify.e2e`): passed on the final commit")
21693 );
21694 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21695 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21696 let prompt = std::fs::read_to_string(captured).unwrap();
21697 assert!(prompt.contains(&format!(
21698 "git diff {} {head}",
21699 f.runner.state.change_summary.as_ref().unwrap().base
21700 )));
21701 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21702 assert_eq!(
21703 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21704 "二回目の内部修正報告です。"
21705 );
21706 }
21707 }
21708
21709 #[tokio::test]
21710 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21711 for review_only in [false, true] {
21712 let mut f = summary_fixture();
21713 if !review_only {
21714 f.runner.state.candidates[0].agent = "alpha".into();
21715 }
21716 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21717 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21718 std::fs::write(&f.reply, "no usable description").unwrap();
21719 let failed = posted(&mut f).await;
21720 assert!(
21721 failed
21722 .body
21723 .contains("No description of what this change does could be generated")
21724 );
21725 assert!(failed.body.is_ascii());
21726 assert!(f.runner.state.github_text.is_none());
21727 assert!(f.runner.state.change_summary.is_none());
21728 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21729 let evidence = crate::pr_outcome::summary_evidence(
21730 &f.runner.state,
21731 crate::pr_outcome::FinalHead::At(&head),
21732 );
21733 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21734 assert!(
21735 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21736 .exists()
21737 );
21738 f.runner.state =
21739 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21740 std::fs::write(
21741 &f.reply,
21742 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21743 )
21744 .unwrap();
21745 let retried = posted(&mut f).await;
21746 assert!(retried.body.contains("Adds the committed own file."));
21747 assert_eq!(
21748 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21749 "日本語の修正報告"
21750 );
21751 }
21752 }
21753
21754 #[tokio::test]
21755 async fn same_head_verification_and_findings_invalidate_public_prose() {
21756 let mut f = summary_fixture();
21757 posted(&mut f).await;
21758 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21759 let original = f
21760 .runner
21761 .state
21762 .change_summary
21763 .as_ref()
21764 .unwrap()
21765 .evidence
21766 .clone();
21767 ok_gate(&mut f.runner.state, Some(&head));
21768 std::fs::write(
21769 &f.reply,
21770 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21771 )
21772 .unwrap();
21773 let verified = posted(&mut f).await;
21774 assert!(verified.body.contains("now verified"));
21775 assert_ne!(
21776 f.runner.state.change_summary.as_ref().unwrap().evidence,
21777 original
21778 );
21779 let mut round = round_with_notes(1, Some("内部報告"));
21780 round.reviews.push(ReviewRecord {
21781 attempts: 0,
21782 reviewer: 1,
21783 agent: "alpha".into(),
21784 summary: "内部レビュー".into(),
21785 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21786 vote: None,
21787 failed: None,
21788 duration_ms: 0,
21789 });
21790 round
21791 .fix
21792 .as_mut()
21793 .unwrap()
21794 .rejected
21795 .push(crate::verdict::Rejection {
21796 id: "R1-1-1".into(),
21797 why: "呼び出し元から到達できない".into(),
21798 });
21799 f.runner.state.reviews = vec![round];
21800 let omitted = posted(&mut f).await;
21802 assert!(omitted.body.contains("No description"));
21803 assert!(omitted.body.contains("R1-1-1"));
21804 assert!(!omitted.body.contains("到達"));
21805 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21806 let described = posted(&mut f).await;
21807 assert!(described.body.contains("unreachable from callers"));
21808 assert!(described.body.contains("R1-1-1"));
21809 assert!(!described.body.contains("到達"));
21810 let evidence = f
21811 .runner
21812 .state
21813 .change_summary
21814 .as_ref()
21815 .unwrap()
21816 .evidence
21817 .clone();
21818 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21819 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21820 assert!(posted(&mut f).await.body.contains("rationale updated"));
21821 assert_ne!(
21822 f.runner.state.change_summary.as_ref().unwrap().evidence,
21823 evidence
21824 );
21825 f.runner
21827 .state
21828 .change_summary
21829 .as_mut()
21830 .unwrap()
21831 .evidence
21832 .clear();
21833 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21834 assert!(posted(&mut f).await.body.contains("current specification"));
21835 }
21836
21837 #[tokio::test]
21838 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21839 let mut f = summary_fixture();
21840 let m = posted(&mut f).await;
21841 assert_eq!(m.title, "feat(own): add the own file");
21842 assert!(
21843 m.body
21844 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21845 "{}",
21846 m.body
21847 );
21848 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21849 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21850 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21851 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21852 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21853 let kept = f.runner.state.change_summary.as_ref().unwrap();
21854 assert_eq!(
21855 (kept.head.as_str(), kept.base.as_str()),
21856 (head.as_str(), base.as_str())
21857 );
21858 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21859 }
21860
21861 #[tokio::test]
21862 async fn review_only_summary_receives_the_configured_github_language() {
21863 let mut f = summary_fixture();
21864 f.runner.state.config.graph.github_language = "ja".into();
21865 let captured = f.repo.join("summary-prompt.txt");
21866 let author = &mut f.runner.state.config.agents[0];
21867 author.command = vec![
21868 "sh".into(),
21869 "-c".into(),
21870 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21871 ];
21872 author.env.insert(
21873 "PROMPT_FILE".into(),
21874 captured.to_string_lossy().into_owned(),
21875 );
21876 std::fs::write(
21877 &f.reply,
21878 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21879 )
21880 .unwrap();
21881 let m = posted(&mut f).await;
21882 let prompt = std::fs::read_to_string(captured).unwrap();
21883 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21884 assert!(!prompt.contains("in English"));
21885 assert_eq!(m.title, "feat(own): ファイルを追加");
21886 assert!(m.body.contains("own.txt を追加します。"));
21887 }
21888
21889 #[tokio::test]
21890 async fn review_fixes_come_after_the_whole_change_summary() {
21891 let mut f = summary_fixture();
21892 f.runner.state.reviews = vec![
21893 round_with_notes(1, Some("handled the PATH case")),
21894 round_with_notes(2, None),
21895 ];
21896 let m = posted(&mut f).await;
21897 let summary = m.body.find("## Summary").expect("summary");
21898 let fixes = m.body.find("## Review rounds").expect("review rounds");
21899 assert!(summary < fixes, "{}", m.body);
21900 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21901 assert!(m.body.contains("## Final verification"), "{}", m.body);
21902 }
21903
21904 #[tokio::test]
21905 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21906 let mut f = summary_fixture();
21907 let first = posted(&mut f).await;
21908 let first_head = f.runner.state.change_summary.clone().unwrap().head;
21909 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21911 let again = posted(&mut f).await;
21912 assert_eq!((again.title, again.body), (first.title, first.body));
21913
21914 let g = |args: &[&str]| {
21915 let out = std::process::Command::new("git")
21916 .args(args)
21917 .current_dir(&f.repo)
21918 .quiet()
21919 .output()
21920 .expect("spawn git");
21921 assert!(out.status.success(), "git {args:?}");
21922 };
21923 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21924 g(&["add", "-A"]);
21925 g(&["commit", "-q", "-m", "fix(own): handle more"]);
21926 let second = posted(&mut f).await;
21927 let kept = f.runner.state.change_summary.clone().unwrap();
21928 assert_ne!(kept.head, first_head);
21929 assert!(second.body.contains("Not read yet."), "{}", second.body);
21930 assert!(!second.body.contains("so the first summary is read"));
21931 assert_eq!(second.title, "feat(own): other");
21932 }
21933
21934 #[tokio::test]
21935 async fn an_unavailable_summary_is_stated_and_never_invented() {
21936 let mut f = summary_fixture();
21937 std::fs::write(&f.reply, "no title line here").unwrap();
21938 let m = posted(&mut f).await;
21939 assert!(
21940 m.body
21941 .contains("No description of what this change does could be generated"),
21942 "{}",
21943 m.body
21944 );
21945 assert!(m.body.contains("internal reports remain in local records"));
21946 assert!(f.runner.state.change_summary.is_none());
21947 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21948 let winner = f.runner.state.candidates[0].clone();
21950 let preview = f
21951 .runner
21952 .guarded_pr_message(&winner, None, false)
21953 .await
21954 .unwrap()
21955 .unwrap();
21956 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21957 }
21958
21959 #[test]
21960 fn pr_message_review_single_commit_uses_its_subject() {
21961 let state = review_state(&["feat(nats): per-role user"]);
21962 let m = pr_message(&state, 'A');
21963 assert_eq!(m.title, "feat(nats): per-role user");
21964 assert!(!m.body.contains("Review the work already"), "{}", m.body);
21965 assert!(m.body.contains("## Commits under review"), "{}", m.body);
21966 }
21967
21968 #[test]
21969 fn pr_message_review_multi_commit_takes_the_oldest() {
21970 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21971 let m = pr_message(&state, 'A');
21972 assert_eq!(m.title, "feat: the change");
21973 for s in ["feat: the change", "fix: typo", "fix: again"] {
21974 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21975 }
21976 }
21977
21978 #[test]
21979 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21980 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21981 let mut state = review_state(&[first, "fix: later fixup"]);
21982 state.candidates[0].branch = "機能/ログイン".to_owned();
21983 let m = pr_message(&state, 'A');
21984 assert!(
21985 m.title.starts_with("chore: land candidate A of run"),
21986 "{}",
21987 m.title
21988 );
21989 }
21990 }
21991
21992 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
21993 BranchFacts {
21994 commits: commits
21995 .iter()
21996 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
21997 .collect(),
21998 stat: stat.to_owned(),
21999 }
22000 }
22001
22002 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22003 let mut r = review_round(true, 0, 1, 1, true, true);
22004 r.round = round;
22005 r.fix = notes.map(|n| FixRecord {
22006 agent: "fixer".to_owned(),
22007 addressed: Vec::new(),
22008 rejected: Vec::new(),
22009 notes: n.to_owned(),
22010 committed: true,
22011 failed: None,
22012 duration_ms: 0,
22013 continuation: None,
22014 });
22015 r
22016 }
22017
22018 #[test]
22019 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22020 let state = review_state(&["ignored"]);
22021 let f = facts(
22022 &[
22023 (
22024 "fix(login): resolve PATH on macOS",
22025 "Login shells skip rc files.",
22026 ),
22027 ("fix: address review", ""),
22028 ],
22029 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22030 );
22031 let m = pr_message_with(&state, 'A', Some(&f));
22032 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22033 assert!(
22034 m.body
22035 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22036 "{}",
22037 m.body
22038 );
22039 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22040 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22041 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22042 for banned in [
22043 "Review the work already",
22044 "no task statement",
22045 "Original task",
22046 ] {
22047 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22048 }
22049 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22050 }
22051
22052 #[test]
22053 fn pr_message_review_truncates_a_huge_first_commit_body() {
22054 let state = review_state(&["ignored"]);
22055 let f = facts(
22056 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22057 "s",
22058 );
22059 let m = pr_message_with(&state, 'A', Some(&f));
22060 assert!(m.body.len() < 30_000, "{}", m.body.len());
22061 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22062 assert!(
22063 m.body.contains("- ... 1 more commit(s)"),
22064 "{}",
22065 m.body.len()
22066 );
22067 assert!(m.body.ends_with("magi:candidate-a\n"));
22068 }
22069
22070 #[test]
22071 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22072 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22073 assert_eq!(m.title, "feat: x");
22074 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22075 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22076 assert!(!m.body.contains("no task statement"), "{}", m.body);
22077 }
22078
22079 #[test]
22080 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22081 let mut state = review_state(&["日本語の件名"]);
22082 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22083 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22084 state.candidates[0].branch = "機能/ログイン".to_owned();
22085 assert!(
22086 pr_message(&state, 'A')
22087 .title
22088 .starts_with("chore: land candidate A")
22089 );
22090 }
22091
22092 #[test]
22093 fn pr_message_review_fixes_survive_a_clean_final_round() {
22094 let mut state = review_state(&["feat: x"]);
22095 state.reviews = vec![
22096 round_with_notes(1, Some("handled the PATH case")),
22097 round_with_notes(2, None),
22098 ];
22099 let body = pr_message(&state, 'A').body;
22100 assert!(
22101 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22102 "{body}"
22103 );
22104 assert!(body.contains("### Round 1"), "{body}");
22105
22106 state.reviews = vec![
22107 round_with_notes(1, Some("first fix")),
22108 round_with_notes(2, Some("")),
22109 round_with_notes(3, Some("second fix")),
22110 round_with_notes(4, None),
22111 ];
22112 let body = pr_message(&state, 'A').body;
22113 assert!(
22114 body.contains("### Round 1") && !body.contains("first fix"),
22115 "{body}"
22116 );
22117 assert!(
22118 body.contains("### Round 3") && !body.contains("second fix"),
22119 "{body}"
22120 );
22121 assert!(body.contains("### Round 2"), "{body}");
22122 }
22123
22124 const FINAL: &str = "abcdef1234567890";
22125
22126 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22127 let mut r = round_with_notes(round, None);
22128 r.verified_head = verified.map(str::to_owned);
22129 r
22130 }
22131
22132 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22133 state.gate = vec![CommandOutcome {
22134 command: "gate".to_owned(),
22135 code: Some(0),
22136 output_tail: String::new(),
22137 duration_ms: 0,
22138 resource_blocked: false,
22139 }];
22140 state.gate_ran = true;
22141 state.gate_head = head.map(str::to_owned);
22142 }
22143
22144 fn body_at(state: &RunState) -> String {
22145 pr_message_raw(
22146 state,
22147 'A',
22148 None,
22149 crate::pr_outcome::FinalHead::At(FINAL),
22150 ChangeNote::NotAsked,
22151 )
22152 .body
22153 }
22154
22155 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22156
22157 #[test]
22158 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22159 let mut state = review_state(&["feat: x"]);
22160 state.reviews = vec![
22161 round_with_notes(1, Some(SANDBOXED)),
22162 passing_round(2, Some(FINAL)),
22163 ];
22164 ok_gate(&mut state, Some(FINAL));
22165 let body = body_at(&state);
22166 assert!(!body.contains("Handled the PATH case."), "{body}");
22167 assert!(!body.contains("could not commit"), "{body}");
22168 assert!(!body.contains("could not run"), "{body}");
22169 assert!(
22170 body.contains("The fix is committed on the branch"),
22171 "{body}"
22172 );
22173 assert!(!body.contains("sandbox"), "{body}");
22174 assert!(
22175 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22176 "{body}"
22177 );
22178 assert!(
22179 body.contains("Gate (`verify.gate`): passed on the final commit"),
22180 "{body}"
22181 );
22182 assert!(
22184 state.reviews[0]
22185 .fix
22186 .as_ref()
22187 .unwrap()
22188 .notes
22189 .contains("could not commit")
22190 );
22191 }
22192
22193 #[test]
22194 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22195 let mut state = review_state(&["feat: x"]);
22196 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22197 r1.fix.as_mut().unwrap().committed = false;
22198 let mut r2 = passing_round(2, Some(FINAL));
22199 r2.e2e[0].code = Some(1);
22200 state.reviews = vec![r1, r2];
22201 let body = body_at(&state);
22202 assert!(!body.contains("could not commit"), "{body}");
22203 assert!(!body.contains("could not run"), "{body}");
22204 assert!(body.contains("This round produced no commit"), "{body}");
22205 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22206 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22207 assert!(!body.contains("passed"), "{body}");
22208 }
22209
22210 #[test]
22211 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22212 let mut state = review_state(&["feat: x"]);
22213 state.reviews = vec![
22214 round_with_notes(1, Some(SANDBOXED)),
22215 passing_round(2, Some("0123456789abcdef")),
22216 ];
22217 ok_gate(&mut state, Some("0123456789abcdef"));
22218 let body = body_at(&state);
22219 assert!(!body.contains("HTTP tests could not run"), "{body}");
22220 assert!(!body.contains("could not commit"), "{body}");
22222 assert!(
22223 body.contains("passed on an earlier commit (`0123456`)"),
22224 "{body}"
22225 );
22226 assert!(!body.contains("passed on the final commit"), "{body}");
22227
22228 ok_gate(&mut state, None);
22230 let body = body_at(&state);
22231 assert!(
22232 body.contains("passed on a commit that was not recorded"),
22233 "{body}"
22234 );
22235 }
22236
22237 #[test]
22238 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22239 let state = state_with_summary(
22240 "add retries to the client",
22241 "TITLE: feat: retries\n\nDid it.",
22242 );
22243 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22244 assert_eq!(m.title, "feat: retries");
22245 assert!(
22246 m.body.contains("<summary>Original task</summary>"),
22247 "{}",
22248 m.body
22249 );
22250 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22251 assert!(
22252 m.body
22253 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22254 "{}",
22255 m.body
22256 );
22257 }
22258
22259 #[test]
22260 fn pr_message_review_bounds_a_long_english_subject() {
22261 let long = format!("feat: {}", "word ".repeat(100));
22262 let m = pr_message(&review_state(&[&long]), 'A');
22263 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22264 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22265 }
22266
22267 #[test]
22268 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22269 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22270 let m = pr_message(&state, 'A');
22271 assert_eq!(m.title, "add retries");
22272 assert!(m.body.contains("<summary>Original task</summary>"));
22273 assert!(!m.body.contains("Commits under review"));
22274 assert_eq!(landing_subject_source(&state), state.instruction);
22275 }
22276
22277 #[test]
22278 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22279 let state = review_state(&["feat: the change", "fix: typo"]);
22280 let source = landing_subject_source(&state);
22281 assert_eq!(land::merge_subject("", &source), "feat: the change");
22282 assert_eq!(
22283 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22284 "feat: the change"
22285 );
22286 assert_eq!(
22288 land::merge_subject("feat: renamed by hand", &source),
22289 "feat: renamed by hand"
22290 );
22291 let blank = review_state(&["日本語"]);
22292 assert!(
22293 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22294 );
22295 }
22296
22297 #[test]
22298 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22299 let state = review_state(&["feat: the change"]);
22300 let source = landing_subject_source(&state);
22301 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22302 assert_eq!(
22303 land::merge_subject(landing_title(&state, old), &source),
22304 "feat: the change"
22305 );
22306 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22307 let task = state_with_summary("add retries", "");
22308 assert_eq!(landing_title(&task, old), old);
22309 }
22310
22311 #[test]
22312 fn pr_message_describes_the_change_not_the_task() {
22313 let state = state_with_summary(
22314 "今回やってほしいこと: results projector を直す",
22315 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22316 );
22317 let m = pr_message(&state, 'A');
22318 assert_eq!(m.title, "fix(web): batch the runs list reads");
22319 assert!(
22320 m.body.starts_with("## Summary\n\n- reads run.json once"),
22321 "{}",
22322 m.body
22323 );
22324 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22325 let task_at = m.body.find("今回やってほしいこと").unwrap();
22326 let details_at = m.body.find("<details>").unwrap();
22327 assert!(
22328 details_at < task_at,
22329 "the task lives inside <details>: {}",
22330 m.body
22331 );
22332 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22333 assert!(m.body.contains("magi:candidate-a"));
22334 }
22335
22336 #[test]
22337 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22338 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22339 let m = pr_message(&state, 'A');
22340 assert_eq!(m.title, "add retries");
22341 assert!(
22342 m.body.contains("## Summary\n\n- did some things"),
22343 "{}",
22344 m.body
22345 );
22346
22347 let none = RunState::new(
22348 PathBuf::from("/repo"),
22349 "main".to_owned(),
22350 "abc1234".to_owned(),
22351 "add retries".to_owned(),
22352 Config::default(),
22353 );
22354 let m = pr_message(&none, 'A');
22355 assert_eq!(m.title, "add retries");
22356 assert!(!m.body.contains("## Summary"), "{}", m.body);
22357 }
22358
22359 #[test]
22360 fn pr_message_refuses_the_candidate_commit_subject() {
22361 for bad in [
22362 "TITLE: magi: candidate A (uncommitted work)",
22363 "TITLE: chore: stuff (uncommitted work)",
22364 "TITLE: ",
22365 ] {
22366 let state = state_with_summary("add retries", bad);
22367 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22368 }
22369 }
22370
22371 #[test]
22372 fn pr_message_bounds_a_very_long_task_and_title() {
22373 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22374 let state = state_with_summary(&long, "- nothing");
22375 let m = pr_message(&state, 'A');
22376 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22377 assert!(!m.title.contains('\n'));
22378
22379 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22380 let m = pr_message(&state, 'A');
22381 assert!(m.title.starts_with("feat: "));
22382 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22383 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22384 }
22385
22386 fn long_title_of(instruction: &str) -> String {
22387 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22388 }
22389
22390 #[test]
22391 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22392 let first = "Make the landing path keep a readable title for long tasks";
22393 let line = format!(
22394 "{first}. {}",
22395 "And then keep going with more words ".repeat(20)
22396 );
22397 let t = long_title_of(&line);
22398 assert_eq!(t, first);
22399 assert!(!t.starts_with("chore: land"));
22400 }
22401
22402 #[test]
22403 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22404 let line = "word ".repeat(200);
22405 let t = long_title_of(&line);
22406 assert!(t.ends_with("word..."), "{t}");
22407 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22408 }
22409
22410 #[test]
22411 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22412 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22413 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22414 }
22415 }
22416
22417 #[test]
22418 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22419 let mut state = state_with_summary("再試行を追加する", "- nothing");
22420 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22421 state.config.graph.github_language = "ja".into();
22422 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22423 }
22424
22425 #[test]
22426 fn pr_message_title_limit_is_exact() {
22427 let at = "a".repeat(PR_TITLE_MAX);
22428 assert_eq!(long_title_of(&at), at);
22429 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22430 assert!(over.ends_with("..."), "{over}");
22431 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22432 }
22433
22434 #[test]
22435 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22436 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22437 let t = long_title_of(&line);
22438 assert!(t.ends_with("..."), "{t}");
22439 assert!(t.is_ascii(), "{t}");
22440 }
22441
22442 #[test]
22443 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22444 let line = format!(
22445 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22446 "plus more filler words ".repeat(20)
22447 );
22448 let t = long_title_of(&line);
22449 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22450 }
22451
22452 #[test]
22453 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22454 let line = format!("\n\n# {}", "title words ".repeat(40));
22455 let state = state_with_summary(&line, "- nothing");
22456 let m = pr_message(&state, 'A');
22457 assert!(m.title.starts_with("title words"), "{}", m.title);
22458 assert_eq!(
22459 land::merge_subject(&m.title, &landing_subject_source(&state)),
22460 m.title
22461 );
22462 assert_eq!(
22464 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22465 "feat: renamed by hand"
22466 );
22467 }
22468
22469 #[tokio::test]
22470 async fn github_text_rewrite_is_bounded_and_falls_back() {
22471 crate::run::pin_test_home();
22472 for (reply, accepted) in [
22473 (
22474 "TITLE: fix: retries\nAdd retries for failed requests.",
22475 true,
22476 ),
22477 (
22478 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22479 false,
22480 ),
22481 (
22482 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22483 true,
22484 ),
22485 (
22486 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22487 true,
22488 ),
22489 (
22490 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22491 true,
22492 ),
22493 (
22494 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22495 false,
22496 ),
22497 ("TITLE: fix: retries\n", false),
22498 ] {
22499 let dir = tempfile::tempdir().unwrap();
22500 let mut runner = runner_at(RunStatus::Gating);
22501 runner.state = state_with_summary(
22502 "add retries",
22503 "TITLE: fix: retries\n日本語の説明を書きます。",
22504 );
22505 runner.state.repo = dir.path().to_owned();
22506 runner.state.candidates[0].worktree = dir.path().to_owned();
22507 let mut author = spec("alpha");
22508 author.command = vec![
22509 "sh".into(),
22510 "-c".into(),
22511 "printf '%s' \"$REWRITE_REPLY\"".into(),
22512 ];
22513 author.env.insert("REWRITE_REPLY".into(), reply.into());
22514 runner.state.config.agents = vec![author];
22515 seed_public_summary(&mut runner);
22516 let winner = runner.state.candidates[0].clone();
22517 let first = runner
22518 .guarded_pr_message(&winner, None, true)
22519 .await
22520 .unwrap();
22521 let message = if accepted {
22522 first.expect("a passing rewrite never asks")
22523 } else {
22524 assert!(first.is_none());
22526 assert!(runner.state.parked);
22527 let store = ask::Questions::open();
22528 let asked: Vec<_> = store
22529 .list()
22530 .into_iter()
22531 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22532 .collect();
22533 assert_eq!(asked.len(), 1);
22534 assert!(!asked[0].detail.contains("secret"));
22535 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22536 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22537 assert_eq!(
22538 asked[0]
22539 .choices
22540 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22541 rewrite.is_some()
22542 );
22543 store
22546 .update(&asked[0].id, |q| {
22547 q.abandon("test");
22548 Ok(())
22549 })
22550 .unwrap();
22551 let again = runner
22552 .guarded_pr_message(&winner, None, true)
22553 .await
22554 .unwrap()
22555 .expect("an abandoned question falls back");
22556 let n = store
22557 .list()
22558 .into_iter()
22559 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22560 .count();
22561 assert_eq!(n, 1, "asked once per rejected text");
22562 again
22563 };
22564 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22565 assert!(!message.body.contains("secret"));
22566 assert!(!message.body.contains("prefix"));
22567 assert!(!message.title.contains("secret"));
22568 assert_eq!(
22569 message.body.contains("Add retries for failed requests."),
22570 accepted
22571 );
22572 assert_eq!(
22573 runner.state.seats["github-text-rewrite"].turns, 1,
22574 "only one rewrite invocation"
22575 );
22576 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22577 if accepted {
22578 assert!(runner.state.github_text.is_none());
22579 assert!(!runner.state.parked);
22580 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22581 assert!(message.body.contains("## Final verification"));
22582 assert!(message.body.contains("Evidence: final diff"));
22583 assert!(
22584 message
22585 .body
22586 .contains(&format!("magi:run/{}", runner.state.id))
22587 );
22588 }
22589 if !accepted {
22590 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22591 assert!(
22592 message
22593 .body
22594 .contains(&format!("magi:run/{}", runner.state.id))
22595 );
22596 }
22597 }
22598 }
22599
22600 fn scripted(id: &str, reply: &str) -> AgentSpec {
22601 let mut a = spec(id);
22602 a.command = vec![
22603 "sh".into(),
22604 "-c".into(),
22605 "printf '%s' \"$REWRITE_REPLY\"".into(),
22606 ];
22607 a.env.insert("REWRITE_REPLY".into(), reply.into());
22608 a
22609 }
22610
22611 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22612 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22613
22614 fn seed_public_summary(runner: &mut Runner) {
22616 let repo = &runner.state.repo;
22617 let g = |args: &[&str]| {
22618 let out = std::process::Command::new("git")
22619 .args(args)
22620 .current_dir(repo)
22621 .quiet()
22622 .output()
22623 .unwrap();
22624 assert!(out.status.success(), "git {args:?}");
22625 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22626 };
22627 g(&["init", "-q", "-b", "main"]);
22628 g(&[
22629 "-c",
22630 "user.name=Test",
22631 "-c",
22632 "user.email=test@example.invalid",
22633 "commit",
22634 "-q",
22635 "--allow-empty",
22636 "-m",
22637 "feat: fixture",
22638 ]);
22639 let head = g(&["rev-parse", "HEAD"]);
22640 g(&["update-ref", "refs/remotes/origin/main", &head]);
22641 let evidence = crate::pr_outcome::summary_evidence(
22642 &runner.state,
22643 crate::pr_outcome::FinalHead::At(&head),
22644 );
22645 let generated = &runner.state.candidates[0].summary;
22646 runner.state.change_summary = Some(crate::run::ChangeSummary {
22647 base: head.clone(),
22648 head,
22649 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22650 title: summary_title(generated).unwrap(),
22651 body: summary_without_title(generated),
22652 });
22653 }
22654
22655 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22656 let mut runner = runner_at(RunStatus::Gating);
22657 runner.state = state_with_summary("add retries", JA_SUMMARY);
22658 if review_only {
22659 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22660 }
22661 runner.state.repo = dir.to_owned();
22662 runner.state.candidates[0].worktree = dir.to_owned();
22663 runner.state.config.agents = agents;
22664 seed_public_summary(&mut runner);
22665 runner
22666 }
22667
22668 fn gate_questions(runner: &Runner) -> usize {
22669 ask::Questions::open()
22670 .list()
22671 .into_iter()
22672 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22673 .count()
22674 }
22675
22676 #[tokio::test]
22677 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22678 crate::run::pin_test_home();
22679 let dir = tempfile::tempdir().unwrap();
22680 let mut runner = rewrite_runner(
22681 dir.path(),
22682 true,
22683 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22684 );
22685 runner.state.config.roles.language_judge =
22686 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22687 let winner = runner.state.candidates[0].clone();
22688 let msg = runner
22689 .guarded_pr_message(&winner, None, true)
22690 .await
22691 .unwrap()
22692 .expect("an English rewrite posts without asking");
22693 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22694 assert!(msg.body.contains("<details>kept</details>"));
22695 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22696 assert_eq!(gate_questions(&runner), 0);
22697 assert!(runner.state.github_text.is_none());
22698 assert!(!runner.state.seats.contains_key("impl-A"));
22699 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22700 }
22701
22702 #[tokio::test]
22703 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22704 crate::run::pin_test_home();
22705 let dir = tempfile::tempdir().unwrap();
22706 let mut runner = rewrite_runner(
22707 dir.path(),
22708 false,
22709 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22710 );
22711 runner.state.config.roles.language_judge =
22712 Some(crate::config::AgentChoice::One("beta".into()));
22713 runner.state.seats.insert(
22714 "impl-A".into(),
22715 SeatState::new("impl-A", "alpha", runner.state.seed),
22716 );
22717 let winner = runner.state.candidates[0].clone();
22718 let msg = runner
22719 .guarded_pr_message(&winner, None, true)
22720 .await
22721 .unwrap()
22722 .expect("rewritten");
22723 assert!(msg.body.contains("<details>kept</details>"));
22724 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22725 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22726 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22727 }
22728
22729 #[tokio::test]
22730 async fn rewrite_reuses_a_resumable_implementer_session() {
22731 crate::run::pin_test_home();
22732 let dir = tempfile::tempdir().unwrap();
22733 let mut runner = rewrite_runner(
22734 dir.path(),
22735 false,
22736 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22737 );
22738 runner.state.config.roles.language_judge =
22739 Some(crate::config::AgentChoice::One("beta".into()));
22740 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22741 seat.turns = 1;
22742 runner.state.seats.insert("impl-A".into(), seat);
22743 let winner = runner.state.candidates[0].clone();
22744 runner
22745 .guarded_pr_message(&winner, None, true)
22746 .await
22747 .unwrap()
22748 .expect("rewritten");
22749 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22750 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22751 }
22752
22753 #[tokio::test]
22754 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22755 crate::run::pin_test_home();
22758 let dir = tempfile::tempdir().unwrap();
22759 let mut exhausted = spec("alpha");
22760 exhausted.command = vec!["false".into()];
22761 let mut runner = rewrite_runner(
22762 dir.path(),
22763 false,
22764 vec![exhausted, scripted("beta", EN_REWRITE)],
22765 );
22766 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22768 "alpha".into(),
22769 "beta".into(),
22770 ]));
22771 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22772 seat.turns = 1;
22773 runner.state.seats.insert("impl-A".into(), seat);
22774 let winner = runner.state.candidates[0].clone();
22775 let msg = runner
22776 .guarded_pr_message(&winner, None, true)
22777 .await
22778 .unwrap()
22779 .expect("the role rewrites after the implementer fails");
22780 assert!(msg.body.contains("`src/net.rs`"));
22781 assert_eq!(
22782 runner.state.seats["impl-A"].turns, 2,
22783 "the implementer was tried once"
22784 );
22785 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22786 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22787 }
22788
22789 #[tokio::test]
22790 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22791 crate::run::pin_test_home();
22792 let dir = tempfile::tempdir().unwrap();
22793 let mut broken = spec("bad");
22794 broken.command = vec!["false".into()];
22795 let mut runner = rewrite_runner(
22796 dir.path(),
22797 true,
22798 vec![
22799 broken,
22800 scripted("noformat", "Add retries."),
22801 scripted("good", EN_REWRITE),
22802 ],
22803 );
22804 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22805 "bad".into(),
22806 "noformat".into(),
22807 "good".into(),
22808 ]));
22809 let winner = runner.state.candidates[0].clone();
22810 let msg = runner
22811 .guarded_pr_message(&winner, None, true)
22812 .await
22813 .unwrap()
22814 .expect("the third agent rewrites");
22815 assert!(msg.body.contains("`src/net.rs`"));
22816 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22817 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22818 }
22819
22820 #[tokio::test]
22821 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22822 crate::run::pin_test_home();
22823 for (agents, role, saved) in [
22825 (
22826 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22827 Some("beta"),
22828 true,
22829 ),
22830 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22831 (Vec::new(), None, false),
22832 ] {
22833 let dir = tempfile::tempdir().unwrap();
22834 let mut runner = rewrite_runner(dir.path(), true, agents);
22835 runner.state.config.roles.language_judge =
22836 role.map(|r| crate::config::AgentChoice::One(r.into()));
22837 let winner = runner.state.candidates[0].clone();
22838 let first = runner
22839 .guarded_pr_message(&winner, None, true)
22840 .await
22841 .unwrap();
22842 assert!(first.is_none(), "withheld text parks the run");
22843 assert!(runner.state.parked);
22844 assert_eq!(gate_questions(&runner), 1);
22845 let g = runner.state.github_text.as_ref().unwrap();
22846 assert_eq!(g.rewrite.is_some(), saved);
22847 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22848 }
22849 }
22850
22851 #[tokio::test]
22852 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22853 crate::run::pin_test_home();
22854 let dir = tempfile::tempdir().unwrap();
22855 let mut runner = runner_at(RunStatus::Gating);
22856 runner.state =
22857 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22858 runner.state.repo = dir.path().to_owned();
22859 runner.state.candidates[0].worktree = dir.path().to_owned();
22860 let mut author = spec("alpha");
22861 author.command = vec![
22862 "sh".into(),
22863 "-c".into(),
22864 "printf '%s' \"$REWRITE_REPLY\"".into(),
22865 ];
22866 author.env.insert(
22867 "REWRITE_REPLY".into(),
22868 format!(
22869 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22870 "English sentence. ".repeat(600)
22871 ),
22872 );
22873 runner.state.config.agents = vec![author];
22874 seed_public_summary(&mut runner);
22875 let winner = runner.state.candidates[0].clone();
22876 assert!(
22877 runner
22878 .guarded_pr_message(&winner, None, true)
22879 .await
22880 .unwrap()
22881 .is_none()
22882 );
22883 let record = runner.state.github_text.clone().unwrap();
22884 let rewrite = record.rewrite.clone().unwrap();
22885 assert_eq!(rewrite.failed, ["body-language"]);
22886 assert!(
22887 rewrite
22888 .body
22889 .ends_with(&format!("magi:run/{}", runner.state.id))
22890 );
22891 let store = ask::Questions::open();
22892 let q = store.get(record.question.as_ref().unwrap()).unwrap();
22893 assert!(
22894 q.choices
22895 .contains(&crate::github_text::USE_REWRITE.to_owned())
22896 );
22897 for label in [
22898 "ORIGINAL",
22899 "FALLBACK",
22900 "REWRITE",
22901 "Rules still failed: body-language",
22902 "truncated",
22903 ] {
22904 assert!(q.detail.contains(label), "{label}");
22905 }
22906 let artifact = std::fs::read_to_string(crate::run::artifact_path(
22907 &runner.state,
22908 &crate::github_text::artifact_name(&record.fingerprint),
22909 ))
22910 .unwrap();
22911 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22912 runner.state =
22914 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22915 store
22916 .update(&q.id, |q| {
22917 q.say("まだ日本語")?;
22918 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22919 })
22920 .unwrap();
22921 assert!(
22922 runner
22923 .guarded_pr_message(&winner, None, true)
22924 .await
22925 .unwrap()
22926 .is_none()
22927 );
22928 let retry = runner.state.github_text.as_ref().unwrap();
22929 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22930 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22931 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22932 assert!(
22933 q.posting_comparison
22934 .as_ref()
22935 .unwrap()
22936 .matches_detail(&q.detail)
22937 );
22938 store
22939 .update(&q.id, |q| {
22940 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22941 })
22942 .unwrap();
22943 let message = runner
22944 .guarded_pr_message(&winner, None, true)
22945 .await
22946 .unwrap()
22947 .unwrap();
22948 assert_eq!(message.title, rewrite.title);
22949 assert_eq!(message.body, rewrite.body);
22950 assert_eq!(
22951 runner.state.github_text.as_ref().unwrap().choice,
22952 Some(GithubTextChoice::Rewrite)
22953 );
22954 let again = runner
22955 .guarded_pr_message(&winner, None, true)
22956 .await
22957 .unwrap()
22958 .unwrap();
22959 assert_eq!(again.title, message.title);
22960 assert_eq!(again.body, message.body);
22961 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22962 assert!(
22963 !runner
22964 .state
22965 .events
22966 .iter()
22967 .any(|e| e.message.contains(&rewrite.title)
22968 || e.message.contains("English sentence."))
22969 );
22970 }
22971
22972 #[tokio::test]
22973 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22974 crate::run::pin_test_home();
22975 let dir = tempfile::tempdir().unwrap();
22976 let mut runner = runner_at(RunStatus::Gating);
22977 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22978 runner.state.repo = dir.path().to_owned();
22979 runner.state.candidates[0].worktree = dir.path().to_owned();
22980 seed_public_summary(&mut runner);
22981 let winner = runner.state.candidates[0].clone();
22982 assert!(
22983 runner
22984 .guarded_pr_message(&winner, None, true)
22985 .await
22986 .unwrap()
22987 .is_none()
22988 );
22989 let record = runner.state.github_text.as_ref().unwrap();
22990 assert!(record.rewrite.is_none());
22991 let mut old = serde_json::to_value(record).unwrap();
22993 old.as_object_mut().unwrap().remove("rewrite");
22994 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
22995 assert!(old.rewrite.is_none());
22996 let id = record.question.clone().unwrap();
22997 ask::Questions::open()
22998 .update(&id, |q| {
22999 q.choices.push(crate::github_text::USE_REWRITE.into());
23000 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23001 })
23002 .unwrap();
23003 let message = runner
23004 .guarded_pr_message(&winner, None, true)
23005 .await
23006 .unwrap()
23007 .unwrap();
23008 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23009 assert_eq!(
23010 runner.state.github_text.as_ref().unwrap().choice,
23011 Some(GithubTextChoice::Fallback)
23012 );
23013 }
23014
23015 #[tokio::test]
23016 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23017 crate::run::pin_test_home();
23018 let dir = tempfile::tempdir().unwrap();
23019 let mut runner = runner_at(RunStatus::Gating);
23020 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23021 runner.state.repo = dir.path().to_owned();
23022 runner.state.candidates[0].worktree = dir.path().to_owned();
23023 seed_public_summary(&mut runner);
23024 let winner = runner.state.candidates[0].clone();
23025 assert!(
23026 runner
23027 .guarded_pr_message(&winner, None, true)
23028 .await
23029 .unwrap()
23030 .is_none()
23031 );
23032 let store = ask::Questions::open();
23033 let find = |run: &str| {
23034 store
23035 .list()
23036 .into_iter()
23037 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23038 .unwrap()
23039 };
23040 let run = runner.state.id.clone();
23041 let q = find(&run);
23042 let fp = runner
23043 .state
23044 .github_text
23045 .as_ref()
23046 .unwrap()
23047 .fingerprint
23048 .clone();
23049 let name = crate::github_text::artifact_name(&fp);
23050 let path = crate::run::artifact_path(&runner.state, &name);
23051 assert!(q.detail.contains(&path.display().to_string()));
23052 assert!(q.detail.contains("title-language: title"));
23053 let saved = std::fs::read_to_string(&path).expect("artifact written");
23054 assert!(saved.contains("日本語のタイトル"));
23055 assert!(q.cwd.is_none());
23056 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23057 let say = |id: &str, text: &str, choice: &str| {
23059 store
23060 .update(id, |q| {
23061 q.thread.push(ask::Turn {
23062 who: ask::Who::Operator,
23063 body: text.to_owned(),
23064 at: jiff::Timestamp::now(),
23065 note: None,
23066 });
23067 q.answer(ask::Answer::Choice(choice.to_owned()))
23068 })
23069 .unwrap();
23070 };
23071 say(&q.id, "まだ日本語", "use my text");
23072 assert!(
23073 runner
23074 .guarded_pr_message(&winner, None, true)
23075 .await
23076 .unwrap()
23077 .is_none()
23078 );
23079 let second = find(&run);
23080 assert_ne!(second.id, q.id);
23081 assert!(second.detail.contains("did not pass"));
23082 assert!(second.detail.contains("日本語のタイトル"));
23085 assert!(!second.detail.contains("まだ日本語"));
23086 assert!(second.detail.contains(&path.display().to_string()));
23087 say(&second.id, "fix: retry failed requests", "use my text");
23088 let message = runner
23089 .guarded_pr_message(&winner, None, true)
23090 .await
23091 .unwrap()
23092 .expect("a vetted replacement is posted");
23093 assert_eq!(message.title, "fix: retry failed requests");
23094 assert_eq!(
23095 runner
23096 .state
23097 .github_text
23098 .as_ref()
23099 .unwrap()
23100 .chosen_title
23101 .as_deref(),
23102 Some("fix: retry failed requests")
23103 );
23104 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23105 let again = runner
23107 .guarded_pr_message(&winner, None, true)
23108 .await
23109 .unwrap()
23110 .unwrap();
23111 assert_eq!(again.title, message.title);
23112 assert_eq!(
23113 store
23114 .list()
23115 .into_iter()
23116 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23117 .count(),
23118 2
23119 );
23120 }
23121
23122 #[test]
23123 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23124 let mut state = state_with_summary(
23128 "add retries",
23129 "TITLE: fix(web): batch reads\n- reads run.json once",
23130 );
23131 state.config.graph.language = "ja".to_owned();
23132 let m = pr_message(&state, 'A');
23133 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23134 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23135
23136 let task = "今回やってほしいこと: results projector を直す";
23139 let mut state = state_with_summary(task, "- no title line");
23140 state.config.graph.language = "ja".to_owned();
23141 let m = pr_message(&state, 'A');
23142 assert_eq!(
23143 m.title,
23144 format!("chore: land candidate A of run {}", state.id)
23145 );
23146 assert!(
23147 m.body.contains(&format!(
23148 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23149 )),
23150 "{}",
23151 m.body
23152 );
23153 }
23154
23155 #[test]
23156 fn pr_message_scrubs_home_paths_and_addresses() {
23157 let state = state_with_summary(
23158 "fix it in /Users/someone/src/x",
23159 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23160 );
23161 let m = pr_message(&state, 'A');
23162 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23163 assert!(!m.body.contains(leak), "{}", m.body);
23164 }
23165 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23166 }
23167
23168 #[test]
23169 fn pr_message_survives_a_task_that_closes_details() {
23170 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23171 let m = pr_message(&state, 'A');
23172 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23173 }
23174
23175 #[test]
23176 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23177 let cmd = manual_merge_command(
23178 MergeStyle::Squash,
23179 Path::new("/repo"),
23180 "b",
23181 "fix: \"quoted\" $(x) `y`\n\nbody",
23182 );
23183 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23184 }
23185
23186 #[test]
23187 fn manual_merge_command_matches_the_configured_style() {
23188 let repo = Path::new("/repo");
23189 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23190
23191 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23192 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23193
23194 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23195 assert_eq!(
23196 squash,
23197 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23198 \"Merge magi run 0832 (candidate A)\""
23199 );
23200
23201 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23202 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23203 }
23204
23205 #[test]
23206 fn a_nudge_gets_a_quarter_of_the_budget() {
23207 assert_eq!(retry_budget(secs(1200), true), secs(300));
23209 assert_eq!(retry_budget(secs(3600), true), secs(900));
23210 }
23211
23212 #[test]
23213 fn a_resent_prompt_keeps_the_whole_budget() {
23214 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23217 assert_eq!(retry_budget(secs(60), false), secs(60));
23218 }
23219
23220 #[test]
23221 fn the_floor_never_exceeds_the_original_budget() {
23222 assert_eq!(retry_budget(secs(60), true), secs(60));
23226 assert_eq!(retry_budget(secs(480), true), secs(120));
23227 assert_eq!(retry_budget(secs(0), true), secs(0));
23228 }
23229
23230 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23231 agent::CommandEvidence {
23232 id: "item1".to_owned(),
23233 description: "cargo test".to_owned(),
23234 exit_code,
23235 result_summary: String::new(),
23236 source: "codex".to_owned(),
23237 }
23238 }
23239
23240 #[test]
23241 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23242 assert!(!has_unconfirmed_command(&[]));
23246 }
23247
23248 #[test]
23249 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23250 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23254 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23255 assert!(!has_unconfirmed_command(&[
23256 evidence(Some(0)),
23257 evidence(Some(101))
23258 ]));
23259 }
23260
23261 #[test]
23262 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23263 assert!(has_unconfirmed_command(&[
23264 evidence(Some(0)),
23265 evidence(None)
23266 ]));
23267 }
23268
23269 #[test]
23270 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23271 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23272 assert_eq!(
23273 verified_noop_claim(true, &[], text).as_deref(),
23274 Some("already fixed by b32cfc4, on main.")
23275 );
23276 }
23277
23278 #[test]
23279 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23280 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23283 assert!(verified_noop_claim(false, &[], text).is_none());
23284 }
23285
23286 #[test]
23287 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23288 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23289 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23290 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23292 }
23293
23294 #[test]
23295 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23296 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23297 }
23298
23299 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23302 runner.state.candidates = shape
23303 .iter()
23304 .enumerate()
23305 .map(|(i, &(empty, verified))| Candidate {
23306 index: i,
23307 label: (b'A' + i as u8) as char,
23308 agent: "sonnet".to_owned(),
23309 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23310 worktree: PathBuf::from(format!("/wt/{i}")),
23311 summary: String::new(),
23312 stat: String::new(),
23313 files: 0,
23314 commits: 0,
23315 empty,
23316 failed: None,
23317 verified_noop: verified.map(str::to_owned),
23318 duration_ms: 0,
23319 folded: false,
23320 })
23321 .collect();
23322 }
23323
23324 #[test]
23325 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23326 ask_test_home();
23327 let mut runner = runner_at(RunStatus::Implementing);
23328 set_candidates(
23329 &mut runner,
23330 &[
23331 (true, Some("already on main at b32cfc4")),
23332 (true, Some("same fix, see the existing test")),
23333 ],
23334 );
23335
23336 runner
23337 .after_implement()
23338 .expect("a verified no-op is not an error");
23339
23340 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23341 }
23342
23343 #[test]
23344 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23345 ask_test_home();
23346 let mut runner = runner_at(RunStatus::Implementing);
23347 set_candidates(
23351 &mut runner,
23352 &[(true, Some("already on main at b32cfc4")), (true, None)],
23353 );
23354
23355 let err = runner
23356 .after_implement()
23357 .expect_err("an unverified empty candidate must still fail the run");
23358
23359 assert!(
23360 err.to_string().contains("no candidate produced a change"),
23361 "{err}"
23362 );
23363 assert_eq!(runner.state.status, RunStatus::Failed);
23364 }
23365
23366 #[test]
23367 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23368 ask_test_home();
23369 let mut runner = runner_at(RunStatus::Implementing);
23370 set_candidates(&mut runner, &[(true, None), (true, None)]);
23371
23372 let err = runner
23373 .after_implement()
23374 .expect_err("no candidate declared anything; this is an ordinary failure");
23375
23376 assert!(
23377 err.to_string().contains("no candidate produced a change"),
23378 "{err}"
23379 );
23380 assert_eq!(runner.state.status, RunStatus::Failed);
23381 }
23382}
23383
23384#[derive(Debug)]
23387struct GraftRestoreFailed(String);
23388
23389impl std::fmt::Display for GraftRestoreFailed {
23390 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23391 write!(
23392 f,
23393 "could not restore the winner to its pre-graft commit: {}",
23394 self.0
23395 )
23396 }
23397}
23398
23399impl std::error::Error for GraftRestoreFailed {}
23400
23401#[cfg(test)]
23405mod resource_wait_tests {
23406 use super::*;
23407 use std::collections::BTreeMap;
23408
23409 fn sh(id: &str, script: &str) -> AgentSpec {
23410 AgentSpec {
23411 id: id.to_owned(),
23412 kind: crate::config::AgentKind::Command,
23413 model: None,
23414 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23415 extra_args: Vec::new(),
23416 env: BTreeMap::new(),
23417 prompt_delivery: None,
23418 max_turn_secs: None,
23419 }
23420 }
23421
23422 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23423 SeatJob {
23424 seat: SeatState::new("impl-A", &spec.id, 1),
23425 spec,
23426 cwd: dir.to_path_buf(),
23427 prompt: "do it".to_owned(),
23428 timeout,
23429 allow_write: true,
23430 sessions: false,
23431 artifacts: dir.join("art"),
23432 stem: "impl-A".to_owned(),
23433 handover: None,
23434 }
23435 }
23436
23437 fn state(resource_wait_secs: u64) -> RunState {
23438 let mut config = Config::default();
23439 config.graph.timeout_resource_wait = resource_wait_secs;
23440 RunState::new(
23441 PathBuf::from("/repo"),
23442 "main".to_owned(),
23443 "abc1234".to_owned(),
23444 "task".to_owned(),
23445 config,
23446 )
23447 }
23448
23449 fn hold(cache: &Path) -> crate::cache::Guard {
23450 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23451 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23452 crate::cache::AcquireOutcome::Acquired(g) => g,
23453 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23454 }
23455 }
23456
23457 async fn run(
23458 jobs: Vec<SeatJob>,
23459 sem: Arc<Semaphore>,
23460 cache: Option<&Path>,
23461 st: &mut RunState,
23462 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23463 let prompts = Prompts::default();
23464 let ctx = WaveCtx {
23465 run: "r",
23466 node: "implement",
23467 prompts: &prompts,
23468 cache,
23469 round: None,
23470 carry_seats: false,
23471 pauses: None,
23472 };
23473 wave(jobs, sem, &ctx, st, 0).await
23474 }
23475
23476 #[tokio::test]
23477 async fn verification_cache_wait_preserves_the_command_budget() {
23478 crate::run::pin_test_home();
23479 let tmp = tempfile::tempdir().unwrap();
23480 let cache = tmp.path().join("target");
23481 let guard = hold(&cache);
23482 let release = tokio::spawn(async move {
23483 tokio::time::sleep(Duration::from_millis(100)).await;
23484 guard.release();
23485 });
23486 let mut st = state(10);
23487 let budget = Duration::from_millis(50);
23488 let wait = repository_wait(&st, None);
23489 let (outcomes, _) = with_cache_lease(
23490 &mut st,
23491 Some(&cache),
23492 "gate",
23493 "gate",
23494 tmp.path(),
23495 "abc1234",
23496 budget,
23497 "test gate",
23498 &wait,
23499 |_, available| async move {
23500 assert_eq!(
23501 available, budget,
23502 "contention must not shorten verification"
23503 );
23504 (Vec::new(), false, Vec::new())
23505 },
23506 )
23507 .await
23508 .unwrap();
23509 release.await.unwrap();
23510 assert!(outcomes.is_empty(), "{outcomes:?}");
23511 assert!(
23512 st.events
23513 .iter()
23514 .any(|event| event.message.contains("waiting for"))
23515 );
23516 assert!(st.resource_wait.is_none());
23517 hold(&cache).release();
23519 }
23520
23521 #[tokio::test]
23522 async fn verification_cache_wait_expires_without_starting_commands() {
23523 crate::run::pin_test_home();
23524 let tmp = tempfile::tempdir().unwrap();
23525 let cache = tmp.path().join("target");
23526 let _held = hold(&cache);
23527 let mut st = state(0);
23528 let wait = repository_wait(&st, None);
23529 let (outcomes, _) = with_cache_lease(
23530 &mut st,
23531 Some(&cache),
23532 "gate",
23533 "gate",
23534 tmp.path(),
23535 "abc1234",
23536 Duration::from_secs(60),
23537 "test gate",
23538 &wait,
23539 |_, _| async { panic!("commands must not start without the cache lease") },
23540 )
23541 .await
23542 .unwrap();
23543 assert_eq!(outcomes.len(), 1);
23544 assert!(outcomes[0].resource_blocked);
23545 assert!(st.resource_wait.is_none());
23546 assert!(matches!(
23547 st.resource_block.unwrap().cause,
23548 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23549 ));
23550 }
23551
23552 #[tokio::test]
23553 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23554 crate::run::pin_test_home();
23555 let tmp = tempfile::tempdir().unwrap();
23556 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23557 let cache = tmp.path().join("cache");
23558 let mut state = state(21600);
23559 let wait = repository_wait(&state, None);
23560 let (outcomes, _) = with_cache_lease(
23561 &mut state,
23562 Some(&cache),
23563 "gate",
23564 "gate",
23565 tmp.path(),
23566 "head",
23567 Duration::from_secs(60),
23568 "gate",
23569 &wait,
23570 |_, _| async { panic!("freshness failure must not start verification") },
23571 )
23572 .await
23573 .unwrap();
23574 assert_eq!(outcomes.len(), 1);
23575 assert!(outcomes[0].resource_blocked);
23576 let block = state.resource_block.unwrap();
23577 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23578 assert!(!block.description().contains("timed out"));
23579 }
23580
23581 #[tokio::test]
23582 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23583 crate::run::pin_test_home();
23584 let tmp = tempfile::tempdir().unwrap();
23585 let cache = tmp.path().join("cache");
23586 let _held = hold(&cache);
23587 let mut state = state(21600);
23588 let pause = Pause::new();
23589 let interrupt = Pause::new();
23590 let cause = ParkCause::Operator;
23591 pause.park_for(cause.clone());
23592 let prompts = Prompts::default();
23593 let ctx = WaveCtx {
23594 run: "test",
23595 node: "fix",
23596 prompts: &prompts,
23597 cache: Some(&cache),
23598 round: None,
23599 carry_seats: false,
23600 pauses: Some((&pause, &interrupt)),
23601 };
23602 let result = wave(
23603 vec![job(
23604 sh("writer", "exit 99"),
23605 tmp.path(),
23606 Duration::from_secs(1),
23607 )],
23608 Arc::new(Semaphore::new(1)),
23609 &ctx,
23610 &mut state,
23611 0,
23612 )
23613 .await;
23614 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23615 assert_eq!(result[0].1.turns, 0);
23616 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23617 assert_eq!(
23618 state.resource_block.unwrap().cause,
23619 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23620 );
23621 assert!(state.active.is_empty());
23622 }
23623
23624 #[tokio::test]
23625 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23626 crate::run::pin_test_home();
23627 let tmp = tempfile::tempdir().unwrap();
23628 let cache = tmp.path().join("target-a");
23629 let guard = hold(&cache);
23630 tokio::spawn(async move {
23632 tokio::time::sleep(Duration::from_millis(1500)).await;
23633 guard.release();
23634 });
23635 let mut st = state(60);
23636 let results = run(
23637 vec![job(
23638 sh("alpha", "echo ok"),
23639 tmp.path(),
23640 Duration::from_secs(1),
23641 )],
23642 Arc::new(Semaphore::new(1)),
23643 Some(&cache),
23644 &mut st,
23645 )
23646 .await;
23647 assert!(
23648 matches!(results[0].2, AgentOutcome::Ok(_)),
23649 "the wait must not be charged to the agent"
23650 );
23651 assert!(st.resource_block.is_none());
23652 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23653 assert!(st.resource_wait.is_none() && st.active.is_empty());
23654 }
23655
23656 #[tokio::test]
23657 #[cfg(unix)]
23658 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23659 use std::os::fd::AsRawFd as _;
23660 crate::run::pin_test_home();
23661 let tmp = tempfile::tempdir().unwrap();
23662 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23663 let common = tmp.path().join(".git");
23664 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23665 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23666 assert_eq!(
23667 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23668 0
23669 );
23670 let mut st = state(1);
23671 st.config.coordination = crate::config::test_coordination();
23672 let mut reader = job(
23673 sh("reader", "echo must-not-run"),
23674 tmp.path(),
23675 Duration::from_secs(30),
23676 );
23677 reader.allow_write = false;
23678 let results = run(
23679 vec![reader.clone()],
23680 Arc::new(Semaphore::new(1)),
23681 None,
23682 &mut st,
23683 )
23684 .await;
23685 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23686 assert_eq!(results[0].1.turns, 0);
23687 assert!(FailClass::of(&results[0].2).is_none());
23688 assert_eq!(
23689 st.resource_block.as_ref().unwrap().kind,
23690 "repository coordination"
23691 );
23692 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23693 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23694 drop(external);
23695 let editor = crate::repo_lock::SeatLease::for_checkout(
23696 tmp.path(),
23697 true,
23698 crate::config::test_coordination().as_ref(),
23699 )
23700 .await
23701 .unwrap();
23702 let pause = Pause::new();
23703 let interrupt = Pause::new();
23704 let prompts = Prompts::default();
23705 let ctx = WaveCtx {
23706 run: "r",
23707 node: "review",
23708 prompts: &prompts,
23709 cache: None,
23710 round: None,
23711 carry_seats: false,
23712 pauses: Some((&pause, &interrupt)),
23713 };
23714 let (results, ()) = tokio::join!(
23715 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23716 async {
23717 tokio::time::sleep(Duration::from_millis(50)).await;
23718 pause.park();
23719 }
23720 );
23721 assert!(
23722 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23723 );
23724 assert_eq!(results[0].1.turns, 0);
23725 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23726 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23727 drop(editor);
23728 }
23729
23730 #[tokio::test]
23731 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23732 crate::run::pin_test_home();
23733 let tmp = tempfile::tempdir().unwrap();
23734 let cache = tmp.path().join("target-b");
23735 let _held = hold(&cache);
23736 let mut st = state(1);
23737 let results = run(
23738 vec![job(
23739 sh("alpha", "echo ok"),
23740 tmp.path(),
23741 Duration::from_secs(1),
23742 )],
23743 Arc::new(Semaphore::new(1)),
23744 Some(&cache),
23745 &mut st,
23746 )
23747 .await;
23748 let (_, seat, out) = &results[0];
23749 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23750 assert!(wave_blocked(&results));
23751 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23752 assert_eq!(seat.turns, 0, "nothing was asked");
23753 assert!(
23754 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23755 "no agent was spawned"
23756 );
23757 assert!(FailClass::of(out).is_none(), "not a failure class");
23758 let block = st.resource_block.as_ref().expect("recorded for the resume");
23759 assert_eq!(block.kind, "shared build cache");
23760 assert!(matches!(
23761 block.cause,
23762 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23763 ));
23764 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23765 assert!(st.active.is_empty());
23766 }
23767
23768 #[tokio::test]
23769 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23770 crate::run::pin_test_home();
23771 let tmp = tempfile::tempdir().unwrap();
23772 let mut st = state(1);
23773 let results = run(
23774 vec![job(
23775 sh("alpha", "echo ok"),
23776 tmp.path(),
23777 Duration::from_secs(1),
23778 )],
23779 Arc::new(Semaphore::new(0)),
23780 None,
23781 &mut st,
23782 )
23783 .await;
23784 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23785 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23786 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23787 }
23788
23789 #[tokio::test]
23790 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23791 crate::run::pin_test_home();
23792 let tmp = tempfile::tempdir().unwrap();
23793 let mut st = state(60);
23794 let results = run(
23795 vec![job(
23796 sh("alpha", "sleep 30"),
23797 tmp.path(),
23798 Duration::from_secs(1),
23799 )],
23800 Arc::new(Semaphore::new(1)),
23801 None,
23802 &mut st,
23803 )
23804 .await;
23805 let out = &results[0].2;
23806 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23807 let class = FailClass::of(out).expect("a failure");
23808 assert_eq!(class, FailClass::Timeout);
23809 assert!(should_hand_over(None, &class), "existing handover policy");
23810 assert!(st.resource_block.is_none());
23811 }
23812}