1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false, None).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true, None).await
1357 }
1358
1359 pub(crate) async fn authorize_base_recovery_control(
1360 home: &Path,
1361 task: &str,
1362 control: &mut crate::base_recovery_control::Transaction,
1363 ) -> Result<()> {
1364 let id = control.request.run.clone();
1365 Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366 }
1367
1368 async fn authorize_base_recovery_request(
1369 home: &Path,
1370 id: &str,
1371 task: &str,
1372 resumption: bool,
1373 mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374 ) -> Result<()> {
1375 let _owner = crate::pr_head::owner_lock(home, id)?;
1376 let mut state = RunState::load_under(id, home)?;
1377 crate::pr_head::ensure_owner(&state)?;
1378 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379 anyhow::ensure!(
1380 queued.execution_runs().last().is_some_and(|run| run == id)
1381 && queued.status != crate::queue::TaskStatus::Done
1382 && !queued.fresh_start
1383 && queued.review_branch.is_none(),
1384 "run task ownership changed"
1385 );
1386 anyhow::ensure!(
1387 state
1388 .origin
1389 .as_ref()
1390 .and_then(|o| o.task.as_deref())
1391 .is_none_or(|owner| owner == task),
1392 "run belongs to another task"
1393 );
1394 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395 anyhow::ensure!(
1396 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397 "run is live or its ownership is unknown"
1398 );
1399 let _execution = crate::driver::maintenance(home, id)?;
1402 anyhow::ensure!(
1403 state.status.resumable()
1404 && state.released_to.is_none()
1405 && state.retired_to.is_none()
1406 && state.pr_taken_over_by.is_none(),
1407 "run ownership is final"
1408 );
1409 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410 anyhow::ensure!(
1411 !state.pr_publication_pending
1412 && !crate::handover::landing_checkpoint(&state)
1413 && state.merge_decision.is_none(),
1414 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415 );
1416 let winner = state.winner().context("run has no retained winner")?;
1417 anyhow::ensure!(
1418 git::is_clean(&winner.worktree).await?,
1419 "retained worktree has uncommitted changes"
1420 );
1421 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422 anyhow::ensure!(
1423 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424 "retained worktree does not match the winner branch"
1425 );
1426 if let Some(control) = control.as_ref() {
1427 anyhow::ensure!(
1428 head.starts_with(&control.request.head),
1429 "retained HEAD changed since the owner request"
1430 );
1431 anyhow::ensure!(
1432 state
1433 .base_sync_recovery
1434 .stop
1435 .as_ref()
1436 .is_none_or(|stop| stop.head == head),
1437 "retained HEAD differs from the recorded stop head; inspect it first"
1438 );
1439 }
1440 state.ensure_base_episode();
1441 state.record_base_sync_stop(&head);
1442 anyhow::ensure!(
1443 state
1444 .base_sync_recovery
1445 .stop
1446 .as_ref()
1447 .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448 "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449 );
1450 let fetched =
1451 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452 anyhow::ensure!(
1453 fetched.ok(),
1454 "cannot authorize base recovery without a fetched base"
1455 );
1456 let approved_tip = git::rev_parse(
1457 &state.repo,
1458 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459 )
1460 .await?;
1461 if let Some(control) = control.as_deref_mut() {
1462 let queue = crate::queue::Queue::at(home.join("queue"));
1463 control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464 if let Some(prior) = state
1465 .base_sync_recovery
1466 .authorizations
1467 .iter()
1468 .find(|a| a.request.as_deref() == Some(&control.key()))
1469 {
1470 anyhow::ensure!(
1471 !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472 "recovery request was consumed or invalidated; it cannot queue again"
1473 );
1474 return control.enqueue(&queue);
1475 }
1476 }
1477 if resumption {
1478 anyhow::ensure!(
1479 crate::daemon::compatible_run_instruction(&state, &queued),
1480 "resumption instruction does not match the retained run"
1481 );
1482 let approval = queued
1483 .resumption
1484 .as_ref()
1485 .context("no operator resumption")?;
1486 anyhow::ensure!(
1487 approval.instruction == queued.instruction,
1488 "resumption approval does not match the current task instruction"
1489 );
1490 state.authorize_base_sync_resumption(task, approval)?;
1491 } else {
1492 state.authorize_base_sync(task)?;
1493 }
1494 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495 state
1496 .base_sync_recovery
1497 .authorizations
1498 .last_mut()
1499 .unwrap()
1500 .scope = Some(crate::base_sync_recovery::Scope {
1501 tip: approved_tip,
1502 instruction: queued.instruction.clone(),
1503 episode,
1504 });
1505 if let Some(control) = control.as_deref_mut() {
1506 let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507 authorization.request = Some(control.key());
1508 authorization.generation = control.generation();
1509 authorization.decision = control.decision();
1510 authorization.question = control
1511 .request
1512 .decision
1513 .as_ref()
1514 .map(|_| control.request.request.clone());
1515 }
1516 state.event(
1517 "base-recovery",
1518 "owner authorized a bounded base synchronization interval; fresh verification required",
1519 );
1520 state.save_under(home)?;
1521 if let Some(control) = control {
1522 control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523 }
1524 Ok(())
1525 }
1526
1527 pub fn resume(id: &str) -> Result<Self> {
1529 let state = RunState::load(id)?;
1530 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531 if state.startup.as_ref().is_some_and(|s| !s.complete)
1532 || (state.status == RunStatus::Prep
1533 && state.instruction.trim().is_empty()
1534 && state.candidates.is_empty()
1535 && state.prep_complete == Some(false))
1536 {
1537 bail!(
1538 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539 );
1540 }
1541 crate::pr_head::ensure_owner(&state)?;
1542 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543 bail!(
1544 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545 );
1546 }
1547 if let Some(to) = &state.released_to {
1548 bail!(
1549 "run {} cannot be resumed: its worktree was released to run {}",
1550 state.short(),
1551 crate::run::short_of(to)
1552 );
1553 }
1554 let roles = state.config.resolve_roles()?;
1555 if let Some(id) = &state.config.graph.selected_implementer
1556 && state.candidates.iter().any(|c| c.agent != *id)
1557 {
1558 bail!(
1559 "replacement implementer pin does not match the saved candidates; task must stay held"
1560 );
1561 }
1562 let max_parallel = state.config.graph.max_parallel.max(1);
1563 Ok(Self {
1564 state,
1565 roles,
1566 sem: Arc::new(Semaphore::new(max_parallel)),
1567 pause: Pause::new(),
1568 interrupt: Pause::new(),
1569 })
1570 }
1571
1572 pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579 Box::pin(self.execute_inner())
1582 }
1583
1584 async fn execute_inner(&mut self) -> Result<()> {
1585 anyhow::ensure!(
1586 self.state.startup.as_ref().is_none_or(|s| s.complete),
1587 "startup incomplete; inspect retained evidence before explicit recovery"
1588 );
1589 if let Some(selection) = &self.state.pr_recovery
1590 && (selection.source_proof.is_some() || selection.source_run.is_some())
1591 {
1592 Box::pin(selection.verify_source(&self.state.repo)).await?;
1593 }
1594 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597 || self.state.land_approval.is_some()
1598 || self.state.merge_decision.is_some()
1599 || !self.state.merge_repairs.is_empty()
1600 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601 {
1602 let claim = FixClaim::acquire(&self.state.dir())?;
1603 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604 let disk = RunState::load(&self.state.id)?;
1605 let checkpoint = |s: &RunState| {
1606 (
1607 s.merge_repairs.len(),
1608 s.merge_repairs
1609 .last()
1610 .map(|r| (r.question.clone(), r.phase)),
1611 s.land_approval
1612 .as_ref()
1613 .map(|a| (a.question.clone(), a.head.clone())),
1614 )
1615 };
1616 if checkpoint(&disk) != checkpoint(&self.state) {
1617 bail!(
1618 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619 );
1620 }
1621 Some(claim)
1622 } else {
1623 None
1624 };
1625 let mut execution = self.claim_driver().await?;
1626 let runner = &mut *execution.runner;
1627 let wait = runner.repository_wait();
1628 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631 let result = match result {
1632 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633 record_repository_block(&mut runner.state, &e);
1634 runner.park_here()?;
1635 Ok(())
1636 }
1637 other => other,
1638 };
1639 execution.driver.finish(&mut runner.state).await?;
1640 let ended = if result.is_err() {
1641 Some({
1642 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643 if let Err(error) = &result {
1644 notice.message = format!(
1645 "Run {} stopped with an error: {}",
1646 runner.state.id,
1647 error.to_string().lines().next().unwrap_or("unknown error")
1648 );
1649 }
1650 notice
1651 })
1652 } else {
1653 crate::notices::run_ended(&runner.state)
1654 };
1655 if let Some(notice) = ended {
1656 crate::notices::raise(notice);
1657 }
1658 result
1659 }
1660
1661 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663 }
1664
1665 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666 let wait = self.repository_wait();
1667 coordinate_operation(&mut self.state, cwd, &wait).await
1668 }
1669
1670 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672 let owner = match crate::pr_head::owner_lock_wait(
1673 &crate::run::home(),
1674 &self.state.id,
1675 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676 )
1677 .await
1678 {
1679 Ok(owner) => owner,
1680 Err(error) => {
1681 let detail = format!("worktree maintenance or ownership update: {error:#}");
1682 self.state.resource_block = Some(crate::run::ResourceBlock {
1686 cause: crate::run::ResourceBlockCause::Other,
1687 node: "start".into(),
1688 kind: "worktree maintenance".into(),
1689 detail: detail.clone(),
1690 at: Timestamp::now(),
1691 });
1692 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693 }
1694 };
1695 let disk = RunState::load(&self.state.id)?;
1696 crate::pr_head::ensure_owner(&disk)?;
1697 let claimed =
1698 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699 anyhow::ensure!(
1704 disk.liveness(claimed) != crate::run::Liveness::Live,
1705 "run already has a live driver; wait for its execution to end"
1706 );
1707 if let Some(to) = &disk.released_to {
1708 bail!(
1709 "run {} cannot continue: its worktree was released to run {}",
1710 self.state.short(),
1711 crate::run::short_of(to)
1712 );
1713 }
1714 anyhow::ensure!(
1715 serde_json::to_value(&disk.base_sync_recovery)?
1716 == serde_json::to_value(&self.state.base_sync_recovery)?,
1717 "base recovery changed since this run was loaded; reload the saved run"
1718 );
1719 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720 let mut execution = DriverExecution {
1721 runner: self,
1722 driver,
1723 };
1724 let _owner = owner;
1727 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728 .await?;
1729 let state = &mut execution.runner.state;
1730 state.parked = false;
1735 state.resource_wait = None;
1736 state.resource_block = None;
1737 state.clear_active();
1744 let pid = std::process::id();
1759 state.driver_pid = Some(pid);
1760 state.driver_started_at = crate::proc::process_started_at(pid);
1761 state.driver_exited = false;
1762 state.driver_claimed = Some(true);
1763 state.begin_pass();
1764 execution.driver.bind(state);
1765 state.save()?;
1766 execution.driver.record(state)?;
1767 Ok(execution)
1768 }
1769
1770 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774 if let Err(error) = &result
1775 && !error.is::<crate::repo_lock::ResourceBlocked>()
1776 && self
1777 .state
1778 .repair_grants
1779 .last()
1780 .is_some_and(|g| g.conflict.is_none())
1781 {
1782 self.state.repair_grants.last_mut().unwrap().phase =
1783 crate::repair_budget::Phase::Complete;
1784 self.state.status = RunStatus::Blocked;
1785 self.state.save()?;
1786 bail!(
1787 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788 self.state.passes.last().map_or(0, |p| p.index)
1789 );
1790 }
1791 result
1792 }
1793
1794 async fn execute_granted_repair(&mut self) -> Result<()> {
1797 use crate::repair_budget::{Kind, Phase};
1798 let index = self.state.repair_grants.len() - 1;
1799 let winner = self
1800 .state
1801 .winner()
1802 .cloned()
1803 .context("repair winner missing")?;
1804 loop {
1805 let grant = self.state.repair_grants[index].clone();
1806 match grant.phase {
1807 Phase::Pending => {
1808 if grant.spent >= grant.receipt.request.rounds {
1809 self.state.repair_grants[index].phase = Phase::Complete;
1810 self.state.status = RunStatus::Blocked;
1811 self.state.save()?;
1812 return Ok(());
1813 }
1814 self.state.repair_grants[index].spent += 1;
1817 self.state.repair_grants[index].phase = Phase::Fixing;
1818 self.state.save()?;
1819 match grant.receipt.request.kind {
1820 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821 Kind::Gate => {
1822 let failed = self.state.gate.clone();
1823 let _ = self.gate_fix_round(&winner, &failed).await?;
1824 }
1825 Kind::Review => {
1826 let mut record = self
1827 .state
1828 .reviews
1829 .last()
1830 .cloned()
1831 .context("review failure missing")?;
1832 let round = record.round;
1833 let step = self.run_fix_round(&winner, &mut record).await?;
1834 self.state.reviews[round - 1] = record;
1835 if matches!(step, FixStep::Recorded { report_lost: true }) {
1836 self.state.repair_grants[index].phase = Phase::Complete;
1837 self.state.status = RunStatus::Blocked;
1838 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839 self.state.save()?;
1840 return Ok(());
1841 }
1842 if matches!(step, FixStep::Resource) {
1843 self.stop_for_resource()?;
1844 return Ok(());
1845 }
1846 self.state.save()?;
1847 }
1848 }
1849 if self.resource_halt()? {
1850 return Ok(());
1851 }
1852 }
1853 Phase::Fixing => {
1854 if grant.receipt.request.kind == Kind::BaseConflict {
1855 self.repair_base_conflict(false).await?;
1856 }
1857 let _coordination = self.coordinate(&winner.worktree).await?;
1860 let rescued = git::rescue_commit(
1861 &winner.worktree,
1862 "magi: preserve interrupted granted repair",
1863 )
1864 .await?;
1865 self.state.note_withheld("repair-grant", &rescued.withheld);
1866 self.state
1869 .verification_history
1870 .push(crate::run::VerificationEvidence {
1871 superseded_by: None,
1872 archived_at: Some(Timestamp::now()),
1873 fixes_spent: Some(self.state.gate_fixes.len()),
1874 gate_head: self.state.gate_head.take(),
1875 gate_ran: std::mem::take(&mut self.state.gate_ran),
1876 gate: std::mem::take(&mut self.state.gate),
1877 e2e: self.state.head_e2e.take(),
1878 });
1879 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880 self.state
1881 .verification_history
1882 .push(crate::run::VerificationEvidence {
1883 superseded_by: None,
1884 archived_at: Some(Timestamp::now()),
1885 fixes_spent: Some(self.state.gate_fixes.len()),
1886 gate_head: None,
1887 gate_ran: false,
1888 gate: Vec::new(),
1889 e2e: Some(e2e),
1890 });
1891 }
1892 self.state.repair_grants[index].review_start = self.state.reviews.len();
1893 self.state.repair_grants[index].head =
1894 git::rev_parse(&winner.worktree, "HEAD").await?;
1895 self.state.repair_grants[index].phase = Phase::Reviewing;
1896 if grant.receipt.request.kind == Kind::BaseConflict {
1897 self.state.base_sync_recovery.stop = None;
1898 if let Some(sync) = &mut self.state.base_sync {
1899 sync.tip = grant
1900 .receipt
1901 .request
1902 .tip
1903 .clone()
1904 .context("repair base tip missing")?;
1905 sync.conflict = None;
1906 sync.behind = 0;
1907 }
1908 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909 }
1910 self.state.save()?;
1911 }
1912 Phase::Reviewing => {
1913 self.review_loop_pass().await?;
1914 if self.resource_halt()? {
1915 return Ok(());
1916 }
1917 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918 let clean = self.state.reviews.len() > grant.review_start
1919 && self.state.reviews.last().is_some_and(|r| {
1920 r.head == head
1921 && r.clean
1922 && r.expected > 0
1923 && r.answered == r.expected
1924 && matches!(
1925 r.e2e_status(),
1926 E2eStatus::Passed | E2eStatus::NotConfigured
1927 )
1928 });
1929 if !clean {
1930 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932 == Kind::Review
1933 && grant.spent < grant.receipt.request.rounds
1934 && self.state.reviews.last().is_some_and(|r| {
1935 r.fix.is_none()
1936 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937 }) {
1938 Phase::Pending
1939 } else {
1940 Phase::Complete
1941 };
1942 self.state.status = RunStatus::Blocked;
1943 self.state.event(
1944 "repair-grant",
1945 "fresh review did not clear the repaired head; no publication",
1946 );
1947 self.state.save()?;
1948 if self.state.repair_grants[index].phase == Phase::Pending {
1949 continue;
1950 }
1951 return Ok(());
1952 }
1953 self.state.repair_grants[index].phase = Phase::Gating;
1954 self.state.save()?;
1955 }
1956 Phase::Gating => {
1957 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958 self.state.reconcile_verification_head(&head);
1959 let wait = self.repository_wait();
1960 if self
1961 .state
1962 .reviews
1963 .last()
1964 .is_none_or(|r| r.head != head || !r.clean)
1965 {
1966 self.state.status = RunStatus::Blocked;
1967 self.state.repair_grants[index].phase = Phase::Complete;
1968 self.state.event(
1969 "repair-grant",
1970 "head changed since fresh review; repair held",
1971 );
1972 self.state.save()?;
1973 return Ok(());
1974 }
1975 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976 return Ok(());
1977 }
1978 let outcomes = self.run_gate(&winner).await?;
1979 if verify_inconclusive(&outcomes) {
1980 return Ok(());
1981 }
1982 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983 if after != head {
1984 self.state.repair_grants[index].review_start = self.state.reviews.len();
1985 self.state.repair_grants[index].head = after;
1986 self.state.repair_grants[index].phase = Phase::Reviewing;
1987 self.state.status = RunStatus::Blocked;
1988 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989 self.state.save()?;
1990 return Ok(());
1991 }
1992 let passed = outcomes.iter().all(CommandOutcome::ok);
1993 if passed
1996 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997 {
1998 return Ok(());
1999 }
2000 self.state.repair_grants[index].succeeded = passed;
2001 self.state.gate = outcomes;
2002 self.state.gate_ran = true;
2003 self.state.status = if passed {
2004 RunStatus::Gating
2005 } else {
2006 RunStatus::Blocked
2007 };
2008 self.state.repair_grants[index].phase = if !passed
2009 && grant.receipt.request.kind == Kind::Gate
2010 && grant.spent < grant.receipt.request.rounds
2011 {
2012 Phase::Pending
2013 } else {
2014 Phase::Complete
2015 };
2016 if passed && self.state.config.merge.mode == MergeMode::Pr {
2017 self.state.pr_publication_pending = true;
2019 } else if passed {
2020 self.state.status = RunStatus::Ready;
2021 }
2022 self.state.save()?;
2023 if !passed {
2024 continue;
2025 }
2026 if self.state.config.merge.mode != MergeMode::Pr {
2030 self.state.status = RunStatus::Ready;
2031 self.state.event(
2032 "repair-grant",
2033 "repair verified locally; grant carries no merge authorization",
2034 );
2035 self.state.save()?;
2036 return Ok(());
2037 }
2038 return self.merge().await;
2039 }
2040 Phase::Complete => return Ok(()),
2041 }
2042 }
2043 }
2044
2045 async fn execute_graph(&mut self) -> Result<()> {
2046 let archive_home = crate::run::home().canonicalize()?;
2050 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051 let repo = self.state.repo.clone();
2052 let _coordination = self.coordinate(&repo).await?;
2053 ensure_setup_config(&mut self.state, &repo).await?;
2054 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055 }
2056 self.state.configure_recovery_resume()?;
2057 if self
2058 .state
2059 .repair_grants
2060 .last()
2061 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062 {
2063 return self.execute_granted_repair().await;
2064 }
2065 if self
2066 .state
2067 .repair_grants
2068 .last()
2069 .is_some_and(|g| !g.succeeded)
2070 {
2071 self.state.status = RunStatus::Blocked;
2072 self.state.save()?;
2073 return Ok(());
2074 }
2075 self.persist_candidates().await?;
2078 if self.state.base_sync_blocked() {
2081 self.sync_to_base().await?;
2082 if !self.state.base_sync_blocked()
2083 && self.state.status != RunStatus::AlreadyInBase
2084 && !self.resource_halt()?
2085 {
2086 self.finish_after_tally().await?;
2087 }
2088 return Ok(());
2089 }
2090 if self
2091 .state
2092 .merge_decision
2093 .as_ref()
2094 .is_some_and(|decision| !decision.closed)
2095 && self.state.status != RunStatus::Landing
2096 {
2097 if crate::merge_decision::resumable_hold(&self.state) {
2098 self.state.merge = Some(MergeOutcome {
2099 mode: MergeMode::Pr,
2100 ok: false,
2101 detail: self
2102 .state
2103 .merge_decision
2104 .as_ref()
2105 .unwrap()
2106 .target
2107 .pr
2108 .clone(),
2109 empty: false,
2110 });
2111 self.run_land().await?;
2112 self.settle_questions();
2113 return Ok(());
2114 }
2115 bail!(
2116 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117 );
2118 }
2119 if self.state.config.graph.selected_implementer.is_some()
2120 && self.state.status == RunStatus::Blocked
2121 && self.state.tally.is_none()
2122 && self.state.candidate_preservation.is_empty()
2123 && self.state.interrupted_implementations.is_empty()
2124 {
2125 bail!(
2126 "selected implementation is held; explicitly release the owning task to retry it"
2127 );
2128 }
2129
2130 if self.state.merge_repairs.last().is_some_and(|r| {
2131 matches!(
2132 r.phase,
2133 crate::run::MergeRepairPhase::Requested
2134 | crate::run::MergeRepairPhase::Fixing
2135 | crate::run::MergeRepairPhase::Reviewing
2136 )
2137 }) {
2138 if self.repair_merge_review().await? {
2139 self.run_land().await?;
2140 }
2141 self.settle_questions();
2142 return Ok(());
2143 }
2144 if self.state.pr_publication_pending {
2147 self.merge().await?;
2148 if self.resource_halt()? {
2149 return Ok(());
2150 }
2151 self.state.save()?;
2152 self.settle_questions();
2153 return Ok(());
2154 }
2155 if self.state.status == RunStatus::Stalled {
2168 if self.recover_stall().await? {
2169 self.finish_after_tally().await?;
2170 } else {
2171 self.state.save()?;
2173 }
2174 return Ok(());
2175 }
2176 if self.state.status == RunStatus::Landing
2186 || crate::handover::landing_checkpoint(&self.state)
2187 {
2188 self.state.land_sync_pending = true;
2189 self.state.save()?;
2190 self.run_land().await?;
2191 self.settle_questions();
2196 return Ok(());
2197 }
2198 if self.state.github_text.is_some()
2203 && self.state.merge.is_none()
2204 && self.state.status == RunStatus::Gating
2205 {
2206 self.merge().await?;
2207 if self.resource_halt()? {
2208 return Ok(());
2209 }
2210 self.state.save()?;
2211 return Ok(());
2212 }
2213 if let Err(e) = self.prep().await {
2214 if e.is::<crate::repo_lock::ResourceBlocked>() {
2215 return Err(e);
2216 }
2217 self.state.status = RunStatus::Blocked;
2218 self.state
2219 .prep_failure
2220 .get_or_insert_with(|| format!("{e:#}"));
2221 self.state
2222 .event("prep", format!("preparation stopped: {e:#}"));
2223 self.state.save()?;
2224 return Err(e);
2225 }
2226 if self.park_here()? {
2227 return Ok(());
2228 }
2229 self.advise().await?;
2230 if self.park_here()? {
2231 return Ok(());
2232 }
2233 self.implement().await?;
2234 if self.park_here()? {
2235 return Ok(());
2236 }
2237 if self.state.status == RunStatus::VerifiedNoop {
2241 return Ok(());
2242 }
2243 self.judge().await?;
2244 if self.park_here()? {
2245 return Ok(());
2246 }
2247 self.deliberate().await?;
2248 if self.park_here()? {
2249 return Ok(());
2250 }
2251 self.vote().await?;
2252 if self.park_here()? {
2253 return Ok(());
2254 }
2255 self.tally()?;
2256 if self.state.status == RunStatus::Stalled {
2261 self.state.save()?;
2265 return Ok(());
2266 }
2267 self.finish_after_tally().await?;
2268 Ok(())
2269 }
2270
2271 fn stop_for_resource(&mut self) -> Result<()> {
2276 if self.pause.parked() || self.interrupt.parked() {
2279 self.park_here()?;
2280 return Ok(());
2281 }
2282 if self.state.resource_block.is_some() {
2283 self.state.status = RunStatus::Blocked;
2284 }
2285 self.state.save()
2286 }
2287
2288 fn resource_halt(&mut self) -> Result<bool> {
2290 if self.state.resource_block.is_none() {
2291 return Ok(false);
2292 }
2293 self.stop_for_resource()?;
2294 Ok(true)
2295 }
2296
2297 fn park_here(&mut self) -> Result<bool> {
2304 let parking = self.pause.parked() || self.interrupt.parked();
2305 if !parking && self.state.resource_block.is_some() {
2306 self.stop_for_resource()?;
2307 return Ok(true);
2308 }
2309 if !self.pause.parked() && !self.interrupt.parked() {
2314 return Ok(false);
2315 }
2316 if self.state.parked {
2317 return Ok(true);
2318 }
2319 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320 if let Some(block) = &mut self.state.resource_block
2321 && matches!(
2322 block.cause,
2323 crate::run::ResourceBlockCause::Cancelled { .. }
2324 )
2325 {
2326 block.cause = crate::run::ResourceBlockCause::Cancelled {
2327 park: cause.clone(),
2328 };
2329 }
2330 let why = match cause.as_ref().map(ParkCause::label) {
2331 Some(reason) => format!(
2332 "parked after `{}` ({reason}) — resume to carry on from here",
2333 self.state.status.as_str()
2334 ),
2335 None => format!(
2336 "parked after `{}` — resume to carry on from here",
2337 self.state.status.as_str()
2338 ),
2339 };
2340 self.state.event("park", why);
2341 self.state
2342 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343 self.state.parked = true;
2344 self.state.save()?;
2345 Ok(true)
2346 }
2347
2348 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350 if self.state.prep_failure.is_none() {
2351 return Ok(());
2352 }
2353 let failed_at = self
2354 .state
2355 .events
2356 .iter()
2357 .rev()
2358 .find(|event| event.node == "prep")
2359 .map_or(self.state.updated_at, |event| event.at);
2360 anyhow::ensure!(
2361 approval_at > failed_at,
2362 "preparation retry requires a new explicit release after the failure"
2363 );
2364 self.state.prep_failure = None;
2365 self.state.prep_complete = Some(false);
2366 self.state
2367 .event("prep-retry", "operator approved retry of preparation");
2368 Ok(())
2369 }
2370
2371 pub fn on_pause(&mut self, pause: Pause) {
2373 self.pause = pause;
2374 }
2375
2376 pub fn watch_interrupt(&mut self, pause: Pause) {
2382 self.interrupt = pause;
2383 }
2384
2385 fn settle_questions(&mut self) {
2405 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407 }
2408 }
2409
2410 fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413 Box::pin(self.finish_after_tally_inner())
2414 }
2415
2416 async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417 self.graft().await?;
2420 if self.resource_halt()? {
2421 return Ok(());
2422 }
2423 self.fold_losers().await?;
2424 self.sync_to_base().await?;
2429 if self.hold_base_sync().await? || self.resource_halt()? {
2430 return Ok(());
2431 }
2432 if self.state.status == RunStatus::AlreadyInBase {
2433 return Ok(());
2434 }
2435 self.review_loop().await?;
2436 if self.resource_halt()? {
2437 return Ok(());
2438 }
2439 self.sync_to_base().await?;
2440 if self.hold_base_sync().await? || self.resource_halt()? {
2441 return Ok(());
2442 }
2443 if self.state.status == RunStatus::AlreadyInBase {
2444 return Ok(());
2445 }
2446 self.gate().await?;
2447 self.merge().await?;
2448 if self.resource_halt()? {
2449 return Ok(());
2450 }
2451 self.state.save()?;
2452 Ok(())
2453 }
2454
2455 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462 let result = match ensure_setup_config(&mut self.state, repo).await {
2463 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464 Err(e) => Err(e),
2465 };
2466 if let Err(e) = result {
2467 git::worktree_remove_clean(repo, wt).await.ok();
2468 if in_prep {
2469 self.rollback_prep(repo).await;
2470 }
2471 self.state.status = RunStatus::Blocked;
2472 self.state.event(
2473 "setup",
2474 format!("worktree setup failed in {}: {e:#}", wt.display()),
2475 );
2476 self.state.save()?;
2477 return Err(e);
2478 }
2479 Ok(())
2480 }
2481
2482 async fn rollback_prep(&mut self, repo: &Path) {
2484 let root = self.state.worktree_root();
2485 for c in std::mem::take(&mut self.state.candidates) {
2486 if matches!(
2487 git::worktree_remove_clean(repo, &c.worktree).await,
2488 Ok(true)
2489 ) {
2490 git::branch_delete(repo, &c.branch).await.ok();
2491 } else {
2492 self.state.candidates.push(c);
2493 }
2494 }
2495 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496 for j in 1..=seats {
2497 remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2498 .await
2499 .ok();
2500 }
2501 for k in 1..=self.state.config.graph.advisors {
2502 remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2503 .await
2504 .ok();
2505 }
2506 }
2507
2508 async fn prep(&mut self) -> Result<()> {
2509 if let Some(reason) = &self.state.prep_failure {
2510 bail!(
2511 "candidate preparation previously failed: {reason}; correct the source before retrying"
2512 );
2513 }
2514 if self
2515 .state
2516 .prep_complete
2517 .unwrap_or(!self.state.candidates.is_empty())
2518 {
2519 return Ok(());
2520 }
2521 let repo = self.state.repo.clone();
2522 let _coordination = self.coordinate(&repo).await?;
2525 let base = self.state.base_commit.clone();
2526 let plan = self.state.sources.plan(&repo, &base).await?;
2527 self.rollback_prep(&repo).await;
2528 if !self.state.candidates.is_empty() {
2529 bail!(
2530 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531 );
2532 }
2533 self.state.prep_complete = Some(false);
2534 self.state.status = RunStatus::Prep;
2535 self.state.save()?;
2536 let repo = self.state.repo.clone();
2537 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538 let root = self.state.worktree_root();
2539 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541 let hooks_dir = self.state.dir().join("hooks");
2544 if self.state.config.blind.commit_msg_hook {
2545 std::fs::create_dir_all(&hooks_dir)
2546 .with_context(|| format!("create {}", hooks_dir.display()))?;
2547 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548 let path = hooks_dir.join("commit-msg");
2549 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550 make_executable(&path)?;
2551 git::acquire_worktree_config(&repo).await?;
2559 self.state.enabled_worktree_config = true;
2560 }
2561
2562 ensure_setup_config(&mut self.state, &repo).await?;
2565
2566 for (index, (spec, label)) in self
2567 .roles
2568 .implementers
2569 .clone()
2570 .into_iter()
2571 .zip(labels)
2572 .enumerate()
2573 {
2574 let branch = self.state.branch_for(label);
2575 let worktree = root.join(format!("cand-{label}"));
2576 self.state.event(
2580 "prep-intent",
2581 serde_json::json!({
2582 "branch": branch,
2583 "worktree": worktree,
2584 "start": start,
2585 "picks": plan.picks,
2586 })
2587 .to_string(),
2588 );
2589 self.state.save()?;
2590 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591 if self.state.config.blind.commit_msg_hook {
2592 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593 }
2594 git::local_exclude(&worktree, "/.magi/").await?;
2595 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596 git::worktree_remove_clean(&repo, &worktree).await.ok();
2599 git::branch_delete(&repo, &branch).await.ok();
2600 self.rollback_prep(&repo).await;
2601 self.state.status = RunStatus::Blocked;
2602 self.state
2603 .event("prep", format!("worktree setup failed: {e:#}"));
2604 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605 self.state.save()?;
2606 return Err(e);
2607 }
2608 self.state.candidates.push(Candidate {
2609 index,
2610 label,
2611 agent: spec.id.clone(),
2612 branch,
2613 worktree,
2614 summary: String::new(),
2615 stat: String::new(),
2616 files: 0,
2617 commits: 0,
2618 empty: false,
2619 failed: None,
2620 verified_noop: None,
2621 duration_ms: 0,
2622 folded: false,
2623 });
2624 self.state.save()?;
2625 for pick in &plan.picks {
2626 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627 {
2628 self.state.status = RunStatus::Blocked;
2629 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630 self.state
2631 .event("prep", format!("cannot apply referenced commit: {e}"));
2632 self.state.save()?;
2633 return Err(e);
2634 }
2635 }
2636 }
2637
2638 for j in 1..=self.roles.judges.len() {
2639 let wt = root.join(format!("judge-{j}"));
2640 if !wt.exists() {
2641 git::worktree_add_detached(&repo, &wt, &base).await?;
2642 self.setup_seat_worktree(&repo, &wt, true).await?;
2643 }
2644 }
2645
2646 if self.state.config.graph.advise {
2654 for k in 1..=self.state.config.graph.advisors {
2655 let wt = root.join(format!("advisor-{k}"));
2656 if !wt.exists() {
2657 git::worktree_add_detached(&repo, &wt, &base).await?;
2658 self.setup_seat_worktree(&repo, &wt, true).await?;
2659 }
2660 }
2661 }
2662
2663 let authors: Vec<&str> = self
2668 .roles
2669 .implementers
2670 .iter()
2671 .map(|a| a.id.as_str())
2672 .collect();
2673 let overlap: Vec<String> = self
2674 .roles
2675 .judges
2676 .iter()
2677 .enumerate()
2678 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680 .collect();
2681 if !overlap.is_empty() {
2682 let note = format!(
2683 "{} also authored a candidate; blind, but the panel is less \
2684 independent than {} distinct agents would be",
2685 overlap.join(", "),
2686 self.roles.judges.len()
2687 );
2688 self.state.event("prep", note);
2689 }
2690
2691 self.state.event(
2692 "prep",
2693 format!(
2694 "{} candidates, {} judges, base {} ({})",
2695 self.state.candidates.len(),
2696 self.roles.judges.len(),
2697 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698 self.state.base_branch
2699 ),
2700 );
2701 self.state.prep_complete = Some(true);
2702 self.state.status = RunStatus::Implementing;
2703 self.state.save()?;
2704 Ok(())
2705 }
2706
2707 async fn advise(&mut self) -> Result<()> {
2740 let implement_untouched = self
2741 .state
2742 .candidates
2743 .iter()
2744 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745 if !self.state.config.graph.advise || self.state.advise_attempted {
2746 return Ok(());
2747 }
2748 if !implement_untouched {
2749 self.state.event(
2750 "advise",
2751 "skipping the design-deliberation stage: at least one \
2752 candidate already shows implementation progress, so this \
2753 run is past the point the stage exists to run before"
2754 .to_owned(),
2755 );
2756 self.state.advise_attempted = true;
2757 self.state.save()?;
2758 return Ok(());
2759 }
2760 let run_id = self.state.id.clone();
2761 let prompts = self.state.config.prompts.clone();
2762 let instruction = self.state.instruction.clone();
2763 let language = self.state.config.graph.language.clone();
2764 let root = self.state.worktree_root();
2765 let n = self.state.config.graph.advisors;
2766 let where_recorded = self.state.dir().join("run.json");
2767
2768 let seats = match self.state.config.advisors() {
2769 Ok(seats) if !seats.is_empty() => seats,
2770 Ok(_) => {
2771 self.state.event(
2772 "advise",
2773 format!(
2774 "[graph] advisors is 0; skipping the design-deliberation \
2775 stage and continuing without a synthesis brief (see {})",
2776 where_recorded.display()
2777 ),
2778 );
2779 self.state.advise_attempted = true;
2780 self.state.save()?;
2781 return Ok(());
2782 }
2783 Err(e) => {
2784 self.state.event(
2785 "advise",
2786 format!(
2787 "could not resolve advisor seats ({e:#}); continuing \
2788 without a design-deliberation brief (see {})",
2789 where_recorded.display()
2790 ),
2791 );
2792 self.state.advise_attempted = true;
2793 self.state.save()?;
2794 return Ok(());
2795 }
2796 };
2797
2798 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799 let artifacts = agent::artifacts_dir(&self.state.dir());
2800 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802 let mut jobs = Vec::new();
2803 for (i, spec) in seats.iter().cloned().enumerate() {
2804 let seat_key = format!("advisor-{}", i + 1);
2805 let seat = self.seat(&seat_key, &spec.id);
2806 jobs.push(SeatJob {
2807 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808 spec,
2809 seat,
2810 cwd: worktrees[i % worktrees.len()].clone(),
2811 timeout,
2812 allow_write: false,
2813 sessions: false,
2814 artifacts: artifacts.clone(),
2815 stem: seat_key,
2816 handover: None,
2817 });
2818 }
2819
2820 self.state.event(
2821 "advise",
2822 format!(
2823 "{} advisor seat(s) sketching a design in parallel",
2824 jobs.len()
2825 ),
2826 );
2827 let mut quota_losses = Vec::new();
2828 let cache = self.state.config.cache_dir();
2829 let ctx = WaveCtx {
2830 carry_seats: false,
2831 pauses: Some((&self.pause, &self.interrupt)),
2832 run: &run_id,
2833 node: "advise",
2834 prompts: &prompts,
2835 cache: cache.as_deref(),
2836 round: None,
2837 };
2838 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839 let results = ask_json_wave::<Proposal>(
2840 jobs,
2841 Arc::clone(&self.sem),
2842 self.state.config.graph.retries,
2843 &advisor_roster,
2844 &ctx,
2845 &mut quota_losses,
2846 &mut self.state,
2847 &|p: &Proposal| p.validate(),
2848 )
2849 .await;
2850 if self.state.resource_block.is_some() {
2851 return self.stop_for_resource();
2852 }
2853 self.state.quota.extend(quota_losses);
2854
2855 let mut records = Vec::with_capacity(results.len());
2856 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857 let agent_id = seat.agent.clone();
2858 self.state.seats.insert(seat.key.clone(), seat);
2859 match res {
2860 Ok((proposal, out)) => {
2861 self.state
2862 .event("advise", format!("advisor-{} proposed a design", i + 1));
2863 records.push(advise::AdvisorRecord::proposed(
2864 i + 1,
2865 agent_id,
2866 proposal,
2867 out.duration_ms,
2868 ));
2869 }
2870 Err(e) => {
2871 self.state.event(
2872 "advise",
2873 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874 );
2875 records.push(advise::AdvisorRecord::failed(
2876 i + 1,
2877 agent_id,
2878 e.to_string(),
2879 ));
2880 }
2881 }
2882 }
2883
2884 let mut advice = advise::Advice {
2885 records,
2886 synthesis: None,
2887 };
2888 if advice.proposals().is_empty() {
2889 self.state.event(
2890 "advise",
2891 "no advisor produced a usable proposal; continuing without a \
2892 synthesis brief"
2893 .to_owned(),
2894 );
2895 } else {
2896 match self
2897 .synthesize_brief(
2898 &advice,
2899 &instruction,
2900 &language,
2901 &worktrees[0],
2902 &artifacts,
2903 &run_id,
2904 &prompts,
2905 cache.as_deref(),
2906 )
2907 .await
2908 {
2909 Ok(Some(text)) => {
2910 self.state.event(
2911 "advise",
2912 "synthesized a design brief for the implementer".to_owned(),
2913 );
2914 advice.synthesis = Some(text);
2915 }
2916 Ok(None) => {
2917 self.state.event(
2918 "advise",
2919 "the synthesis seat produced nothing usable; continuing \
2920 without a design brief"
2921 .to_owned(),
2922 );
2923 }
2924 Err(e) => {
2925 if e.is::<crate::repo_lock::ResourceBlocked>() {
2926 return Err(e);
2927 }
2928 self.state.event(
2929 "advise",
2930 format!("could not synthesize a design brief: {e:#}"),
2931 );
2932 }
2933 }
2934 }
2935 advise::apply_reflection(&mut advice);
2936
2937 self.state.advice = Some(advice);
2938 self.state.advise_attempted = true;
2939 self.state.save()?;
2940 Ok(())
2941 }
2942
2943 #[allow(clippy::too_many_arguments)]
2955 async fn synthesize_brief(
2956 &mut self,
2957 advice: &advise::Advice,
2958 instruction: &str,
2959 language: &str,
2960 cwd: &Path,
2961 artifacts: &Path,
2962 run_id: &str,
2963 prompts: &Prompts,
2964 cache: Option<&Path>,
2965 ) -> Result<Option<String>> {
2966 let chain = agent::pick_chain(
2967 &self.state.config.agents,
2968 self.state.config.roles.synthesizer.as_ref(),
2969 &agent::installed,
2970 "synthesizer",
2971 )?;
2972 let proposals = advice.proposals();
2973 let mut prompt = prompt::with_overlay(
2974 prompt::synthesize_brief(instruction, &proposals, language),
2975 prompts.overlay("advise"),
2976 );
2977 if cache.is_some() {
2978 prompt.push('\n');
2983 prompt.push_str(&prompt::build_cache_note("advise", false));
2984 }
2985 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986 let mut last = None;
2991 for (n, spec) in chain.iter().enumerate() {
2992 if n > 0 {
2993 self.state
2994 .event("advise", format!("synthesis falling back to {}", spec.id));
2995 }
2996 let mut seat = self.seat("advise-synthesis", &spec.id);
2997 let wait = self.repository_wait();
2998 let outcome = wait
2999 .scope(agent::invoke(
3000 spec,
3001 &mut seat,
3002 &Invocation {
3003 coordination: self.state.config.coordination.as_ref(),
3004 fetched_base: None,
3005 cwd,
3006 prompt: &prompt,
3007 timeout,
3008 allow_write: false,
3009 sessions: false,
3010 artifacts,
3011 stem: &if n == 0 {
3012 "advise-synthesis".to_owned()
3013 } else {
3014 format!("advise-synthesis-{}", spec.id)
3015 },
3016 run: run_id,
3017 node: "advise",
3018 cache_dir: None,
3019 attachments: &[],
3020 writable: &[],
3021 },
3022 ))
3023 .await;
3024 if outcome.is_ok() {
3025 self.state.seats.insert(seat.key.clone(), seat);
3026 }
3027 let advance = agent::chain_advances(&outcome);
3028 last = Some(outcome);
3029 if !advance {
3030 break;
3031 }
3032 }
3033 let out = last.expect("a chain holds at least one agent")?;
3036 if !out.usable() {
3037 return Ok(None);
3038 }
3039 let text =
3040 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041 Ok((!text.trim().is_empty()).then_some(text))
3042 }
3043
3044 async fn implement(&mut self) -> Result<()> {
3047 self.persist_candidates().await?;
3048 let run_id = self.state.id.clone();
3053 let prompts = self.state.config.prompts.clone();
3054 let todo: Vec<usize> = self
3055 .state
3056 .candidates
3057 .iter()
3058 .enumerate()
3059 .filter(|(_, c)| {
3060 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061 && c.failed.is_none()
3062 && !c.empty
3063 })
3064 .map(|(i, _)| i)
3065 .collect();
3066 if todo.is_empty() {
3067 return self.after_implement();
3068 }
3069 self.state.status = RunStatus::Implementing;
3070
3071 let language = self.state.config.graph.language.clone();
3072 let github_language = self.state.config.graph.github_language.clone();
3073 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074 let sessions = self.state.config.graph.sessions;
3075 let artifacts = agent::artifacts_dir(&self.state.dir());
3076 let brief = self
3080 .state
3081 .advice
3082 .as_ref()
3083 .and_then(|a| a.synthesis.as_deref())
3084 .map(str::to_owned);
3085 let attachments = self.state.attachments.clone();
3086
3087 let mut jobs = Vec::new();
3088 for &i in &todo {
3089 let (index, label, worktree) = {
3090 let c = &self.state.candidates[i];
3091 (c.index, c.label, c.worktree.clone())
3092 };
3093 let spec = self.roles.implementers[index].clone();
3094 let seat_key = format!("impl-{label}");
3095 let seat = self.seat(&seat_key, &spec.id);
3096 let instruction = seeded_instruction(&self.state);
3097 jobs.push(SeatJob {
3098 spec,
3099 seat,
3100 prompt: prompt::implement_in(
3101 &instruction,
3102 &worktree.to_string_lossy(),
3103 &language,
3104 &github_language,
3105 brief.as_deref(),
3106 &attachments,
3107 ),
3108 cwd: worktree,
3109 timeout,
3110 allow_write: true,
3111 sessions,
3112 artifacts: artifacts.clone(),
3113 stem: format!("impl-{label}"),
3114 handover: None,
3115 });
3116 }
3117
3118 self.state.event(
3119 "implement",
3120 format!("{} candidates in parallel", jobs.len()),
3121 );
3122 let mut sent = jobs.clone();
3128 let cache = self.state.config.cache_dir();
3129 let ctx = WaveCtx {
3130 carry_seats: false,
3131 pauses: Some((&self.pause, &self.interrupt)),
3132 run: &run_id,
3133 node: "implement",
3134 prompts: &prompts,
3135 cache: cache.as_deref(),
3136 round: None,
3137 };
3138 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139 if !wave_blocked(&results) {
3140 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141 .await;
3142 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143 .await?;
3144 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145 .await;
3146 }
3147 if !results.is_empty()
3150 && results
3151 .iter()
3152 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153 {
3154 let losses: Vec<_> = results
3155 .iter()
3156 .map(|(_, seat, out)| {
3157 let AgentOutcome::Quota(o) = out else {
3158 unreachable!()
3159 };
3160 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161 })
3162 .collect();
3163 self.state.quota.extend(losses.clone());
3164 for (_, seat, _) in results {
3165 self.state.seats.insert(seat.key.clone(), seat);
3166 }
3167 for &i in &todo {
3168 let label = self.state.candidates[i].label;
3169 self.state.interrupted_implementations.insert(label);
3170 self.state
3171 .candidate_preservation
3172 .insert(label, "machine outage awaiting persistence".into());
3173 }
3174 block_machine_outage(&mut self.state, "implement", &losses);
3177 self.state.save()?;
3178 self.persist_candidates().await?;
3179 for &i in &todo {
3180 self.state
3181 .interrupted_implementations
3182 .insert(self.state.candidates[i].label);
3183 }
3184 return self.stop_for_resource();
3185 }
3186 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189 && results
3190 .iter()
3191 .any(|(_, _, out)| FailClass::of(out).is_some());
3192 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194 if self
3198 .state
3199 .candidate_preservation
3200 .contains_key(&self.state.candidates[i].label)
3201 {
3202 self.state.seats.insert(seat.key.clone(), seat);
3203 }
3204 continue;
3205 }
3206 let seat_key = seat.key.clone();
3207 let agent = seat.agent.clone();
3216 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217 self.state.seats.insert(seat.key.clone(), seat);
3218 let label = self.state.candidates[i].label;
3219 self.state.interrupted_implementations.remove(&label);
3220
3221 let (summary, duration, failed, verified_claim) = match out {
3222 AgentOutcome::Ok(o) => {
3223 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224 let failed = (!o.usable()).then(|| {
3225 if o.timed_out {
3226 "agent timed out".to_owned()
3227 } else {
3228 format!(
3229 "agent exited with {}",
3230 crate::notices::exit_status(o.exit_code)
3231 )
3232 }
3233 });
3234 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235 (text, o.duration_ms, failed, verified_claim)
3236 }
3237 AgentOutcome::Dropped(o) => {
3243 let why = o
3244 .dropped
3245 .as_ref()
3246 .map(|d| d.why.as_str())
3247 .unwrap_or("the CLI ended the stream without delivering its answer");
3248 (
3249 String::new(),
3250 o.duration_ms,
3251 Some(format!("the CLI dropped the stream ({why})")),
3252 None,
3253 )
3254 }
3255 AgentOutcome::Quota(o) => {
3256 self.state.quota.push(QuotaLoss::from_report(
3257 seat_key,
3258 "implement",
3259 o.quota.as_ref(),
3260 ));
3261 (
3262 String::new(),
3263 o.duration_ms,
3264 Some("rate limited (quota); produced no change".to_owned()),
3265 None,
3266 )
3267 }
3268 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269 (String::new(), 0, Some(e), None)
3270 }
3271 };
3272
3273 let c = &mut self.state.candidates[i];
3276 if !exhausted_the_fallback_chain {
3277 c.agent = agent;
3278 }
3279 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280 c.duration_ms = duration;
3281 c.failed = failed;
3282 c.verified_noop = verified_claim;
3283 self.state
3284 .candidate_preservation
3285 .insert(label, "awaiting persistence".to_owned());
3286 }
3287 self.state.save()?;
3288 self.persist_candidates().await?;
3289
3290 if resource_blocked {
3291 return self.stop_for_resource();
3292 }
3293 if selected_failed {
3294 self.state.status = RunStatus::Blocked;
3295 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296 self.state.save()?;
3297 bail!(
3298 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299 );
3300 }
3301 self.after_implement()
3302 }
3303
3304 async fn persist_candidates(&mut self) -> Result<()> {
3306 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307 for label in labels {
3308 let i = self
3309 .state
3310 .candidates
3311 .iter()
3312 .position(|c| c.label == label)
3313 .context("pending candidate is missing")?;
3314 let c = self.state.candidates[i].clone();
3315 let interrupted = self.state.interrupted_implementations.contains(&label);
3316 let result = async {
3317 let wait = crate::repo_lock::WaitPolicy::new(
3318 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319 || false,
3320 );
3321 let _coordination =
3322 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323 if let Err(e) = git::snapshot_uncommitted(
3326 &c.worktree,
3327 &self
3328 .state
3329 .dir()
3330 .join("artifacts")
3331 .join(format!("candidate-{label}-recovery")),
3332 )
3333 .await
3334 {
3335 self.state.event(
3336 "preserve",
3337 format!("candidate {label} snapshot incomplete: {e:#}"),
3338 );
3339 }
3340 let rescued = git::rescue_commit(
3341 &c.worktree,
3342 &format!("magi: candidate {label} (uncommitted work)"),
3343 )
3344 .await?;
3345 self.state.note_withheld("implement", &rescued.withheld);
3346 let base = &self.state.base_commit;
3347 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352 if c.empty && commits > 0 && !patch.trim().is_empty()
3353 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355 || self.state.tally.is_some())
3356 {
3357 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361 }
3362 let c = &mut self.state.candidates[i];
3363 c.commits = commits;
3364 c.stat = stat;
3365 c.files = files;
3366 let no_changes = commits == 0 || patch.trim().is_empty();
3367 c.empty = no_changes && !interrupted;
3368 if interrupted && no_changes {
3369 c.failed = None;
3370 c.verified_noop = None;
3371 }
3372 if !c.empty {
3373 c.failed = None;
3374 c.verified_noop = None;
3375 }
3376 self.state.event(
3377 "preserve",
3378 format!("candidate {label}: {files} files, {commits} commits"),
3379 );
3380 Ok::<_, anyhow::Error>(())
3381 }
3382 .await;
3383 match result {
3384 Ok(()) => {
3385 self.state.candidate_preservation.remove(&label);
3386 if !interrupted || self.state.candidates[i].files > 0 {
3387 self.state.interrupted_implementations.remove(&label);
3388 }
3389 }
3390 Err(e) => {
3391 let reason = format!("candidate {label} preservation failed: {e:#}");
3392 self.state.event("preserve", &reason);
3393 self.state.candidate_preservation.insert(label, reason);
3394 }
3395 }
3396 if self.state.candidate_preservation.is_empty()
3397 && self.state.status == RunStatus::Blocked
3398 && self.state.candidates.iter().all(|c| c.failed.is_none())
3399 {
3400 self.state.status = RunStatus::Implementing;
3404 self.state.event(
3405 "preserve",
3406 "candidate persistence recovered; implementation can continue",
3407 );
3408 }
3409 self.state.save()?;
3410 }
3411 if !self.state.candidate_preservation.is_empty() {
3412 self.state.status = RunStatus::Blocked;
3413 self.state.save()?;
3414 bail!(
3415 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416 );
3417 }
3418 Ok(())
3419 }
3420
3421 async fn resume_undelivered(
3449 &mut self,
3450 results: &mut [(usize, SeatState, AgentOutcome)],
3451 sent: &[SeatJob],
3452 prompts: &Prompts,
3453 run_id: &str,
3454 ) {
3455 for (wi, seat, out) in results.iter_mut() {
3456 let Some(dropped) = (match &*out {
3457 AgentOutcome::Dropped(o) => o.dropped.clone(),
3458 _ => None,
3459 }) else {
3460 continue;
3461 };
3462 let Some(job) = sent.get(*wi) else { continue };
3463 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465 self.state.event(
3466 "implement",
3467 format!(
3468 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469 work is in the tree",
3470 seat.key, dropped.output_tokens, dropped.why
3471 ),
3472 );
3473 continue;
3474 }
3475 if !has_context(&job.spec, seat, job.sessions) {
3483 self.state.event(
3484 "implement",
3485 format!(
3486 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487 is no session left to resume",
3488 seat.key, dropped.output_tokens, dropped.why
3489 ),
3490 );
3491 continue;
3492 }
3493 self.state.event(
3494 "implement",
3495 format!(
3496 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497 conversation",
3498 seat.key, dropped.output_tokens, dropped.why
3499 ),
3500 );
3501 let mut retry = job.clone();
3502 retry.seat = seat.clone();
3503 retry.prompt = prompt::resume_after_drop(&dropped.why);
3504 retry.timeout = retry_budget(job.timeout, true);
3505 retry.stem = format!("{}-resume", job.stem);
3506 let cache = self.state.config.cache_dir();
3507 let ctx = WaveCtx {
3508 carry_seats: false,
3509 pauses: Some((&self.pause, &self.interrupt)),
3510 run: run_id,
3511 node: "implement",
3512 prompts,
3513 cache: cache.as_deref(),
3514 round: None,
3515 };
3516 let (resumed_seat, resumed) =
3517 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518 *seat = resumed_seat;
3519 *out = resumed;
3520 }
3521 }
3522
3523 async fn resume_seat_handovers(
3588 &mut self,
3589 results: &mut [(usize, SeatState, AgentOutcome)],
3590 sent: &mut [SeatJob],
3591 prompts: &Prompts,
3592 run_id: &str,
3593 ) -> Result<()> {
3594 let instruction = seeded_instruction(&self.state);
3595 let language = self.state.config.graph.language.clone();
3596 let github_language = self.state.config.graph.github_language.clone();
3597 let brief = self
3598 .state
3599 .advice
3600 .as_ref()
3601 .and_then(|a| a.synthesis.as_deref())
3602 .map(str::to_owned);
3603 let attachments = self.state.attachments.clone();
3604 for (wi, seat, out) in results.iter_mut() {
3605 let others: BTreeSet<String> = sent
3608 .iter()
3609 .enumerate()
3610 .filter(|(j, _)| j != wi)
3611 .map(|(_, j)| j.spec.id.clone())
3612 .collect();
3613 let Some(job) = sent.get_mut(*wi) else {
3614 continue;
3615 };
3616 let start = self
3621 .roles
3622 .implementer_roster
3623 .iter()
3624 .position(|s| s.id == job.spec.id)
3625 .unwrap_or(0);
3626 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627 let mut fallback_attempt = 0usize;
3628 let mut prev: Option<FailClass> = None;
3629 while let Some(cur) = FailClass::of(&*out) {
3630 if !should_hand_over(prev.as_ref(), &cur) {
3631 break;
3632 }
3633 let Some(next) =
3634 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635 .cloned()
3636 else {
3637 break;
3638 };
3639 tried.insert(next.id.clone());
3640 fallback_attempt += 1;
3641
3642 let coordination = self.coordinate(&job.cwd).await?;
3643 if let Ok(r) = git::rescue_commit(
3644 &job.cwd,
3645 &format!(
3646 "magi: candidate {} (uncommitted work before {} fallback)",
3647 seat.key,
3648 if cur == FailClass::Quota {
3649 "quota"
3650 } else {
3651 "handover"
3652 }
3653 ),
3654 )
3655 .await
3656 {
3657 self.state.note_withheld("implement", &r.withheld);
3658 }
3659 drop(coordination);
3660
3661 record_handover(
3662 &mut self.state,
3663 "implement",
3664 &seat.key,
3665 &seat.agent,
3666 &next.id,
3667 &cur,
3668 &fail_reason(&*out),
3669 );
3670 prev = Some(cur.clone());
3671
3672 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674 job.spec = next.clone();
3682 let mut retry = job.clone();
3683 retry.seat = new_seat;
3684 retry.prompt = prompt::implement_in(
3685 &instruction,
3686 &job.cwd.to_string_lossy(),
3687 &language,
3688 &github_language,
3689 brief.as_deref(),
3690 &attachments,
3691 );
3692 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693 let cache = self.state.config.cache_dir();
3694 let ctx = WaveCtx {
3695 carry_seats: false,
3696 pauses: Some((&self.pause, &self.interrupt)),
3697 run: run_id,
3698 node: "implement",
3699 prompts,
3700 cache: cache.as_deref(),
3701 round: None,
3702 };
3703 let (fallback_seat, fallback_out) = run_one(
3704 retry,
3705 Arc::clone(&self.sem),
3706 &ctx,
3707 &mut self.state,
3708 fallback_attempt,
3709 )
3710 .await;
3711 *seat = fallback_seat;
3712 *out = fallback_out;
3713 }
3714 }
3715 Ok(())
3716 }
3717
3718 async fn resume_unconfirmed_commands(
3742 &mut self,
3743 results: &mut [(usize, SeatState, AgentOutcome)],
3744 sent: &[SeatJob],
3745 prompts: &Prompts,
3746 run_id: &str,
3747 ) {
3748 for (wi, seat, out) in results.iter_mut() {
3749 let AgentOutcome::Ok(o) = &*out else {
3750 continue;
3751 };
3752 if !has_unconfirmed_command(&o.commands) {
3753 continue;
3754 }
3755 let Some(job) = sent.get(*wi) else { continue };
3756 if !has_context(&job.spec, seat, job.sessions) {
3757 self.state.event(
3758 "implement",
3759 format!(
3760 "{}: the reply named a command whose own CLI never confirmed the exit \
3761 status of, but there is no session left to resume",
3762 seat.key
3763 ),
3764 );
3765 continue;
3766 }
3767 self.state.event(
3768 "implement",
3769 format!(
3770 "{}: the reply named a command whose own CLI never confirmed the exit \
3771 status of; resuming the conversation",
3772 seat.key
3773 ),
3774 );
3775 let mut retry = job.clone();
3776 retry.seat = seat.clone();
3777 retry.prompt = prompt::resume_incomplete(
3778 "a command in your last reply had no confirmed exit status",
3779 );
3780 retry.timeout = retry_budget(job.timeout, true);
3781 retry.stem = format!("{}-confirm", job.stem);
3782 let cache = self.state.config.cache_dir();
3783 let ctx = WaveCtx {
3784 carry_seats: false,
3785 pauses: Some((&self.pause, &self.interrupt)),
3786 run: run_id,
3787 node: "implement",
3788 prompts,
3789 cache: cache.as_deref(),
3790 round: None,
3791 };
3792 let (resumed_seat, resumed) =
3793 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794 *seat = resumed_seat;
3795 *out = resumed;
3796 }
3797 }
3798
3799 async fn continue_fix_report(
3820 &mut self,
3821 mut seat: SeatState,
3822 parse_err: String,
3823 job: &SeatJob,
3824 prompts: &Prompts,
3825 run_id: &str,
3826 round: usize,
3827 ) -> (
3828 SeatState,
3829 Option<FixReport>,
3830 Option<String>,
3831 ContinuationRecord,
3832 ) {
3833 let mut last_err = parse_err;
3834 let mut cumulative_wait_ms = 0u64;
3835 let mut attempts = 0usize;
3836 loop {
3837 if !has_context(&job.spec, &seat, job.sessions) {
3838 self.state.event(
3839 "fix",
3840 format!(
3841 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842 session left to resume into"
3843 ),
3844 );
3845 let outcome = if attempts == 0 {
3846 ContinuationOutcome::NoSession
3847 } else {
3848 ContinuationOutcome::Exhausted
3849 };
3850 return (
3851 seat,
3852 None,
3853 Some(format!("unparsable fix report: {last_err}")),
3854 ContinuationRecord {
3855 attempts,
3856 cumulative_wait_ms,
3857 outcome,
3858 },
3859 );
3860 }
3861 if attempts >= MAX_FIX_CONTINUATIONS {
3862 self.state.event(
3863 "fix",
3864 format!(
3865 "round {round}: fixer's reply still had no adoption report after \
3866 {attempts} continuation(s) ({last_err}); giving up"
3867 ),
3868 );
3869 return (
3870 seat,
3871 None,
3872 Some(format!(
3873 "unparsable fix report after {attempts} continuation(s): {last_err}"
3874 )),
3875 ContinuationRecord {
3876 attempts,
3877 cumulative_wait_ms,
3878 outcome: ContinuationOutcome::Exhausted,
3879 },
3880 );
3881 }
3882 attempts += 1;
3883 self.state.event(
3884 "fix",
3885 format!(
3886 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888 ),
3889 );
3890 let mut retry = job.clone();
3891 retry.seat = seat.clone();
3892 retry.prompt = prompt::resume_incomplete(&last_err);
3893 retry.timeout = retry_budget(job.timeout, true);
3894 retry.stem = format!("{}-continue{attempts}", job.stem);
3895 let cache = self.state.config.cache_dir();
3896 let ctx = WaveCtx {
3897 carry_seats: false,
3898 pauses: Some((&self.pause, &self.interrupt)),
3899 run: run_id,
3900 node: "fix",
3901 prompts,
3902 cache: cache.as_deref(),
3903 round: Some(round),
3904 };
3905 let (resumed_seat, resumed_out) = run_one(
3906 retry,
3907 Arc::clone(&self.sem),
3908 &ctx,
3909 &mut self.state,
3910 attempts,
3911 )
3912 .await;
3913 seat = resumed_seat;
3914 match resumed_out {
3915 AgentOutcome::Ok(o) => {
3916 cumulative_wait_ms += o.duration_ms;
3917 match verdict::extract_json::<FixReport>(&o.text) {
3918 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919 self.state.event(
3920 "fix",
3921 format!(
3922 "round {round}: fixer's adoption report recovered after \
3923 {attempts} continuation(s)"
3924 ),
3925 );
3926 return (
3927 seat,
3928 Some(report),
3929 None,
3930 ContinuationRecord {
3931 attempts,
3932 cumulative_wait_ms,
3933 outcome: ContinuationOutcome::Resumed,
3934 },
3935 );
3936 }
3937 Ok(_) => {
3945 last_err = "the reply parsed, but it reported a command whose own CLI \
3946 never confirmed an exit status"
3947 .to_owned();
3948 }
3949 Err(e) => last_err = e.to_string(),
3950 }
3951 }
3952 AgentOutcome::Quota(o) => {
3953 cumulative_wait_ms += o.duration_ms;
3954 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956 self.state.quota.push(loss);
3957 self.state.event(
3958 "fix",
3959 format!(
3960 "round {round}: continuation rate limited (quota); not retrying now"
3961 ),
3962 );
3963 return (
3964 seat,
3965 None,
3966 Some("rate limited (quota) while recovering the fix report".to_owned()),
3967 ContinuationRecord {
3968 attempts,
3969 cumulative_wait_ms,
3970 outcome: ContinuationOutcome::QuotaLost,
3971 },
3972 );
3973 }
3974 AgentOutcome::Dropped(o) => {
3975 cumulative_wait_ms += o.duration_ms;
3976 let why = o
3977 .dropped
3978 .as_ref()
3979 .map(|d| d.why.as_str())
3980 .unwrap_or("the CLI ended the stream without delivering its answer");
3981 last_err = format!("the CLI dropped the stream ({why})");
3982 }
3983 AgentOutcome::Failed(e) => last_err = e,
3984 AgentOutcome::ResourceBlocked(e) => {
3987 return (
3988 seat,
3989 None,
3990 Some(format!("the continuation was not started: {e}")),
3991 ContinuationRecord {
3992 attempts: attempts.saturating_sub(1),
3993 cumulative_wait_ms,
3994 outcome: ContinuationOutcome::NoSession,
3995 },
3996 );
3997 }
3998 }
3999 }
4000 }
4001
4002 fn after_implement(&mut self) -> Result<()> {
4003 if !self.state.candidate_preservation.is_empty() {
4004 bail!("candidate preservation incomplete; judging is unsafe");
4005 }
4006 if self.state.leaks.is_empty() {
4008 let cfg = self.state.config.blind.clone();
4009 let mut leaks = Vec::new();
4010 for c in &self.state.candidates {
4011 let Some(patch) =
4012 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013 else {
4014 continue;
4015 };
4016 leaks.extend(blind::scan(
4017 &format!("candidate {} patch", c.label),
4018 &patch,
4019 &cfg.vendor_tokens,
4020 ));
4021 }
4022 if !leaks.is_empty() {
4023 let summary = leaks
4024 .iter()
4025 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026 .collect::<Vec<_>>()
4027 .join(", ");
4028 match cfg.on_leak {
4029 LeakPolicy::Fail => {
4030 self.state.status = RunStatus::Failed;
4031 self.state
4032 .event("blind", format!("vendor text in a patch: {summary}"));
4033 self.state.leaks = leaks;
4034 self.state.save()?;
4035 self.settle_questions();
4036 bail!(
4037 "blind.on_leak = \"fail\" and vendor text reached a \
4038 judged patch: {summary}"
4039 );
4040 }
4041 LeakPolicy::Redact => self.state.event(
4042 "blind",
4043 format!("redacting vendor text for judging: {summary}"),
4044 ),
4045 LeakPolicy::Warn => self.state.event(
4046 "blind",
4047 format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048 ),
4049 }
4050 self.state.leaks = leaks;
4051 }
4052 }
4053
4054 if self.state.viable().is_empty() {
4055 if self.state.all_candidates_verified_noop() {
4056 self.state.status = RunStatus::VerifiedNoop;
4067 self.state.save()?;
4068 self.settle_questions();
4069 return Ok(());
4070 }
4071 self.state.status = RunStatus::Failed;
4072 self.state.save()?;
4073 self.settle_questions();
4074 bail!("no candidate produced a change; nothing to judge");
4075 }
4076 self.state.status = RunStatus::Judging;
4077 self.state.save()?;
4078 Ok(())
4079 }
4080
4081 async fn judge(&mut self) -> Result<()> {
4084 let run_id = self.state.id.clone();
4089 let prompts = self.state.config.prompts.clone();
4090 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091 return Ok(());
4092 }
4093 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094 if viable.len() == 1 {
4095 self.state.judge_skipped = true;
4102 self.state.event(
4103 "judge",
4104 format!(
4105 "only candidate {} produced a change; judging skipped",
4106 viable[0].label
4107 ),
4108 );
4109 self.state.save()?;
4110 return Ok(());
4111 }
4112 self.state.status = RunStatus::Judging;
4113
4114 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115 let language = self.state.config.graph.language.clone();
4116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117 let sessions = self.state.config.graph.sessions;
4118 let artifacts = agent::artifacts_dir(&self.state.dir());
4119 let root = self.state.worktree_root();
4120 let base_short = short(&self.state.base_commit);
4121
4122 let mut jobs = Vec::new();
4123 let mut orders = Vec::new();
4124 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128 let seat_key = format!("judge-{}", j + 1);
4129 let seat = self.seat(&seat_key, &spec.id);
4130 jobs.push(SeatJob {
4131 prompt: prompt::judge(
4132 &self.state.instruction,
4133 &views,
4134 self.roles.judges.len(),
4135 &base_short,
4136 &language,
4137 ),
4138 spec,
4139 seat,
4140 cwd: root.join(format!("judge-{}", j + 1)),
4141 timeout,
4142 allow_write: false,
4143 sessions,
4144 artifacts: artifacts.clone(),
4145 stem: format!("judge-{}", j + 1),
4146 handover: None,
4147 });
4148 }
4149
4150 self.state.event(
4151 "judge",
4152 format!(
4153 "{} judges ranking {} candidates blind",
4154 jobs.len(),
4155 viable.len()
4156 ),
4157 );
4158 let labels_for_check = labels.clone();
4159 let mut quota_losses = Vec::new();
4160 let cache = self.state.config.cache_dir();
4161 let ctx = WaveCtx {
4162 carry_seats: false,
4163 pauses: Some((&self.pause, &self.interrupt)),
4164 run: &run_id,
4165 node: "judge",
4166 prompts: &prompts,
4167 cache: cache.as_deref(),
4168 round: None,
4169 };
4170 let results = ask_json_wave::<Ranking>(
4171 jobs,
4172 Arc::clone(&self.sem),
4173 self.state.config.graph.retries,
4174 &self.roles.judge_roster,
4175 &ctx,
4176 &mut quota_losses,
4177 &mut self.state,
4178 &move |r: &Ranking| r.validate(&labels_for_check),
4179 )
4180 .await;
4181 if self.state.resource_block.is_some() {
4182 return self.stop_for_resource();
4183 }
4184 self.state.quota.extend(quota_losses);
4185
4186 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187 let agent_id = seat.agent.clone();
4188 self.state.seats.insert(seat.key.clone(), seat);
4189 let mut record = Judgement {
4190 judge: j + 1,
4191 seat: format!("judge-{}", j + 1),
4192 agent: agent_id,
4193 ranking: Vec::new(),
4194 reasons: BTreeMap::new(),
4195 confidence: None,
4196 strengths: BTreeMap::new(),
4197 order: orders[j].clone(),
4198 failed: None,
4199 duration_ms: 0,
4200 };
4201 match res {
4202 Ok((ranking, out)) => {
4203 record.ranking = ranking.normalized();
4204 record.reasons = ranking.reasons;
4205 record.strengths = ranking.strengths;
4206 record.confidence = ranking.confidence;
4207 record.duration_ms = out.duration_ms;
4208 self.state.event(
4209 "judge",
4210 format!(
4211 "judge {} ranked {}",
4212 j + 1,
4213 record.ranking.iter().collect::<String>()
4214 ),
4215 );
4216 }
4217 Err(e) => {
4218 record.failed = Some(e.to_string());
4219 self.state
4220 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221 }
4222 }
4223 self.state.judgements.push(record);
4224 self.state.save()?;
4225 }
4226 Ok(())
4227 }
4228
4229 async fn deliberate(&mut self) -> Result<()> {
4232 if self.hold_base_sync().await? {
4233 return Ok(());
4234 }
4235 let run_id = self.state.id.clone();
4240 let prompts = self.state.config.prompts.clone();
4241 let rounds = self.state.config.graph.deliberate_rounds;
4242 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244 return Ok(());
4245 }
4246 let tops: Vec<char> = self
4247 .state
4248 .judgements
4249 .iter()
4250 .filter_map(|j| j.ranking.first().copied())
4251 .collect();
4252 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254 self.state.event(
4255 "deliberate",
4256 format!("judges agreed on {} outright; no deliberation", tops[0]),
4257 );
4258 }
4259 self.state.status = RunStatus::Voting;
4260 self.state.save()?;
4261 return Ok(());
4262 }
4263
4264 self.state.status = RunStatus::Deliberating;
4265 self.state.event(
4266 "deliberate",
4267 format!(
4268 "split: first choices were {} — opening {rounds} round(s)",
4269 tops.iter().collect::<String>()
4270 ),
4271 );
4272
4273 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274 let language = self.state.config.graph.language.clone();
4275 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276 let sessions = self.state.config.graph.sessions;
4277 let artifacts = agent::artifacts_dir(&self.state.dir());
4278 let root = self.state.worktree_root();
4279 let base_short = short(&self.state.base_commit);
4280
4281 for round in self.state.deliberation.len() + 1..=rounds {
4285 let mut turns: Vec<DeliberationTurn> = Vec::new();
4286 let mut round_losses = Vec::new();
4287 let mut asked = 0;
4288 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289 if self.state.judgements[j].failed.is_some() {
4290 continue;
4291 }
4292 let seat_key = format!("judge-{}", j + 1);
4293 let spec = self.occupant(&seat_key, spec);
4294 let mut seat = self.seat(&seat_key, &spec.id);
4295 let transcript = self.transcript(&turns, j);
4296 let build = |context: Option<&str>| {
4297 prompt::deliberate(
4298 &self.state.instruction,
4299 context,
4300 &transcript,
4301 round,
4302 rounds,
4303 &language,
4304 )
4305 };
4306 let block = self.candidate_block(&viable, &base_short);
4307 let full = build(Some(&block));
4308 let text = if has_context(&spec, &seat, sessions) {
4309 build(None)
4310 } else {
4311 full.clone()
4312 };
4313 let job = SeatJob {
4314 spec,
4315 seat: seat.clone(),
4316 prompt: text,
4317 cwd: root.join(format!("judge-{}", j + 1)),
4318 timeout,
4319 allow_write: false,
4320 sessions,
4321 artifacts: artifacts.clone(),
4322 stem: format!("delib-{round}-judge-{}", j + 1),
4323 handover: Some(full),
4324 };
4325 let cache = self.state.config.cache_dir();
4326 let ctx = WaveCtx {
4327 carry_seats: false,
4328 pauses: Some((&self.pause, &self.interrupt)),
4329 run: &run_id,
4330 node: "deliberate",
4331 prompts: &prompts,
4332 cache: cache.as_deref(),
4333 round: None,
4334 };
4335 let mut losses = Vec::new();
4339 asked += 1;
4340 let mut results = ask_wave_with::<String>(
4341 vec![job],
4342 Arc::clone(&self.sem),
4343 0,
4344 &self.roles.judge_roster,
4345 &ctx,
4346 &mut losses,
4347 &mut self.state,
4348 &|text: &str| {
4349 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350 },
4351 )
4352 .await;
4353 if self.state.resource_block.is_some() {
4354 return self.stop_for_resource();
4355 }
4356 round_losses.extend(losses.iter().cloned());
4357 self.state.quota.extend(losses);
4358 let (updated, res, _) = results.pop().expect("one job in, one result out");
4359 seat = updated;
4360 let agent_id = seat.agent.clone();
4361 self.state.seats.insert(seat.key.clone(), seat);
4362 let body = match res {
4363 Ok((body, _)) => body,
4364 Err(e) => {
4367 self.state
4368 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369 continue;
4370 }
4371 };
4372 let tentative = verdict::extract_json::<Position>(&body)
4373 .ok()
4374 .and_then(|p| p.tentative)
4375 .and_then(|s| s.trim().chars().next())
4376 .map(|c| c.to_ascii_uppercase());
4377 self.state.event(
4378 "deliberate",
4379 format!(
4380 "round {round}: judge {} now favours {}",
4381 j + 1,
4382 tentative.map_or("—".to_owned(), |c| c.to_string())
4383 ),
4384 );
4385 turns.push(DeliberationTurn {
4386 judge: j + 1,
4387 agent: agent_id,
4388 body: blind::sanitize_prose(&body, &self.state.config.blind),
4389 tentative,
4390 });
4391 }
4392 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394 return self.stop_for_resource();
4395 }
4396 self.state
4397 .deliberation
4398 .push(DeliberationRound { round, turns });
4399 self.state.save()?;
4400 }
4401
4402 self.state.status = RunStatus::Voting;
4403 self.state.save()?;
4404 Ok(())
4405 }
4406
4407 async fn vote(&mut self) -> Result<()> {
4410 if self.hold_base_sync().await? {
4411 return Ok(());
4412 }
4413 let run_id = self.state.id.clone();
4418 let prompts = self.state.config.prompts.clone();
4419 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420 return Ok(());
4421 }
4422 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423 if viable.len() == 1 {
4424 return Ok(());
4425 }
4426 self.state.status = RunStatus::Voting;
4427
4428 let language = self.state.config.graph.language.clone();
4429 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430 let sessions = self.state.config.graph.sessions;
4431 let artifacts = agent::artifacts_dir(&self.state.dir());
4432 let root = self.state.worktree_root();
4433 let base_short = short(&self.state.base_commit);
4434 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436 let mut jobs = Vec::new();
4437 let mut seats_at = Vec::new();
4438 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439 if self
4440 .state
4441 .judgements
4442 .get(j)
4443 .is_some_and(|r| r.failed.is_some())
4444 {
4445 continue;
4446 }
4447 let seat_key = format!("judge-{}", j + 1);
4448 let spec = self.occupant(&seat_key, spec);
4449 let seat = self.seat(&seat_key, &spec.id);
4450 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451 let text = if has_context(&spec, &seat, sessions) {
4452 prompt::final_vote(&viable, &language)
4453 } else {
4454 full.clone()
4455 };
4456 jobs.push(SeatJob {
4457 spec,
4458 seat,
4459 prompt: text,
4460 cwd: root.join(format!("judge-{}", j + 1)),
4461 timeout,
4462 allow_write: false,
4463 sessions,
4464 artifacts: artifacts.clone(),
4465 stem: format!("vote-judge-{}", j + 1),
4466 handover: Some(full),
4467 });
4468 seats_at.push(j);
4469 }
4470
4471 self.state.event(
4472 "vote",
4473 format!(
4474 "collecting {} final votes one by one, privately",
4475 jobs.len()
4476 ),
4477 );
4478 let allowed = viable.clone();
4479 let mut quota_losses = Vec::new();
4480 let cache = self.state.config.cache_dir();
4481 let ctx = WaveCtx {
4482 carry_seats: false,
4483 pauses: Some((&self.pause, &self.interrupt)),
4484 run: &run_id,
4485 node: "vote",
4486 prompts: &prompts,
4487 cache: cache.as_deref(),
4488 round: None,
4489 };
4490 let results = ask_json_wave::<FinalVote>(
4491 jobs,
4492 Arc::clone(&self.sem),
4493 self.state.config.graph.retries,
4494 &self.roles.judge_roster,
4495 &ctx,
4496 &mut quota_losses,
4497 &mut self.state,
4498 &move |v: &FinalVote| match v.label() {
4499 Some(c) if allowed.contains(&c) => Ok(()),
4500 other => bail!("vote {other:?} is not one of {allowed:?}"),
4501 },
4502 )
4503 .await;
4504 if self.state.resource_block.is_some() {
4505 return self.stop_for_resource();
4506 }
4507 self.state.quota.extend(quota_losses);
4508
4509 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510 let agent_id = seat.agent.clone();
4511 self.state.seats.insert(seat.key.clone(), seat);
4512 let initial = self
4513 .state
4514 .judgements
4515 .get(j)
4516 .and_then(|r| r.ranking.first().copied());
4517 let mut record = VoteRecord {
4518 judge: j + 1,
4519 agent: agent_id,
4520 vote: None,
4521 reason: String::new(),
4522 changed: false,
4523 };
4524 match res {
4525 Ok((v, _)) => {
4526 record.vote = v.label();
4527 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529 self.state.event(
4530 "vote",
4531 format!(
4532 "judge {} voted {}{}",
4533 j + 1,
4534 record.vote.unwrap_or('?'),
4535 if record.changed { " (changed)" } else { "" }
4536 ),
4537 );
4538 }
4539 Err(e) => {
4540 self.state
4541 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542 }
4543 }
4544 self.state.votes.push(record);
4545 self.state.save()?;
4546 }
4547 Ok(())
4548 }
4549
4550 fn tally(&mut self) -> Result<()> {
4553 if self.state.tally.is_some() {
4554 return Ok(());
4555 }
4556 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557 let tops: Vec<char> = self
4558 .state
4559 .judgements
4560 .iter()
4561 .filter_map(|j| j.ranking.first().copied())
4562 .collect();
4563 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568 let mut cast: Vec<char> = Vec::new();
4569 for (i, j) in self.state.judgements.iter().enumerate() {
4570 let vote = self
4571 .state
4572 .votes
4573 .iter()
4574 .find(|v| v.judge == i + 1)
4575 .and_then(|v| v.vote)
4576 .or_else(|| j.ranking.first().copied());
4577 if let Some(v) = vote {
4578 *first_choice.entry(v).or_insert(0) += 1;
4579 cast.push(v);
4580 }
4581 }
4582
4583 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584 for j in &self.state.judgements {
4585 let n = j.ranking.len();
4586 for (pos, label) in j.ranking.iter().enumerate() {
4587 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588 }
4589 }
4590
4591 let best = first_choice.values().copied().max().unwrap_or(0);
4592 let mut leaders: Vec<char> = first_choice
4593 .iter()
4594 .filter(|(_, v)| **v == best)
4595 .map(|(k, _)| *k)
4596 .collect();
4597 let mut tie_break = None;
4598 if leaders.len() > 1 {
4599 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600 let borda_leaders: Vec<char> = leaders
4601 .iter()
4602 .copied()
4603 .filter(|l| borda[l] == top_borda)
4604 .collect();
4605 tie_break = Some(if borda_leaders.len() == 1 {
4606 format!(
4607 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608 leaders.len()
4609 )
4610 } else {
4611 format!(
4612 "{} way tie on both first-choice votes and Borda points, broken by label order",
4613 leaders.len()
4614 )
4615 });
4616 leaders = borda_leaders;
4617 leaders.sort_unstable();
4618 }
4619 let winner = *leaders
4620 .first()
4621 .or(viable.first())
4622 .context("no candidate to declare a winner from")?;
4623
4624 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626 let deliberated = !self.state.deliberation.is_empty();
4627
4628 let quota_seats: std::collections::BTreeSet<&str> =
4632 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633 let mut present = 0usize;
4634 for (i, j) in self.state.judgements.iter().enumerate() {
4635 if quota_seats.contains(j.seat.as_str()) {
4636 continue;
4637 }
4638 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639 let voted = self
4640 .state
4641 .votes
4642 .iter()
4643 .any(|v| v.judge == i + 1 && v.vote.is_some());
4644 if ranked || voted {
4645 present += 1;
4646 }
4647 }
4648 let needs_quorum = viable.len() > 1;
4654 let judges_total = if needs_quorum {
4655 self.roles.judges.len()
4656 } else {
4657 0
4658 };
4659 let quorum = if needs_quorum {
4660 judges_total / 2 + 1
4661 } else {
4662 0
4663 };
4664 let met_quorum = !needs_quorum || present >= quorum;
4665 let uncontested = (!needs_quorum).then(|| {
4666 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667 });
4668
4669 self.state.event(
4670 "tally",
4671 match &uncontested {
4672 Some(reason) => format!("winner {winner} — {reason}"),
4673 None => format!(
4674 "winner {winner} — votes {} | initial {} | {} changed | \
4675 {present}/{judges_total} judges{}",
4676 first_choice
4677 .iter()
4678 .map(|(k, v)| format!("{k}:{v}"))
4679 .collect::<Vec<_>>()
4680 .join(" "),
4681 if unanimous_initial {
4682 "unanimous"
4683 } else {
4684 "split"
4685 },
4686 changed_votes,
4687 if met_quorum {
4688 String::new()
4689 } else {
4690 format!(" — below quorum ({quorum} required)")
4691 },
4692 ),
4693 },
4694 );
4695 if !met_quorum {
4696 self.state.event(
4697 "stall",
4698 format!(
4699 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700 the run stops here, resumable"
4701 ),
4702 );
4703 }
4704 self.state.tally = Some(Tally {
4705 first_choice,
4706 borda,
4707 winner,
4708 rankings: tops.len(),
4709 unanimous_initial,
4710 deliberated,
4711 changed_votes,
4712 unanimous_final,
4713 tie_break,
4714 judges: judges_total,
4715 present,
4716 quorum,
4717 met_quorum,
4718 uncontested,
4719 });
4720 self.state.status = if met_quorum {
4721 RunStatus::Reviewing
4722 } else {
4723 RunStatus::Stalled
4724 };
4725 self.state.save()?;
4726 Ok(())
4727 }
4728
4729 #[allow(clippy::too_many_lines)]
4750 async fn recover_stall(&mut self) -> Result<bool> {
4751 let run_id = self.state.id.clone();
4756 let prompts = self.state.config.prompts.clone();
4757 let quota_seats: BTreeSet<&str> =
4762 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763 let absent: Vec<String> = self
4764 .state
4765 .judgements
4766 .iter()
4767 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768 .map(|j| j.seat.clone())
4769 .collect();
4770 if absent.is_empty() {
4771 return Ok(false);
4772 }
4773 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774 if viable.len() <= 1 {
4775 return Ok(false);
4776 }
4777 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778 let language = self.state.config.graph.language.clone();
4779 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780 let sessions = self.state.config.graph.sessions;
4781 let artifacts = agent::artifacts_dir(&self.state.dir());
4782 let root = self.state.worktree_root();
4783 let base_short = short(&self.state.base_commit);
4784 let candidates: Vec<Candidate> = viable.clone();
4785
4786 let mut positions: Vec<usize> = absent
4788 .iter()
4789 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790 .collect();
4791 if positions.is_empty() {
4792 return Ok(false);
4793 }
4794 positions.sort_unstable();
4795 positions.dedup();
4796
4797 let mut judge_jobs = Vec::new();
4799 for &j in &positions {
4800 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802 let seat_key = format!("judge-{}", j + 1);
4803 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804 let seat = self.seat(&seat_key, &spec.id);
4805 judge_jobs.push(SeatJob {
4806 spec,
4807 seat,
4808 prompt: prompt::judge(
4809 &self.state.instruction,
4810 &views,
4811 self.roles.judges.len(),
4812 &base_short,
4813 &language,
4814 ),
4815 cwd: root.join(seat_key),
4816 timeout,
4817 allow_write: false,
4818 sessions,
4819 artifacts: artifacts.clone(),
4820 stem: format!("judge-{}-recover", j + 1),
4821 handover: None,
4822 });
4823 }
4824
4825 let labels_for_check = labels.clone();
4826 let mut judge_losses = Vec::new();
4827 let retries = self.state.config.graph.retries;
4828 let cache = self.state.config.cache_dir();
4829 let ctx = WaveCtx {
4830 carry_seats: false,
4831 pauses: Some((&self.pause, &self.interrupt)),
4832 run: &run_id,
4833 node: "judge",
4834 prompts: &prompts,
4835 cache: cache.as_deref(),
4836 round: None,
4837 };
4838 let results = ask_json_wave::<Ranking>(
4839 judge_jobs,
4840 Arc::clone(&self.sem),
4841 retries,
4842 &self.roles.judge_roster,
4843 &ctx,
4844 &mut judge_losses,
4845 &mut self.state,
4846 &move |r: &Ranking| r.validate(&labels_for_check),
4847 )
4848 .await;
4849 if self.state.resource_block.is_some() {
4850 self.stop_for_resource()?;
4851 return Ok(false);
4852 }
4853
4854 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857 let agent_id = seat.agent.clone();
4858 self.state.seats.insert(seat.key.clone(), seat);
4859 let record = &mut self.state.judgements[j];
4860 match res {
4861 Ok((ranking, out)) => {
4862 record.agent = agent_id;
4863 record.ranking = ranking.normalized();
4864 record.reasons = ranking.reasons;
4865 record.strengths = ranking.strengths;
4866 record.confidence = ranking.confidence;
4867 record.failed = None;
4868 record.duration_ms = out.duration_ms;
4869 recovered.insert(j);
4870 self.state.event(
4871 "recover",
4872 format!("judge {} ranked again after the limit", j + 1),
4873 );
4874 }
4875 Err(e) => {
4876 self.state
4877 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878 }
4879 }
4880 }
4881
4882 let mut vote_jobs = Vec::new();
4884 let mut vote_pos: Vec<usize> = Vec::new();
4885 for &j in &recovered {
4886 let seat_key = format!("judge-{}", j + 1);
4887 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888 let seat = self.seat(&seat_key, &spec.id);
4889 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890 let text = if has_context(&spec, &seat, sessions) {
4891 prompt::final_vote(&labels, &language)
4892 } else {
4893 full.clone()
4894 };
4895 vote_jobs.push(SeatJob {
4896 spec,
4897 seat,
4898 prompt: text,
4899 cwd: root.join(seat_key),
4900 timeout,
4901 allow_write: false,
4902 sessions,
4903 artifacts: artifacts.clone(),
4904 stem: format!("vote-judge-{}-recover", j + 1),
4905 handover: Some(full),
4906 });
4907 vote_pos.push(j);
4908 }
4909 let allowed = labels.clone();
4910 let mut vote_losses = Vec::new();
4911 let vote_retries = self.state.config.graph.retries;
4912 let vote_cache = self.state.config.cache_dir();
4913 let ctx = WaveCtx {
4914 carry_seats: false,
4915 pauses: Some((&self.pause, &self.interrupt)),
4916 run: &run_id,
4917 node: "vote",
4918 prompts: &prompts,
4919 cache: vote_cache.as_deref(),
4920 round: None,
4921 };
4922 let votes = ask_json_wave::<FinalVote>(
4923 vote_jobs,
4924 Arc::clone(&self.sem),
4925 vote_retries,
4926 &self.roles.judge_roster,
4927 &ctx,
4928 &mut vote_losses,
4929 &mut self.state,
4930 &move |v: &FinalVote| match v.label() {
4931 Some(c) if allowed.contains(&c) => Ok(()),
4932 other => bail!("vote {other:?} is not one of {allowed:?}"),
4933 },
4934 )
4935 .await;
4936 if self.state.resource_block.is_some() {
4937 self.stop_for_resource()?;
4938 return Ok(false);
4939 }
4940 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941 let agent_id = seat.agent.clone();
4942 self.state.seats.insert(seat.key.clone(), seat);
4943 match res {
4944 Ok((v, _)) => {
4945 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946 rec.vote = v.label();
4947 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948 } else {
4949 self.state.votes.push(VoteRecord {
4950 judge: j + 1,
4951 agent: agent_id,
4952 vote: v.label(),
4953 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954 changed: false,
4955 });
4956 }
4957 self.state.event(
4958 "recover",
4959 format!("judge {} voted again after the limit", j + 1),
4960 );
4961 }
4962 Err(e) => {
4963 self.state
4964 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965 }
4966 }
4967 }
4968
4969 let recovered_keys: BTreeSet<String> = recovered
4973 .iter()
4974 .map(|&j| format!("judge-{}", j + 1))
4975 .collect();
4976 self.state
4977 .quota
4978 .retain(|q| !recovered_keys.contains(&q.seat));
4979 for loss in judge_losses.into_iter().chain(vote_losses) {
4983 if recovered_keys.contains(&loss.seat) {
4984 continue;
4985 }
4986 self.state.quota.retain(|q| q.seat != loss.seat);
4987 self.state.quota.push(loss);
4988 }
4989
4990 self.state.tally = None;
4992 self.tally()?;
4993 Ok(self
4994 .state
4995 .tally
4996 .as_ref()
4997 .map(|t| t.met_quorum)
4998 .unwrap_or(false))
4999 }
5000
5001 async fn graft(&mut self) -> Result<()> {
5017 let Err(e) = self.graft_inner().await else {
5018 return Ok(());
5019 };
5020 if e.is::<crate::repo_lock::ResourceBlocked>() {
5021 return Err(e);
5022 }
5023 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024 self.state.status = RunStatus::Blocked;
5025 self.state.event("graft", format!("blocked: {e:#}"));
5026 self.state.save()?;
5027 return Err(e);
5028 }
5029 if let Some(rec) = self.state.graft.clone()
5034 && !rec.outcome.settled()
5035 {
5036 if let Some(winner) = self.state.winner().cloned() {
5037 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038 if re.is::<crate::repo_lock::ResourceBlocked>() {
5039 return Err(re);
5040 }
5041 self.state.status = RunStatus::Blocked;
5042 self.state.event("graft", format!("blocked: {re:#}"));
5043 self.state.save()?;
5044 return Err(re);
5045 }
5046 }
5047 if let Some(g) = self.state.graft.as_mut() {
5048 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049 }
5050 }
5051 self.state.event("graft", format!("skipped: {e:#}"));
5052 if let Err(e) = self.state.save() {
5053 tracing::warn!("save after graft: {e:#}");
5054 }
5055 Ok(())
5056 }
5057
5058 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061 let _coordination = self.coordinate(wt).await?;
5062 crate::recovery::restore_repair(wt, pre)
5063 .await
5064 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065 }
5066
5067 fn graft_skipped(&mut self, why: &str) {
5069 let message = format!("skipped: {why}");
5070 if !self
5071 .state
5072 .events
5073 .iter()
5074 .any(|e| e.node == "graft" && e.message == message)
5075 {
5076 self.state.event("graft", message);
5077 }
5078 }
5079
5080 async fn graft_inner(&mut self) -> Result<()> {
5081 if let Some(rec) = self.state.graft.clone()
5084 && !rec.outcome.settled()
5085 {
5086 if self.state.reviews.is_empty()
5090 && let Some(winner) = self.state.winner().cloned()
5091 {
5092 self.graft_restore(&winner.worktree, &rec.pre).await?;
5093 }
5094 if rec.outcome == GraftOutcome::WaitingForRestore {
5095 self.state.graft = None;
5096 self.state.save()?;
5097 } else {
5098 if let Some(g) = self.state.graft.as_mut() {
5099 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100 }
5101 self.state.event(
5102 "graft",
5103 "rolled back: the previous round was interrupted; the winner is untouched",
5104 );
5105 self.state.save()?;
5106 return Ok(());
5107 }
5108 }
5109 let plan = match graft::plan(&self.state) {
5110 Ok(plan) => plan,
5111 Err(skip) => {
5112 if skip != graft::Skip::AlreadyDone {
5113 self.graft_skipped(skip.reason());
5114 self.state.save()?;
5115 }
5116 return Ok(());
5117 }
5118 };
5119 let Some(winner) = self.state.winner().cloned() else {
5120 return Ok(());
5121 };
5122 let wt = winner.worktree.clone();
5123 if !git::is_clean(&wt).await? {
5124 self.graft_skipped("the winner's worktree has uncommitted changes");
5125 self.state.save()?;
5126 return Ok(());
5127 }
5128 let pre = git::rev_parse(&wt, "HEAD").await?;
5129 self.state.graft = Some(GraftRecord {
5132 pre: pre.clone(),
5133 post: None,
5134 lines: None,
5135 limit: None,
5136 agent: None,
5137 outcome: GraftOutcome::Running,
5138 requested: graft::requested(&plan),
5139 adoption: None,
5140 adoption_error: None,
5141 });
5142 self.state.event(
5143 "graft",
5144 format!(
5145 "porting strengths of candidate(s) {} onto {}",
5146 plan.strengths
5147 .iter()
5148 .map(|(l, _, _)| l.to_string())
5149 .collect::<Vec<_>>()
5150 .join(", "),
5151 plan.winner
5152 ),
5153 );
5154 self.state.save()?;
5155
5156 let fix_prompt = prompt::graft(
5157 &self.state.instruction,
5158 plan.winner,
5159 &plan.strengths,
5160 &self.state.config.graph.language,
5161 );
5162 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163 let sessions = self.state.config.graph.sessions;
5164 let artifacts = agent::artifacts_dir(&self.state.dir());
5165 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166 let (_, seat, out) = self
5167 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168 SeatJob {
5169 prompt: fix_prompt.clone(),
5170 spec,
5171 seat,
5172 cwd: wt.clone(),
5173 timeout,
5174 allow_write: true,
5175 sessions,
5176 artifacts: artifacts.clone(),
5177 stem: "graft".to_owned(),
5178 handover: None,
5179 }
5180 })
5181 .await;
5182 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183 self.state.graft.as_mut().expect("graft record").outcome =
5186 GraftOutcome::WaitingForRestore;
5187 return self.stop_for_resource();
5188 }
5189 let mut why: Option<String> = None;
5190 match &out {
5191 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192 why = Some("the fixer produced nothing usable".to_owned());
5193 }
5194 AgentOutcome::Ok(_) => {}
5195 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196 AgentOutcome::Quota(o) => {
5197 self.state.quota.push(QuotaLoss::from_report(
5198 seat.key.clone(),
5199 "graft",
5200 o.quota.as_ref(),
5201 ));
5202 why = Some("rate limited (quota)".to_owned());
5203 }
5204 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205 why = Some(format!(
5206 "the fixer failed: {}",
5207 e.lines()
5208 .next()
5209 .unwrap_or("")
5210 .chars()
5211 .take(160)
5212 .collect::<String>()
5213 ));
5214 }
5215 }
5216 let agent_id = seat.agent.clone();
5217 self.state.seats.insert(seat.key.clone(), seat);
5218
5219 let mut post = None;
5220 let mut lines = None;
5221 let _coordination = self.coordinate(&wt).await?;
5222 if why.is_none() {
5223 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224 if let Ok(r) = &rescued {
5225 self.state.note_withheld("graft", &r.withheld);
5226 }
5227 let head = git::rev_parse(&wt, "HEAD").await?;
5228 let dirty = match &rescued {
5232 Ok(r) => git::status_porcelain(&wt)
5233 .await
5234 .map(|st| {
5235 st.lines().any(|l| {
5236 !r.withheld
5237 .iter()
5238 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239 })
5240 })
5241 .unwrap_or(true),
5242 Err(_) => true,
5243 };
5244 if dirty {
5245 why = Some("the fixer left changes that could not be committed".to_owned());
5246 } else if head == pre {
5247 why = Some("the fixer changed nothing".to_owned());
5248 } else {
5249 lines = git::changed_lines(&wt, &pre, &head).await;
5250 post = Some(head);
5251 }
5252 }
5253 let (adoption, adoption_error) = match &out {
5254 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255 _ => (None, None),
5256 };
5257 let requested = graft::requested(&plan);
5258 let outcome = match why {
5259 None => {
5260 self.state.event(
5261 "graft",
5262 format!(
5263 "applied: {}",
5264 lines.map_or_else(
5265 || "changed lines unavailable".to_owned(),
5266 |n| format!("{n} changed lines")
5267 )
5268 ),
5269 );
5270 GraftOutcome::Applied
5271 }
5272 Some(why) => {
5273 crate::recovery::restore_repair(&wt, &pre)
5276 .await
5277 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278 self.state.event(
5279 "graft",
5280 format!("rolled back: {why}; the winner is untouched"),
5281 );
5282 GraftOutcome::RolledBack(why)
5283 }
5284 };
5285 self.state.graft = Some(GraftRecord {
5286 pre,
5287 post,
5288 lines,
5289 limit: None,
5290 agent: Some(agent_id),
5291 outcome,
5292 requested,
5293 adoption,
5294 adoption_error,
5295 });
5296 self.state.save()?;
5297 Ok(())
5298 }
5299
5300 async fn fold_losers(&mut self) -> Result<()> {
5303 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304 return Ok(());
5305 };
5306 if !self.state.candidate_preservation.is_empty() {
5307 bail!("candidate preservation incomplete; refusing cleanup");
5308 }
5309 let repo = self.state.repo.clone();
5310 let _coordination = self.coordinate(&repo).await?;
5311 let mut folded = Vec::new();
5312 for i in 0..self.state.candidates.len() {
5313 let c = &self.state.candidates[i];
5314 if c.label == winner || c.folded {
5315 continue;
5316 }
5317 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318 remove_preserved_worktree(&repo, &wt).await?;
5319 if git::branch_exists(&repo, &branch).await?
5320 && !git::branch_delete(&repo, &branch).await?
5321 {
5322 bail!("candidate branch removal refused");
5323 }
5324 self.state.candidates[i].folded = true;
5325 self.state.save()?;
5326 folded.push(label.to_string());
5327 }
5328 let root = self.state.worktree_root();
5330 for j in 1..=self.roles.judges.len() {
5331 let wt = root.join(format!("judge-{j}"));
5332 if wt.exists() {
5333 remove_read_seat_worktree(&mut self.state, &wt).await?;
5334 self.state.save()?;
5335 }
5336 }
5337 if self.state.config.graph.advise {
5340 for k in 1..=self.state.config.graph.advisors {
5341 let wt = root.join(format!("advisor-{k}"));
5342 if wt.exists() {
5343 remove_read_seat_worktree(&mut self.state, &wt).await?;
5344 self.state.save()?;
5345 }
5346 }
5347 }
5348 if !folded.is_empty() {
5349 self.state
5350 .event("fold", format!("folded candidates {}", folded.join(", ")));
5351 self.state.save()?;
5352 }
5353 Ok(())
5354 }
5355
5356 async fn hold_base_sync(&mut self) -> Result<bool> {
5359 self.hold_base_sync_as(None).await
5360 }
5361
5362 async fn hold_base_sync_kind(
5363 &mut self,
5364 kind: crate::base_sync_recovery::StopKind,
5365 ) -> Result<()> {
5366 self.hold_base_sync_as(Some(kind)).await?;
5367 Ok(())
5368 }
5369
5370 async fn hold_base_sync_as(
5371 &mut self,
5372 kind: Option<crate::base_sync_recovery::StopKind>,
5373 ) -> Result<bool> {
5374 if !self.state.base_sync_blocked() {
5375 self.state.base_sync_recovery.stop = None;
5376 return Ok(false);
5377 }
5378 let head = match self.state.winner() {
5379 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5380 .await
5381 .unwrap_or_default(),
5382 None => String::new(),
5383 };
5384 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5385 && !authorization.consumed
5386 && authorization.stop.head != head
5387 {
5388 authorization.refuse(format!(
5389 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5390 authorization.stop.head
5391 ));
5392 }
5393 self.state.record_base_sync_stop(&head);
5394 if let Some(kind) = kind {
5395 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5396 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5397 stop.kind = kind;
5398 }
5399 }
5400 self.state.save()?;
5401 Ok(true)
5402 }
5403
5404 async fn sync_to_base(&mut self) -> Result<()> {
5446 if self.state.status == RunStatus::AlreadyInBase {
5447 return Ok(());
5448 }
5449 self.hold_base_sync().await?;
5450 let conflicted = self
5451 .state
5452 .base_sync
5453 .as_ref()
5454 .is_some_and(|s| s.conflict.is_some());
5455 let Some(winner) = self.state.winner().cloned() else {
5456 return Ok(());
5457 };
5458
5459 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5460
5461 let repo = self.state.repo.clone();
5462 let remote = self.state.config.merge.remote.clone();
5463 let base_branch = self.state.base_branch.clone();
5464 let tracking = format!("{remote}/{base_branch}");
5465
5466 let fetched_base =
5467 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5468 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5473 return Ok(());
5474 };
5475
5476 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5477 self.state.ensure_base_episode();
5478 let mut pending = self
5479 .state
5480 .base_sync_recovery
5481 .operations
5482 .last()
5483 .filter(|o| !o.finished)
5484 .cloned();
5485 let scratch = self.state.dir().join("base-sync");
5486 if git::rebase_in_progress(&scratch).await {
5487 if pending.is_none()
5490 && self.state.base_sync_recovery.operations.is_empty()
5491 && !conflicted
5492 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5493 .await
5494 .ok()
5495 .as_deref()
5496 == Some(head.as_str())
5497 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5498 && git::is_clean(&winner.worktree).await?
5499 {
5500 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5501 if self.state.base_sync.is_none() {
5502 self.state.base_sync = Some(BaseSync {
5503 tip: saved_tip.clone(),
5504 behind: 0,
5505 attempts: 0,
5506 conflict: None,
5507 already_in: None,
5508 });
5509 }
5510 self.state.reserve_base_operation(&head, &saved_tip);
5511 self.state.event(
5512 "base-recovery",
5513 "imported standing legacy rebase as a spent synchronization reservation",
5514 );
5515 self.state.save()?;
5516 pending = self.state.base_sync_recovery.operations.last().cloned();
5517 }
5518 let expected = pending.as_ref().map(|op| op.tip.as_str());
5519 if expected.is_none()
5520 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5521 {
5522 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5523 if self.state.base_sync.is_none() {
5524 self.state.base_sync = Some(BaseSync {
5525 tip: tip.clone(),
5526 behind: 0,
5527 attempts: 0,
5528 conflict: None,
5529 already_in: None,
5530 });
5531 }
5532 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5533 self.state.event("land", why);
5534 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535 .await?;
5536 return Ok(());
5537 }
5538 }
5539 if let Some(op) = &pending {
5540 let mut replay_proven =
5542 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5543 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5544 let wait = repository_wait(&self.state, None);
5545 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5546 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5547 Ok(true) => {
5548 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5549 self.state.save()?;
5550 git::worktree_remove(&repo, &scratch).await.ok();
5551 replay_proven = true;
5552 }
5553 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5554 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5555 }
5556 }
5557 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5558 if self.state.foreign_checkout.is_some() {
5559 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5560 .await
5561 .ok()
5562 .as_deref()
5563 == Some(op.head.as_str())
5564 && git::rev_parse(&repo, &format!("{head}^2"))
5565 .await
5566 .ok()
5567 .as_deref()
5568 == Some(op.tip.as_str());
5569 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5570 .await
5571 .is_ok_and(|m| m.is_empty())
5572 && let Ok(added) =
5573 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5574 {
5575 replay_proven = true;
5576 for commit in added {
5577 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5578 replay_proven = false;
5579 break;
5580 }
5581 }
5582 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5583 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5584 }
5585 }
5586 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590 || (head != op.head
5591 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5592 .await?
5593 .is_empty())
5594 || (!git::is_clean(&winner.worktree).await?
5595 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5596 .await
5597 .is_ok_and(|o| o.ok()))
5598 || !replay_proven
5599 {
5600 let why = format!(
5601 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5602 op.id
5603 );
5604 let sync = self
5605 .state
5606 .base_sync
5607 .as_mut()
5608 .context("reserved synchronization has no lifetime accounting")?;
5609 sync.conflict = Some(why.clone());
5610 self.state.event("land", why);
5611 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5612 .await?;
5613 return Ok(());
5614 }
5615 if head != op.head {
5616 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5617 }
5618 tip = op.tip.clone();
5619 }
5620 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5621 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5622 if pending.is_none()
5623 && !conflicted
5624 && fetched_base
5625 && self
5626 .state
5627 .base_sync_recovery
5628 .episodes
5629 .last()
5630 .is_some_and(|e| e.boundary.is_some())
5631 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5632 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5633 && owner.execution_runs().last() == Some(&self.state.id)
5634 && owner.status != crate::queue::TaskStatus::Done
5635 && !owner.fresh_start
5636 && owner.review_branch.is_none()
5637 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5638 && let Some(approval) = owner.resumption.as_ref()
5639 && approval.instruction == owner.instruction
5640 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5641 && git::is_clean(&winner.worktree).await?
5642 && self.state.renew_base_episode(&task, approval, &head, &tip)
5643 {
5644 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5645 self.state.save()?;
5646 }
5647
5648 if (behind > 0 || conflicted || head == tip)
5657 && self
5658 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5659 .await?
5660 {
5661 return Ok(());
5662 }
5663 if conflicted && self.state.base_recovery_pending() {
5664 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5665 let owner =
5666 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5667 let owner_valid = owner.as_ref().is_ok_and(|t| {
5668 t.execution_runs().last() == Some(&self.state.id)
5669 && t.status != crate::queue::TaskStatus::Done
5670 && !t.fresh_start
5671 && t.review_branch.is_none()
5672 });
5673 let invalidation = if !owner_valid {
5674 Some("latest task ownership changed")
5675 } else if !owner.as_ref().is_ok_and(|t| {
5676 authorization
5677 .generation
5678 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5679 }) {
5680 Some("owner approval generation changed")
5681 } else if authorization.question.as_ref().is_some_and(|question| {
5682 !crate::base_recovery_control::decision_current(
5683 &crate::queue::Queue::at(crate::run::home().join("queue")),
5684 &self.state,
5685 &authorization.task,
5686 question,
5687 )
5688 }) {
5689 Some("recovery decision ownership, routing or stop evidence changed")
5690 } else if authorization.request.is_some()
5691 && !owner
5692 .as_ref()
5693 .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5694 {
5695 Some("owner decision changed")
5696 } else if let Some(scope) = &authorization.scope {
5697 if scope.tip != tip {
5698 Some("fetched base changed")
5699 } else if !self
5700 .state
5701 .base_sync_recovery
5702 .episodes
5703 .last()
5704 .is_some_and(|e| e.id == scope.episode)
5705 {
5706 Some("active synchronization interval changed")
5707 } else if !owner
5708 .as_ref()
5709 .is_ok_and(|t| t.instruction == scope.instruction)
5710 {
5711 Some("task instruction changed")
5712 } else {
5713 None
5714 }
5715 } else {
5716 None
5717 };
5718 if let Some(reason) = invalidation {
5719 let authorization = self
5720 .state
5721 .base_sync_recovery
5722 .authorizations
5723 .last_mut()
5724 .unwrap();
5725 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5726 self.state.save()?;
5727 return Ok(());
5728 }
5729 if !fetched_base
5730 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5731 || !git::is_clean(&winner.worktree).await?
5732 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5733 {
5734 return Ok(());
5735 }
5736 }
5737 let recovery = conflicted && self.state.consume_base_recovery(&head);
5738 if conflicted {
5739 self.state.save()?;
5741 if !recovery {
5742 return Ok(());
5743 }
5744 }
5745
5746 if behind == 0 {
5747 if let Some(from) = self
5756 .state
5757 .rebase_fixes
5758 .iter()
5759 .rev()
5760 .find_map(|r| r.from.clone())
5761 && from != head
5762 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5763 .await
5764 .is_ok_and(|o| o.ok())
5765 {
5766 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5767 }
5768 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5774 if let Some(why) = &conflict {
5775 self.state.status = RunStatus::Blocked;
5776 self.state.event("land", why.clone());
5777 }
5778 self.state.base_sync = Some(BaseSync {
5779 tip,
5780 behind: 0,
5781 attempts,
5782 conflict,
5783 already_in: None,
5784 });
5785 if pending.is_some() {
5786 self.state.finish_base_operation(Some(&head));
5787 }
5788 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5789 .await?;
5790 self.state.save()?;
5791 return Ok(());
5792 }
5793
5794 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5795 let why = format!(
5796 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5797 synchronization(s); synchronizing again would only race it",
5798 winner.branch
5799 );
5800 self.state.status = RunStatus::Blocked;
5801 self.state.base_sync = Some(BaseSync {
5802 tip,
5803 behind,
5804 attempts,
5805 conflict: Some(why.clone()),
5806 already_in: None,
5807 });
5808 self.state.event("land", why);
5809 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5810 .await?;
5811 self.state.save()?;
5812 return Ok(());
5813 }
5814
5815 if self.state.foreign_checkout.is_some() {
5816 let wait = repository_wait(&self.state, None);
5819 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5820 if self.state.base_sync.is_none() {
5821 self.state.base_sync = Some(BaseSync {
5822 tip: tip.clone(),
5823 behind,
5824 attempts: 0,
5825 conflict: None,
5826 already_in: None,
5827 });
5828 }
5829 self.state.reserve_base_operation(&head, &tip);
5830 self.state.save()?;
5831 let merged = git::git_raw(
5832 &winner.worktree,
5833 &[
5834 "-c",
5835 "core.hooksPath=/dev/null",
5836 "merge",
5837 "--no-ff",
5838 "--no-autostash",
5839 "--no-edit",
5840 "-m",
5841 &format!("Merge {tracking} before verifying"),
5842 &tip,
5843 ],
5844 )
5845 .await?;
5846 let conflict = if merged.ok() {
5847 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5848 self.state.event(
5849 "land",
5850 format!("merged {tracking} into {} before verifying", winner.branch),
5851 );
5852 None
5853 } else {
5854 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5855 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5856 }
5857 let why = format!(
5858 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5859 winner.branch,
5860 merged.stderr.chars().take(600).collect::<String>()
5861 );
5862 self.state.status = RunStatus::Blocked;
5863 self.state.event("land", &why);
5864 Some(why)
5865 };
5866 let synced = conflict.is_none();
5867 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5868 self.state
5869 .finish_base_operation(synced.then_some(fresh.as_str()));
5870 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5871 self.state.base_sync = Some(BaseSync {
5872 tip,
5873 behind: if synced { 0 } else { behind },
5874 attempts,
5875 conflict,
5876 already_in: None,
5877 });
5878 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5879 .await?;
5880 self.state.save()?;
5881 return Ok(());
5882 }
5883
5884 self.state.event(
5885 "land",
5886 format!(
5887 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5888 winner.branch
5889 ),
5890 );
5891 self.state.save()?;
5892
5893 let branch_tracking = format!("{remote}/{}", winner.branch);
5899 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5900 let remote_tip = if self.state.task_pr_branch().is_none()
5901 && matches!(&fetched_branch, Ok(o) if o.ok())
5902 {
5903 git::rev_parse(&repo, &branch_tracking).await.ok()
5904 } else {
5905 None
5906 };
5907 if let Some(theirs) = &remote_tip
5911 && !git::is_ancestor(&repo, theirs, &head).await
5912 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5913 .await
5914 .is_ok_and(|missing| missing.is_empty())
5915 {
5916 let why = format!(
5917 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5918 them",
5919 short(theirs),
5920 winner.branch
5921 );
5922 self.state.status = RunStatus::Blocked;
5923 self.state.base_sync = Some(BaseSync {
5924 tip,
5925 behind,
5926 attempts,
5927 conflict: Some(why.clone()),
5928 already_in: None,
5929 });
5930 self.state.event("land", why);
5931 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5932 .await?;
5933 self.state.save()?;
5934 return Ok(());
5935 }
5936
5937 if self.state.base_sync.is_none() {
5938 self.state.base_sync = Some(BaseSync {
5939 tip: tip.clone(),
5940 behind,
5941 attempts: 0,
5942 conflict: None,
5943 already_in: None,
5944 });
5945 }
5946 let scratch = self.state.dir().join("base-sync");
5947 if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5948 {
5949 if error.is::<crate::repo_lock::ResourceBlocked>() {
5950 return Ok(());
5951 }
5952 return Err(error);
5953 }
5954 self.state.reserve_base_operation(&head, &tip);
5955 self.state.save()?;
5956 let rebased =
5957 match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
5958 .await
5959 {
5960 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5961 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5962 Err(e) => Err(e),
5963 };
5964 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5965 let stop_kind = match &rebased {
5966 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5967 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5968 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5969 };
5970 match rebased {
5971 Ok(None) => {
5972 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5976 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5977 self.state.finish_base_operation(Some(&fresh));
5978 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5979 let mut conflict = None;
5980 if let Some(pinned) = &remote_tip {
5981 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5982 match pushed {
5983 Ok(o) if o.ok() => self.state.event(
5984 "land",
5985 format!("pushed rebased {} to {remote}", winner.branch),
5986 ),
5987 Ok(o) => {
5988 conflict = Some(format!(
5989 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5990 winner.branch,
5991 short(pinned),
5992 o.stderr.chars().take(600).collect::<String>()
5993 ));
5994 }
5995 Err(e) => {
5996 conflict = Some(format!(
5997 "rebased {} locally but could not push it: {e:#}",
5998 winner.branch
5999 ));
6000 }
6001 }
6002 }
6003 if let Some(why) = &conflict {
6004 self.state.status = RunStatus::Blocked;
6005 self.state.event("land", why.clone());
6006 }
6007 self.state.base_sync = Some(BaseSync {
6008 tip: tip.clone(),
6009 behind: 0,
6010 attempts,
6011 conflict,
6012 already_in: None,
6013 });
6014 self.state
6015 .event("land", format!("rebased {} onto {tracking}", winner.branch));
6016 }
6017 Ok(Some(conflict)) => {
6018 self.state.finish_base_operation(None);
6019 let why = format!(
6020 "{} conflicts with {tracking} and did not rebase: {}",
6021 winner.branch,
6022 conflict.chars().take(600).collect::<String>()
6023 );
6024 self.state.status = RunStatus::Blocked;
6025 self.state.base_sync = Some(BaseSync {
6026 tip,
6027 behind,
6028 attempts,
6029 conflict: Some(why.clone()),
6030 already_in: None,
6031 });
6032 self.state.event("land", why);
6033 }
6034 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6038 self.state.save()?;
6039 return Ok(());
6040 }
6041 Err(e) => {
6042 self.state.finish_base_operation(None);
6043 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6044 self.state.status = RunStatus::Blocked;
6045 self.state.base_sync = Some(BaseSync {
6046 tip,
6047 behind,
6048 attempts,
6049 conflict: Some(why.clone()),
6050 already_in: None,
6051 });
6052 self.state.event("land", why);
6053 }
6054 }
6055 if !self.state.base_sync_blocked() {
6056 self.state.base_sync_recovery.stop = None;
6057 }
6058 self.hold_base_sync_kind(stop_kind).await?;
6059 self.state.save()?;
6060 Ok(())
6061 }
6062
6063 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6069 if self.state.task_pr_branch().is_some() {
6072 return None;
6073 }
6074 let repo = self.state.repo.clone();
6075 let remote = self.state.config.merge.remote.clone();
6076 let fetched = git::fetch(&repo, &remote, branch).await;
6077 if !matches!(&fetched, Ok(o) if o.ok()) {
6078 return None;
6079 }
6080 let branch_tracking = format!("{remote}/{branch}");
6081 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6082 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6083 return None;
6084 }
6085 if !crate::reconcile::origin_missing(&repo, head, &theirs)
6086 .await
6087 .is_ok_and(|missing| missing.is_empty())
6088 {
6089 return Some(format!(
6090 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6091 them",
6092 short(&theirs)
6093 ));
6094 }
6095 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6096 Ok(o) if o.ok() => {
6097 self.state
6098 .event("land", format!("pushed rebased {branch} to {remote}"));
6099 None
6100 }
6101 Ok(o) => Some(format!(
6102 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6103 someone may have pushed): {}",
6104 short(&theirs),
6105 o.stderr.chars().take(600).collect::<String>()
6106 )),
6107 Err(e) => Some(format!(
6108 "{branch} is rebased locally but could not be pushed: {e:#}"
6109 )),
6110 }
6111 }
6112
6113 async fn settle_already_in(
6130 &mut self,
6131 branch: &str,
6132 tip: &str,
6133 head: &str,
6134 attempts: usize,
6135 behind: usize,
6136 ) -> Result<bool> {
6137 let repo = self.state.repo.clone();
6138 let remote = self.state.config.merge.remote.clone();
6139 let start = self.state.base_commit.clone();
6140 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6141 Ok(Some(e)) => e,
6142 Ok(None) => return Ok(false),
6143 Err(e) => {
6144 tracing::warn!("already-in-base check for {branch}: {e:#}");
6145 return Ok(false);
6146 }
6147 };
6148 let mut verified = vec![head.to_owned()];
6149 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6150 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6151 if matches!(&fetched, Ok(o) if o.ok())
6152 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6153 && theirs != head
6154 {
6155 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6156 Ok(Some(_)) => verified.push(theirs),
6157 _ => return Ok(false),
6158 }
6159 }
6160 let base_branch = self.state.base_branch.clone();
6161 let message = format!(
6162 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6163 land",
6164 evidence.names(),
6165 evidence.proof.as_str()
6166 );
6167 let closed =
6168 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6169 .await;
6170 match closed {
6171 Ok(Ok(url)) => self
6172 .state
6173 .event("land", format!("closed {url}: superseded on {base_branch}")),
6174 Ok(Err(why)) => self
6175 .state
6176 .event("land", format!("did not close a pull request: {why}")),
6177 Err(e) => {
6178 let why = format!(
6179 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6180 not be closed ({e:#}); resume to retry"
6181 );
6182 self.state.status = RunStatus::Blocked;
6183 self.state.base_sync = Some(BaseSync {
6184 tip: tip.to_owned(),
6185 behind,
6186 attempts,
6187 conflict: Some(why.clone()),
6188 already_in: None,
6189 });
6190 self.state.event("land", why);
6191 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6192 .await?;
6193 self.state.save()?;
6194 return Ok(true);
6195 }
6196 }
6197 self.state.status = RunStatus::AlreadyInBase;
6198 self.state.base_sync_recovery.stop = None;
6199 self.state.base_sync = Some(BaseSync {
6200 tip: tip.to_owned(),
6201 behind,
6202 attempts,
6203 conflict: None,
6204 already_in: Some(evidence),
6205 });
6206 self.state.event("land", message);
6207 self.state.save()?;
6208 self.settle_questions();
6209 Ok(true)
6210 }
6211
6212 fn landing_base(&self) -> String {
6222 self.state
6223 .base_sync
6224 .as_ref()
6225 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6226 }
6227
6228 pub async fn fix_selected(
6261 &mut self,
6262 ids: &[String],
6263 reason: &str,
6264 allow_stale: bool,
6265 ) -> Result<()> {
6266 let reason = reason.trim();
6267 if reason.is_empty() {
6268 bail!("a fix request needs a reason — that is the operator's own record of why");
6269 }
6270 if ids.is_empty() {
6271 bail!("no finding id given");
6272 }
6273 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6274 bail!(
6275 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6276 has already concluded — can be given a targeted fix. A run still \
6277 in progress should simply be resumed; a `merged` run's branch has \
6278 already landed, so its answer is a fresh `magi review <branch>`, \
6279 not reopening this run's own record",
6280 self.state.id,
6281 self.state.status.as_str()
6282 );
6283 }
6284 let _claim = FixClaim::acquire(&self.state.dir())?;
6287 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6288 let Some(winner) = self.state.winner().cloned() else {
6289 bail!("run {} has no winning candidate to fix", self.state.id);
6290 };
6291 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6292 bail!(
6293 "branch `{}` no longer exists; this run cannot be extended",
6294 winner.branch
6295 );
6296 }
6297 let home = crate::run::home();
6298 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6299 bail!(
6300 "run {} is currently being worked on by another magi process",
6301 self.state.id
6302 );
6303 }
6304 let mut seen = BTreeSet::new();
6308 let mut findings = Vec::new();
6309 let mut missing = Vec::new();
6310 for id in ids {
6311 if !seen.insert(id.clone()) {
6312 continue;
6313 }
6314 match self.state.finding(id) {
6315 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6316 id: f.id.clone(),
6317 severity: f.severity,
6318 reviewer_vote: rec.vote,
6319 round: round.round,
6320 round_head: round.head.clone(),
6321 reviewer: rec.reviewer,
6322 agent: rec.agent.clone(),
6323 file: f.file.clone(),
6324 line: f.line,
6325 title: f.title.clone(),
6326 detail: f.detail.clone(),
6327 outcome: OperatorFixOutcome::Pending,
6328 }),
6329 None => missing.push(id.clone()),
6330 }
6331 }
6332 if !missing.is_empty() {
6333 bail!(
6334 "unknown finding id(s): {}; nothing was changed",
6335 missing.join(", ")
6336 );
6337 }
6338
6339 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6340 let stale_details: Vec<(String, String)> = findings
6341 .iter()
6342 .filter(|f| f.round_head != head_at_request)
6343 .map(|f| (f.id.clone(), f.round_head.clone()))
6344 .collect();
6345 let stale = !stale_details.is_empty();
6346 if stale && !allow_stale {
6347 bail!(
6348 "the branch has moved since some finding(s) were raised — {} — now \
6349 at {}; pass --allow-stale to fix anyway, or re-run review first",
6350 stale_details
6351 .iter()
6352 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6353 .collect::<Vec<_>>()
6354 .join(", "),
6355 short(&head_at_request)
6356 );
6357 }
6358
6359 let request = OperatorFixRequest {
6360 requested_at: Timestamp::now(),
6361 reason: reason.to_owned(),
6362 findings,
6363 head_at_request: head_at_request.clone(),
6364 allow_stale,
6365 stale,
6366 fix: None,
6367 result_head: None,
6368 follow_up_review_run: None,
6369 };
6370 self.state.event(
6371 "fix",
6372 format!(
6373 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6374 request.findings.len(),
6375 request
6376 .findings
6377 .iter()
6378 .map(|f| f.id.as_str())
6379 .collect::<Vec<_>>()
6380 .join(", "),
6381 ),
6382 );
6383 self.state.operator_fixes.push(request);
6390 self.state.save()?;
6391 let request_index = self.state.operator_fixes.len() - 1;
6392
6393 let repo = self.state.repo.clone();
6402 let coordination = self.coordinate(&repo).await?;
6403 if winner.worktree.exists() {
6404 let dirty = git::git(
6407 &winner.worktree,
6408 &["status", "--porcelain", "--untracked-files=all"],
6409 )
6410 .await?;
6411 let only_withheld = dirty.lines().all(|l| {
6412 l.strip_prefix("?? ")
6413 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6414 });
6415 if !only_withheld {
6416 bail!(
6417 "`{}` has uncommitted changes; refusing to touch it — commit or \
6418 discard them first",
6419 winner.worktree.display()
6420 );
6421 }
6422 git::worktree_remove(&self.state.repo, &winner.worktree)
6423 .await
6424 .ok();
6425 }
6426 let fix_worktree = self.state.worktree_root().join("operator-fix");
6427 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6428 git::git(
6429 &self.state.repo,
6430 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6431 )
6432 .await
6433 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6434 if !git::is_clean(&fix_worktree).await? {
6435 git::worktree_remove(&self.state.repo, &fix_worktree)
6436 .await
6437 .ok();
6438 bail!(
6439 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6440 winner.branch
6441 );
6442 }
6443 let repo_for_setup = self.state.repo.clone();
6444 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6445 Ok(()) => {
6446 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6447 }
6448 Err(e) => Err(e),
6449 };
6450 if let Err(e) = setup {
6451 git::worktree_remove(&self.state.repo, &fix_worktree)
6452 .await
6453 .ok();
6454 return Err(e);
6455 }
6456
6457 self.state.resource_block = None;
6460
6461 let run_id = self.state.id.clone();
6462 let prompts = self.state.config.prompts.clone();
6463 let language = self.state.config.graph.language.clone();
6464 let sessions = self.state.config.graph.sessions;
6465 let artifacts = agent::artifacts_dir(&self.state.dir());
6466 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6467 .findings
6468 .iter()
6469 .map(|f| Finding {
6470 id: f.id.clone(),
6471 severity: f.severity,
6472 file: f.file.clone(),
6473 line: f.line,
6474 title: f.title.clone(),
6475 detail: f.detail.clone(),
6476 })
6477 .collect();
6478 let fix_prompt = prompt::operator_fix_in(
6479 &self.state.instruction,
6480 &finding_list,
6481 reason,
6482 &stale_details,
6483 &head_at_request,
6484 &language,
6485 &self.state.config.graph.github_language,
6486 );
6487 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6488 drop(coordination);
6489 let (job, seat, out) = self
6490 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6491 prompt: fix_prompt.clone(),
6492 spec,
6493 seat,
6494 cwd: fix_worktree.clone(),
6495 timeout,
6496 allow_write: true,
6497 sessions,
6498 artifacts: artifacts.clone(),
6499 stem: "operator-fix".to_owned(),
6500 handover: None,
6501 })
6502 .await;
6503 let agent_id = seat.agent.clone();
6504
6505 let mut fix = FixRecord {
6506 agent: agent_id,
6507 addressed: Vec::new(),
6508 rejected: Vec::new(),
6509 notes: String::new(),
6510 committed: false,
6511 failed: None,
6512 duration_ms: 0,
6513 continuation: None,
6514 };
6515 let mut final_seat = seat.clone();
6516 match out {
6517 AgentOutcome::Ok(o) => {
6518 fix.duration_ms = o.duration_ms;
6519 let parsed = verdict::extract_json::<FixReport>(&o.text);
6520 let incomplete_reason = match &parsed {
6521 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6522 "the reply parsed, but it reported a command whose own CLI \
6523 never confirmed an exit status"
6524 .to_owned(),
6525 ),
6526 Ok(_) => None,
6527 Err(e) => Some(e.to_string()),
6528 };
6529 match incomplete_reason {
6530 None => {
6531 let report = parsed.expect("checked Ok above");
6532 fix.addressed = report.addressed;
6533 fix.rejected = report.rejected;
6534 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6535 }
6536 Some(reason) => {
6537 let (resumed_seat, resolved, failure, cont) = self
6538 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6539 .await;
6540 fix.duration_ms += cont.cumulative_wait_ms;
6541 fix.continuation = Some(cont);
6542 final_seat = resumed_seat;
6543 match resolved {
6544 Some(report) => {
6545 fix.addressed = report.addressed;
6546 fix.rejected = report.rejected;
6547 fix.notes =
6548 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6549 }
6550 None => fix.failed = failure,
6551 }
6552 }
6553 }
6554 }
6555 AgentOutcome::Dropped(o) => {
6556 fix.duration_ms = o.duration_ms;
6557 let why = o
6558 .dropped
6559 .as_ref()
6560 .map(|d| d.why.as_str())
6561 .unwrap_or("the CLI ended the stream without delivering its answer");
6562 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6563 }
6564 AgentOutcome::Quota(o) => {
6565 self.state.quota.push(QuotaLoss::from_report(
6566 final_seat.key.clone(),
6567 "fix",
6568 o.quota.as_ref(),
6569 ));
6570 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6571 }
6572 AgentOutcome::Failed(e) => fix.failed = Some(e),
6573 AgentOutcome::ResourceBlocked(e) => {
6574 fix.failed = Some(format!("the fixer was not started: {e}"));
6575 }
6576 }
6577 if fix.continuation.is_none() {
6580 fix.continuation = Some(ContinuationRecord::not_needed());
6581 }
6582 self.state.seats.insert(final_seat.key.clone(), final_seat);
6583
6584 let rescue_message = format!(
6585 "magi: operator-selected fix ({}) (uncommitted work)",
6586 self.state.operator_fixes[request_index]
6587 .findings
6588 .iter()
6589 .map(|f| f.id.as_str())
6590 .collect::<Vec<_>>()
6591 .join(", ")
6592 );
6593 let repo = self.state.repo.clone();
6594 let coordination = self.coordinate(&repo).await?;
6595 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6596 self.state.note_withheld("fix", &r.withheld);
6597 }
6598 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6599 fix.committed = after != head_at_request;
6600 git::worktree_remove(&self.state.repo, &fix_worktree)
6601 .await
6602 .ok();
6603 drop(coordination);
6604
6605 self.state.event(
6606 "fix",
6607 match &fix.failed {
6608 Some(reason) => format!(
6609 "operator fix: adoption report was lost ({reason}); {}",
6610 if fix.committed {
6611 "committed"
6612 } else {
6613 "NO new commit"
6614 }
6615 ),
6616 None => format!(
6617 "operator fix: {} addressed, {} rejected, {}",
6618 fix.addressed.len(),
6619 fix.rejected.len(),
6620 if fix.committed {
6621 "committed"
6622 } else {
6623 "NO new commit"
6624 }
6625 ),
6626 },
6627 );
6628
6629 for f in &mut self.state.operator_fixes[request_index].findings {
6636 f.outcome = if fix.failed.is_some() {
6637 OperatorFixOutcome::Unreported
6638 } else if fix.addressed.contains(&f.id) {
6639 OperatorFixOutcome::Addressed
6640 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6641 OperatorFixOutcome::Rejected { why: r.why.clone() }
6642 } else {
6643 OperatorFixOutcome::Unreported
6644 };
6645 }
6646
6647 let committed = fix.committed;
6648 if committed {
6649 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6650 }
6651 self.state.operator_fixes[request_index].fix = Some(fix);
6652 self.state.save()?;
6655 if self.state.resource_block.is_some() {
6656 return self.stop_for_resource();
6657 }
6658
6659 if committed {
6660 self.state.event(
6661 "fix",
6662 format!(
6663 "operator fix committed {}; opening a follow-up review-only run",
6664 short(&after)
6665 ),
6666 );
6667 let origin =
6670 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6671 match Self::review(
6672 &self.state.repo,
6673 &winner.branch,
6674 self.state.config.clone(),
6675 origin,
6676 )
6677 .await
6678 {
6679 Ok(mut follow_up) => {
6680 follow_up.state.event(
6681 "start",
6682 format!(
6683 "requested by an operator fix on run {} for finding(s) {}",
6684 self.state.id,
6685 self.state.operator_fixes[request_index]
6686 .findings
6687 .iter()
6688 .map(|f| f.id.as_str())
6689 .collect::<Vec<_>>()
6690 .join(", "),
6691 ),
6692 );
6693 follow_up.state.save()?;
6694 let follow_up_id = follow_up.state.id.clone();
6695 let adopted = match crate::direct::adopt(
6698 &follow_up.state,
6699 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6700 ) {
6701 Ok(a) => a,
6702 Err(e) => {
6703 self.state.event(
6706 "fix",
6707 format!(
6708 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6709 ),
6710 );
6711 self.state.save()?;
6712 return Ok(());
6713 }
6714 };
6715 let executed = follow_up.execute().await;
6716 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6717 if let Some(a) = adopted {
6718 a.finish(&follow_up.state, executed);
6719 }
6720 if let Some(e) = failure {
6721 self.state.event(
6722 "fix",
6723 format!(
6724 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6725 ),
6726 );
6727 }
6728 self.state.operator_fixes[request_index].follow_up_review_run =
6729 Some(follow_up_id);
6730 }
6731 Err(e) => {
6732 self.state.event(
6733 "fix",
6734 format!("committed the fix but could not open a follow-up review: {e:#}"),
6735 );
6736 }
6737 }
6738 self.state.save()?;
6739 }
6740
6741 Ok(())
6742 }
6743
6744 async fn ask_fixer(
6764 &mut self,
6765 winner: &Candidate,
6766 node: &'static str,
6767 round: Option<usize>,
6768 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6769 ) -> (SeatJob, SeatState, AgentOutcome) {
6770 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6771 self.ask_fixer_among(attempts, node, round, None, build)
6772 .await
6773 }
6774
6775 async fn ask_fixer_among(
6780 &mut self,
6781 attempts: Vec<(AgentSpec, String)>,
6782 node: &'static str,
6783 round: Option<usize>,
6784 restore: Option<(&Path, &str)>,
6785 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6786 ) -> (SeatJob, SeatState, AgentOutcome) {
6787 let run_id = self.state.id.clone();
6788 let prompts = self.state.config.prompts.clone();
6789 let cache = self.state.config.cache_dir();
6790 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6791 let mut last = None;
6792 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6793 let seat = self.seat(&key, &spec.id);
6794 let mut job = build(spec.clone(), seat);
6795 if i > 0 {
6796 job.stem = format!("{}-{}", job.stem, spec.id);
6797 }
6798 let ctx = WaveCtx {
6799 carry_seats: false,
6800 pauses: Some((&self.pause, &self.interrupt)),
6801 run: &run_id,
6802 node,
6803 prompts: &prompts,
6804 cache: cache.as_deref(),
6805 round,
6806 };
6807 let (seat, out) =
6808 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6809 let advances = match &out {
6812 AgentOutcome::Ok(o) => agent::output_advances(o),
6813 AgentOutcome::ResourceBlocked(_) => false,
6814 _ => true,
6815 };
6816 if let Some(next) = ids.get(i + 1)
6817 && advances
6818 {
6819 self.state.seats.insert(seat.key.clone(), seat.clone());
6820 let class =
6821 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6822 let reason = match &out {
6823 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6824 other => fail_reason(other),
6825 };
6826 if let Some((wt, rev)) = restore
6830 && let Err(e) = self.graft_restore(wt, rev).await
6831 {
6832 let why =
6833 format!("could not restore the worktree before the next fixer: {e:#}");
6834 tracing::warn!("{why}");
6835 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6836 AgentOutcome::ResourceBlocked(why)
6837 } else {
6838 AgentOutcome::Failed(why)
6839 };
6840 last = Some((job, seat, out));
6841 break;
6842 }
6843 record_handover(
6844 &mut self.state,
6845 node,
6846 &seat.key,
6847 &spec.id,
6848 next,
6849 &class,
6850 &reason,
6851 );
6852 continue;
6853 }
6854 last = Some((job, seat, out));
6855 break;
6856 }
6857 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6858 if let AgentOutcome::Quota(output) = &out
6859 && node != "graft"
6860 {
6861 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6866 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6867 self.state.quota.push(loss);
6868 self.state.seats.insert(seat.key.clone(), seat.clone());
6869 self.state.status = RunStatus::Blocked;
6870 return (
6871 job,
6872 seat,
6873 AgentOutcome::ResourceBlocked(
6874 "fixer unavailable; restore agent availability and resume".into(),
6875 ),
6876 );
6877 }
6878 (job, seat, out)
6879 }
6880
6881 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6892 use crate::recovery::{Mode, Repair};
6893 if !self.state.reviews.is_empty() {
6894 return Ok(());
6895 }
6896 let Some(mut rec) = self.state.recovery.clone() else {
6897 return Ok(());
6898 };
6899 if rec.mode != Mode::VerifyRepair {
6900 return Ok(());
6901 }
6902 let wt = winner.worktree.clone();
6903 let mut head = git::rev_parse(&wt, "HEAD").await?;
6904 match rec.repair.clone() {
6905 Repair::Done { .. } => return Ok(()),
6906 Repair::WaitingForRestore { before } => {
6907 self.graft_restore(&wt, &before).await?;
6908 head = before;
6909 rec.repair = Repair::NotStarted;
6910 self.state.recovery = Some(rec.clone());
6911 self.state.save()?;
6912 }
6913 Repair::Started { before } => {
6914 if let Err(e) = self.graft_restore(&wt, &before).await {
6918 if e.is::<crate::repo_lock::ResourceBlocked>() {
6919 return Err(e);
6920 }
6921 return self
6922 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6923 .await;
6924 }
6925 rec.repair = Repair::Done {
6926 before: before.clone(),
6927 after: before,
6928 committed: false,
6929 failed: Some("interrupted; rolled back".to_owned()),
6930 };
6931 self.state
6932 .event("recovery", "the repair was interrupted and rolled back");
6933 self.state.recovery = Some(rec);
6934 self.state.save()?;
6935 return Ok(());
6936 }
6937 Repair::NotStarted => {}
6938 }
6939 if crate::recovery::recheck(&mut rec, &head) {
6940 self.state.event("recovery", rec.summary());
6941 self.state.recovery = Some(rec);
6942 self.state.save()?;
6943 return Ok(());
6944 }
6945 rec.repair = Repair::Started {
6946 before: head.clone(),
6947 };
6948 self.state.recovery = Some(rec.clone());
6949 self.state.save()?;
6950
6951 let language = self.state.config.graph.language.clone();
6952 let sessions = self.state.config.graph.sessions;
6953 let artifacts = agent::artifacts_dir(&self.state.dir());
6954 let fix_prompt = prompt::recovery_fix(
6955 &self.state.instruction,
6956 &rec,
6957 self.state.config.graph.review_rounds,
6958 &language,
6959 );
6960 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6961 self.state.event(
6962 "recovery",
6963 format!(
6964 "repair: the fixer starts from the failed verification on {}",
6965 short(&head)
6966 ),
6967 );
6968 let (_job, seat, out) = self
6969 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6970 prompt: fix_prompt.clone(),
6971 spec,
6972 seat,
6973 cwd: wt.clone(),
6974 timeout,
6975 allow_write: true,
6976 sessions,
6977 artifacts: artifacts.clone(),
6978 stem: "fix-recovery".to_owned(),
6979 handover: None,
6980 })
6981 .await;
6982 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6983 rec.repair = Repair::WaitingForRestore { before: head };
6984 self.state.recovery = Some(rec);
6985 return self.stop_for_resource();
6986 }
6987 let failed = match &out {
6988 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6989 AgentOutcome::Quota(o) => {
6990 self.state.quota.push(QuotaLoss::from_report(
6991 seat.key.clone(),
6992 "fix",
6993 o.quota.as_ref(),
6994 ));
6995 Some("rate limited (quota); the fixer could not run".to_owned())
6996 }
6997 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6998 AgentOutcome::Failed(e) => Some(e.clone()),
6999 };
7000 self.state.seats.insert(seat.key.clone(), seat);
7001 let coordination = self.coordinate(&wt).await?;
7002 let failed = match failed {
7003 Some(why) => {
7005 if let Err(e) = git::restore_to(&wt, &head).await {
7006 return self
7007 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7008 .await;
7009 }
7010 Some(why)
7011 }
7012 None => {
7013 if let Ok(r) = git::rescue_commit(
7014 &wt,
7015 "magi: recovery repair of the failed verification (uncommitted work)",
7016 )
7017 .await
7018 {
7019 self.state.note_withheld("fix", &r.withheld);
7020 }
7021 None
7022 }
7023 };
7024 let after = git::rev_parse(&wt, "HEAD").await?;
7025 drop(coordination);
7026 self.state.event(
7027 "recovery",
7028 match &failed {
7029 Some(why) => format!("repair: nothing kept ({why})"),
7030 None if after != head => {
7031 format!("repair: committed {}", short(&after))
7032 }
7033 None => "repair: the fixer made no commit".to_owned(),
7034 },
7035 );
7036 rec.repair = Repair::Done {
7037 before: head.clone(),
7038 committed: after != head,
7039 after,
7040 failed,
7041 };
7042 self.state.recovery = Some(rec);
7043 self.state.save()?;
7044 Ok(())
7045 }
7046
7047 async fn stop_recovery(
7050 &mut self,
7051 mut rec: crate::recovery::RecoveryRecord,
7052 head: &str,
7053 why: String,
7054 ) -> Result<()> {
7055 rec.repair = crate::recovery::Repair::Started {
7057 before: head.to_owned(),
7058 };
7059 self.state.recovery = Some(rec);
7060 self.state
7061 .event("recovery", format!("repair could not be undone: {why}"));
7062 self.state.status = RunStatus::Blocked;
7063 self.state.save()?;
7064 bail!("recovery repair could not be rolled back: {why}")
7065 }
7066
7067 async fn review_loop(&mut self) -> Result<()> {
7068 if self.hold_base_sync().await? {
7069 return Ok(());
7070 }
7071 loop {
7072 if self
7073 .state
7074 .base_sync
7075 .as_ref()
7076 .is_some_and(|s| s.conflict.is_some())
7077 {
7078 return Ok(());
7079 }
7080 if self.catchup_repairable() {
7081 let record = self.state.reviews.last().expect("repairable round").clone();
7082 let winner = self
7083 .state
7084 .winner()
7085 .cloned()
7086 .context("catch-up repair has no winner")?;
7087 if !self.fix_review_round(&winner, record).await? {
7088 return Ok(());
7089 }
7090 }
7091 self.review_loop_pass().await?;
7092 if !self.catchup_repairable() {
7093 return Ok(());
7094 }
7095 }
7096 }
7097
7098 fn catchup_repairable(&self) -> bool {
7101 self.state.reviews.len() < self.state.config.graph.review_rounds
7102 && self.state.reviews.last().is_some_and(|r| {
7103 !r.clean
7104 && r.blocking == 0
7105 && r.fix.is_none()
7106 && r.e2e_status() == E2eStatus::Failed
7107 })
7108 }
7109
7110 async fn review_loop_pass(&mut self) -> Result<()> {
7111 if self
7116 .state
7117 .base_sync
7118 .as_ref()
7119 .is_some_and(|s| s.conflict.is_some())
7120 {
7121 return Ok(());
7122 }
7123 let run_id = self.state.id.clone();
7128 let prompts = self.state.config.prompts.clone();
7129 let Some(winner) = self.state.winner().cloned() else {
7130 return Ok(());
7131 };
7132 let granted_review = self
7133 .state
7134 .repair_grants
7135 .last()
7136 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7137 .cloned();
7138 let fresh_review = granted_review
7139 .as_ref()
7140 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7141 let max_rounds = granted_review
7142 .as_ref()
7143 .map_or(self.state.config.graph.review_rounds, |g| {
7144 g.review_start.saturating_add(1)
7145 });
7146 if let Some(status) =
7156 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7157 {
7158 if status == RunStatus::Gating {
7161 self.record_contested_handoff();
7162 }
7163 self.state.status = status;
7164 self.state.save()?;
7165 return Ok(());
7166 }
7167 self.state.status = RunStatus::Reviewing;
7168 if !fresh_review
7178 && self.state.reviews.last().is_some_and(|r| {
7179 r.e2e_status() == E2eStatus::ResourceBlocked
7180 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7181 })
7182 {
7183 let shell = self.state.config.shell();
7184 return self
7185 .stop_unresolved_review(
7186 "the last round's own verification never resolved",
7187 &shell,
7188 &winner.worktree,
7189 )
7190 .await;
7191 }
7192
7193 if granted_review.is_none() {
7194 self.recovery_repair(&winner).await?;
7195 }
7196 if self.resource_halt()? {
7197 return Ok(());
7198 }
7199
7200 let repo = self.state.repo.clone();
7201 let root = self.state.worktree_root();
7202 let language = self.state.config.graph.language.clone();
7203 let sessions = self.state.config.graph.sessions;
7204 let artifacts = self.merge_repair_artifacts();
7205 let base = self.landing_base();
7206 let base_short = short(&base);
7207 let reviewers = self.roles.reviewers.clone();
7208 let shell = self.state.config.shell();
7209
7210 for round in (self.state.reviews.len() + 1)..=max_rounds {
7211 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7212 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7213 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7214 let prev_verification = self
7223 .state
7224 .reviews
7225 .last()
7226 .and_then(|r| r.verification_summary(&head));
7227 let prior_section = match (&self.state.recovery, round) {
7230 (Some(rec), 1) if self.state.reviews.is_empty() => {
7231 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7232 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7233 Some(crate::recovery::head_of(
7234 &git::diff(&winner.worktree, reviewed, "HEAD")
7235 .await
7236 .unwrap_or_default(),
7237 crate::recovery::SINCE_DIFF_MAX,
7238 ))
7239 }
7240 _ => None,
7241 };
7242 Some(prompt::recovery_review(rec, since.as_deref()))
7243 }
7244 _ => None,
7245 };
7246
7247 let mut jobs = Vec::new();
7251 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7252 self.coordinate(&repo).await?
7253 } else {
7254 None
7255 };
7256 for (r, spec) in reviewers.iter().cloned().enumerate() {
7257 let wt = root.join(format!("review-{}", r + 1));
7258 if wt.exists() {
7261 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7262 } else {
7263 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7264 None
7265 } else {
7266 self.coordinate(&repo).await?
7267 };
7268 git::worktree_add_detached(&repo, &wt, &head).await?;
7269 }
7270 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7271 None
7272 } else {
7273 self.coordinate(&repo).await?
7274 };
7275 self.setup_seat_worktree(&repo, &wt, false).await?;
7276 let seat_key = format!("review-{}", r + 1);
7277 let spec = pick_start_spec(
7280 &self.roles.reviewer_roster,
7281 spec,
7282 self.state.seat_history.get(&seat_key),
7283 );
7284 let seat = self.seat(&seat_key, &spec.id);
7285 jobs.push(SeatJob {
7286 prompt: prompt::review(&prompt::ReviewCtx {
7287 instruction: &self.state.instruction,
7288 branch: &winner.branch,
7289 base_short: &base_short,
7290 stat: &stat,
7291 patch: &patch,
7292 verification: prev_verification.as_ref(),
7293 prior: prior_section.as_deref(),
7294 reviewers: reviewers.len(),
7295 round,
7296 rounds: max_rounds,
7297 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7300 lens: Lens::for_seat(r),
7301 language: &language,
7302 github_language: &self.state.config.graph.github_language,
7303 }),
7304 spec,
7305 seat,
7306 cwd: wt,
7307 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7308 allow_write: false,
7309 sessions,
7310 artifacts: artifacts.clone(),
7311 stem: format!("review-{round}-{}", r + 1),
7312 handover: None,
7313 });
7314 }
7315
7316 drop(coordination);
7317 self.state.event(
7318 "review",
7319 format!(
7320 "round {round}: {} reviewers on {}",
7321 jobs.len(),
7322 short(&head)
7323 ),
7324 );
7325 let mut quota_losses = Vec::new();
7326 let review_retries = self.state.config.graph.retries;
7327 let review_cache = self.state.config.cache_dir();
7328 let ctx = WaveCtx {
7329 carry_seats: true,
7330 pauses: Some((&self.pause, &self.interrupt)),
7331 run: &run_id,
7332 node: "review",
7333 prompts: &prompts,
7334 cache: review_cache.as_deref(),
7335 round: Some(round),
7336 };
7337 let results = ask_json_wave::<Review>(
7338 jobs,
7339 Arc::clone(&self.sem),
7340 review_retries,
7341 &self.roles.reviewer_roster,
7342 &ctx,
7343 &mut quota_losses,
7344 &mut self.state,
7345 &|_: &Review| Ok(()),
7346 )
7347 .await;
7348 if self.state.resource_block.is_some() {
7349 return self.stop_for_resource();
7350 }
7351 let round_quota_missing = quota_losses.len();
7355 self.state.quota.extend(quota_losses);
7356
7357 let mut records = Vec::new();
7358 let mut all_findings = Vec::new();
7359 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7360 let agent_id = seat.agent.clone();
7361 self.state.seats.insert(seat.key.clone(), seat);
7362 let mut record = ReviewRecord {
7363 reviewer: r + 1,
7364 agent: agent_id,
7365 summary: String::new(),
7366 findings: Vec::new(),
7367 vote: None,
7368 failed: None,
7369 duration_ms: 0,
7370 attempts,
7376 };
7377 match res {
7378 Ok((review, out)) => {
7379 record.summary =
7387 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7388 record.vote = Some(review.vote);
7389 record.duration_ms = out.duration_ms;
7390 for (n, mut f) in review.findings.into_iter().enumerate() {
7391 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7394 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7395 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7396 f.file = f
7402 .file
7403 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7404 all_findings.push(f.clone());
7405 record.findings.push(f);
7406 }
7407 self.state.event(
7408 "review",
7409 format!(
7410 "round {round}: reviewer {} voted {} with {} finding(s)",
7411 r + 1,
7412 review.vote.label(),
7413 record.findings.len()
7414 ),
7415 );
7416 }
7417 Err(e) => {
7418 record.failed = Some(e.to_string());
7419 self.state.event(
7420 "review",
7421 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7422 );
7423 }
7424 }
7425 records.push(record);
7426 }
7427
7428 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7435 let vote_split =
7436 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7437 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7438 if vote_split {
7439 self.state.event(
7440 "review",
7441 format!(
7442 "round {round}: votes split ({}) — one round of reconsideration",
7443 initial_votes
7444 .iter()
7445 .map(|v| v.label())
7446 .collect::<Vec<_>>()
7447 .join(", ")
7448 ),
7449 );
7450 let panel: Vec<ReviewSeatReport<'_>> = records
7453 .iter()
7454 .filter_map(|r| {
7455 r.vote.map(|vote| ReviewSeatReport {
7456 reviewer: r.reviewer,
7457 vote,
7458 summary: &r.summary,
7459 findings: &r.findings,
7460 })
7461 })
7462 .collect();
7463
7464 let mut jobs = Vec::new();
7465 let mut seats_at = Vec::new();
7466 for (r, spec) in reviewers.iter().cloned().enumerate() {
7467 if records[r].vote.is_none() {
7471 continue;
7472 }
7473 let wt = root.join(format!("review-{}", r + 1));
7474 let seat_key = format!("review-{}", r + 1);
7475 let spec = self.occupant(&seat_key, spec);
7476 let seat = self.seat(&seat_key, &spec.id);
7477 let build = |with_patch: bool| {
7485 prompt::review_reconsider(&ReviewReconsiderCtx {
7486 instruction: &self.state.instruction,
7487 reviewer: r + 1,
7488 lens: Lens::for_seat(r),
7489 panel: &panel,
7490 patch: with_patch.then_some(ReviewPatch {
7491 branch: &winner.branch,
7492 base_short: &base_short,
7493 stat: &stat,
7494 patch: &patch,
7495 }),
7496 round,
7497 rounds: max_rounds,
7498 language: &language,
7499 })
7500 };
7501 let full = build(true);
7502 let prompt = if has_context(&spec, &seat, sessions) {
7503 build(false)
7504 } else {
7505 full.clone()
7506 };
7507 jobs.push(SeatJob {
7508 prompt,
7509 spec,
7510 seat,
7511 cwd: wt,
7512 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7513 allow_write: false,
7514 sessions,
7515 artifacts: artifacts.clone(),
7516 stem: format!("review-{round}-reconsider-{}", r + 1),
7517 handover: Some(full),
7518 });
7519 seats_at.push(r);
7520 }
7521
7522 let mut recon_quota_losses = Vec::new();
7523 let recon_cache = self.state.config.cache_dir();
7524 let recon_ctx = WaveCtx {
7525 carry_seats: true,
7526 pauses: Some((&self.pause, &self.interrupt)),
7527 run: &run_id,
7528 node: "review",
7529 prompts: &prompts,
7530 cache: recon_cache.as_deref(),
7531 round: Some(round),
7532 };
7533 let recon_results = ask_json_wave::<ReviewRevote>(
7534 jobs,
7535 Arc::clone(&self.sem),
7536 review_retries,
7537 &self.roles.reviewer_roster,
7538 &recon_ctx,
7539 &mut recon_quota_losses,
7540 &mut self.state,
7541 &|_: &ReviewRevote| Ok(()),
7542 )
7543 .await;
7544 if self.state.resource_block.is_some() {
7545 return self.stop_for_resource();
7546 }
7547 self.state.quota.extend(recon_quota_losses);
7548
7549 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7550 let agent_id = seat.agent.clone();
7551 self.state.seats.insert(seat.key.clone(), seat);
7552 let mut rec = ReviewRevoteRecord {
7553 reviewer: r + 1,
7554 agent: agent_id,
7555 vote: None,
7556 reason: String::new(),
7557 failed: None,
7558 };
7559 match res {
7560 Ok((rv, _)) => {
7561 rec.vote = Some(rv.vote);
7562 rec.reason =
7563 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7564 self.state.event(
7565 "review",
7566 format!(
7567 "round {round}: reviewer {} revoted {}",
7568 r + 1,
7569 rv.vote.label()
7570 ),
7571 );
7572 }
7573 Err(e) => {
7574 rec.failed = Some(e.to_string());
7575 self.state.event(
7576 "review",
7577 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7578 );
7579 }
7580 }
7581 reconsideration.push(rec);
7582 }
7583 } else if initial_votes.len() > 1 {
7584 self.state.event(
7585 "review",
7586 format!(
7587 "round {round}: votes agreed ({}) — no reconsideration",
7588 initial_votes[0].label()
7589 ),
7590 );
7591 }
7592
7593 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7594 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7595 let defer_e2e =
7606 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7607 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7608 let reason =
7609 format!("{blocking} blocking finding(s) already required a fix this round");
7610 self.state.event(
7611 "verify",
7612 format!(
7613 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7614 {}); it will run once a round has none left",
7615 short(&head)
7616 ),
7617 );
7618 (Vec::new(), false, true, Some(reason))
7619 } else {
7620 let e2e_commands = self.state.config.verify.e2e.clone();
7621 let cache_dir = self.state.config.cache_dir();
7622 let context = format!("round {round}");
7623 let wait = self.repository_wait();
7624 let (e2e, verify_retried) = with_cache_lease(
7625 &mut self.state,
7626 cache_dir.as_deref(),
7627 "e2e",
7628 "e2e",
7629 &winner.worktree,
7630 &head,
7631 verify_timeout,
7632 &context,
7633 &wait,
7634 |state, budget| {
7635 let shell = shell.clone();
7636 let e2e_commands = e2e_commands.clone();
7637 let worktree = winner.worktree.clone();
7638 let context = context.clone();
7639 let winner = winner.clone();
7640 async move {
7641 if !e2e_commands.is_empty() {
7642 Self::run_pre_gate(state, &winner).await;
7646 }
7647 run_e2e_with_retry(
7648 state,
7649 &shell,
7650 &e2e_commands,
7651 &worktree,
7652 budget,
7653 &context,
7654 )
7655 .await
7656 }
7657 },
7658 )
7659 .await?;
7660 (e2e, verify_retried, false, None)
7661 };
7662
7663 let expected = records.len();
7664 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7665 let incomplete = answered < expected;
7666 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7667 let policy = self.state.config.graph.incomplete_review;
7668 let clean = round_is_clean(
7669 blocking,
7670 e2e_ok,
7671 answered,
7672 expected,
7673 round_quota_missing,
7674 policy,
7675 );
7676
7677 let mut round_record = ReviewRound {
7678 round,
7679 head: head.clone(),
7680 verified_head: None,
7681 verified_at: None,
7682 reviews: records,
7683 e2e,
7684 verify_retried,
7685 e2e_deferred,
7686 e2e_defer_reason,
7687 fix: None,
7688 blocking,
7689 answered,
7690 expected,
7691 clean,
7692 progressed: false,
7693 vote_split,
7694 reconsideration,
7695 verdict: None,
7696 };
7697 round_record.verdict = ReviewVote::worst(
7701 round_record
7702 .final_votes()
7703 .into_iter()
7704 .map(|(_, _, vote)| vote),
7705 );
7706 if !matches!(
7717 round_record.e2e_status(),
7718 E2eStatus::Deferred | E2eStatus::NotConfigured
7719 ) {
7720 round_record.verified_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
7723 round_record.verified_at = Some(Timestamp::now());
7724 }
7725
7726 if incomplete {
7727 let missing: Vec<String> = round_record
7728 .reviews
7729 .iter()
7730 .filter(|r| r.failed.is_some())
7731 .map(|r| format!("review-{}", r.reviewer))
7732 .collect();
7733 self.state.event(
7734 "review",
7735 format!(
7736 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7737 missing.join(", ")
7738 ),
7739 );
7740 }
7741
7742 if clean {
7743 self.state.event(
7744 "review",
7745 if incomplete && policy == IncompleteReviewPolicy::Warn {
7746 format!(
7747 "round {round}: clean (warn policy, incomplete panel) — no \
7748 blocking findings from the seats that answered, verification green"
7749 )
7750 } else if incomplete {
7751 format!(
7752 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7753 quorum) — no blocking findings from the seats that answered, \
7754 verification green",
7755 expected - answered
7756 )
7757 } else {
7758 format!("round {round}: clean — no blocking findings, verification green")
7759 },
7760 );
7761 self.state.reviews.push(round_record);
7762 self.state.status = RunStatus::Gating;
7763 self.state.save()?;
7764 return Ok(());
7765 }
7766
7767 if incomplete && blocking == 0 && e2e_ok {
7775 self.state.reviews.push(round_record);
7776 self.state.save()?;
7777 if round == max_rounds {
7778 self.state.status = RunStatus::Blocked;
7779 self.state.event(
7780 "review",
7781 format!(
7782 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7783 refusing to call it clean",
7784 expected - answered
7785 ),
7786 );
7787 return Ok(());
7788 }
7789 continue;
7790 }
7791
7792 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7804 self.state.reviews.push(round_record);
7805 return self
7806 .stop_unresolved_review(
7807 "the round's own verification could not run",
7808 &shell,
7809 &winner.worktree,
7810 )
7811 .await;
7812 }
7813
7814 if round == max_rounds {
7815 self.state.reviews.push(round_record);
7816 return self
7817 .stop_reviewing(
7818 &format!(
7819 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7820 ),
7821 &shell,
7822 &winner.worktree,
7823 )
7824 .await;
7825 }
7826
7827 if !self.fix_review_round(&winner, round_record).await? {
7828 return Ok(());
7829 }
7830 }
7831 Ok(())
7832 }
7833
7834 async fn fix_review_round(
7837 &mut self,
7838 winner: &Candidate,
7839 mut round_record: ReviewRound,
7840 ) -> Result<bool> {
7841 let round = round_record.round;
7842 let shell = self.state.config.shell();
7843 let step = self.run_fix_round(winner, &mut round_record).await?;
7844 if matches!(step, FixStep::Resource) {
7845 self.stop_for_resource()?;
7846 return Ok(false);
7847 }
7848 if self.state.reviews.len() < round {
7849 self.state.reviews.push(round_record);
7850 } else {
7851 self.state.reviews[round - 1] = round_record;
7852 }
7853 self.state.save()?;
7854
7855 if matches!(step, FixStep::Recorded { report_lost: true }) {
7868 self.stop_reviewing(
7869 "the fixer's adoption report never came back, even after resuming its \
7870 own seat; refusing to start another round against the same worktree \
7871 while that is unresolved",
7872 &shell,
7873 &winner.worktree,
7874 )
7875 .await?;
7876 return Ok(false);
7877 }
7878
7879 let streak = self
7880 .state
7881 .reviews
7882 .iter()
7883 .rev()
7884 .take_while(|r| !r.progressed)
7885 .count();
7886 if streak >= STAGNANT_LIMIT {
7887 self.stop_reviewing(
7888 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7889 &shell,
7890 &winner.worktree,
7891 )
7892 .await?;
7893 return Ok(false);
7894 }
7895 Ok(true)
7896 }
7897
7898 async fn run_fix_round(
7900 &mut self,
7901 winner: &Candidate,
7902 round_record: &mut ReviewRound,
7903 ) -> Result<FixStep> {
7904 let round = round_record.round;
7905 let max_rounds = self.state.config.graph.review_rounds;
7906 let run_id = self.state.id.clone();
7907 let prompts = self.state.config.prompts.clone();
7908 let language = self.state.config.graph.language.clone();
7909 let sessions = self.state.config.graph.sessions;
7910 let artifacts = self.merge_repair_artifacts();
7911 let base = self.landing_base();
7912 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7913 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7914 let this_round_verification = round_record.verification_summary(&head);
7915 let all_findings: Vec<_> = round_record
7916 .reviews
7917 .iter()
7918 .flat_map(|r| r.findings.iter().cloned())
7919 .collect();
7920 let blocking_findings: Vec<_> = all_findings
7923 .iter()
7924 .filter(|f| f.severity.blocks())
7925 .cloned()
7926 .collect();
7927 let fix_prompt = prompt::fix_in(
7928 &self.state.instruction,
7929 &blocking_findings,
7930 this_round_verification.as_ref(),
7931 round,
7932 max_rounds,
7933 &language,
7934 &self.state.config.graph.github_language,
7935 );
7936 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7937 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7938 let (job, seat, out) = self
7939 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7940 prompt: fix_prompt.clone(),
7941 spec,
7942 seat,
7943 cwd: winner.worktree.clone(),
7944 timeout,
7945 allow_write: true,
7946 sessions,
7947 artifacts: artifacts.clone(),
7948 stem: format!("fix-{round}"),
7949 handover: None,
7950 })
7951 .await;
7952 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7955 return Ok(FixStep::Resource);
7956 }
7957 let agent_id = seat.agent.clone();
7958
7959 let mut fix = FixRecord {
7960 agent: agent_id,
7961 addressed: Vec::new(),
7962 rejected: Vec::new(),
7963 notes: String::new(),
7964 committed: false,
7965 failed: None,
7966 duration_ms: 0,
7967 continuation: None,
7968 };
7969 let mut continuation = ContinuationRecord::not_needed();
7970 let mut final_seat = seat.clone();
7971 match out {
7972 AgentOutcome::Ok(o) => {
7973 fix.duration_ms = o.duration_ms;
7974 let parsed = verdict::extract_json::<FixReport>(&o.text);
7975 let incomplete_reason = match &parsed {
7982 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7983 "the reply parsed, but it reported a command whose own CLI never \
7984 confirmed an exit status"
7985 .to_owned(),
7986 ),
7987 Ok(_) => None,
7988 Err(e) => Some(e.to_string()),
7989 };
7990 match incomplete_reason {
7991 None => {
7992 let report = parsed.expect("checked Ok above");
7993 fix.addressed = report.addressed;
7994 fix.rejected = report.rejected;
7995 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7996 }
7997 Some(reason) => {
7998 let (resumed_seat, resolved, failure, cont) = self
7999 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
8000 .await;
8001 fix.duration_ms += cont.cumulative_wait_ms;
8002 continuation = cont;
8003 final_seat = resumed_seat;
8004 match resolved {
8005 Some(report) => {
8006 fix.addressed = report.addressed;
8007 fix.rejected = report.rejected;
8008 fix.notes =
8009 blind::sanitize_prose(&report.notes, &self.state.config.blind);
8010 }
8011 None => fix.failed = failure,
8012 }
8013 }
8014 }
8015 }
8016 AgentOutcome::Dropped(o) => {
8018 fix.duration_ms = o.duration_ms;
8019 let why = o
8020 .dropped
8021 .as_ref()
8022 .map(|d| d.why.as_str())
8023 .unwrap_or("the CLI ended the stream without delivering its answer");
8024 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8025 }
8026 AgentOutcome::Quota(o) => {
8027 self.state.quota.push(QuotaLoss::from_report(
8028 final_seat.key.clone(),
8029 "fix",
8030 o.quota.as_ref(),
8031 ));
8032 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8033 }
8034 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8035 }
8036 if self.state.resource_block.is_some() {
8037 self.state.seats.insert(final_seat.key.clone(), final_seat);
8038 return Ok(FixStep::Resource);
8039 }
8040 fix.continuation = Some(continuation);
8041 self.state.seats.insert(final_seat.key.clone(), final_seat);
8042 let coordination = self.coordinate(&winner.worktree).await?;
8043 if let Ok(r) = git::rescue_commit(
8044 &winner.worktree,
8045 &format!("magi: review round {round} fixes (uncommitted work)"),
8046 )
8047 .await
8048 {
8049 self.state.note_withheld("fix", &r.withheld);
8050 }
8051 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8052 fix.committed = after != before;
8053 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8061 drop(coordination);
8062 let progressed = diff_after != patch;
8063 let commit_note = if fix.committed {
8064 "committed"
8065 } else {
8066 "NO new commit"
8067 };
8068 let tree_note = if progressed {
8069 "changed vs base"
8070 } else {
8071 "unchanged vs base"
8072 };
8073 self.state.event(
8074 "fix",
8075 match &fix.failed {
8076 Some(reason) => {
8082 format!(
8083 "round {round}: fixer's adoption report was lost ({reason}); \
8084 {commit_note}, tree {tree_note}"
8085 )
8086 }
8087 None => format!(
8088 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8089 {tree_note}{}",
8090 fix.addressed.len(),
8091 fix.rejected.len(),
8092 if continuation.outcome == ContinuationOutcome::Resumed {
8093 format!(
8094 " (adoption report recovered after {} continuation(s))",
8095 continuation.attempts
8096 )
8097 } else {
8098 String::new()
8099 },
8100 ),
8101 },
8102 );
8103 round_record.fix = Some(fix);
8104 round_record.progressed = progressed;
8105 Ok(FixStep::Recorded {
8106 report_lost: matches!(
8107 continuation.outcome,
8108 ContinuationOutcome::Exhausted
8109 | ContinuationOutcome::QuotaLost
8110 | ContinuationOutcome::NoSession
8111 ),
8112 })
8113 }
8114
8115 fn record_contested_handoff(&mut self) {
8149 if self.state.contested_handoff.is_some() {
8150 return;
8151 }
8152 let Some(contested) = self
8153 .state
8154 .reviews
8155 .last()
8156 .and_then(ReviewRound::contested_handoff)
8157 else {
8158 return;
8159 };
8160 self.state.event(
8161 "review",
8162 format!(
8163 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8164 wait for the owner's approval",
8165 contested.findings.len(),
8166 contested.rejecters.len()
8167 ),
8168 );
8169 self.state.contested_handoff = Some(contested);
8170 }
8171
8172 async fn stop_unresolved_review(
8173 &mut self,
8174 why: &str,
8175 shell: &[String],
8176 worktree: &Path,
8177 ) -> Result<()> {
8178 self.stop_reviewing_inner(why, shell, worktree, true).await
8179 }
8180
8181 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8182 self.stop_reviewing_inner(why, shell, worktree, false).await
8183 }
8184
8185 async fn stop_reviewing_inner(
8186 &mut self,
8187 why: &str,
8188 shell: &[String],
8189 worktree: &Path,
8190 repairable: bool,
8191 ) -> Result<()> {
8192 let round_idx = self.state.reviews.len() - 1;
8193 let needs_catchup_run = matches!(
8201 self.state.reviews[round_idx].e2e_status(),
8202 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8203 );
8204 if needs_catchup_run {
8205 let round = self.state.reviews[round_idx].round;
8206 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8207 let commands = self.state.config.verify.e2e.clone();
8208 let before_head = git::rev_parse(worktree, "HEAD").await?;
8209 let winner = self.state.winner().cloned();
8210 let cache_dir = self.state.config.cache_dir();
8211 let context = format!(
8212 "round {round}: verification unresolved, catching up before the final decision"
8213 );
8214 let wait = self.repository_wait();
8215 let (outcomes, verify_retried) = with_cache_lease(
8216 &mut self.state,
8217 cache_dir.as_deref(),
8218 "e2e",
8219 "e2e",
8220 worktree,
8221 &before_head,
8222 timeout,
8223 &context,
8224 &wait,
8225 |state, budget| {
8226 let shell = shell.to_vec();
8227 let commands = commands.clone();
8228 let context = context.clone();
8229 async move {
8230 if let Some(winner) = winner.filter(|_| !commands.is_empty()) {
8231 Self::run_pre_gate(state, &winner).await;
8234 }
8235 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8236 .await
8237 }
8238 },
8239 )
8240 .await?;
8241 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8243 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8244 self.state.record_base_progress(&attempted_head);
8245 }
8246 let last = &mut self.state.reviews[round_idx];
8247 last.e2e = outcomes;
8248 last.verify_retried = verify_retried;
8249 last.verified_head = Some(attempted_head);
8256 last.verified_at = Some(Timestamp::now());
8257 if verify_inconclusive(&last.e2e) {
8258 self.state.save()?;
8265 return Ok(());
8266 }
8267 last.e2e_deferred = false;
8268 if last.answered == last.expected {
8273 last.clean = round_is_clean(
8274 last.blocking,
8275 last.e2e.iter().all(CommandOutcome::ok),
8276 last.answered,
8277 last.expected,
8278 0,
8279 self.state.config.graph.incomplete_review,
8280 );
8281 }
8282 }
8283 let last = &self.state.reviews[round_idx];
8284 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8285
8286 match last.e2e_status() {
8287 E2eStatus::Failed => {
8288 let red: Vec<String> = last
8289 .e2e
8290 .iter()
8291 .filter(|o| !o.ok())
8292 .map(|o| {
8293 format!(
8294 "`{}` -> {:?}\n{}",
8295 o.command,
8296 o.code,
8297 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8298 )
8299 })
8300 .collect();
8301 let can_repair = last.blocking == 0 && last.fix.is_none();
8302 self.state.reviews[round_idx].clean = false;
8303 self.state
8304 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8305 self.state.status = RunStatus::Blocked;
8306 if repairable
8307 && can_repair
8308 && self.state.reviews.len() < self.state.config.graph.review_rounds
8309 {
8310 self.state.status = RunStatus::Reviewing;
8311 }
8312 }
8313 E2eStatus::ResourceBlocked => {
8318 self.state.event(
8319 "review",
8320 format!(
8321 "{why}; e2e could not run (resource unavailable); not \
8322 deciding yet"
8323 ),
8324 );
8325 }
8326 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8327 self.state.event(
8328 "review",
8329 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8330 );
8331 self.record_contested_handoff();
8332 self.state.status = RunStatus::Gating;
8333 }
8334 }
8335 self.state.save()?;
8336 Ok(())
8337 }
8338
8339 async fn gate(&mut self) -> Result<()> {
8342 if self.hold_base_sync().await? {
8343 return Ok(());
8344 }
8345 if self.state.status == RunStatus::Failed
8357 || self
8358 .state
8359 .base_sync
8360 .as_ref()
8361 .is_some_and(|s| s.conflict.is_some())
8362 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8363 != Some(RunStatus::Gating)
8364 {
8365 return Ok(());
8366 }
8367 let Some(winner) = self.state.winner().cloned() else {
8368 return Ok(());
8369 };
8370 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8371 self.state.reconcile_verification_head(&head);
8372 self.state.save()?;
8373 let wait = self.repository_wait();
8374 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8375 return Ok(());
8376 }
8377 if self.state.gate_ran {
8378 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8389 self.state.status = RunStatus::Blocked;
8390 self.state.save()?;
8391 }
8392 return Ok(());
8393 }
8394 self.state.status = RunStatus::Gating;
8395 let mut outcomes = self.run_gate(&winner).await?;
8396 let mut attempted_head = self.state.gate_head.clone();
8397 loop {
8398 if verify_inconclusive(&outcomes) {
8409 self.state.save()?;
8410 return Ok(());
8411 }
8412 if outcomes.iter().all(CommandOutcome::ok) {
8413 break;
8414 }
8415 let fixes_before = self.state.gate_fixes.len();
8416 match self.gate_fix_round(&winner, &outcomes).await? {
8417 GateFix::Retry => {
8418 outcomes = self.run_gate(&winner).await?;
8419 attempted_head = self.state.gate_head.clone();
8420 }
8421 GateFix::Stop => {
8422 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8423 if self.state.gate_fixes.len() > fixes_before
8424 && attempted_head.as_deref() != Some(current.as_str())
8425 && verify_current_head(&mut self.state, &winner, &wait).await?
8426 {
8427 outcomes = self.run_gate(&winner).await?;
8430 attempted_head = self.state.gate_head.clone();
8431 } else {
8432 break;
8433 }
8434 }
8435 GateFix::Defer => {
8436 self.state.save()?;
8437 return Ok(());
8438 }
8439 }
8440 }
8441 let passed = outcomes.iter().all(CommandOutcome::ok);
8442 self.state.gate = outcomes;
8443 self.state.gate_ran = true;
8444 self.state.gate_head = attempted_head;
8445 if !self.state.gate.is_empty()
8446 && !verify_inconclusive(&self.state.gate)
8447 && let Some(head) = self.state.gate_head.clone()
8448 {
8449 self.state.record_base_progress(&head);
8450 }
8451 self.state.save()?;
8452 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8453 return Ok(());
8454 }
8455 if !passed {
8456 self.state.status = RunStatus::Blocked;
8457 let spent = self.state.gate_fixes.len();
8458 self.state.event(
8459 "gate",
8460 if spent == 0 {
8461 "gate failed; not merging".to_owned()
8462 } else {
8463 format!("gate failed after {spent} gate-fix round(s); not merging")
8464 },
8465 );
8466 }
8467 self.state.save()?;
8468 Ok(())
8469 }
8470
8471 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8489 let commands = state.config.verify.pre_gate.clone();
8490 if commands.is_empty() {
8491 return;
8492 }
8493 let shell = state.config.shell();
8494 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8495 let (outcomes, _) = run_commands(
8496 state,
8497 "pre_gate",
8498 "pre_gate",
8499 0,
8500 &shell,
8501 &commands,
8502 &winner.worktree,
8503 timeout,
8504 )
8505 .await;
8506 for o in &outcomes {
8507 if !o.ok() {
8508 tracing::warn!(
8509 "pre_gate `{}` failed ({:?}); the gate decides",
8510 o.command,
8511 o.code
8512 );
8513 }
8514 state.event(
8515 "pre_gate",
8516 format!(
8517 "`{}` -> {}",
8518 o.command,
8519 if o.ok() {
8520 "pass".to_owned()
8521 } else {
8522 format!(
8523 "FAIL ({:?})\n{}",
8524 o.code,
8525 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8526 )
8527 }
8528 ),
8529 );
8530 }
8531 state.pre_gate = outcomes;
8532 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8533 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8534 Ok(head) => {
8535 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8536 state.pre_gate_commit = Some(head);
8537 }
8538 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8539 },
8540 Ok(false) => {}
8541 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8542 }
8543 if let Err(e) = state.save() {
8544 tracing::warn!("could not persist the pre_gate record: {e:#}");
8545 }
8546 }
8547
8548 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8551 let wait = self.repository_wait();
8552 run_landing_gate(&mut self.state, winner, &wait).await
8553 }
8554
8555 async fn gate_fix_round(
8568 &mut self,
8569 winner: &Candidate,
8570 outcomes: &[CommandOutcome],
8571 ) -> Result<GateFix> {
8572 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8573 g.phase == crate::repair_budget::Phase::Fixing
8574 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8575 }) {
8576 self.state.gate_fixes.len().saturating_add(1)
8577 } else {
8578 self.state.config.graph.gate_fix_rounds
8579 };
8580 let spent = self.state.gate_fixes.len();
8581 if spent >= cap {
8582 if cap > 0 {
8583 self.state.event(
8584 "gate",
8585 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8586 );
8587 }
8588 return Ok(GateFix::Stop);
8589 }
8590 if !gate_fixable(outcomes) {
8591 self.state.event(
8592 "gate",
8593 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8594 command or similar); not spending a fix round on it",
8595 );
8596 return Ok(GateFix::Stop);
8597 }
8598 let min_free = self.state.config.disk.min_free_bytes;
8599 if min_free > 0 {
8600 match crate::disk::free_bytes(&winner.worktree) {
8601 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8602 Ok(free) => {
8603 self.state.event(
8604 "gate",
8605 format!(
8606 "only {free} bytes free ({min_free} required by `[disk] \
8607 min_free_bytes`); not spending a fix round on a failure the disk \
8608 may explain"
8609 ),
8610 );
8611 return Ok(GateFix::Stop);
8612 }
8613 Err(e) => {
8614 self.state.event(
8615 "gate",
8616 format!("free disk space could not be measured ({e:#}); no fix round"),
8617 );
8618 return Ok(GateFix::Stop);
8619 }
8620 }
8621 }
8622
8623 let attempt = spent + 1;
8624 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8625 let base = self.landing_base();
8626 let fix_prompt = prompt::gate_fix_in(
8627 &self.state.instruction,
8628 &failed,
8629 attempt,
8630 cap,
8631 &self.state.config.graph.language,
8632 &self.state.config.graph.github_language,
8633 );
8634 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8635 let sessions = self.state.config.graph.sessions;
8636 let artifacts = self.merge_repair_artifacts();
8637 self.state.event(
8638 "gate",
8639 format!("gate failed; gate-fix round {attempt} of {cap}"),
8640 );
8641 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8642 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8643 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8646 head: before.clone(),
8647 passed: false,
8648 outcomes: Vec::new(),
8649 });
8650 self.state.save()?;
8651 let (_, seat, out) = self
8652 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8653 prompt: fix_prompt.clone(),
8654 spec,
8655 seat,
8656 cwd: winner.worktree.clone(),
8657 timeout,
8658 allow_write: true,
8659 sessions,
8660 artifacts: artifacts.clone(),
8661 stem: format!("gate-fix-{attempt}"),
8662 handover: None,
8663 })
8664 .await;
8665 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8668 return Ok(GateFix::Defer);
8669 }
8670 let mut record = GateFixRecord {
8671 agent: seat.agent.clone(),
8672 failed,
8673 notes: String::new(),
8674 committed: false,
8675 error: None,
8676 };
8677 match out {
8678 AgentOutcome::Ok(o) => {
8679 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8682 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8683 }
8684 }
8685 AgentOutcome::Dropped(_) => {
8686 record.error = Some("the CLI dropped the stream".to_owned());
8687 }
8688 AgentOutcome::Quota(o) => {
8689 self.state.quota.push(QuotaLoss::from_report(
8690 seat.key.clone(),
8691 "gate-fix",
8692 o.quota.as_ref(),
8693 ));
8694 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8695 }
8696 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8697 }
8698 self.state.seats.insert(seat.key.clone(), seat);
8699 let coordination = self.coordinate(&winner.worktree).await?;
8700 if let Ok(r) = git::rescue_commit(
8701 &winner.worktree,
8702 &format!("magi: gate fix {attempt} (uncommitted work)"),
8703 )
8704 .await
8705 {
8706 self.state.note_withheld("gate-fix", &r.withheld);
8707 }
8708 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8709 record.committed = after != before;
8710 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8711 let note = record.error.clone();
8712 self.state.gate_fixes.push(record);
8713 self.state.save()?;
8714 if !changed {
8715 self.state.event(
8716 "gate",
8717 match note {
8718 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8719 None => format!("gate-fix round {attempt}: the tree did not change"),
8720 },
8721 );
8722 return Ok(GateFix::Stop);
8723 }
8724 self.state.event(
8725 "gate",
8726 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8727 );
8728
8729 drop(coordination);
8730 let wait = self.repository_wait();
8731 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8732 }
8733
8734 async fn guarded_pr_message(
8741 &mut self,
8742 winner: &Candidate,
8743 facts: Option<&BranchFacts>,
8744 posting: bool,
8745 ) -> Result<Option<PrMessage>> {
8746 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8747 let head = match final_head.as_deref() {
8748 Some(h) => crate::pr_outcome::FinalHead::At(h),
8749 None => crate::pr_outcome::FinalHead::Unknown,
8750 };
8751 let summarized = if posting {
8753 Some(self.summarize_change(winner, final_head.as_deref()).await)
8754 } else {
8755 None
8756 };
8757 if self.state.resource_block.is_some() {
8762 self.state.event(
8763 "pr-summary",
8764 "a shared resource was busy; nothing was published, resume to summarize again",
8765 );
8766 return Ok(None);
8767 }
8768 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8769 anyhow::bail!(
8770 "the final commit moved during public description generation; retry before publishing"
8771 );
8772 }
8773 let note = match (&summarized, &self.state.change_summary) {
8774 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8775 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8776 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8777 (None, _) => ChangeNote::NotAsked,
8778 };
8779 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8780 if !posting {
8781 let identity = crate::scrub::Identity::current();
8782 return Ok(Some(PrMessage {
8783 title: crate::scrub::scrub(&pr.title, &identity),
8784 body: crate::scrub::scrub(&pr.body, &identity),
8785 }));
8786 }
8787 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8790 if let Some(known) = self
8791 .state
8792 .github_text
8793 .clone()
8794 .filter(|g| g.fingerprint == raw_fp)
8795 {
8796 return self.settle_withheld(winner, &pr, known).await;
8797 }
8798 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8799 let mut violations =
8800 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8801 let identity = crate::scrub::Identity::current();
8802 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8803 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8804 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8805 {
8806 violations.push(crate::github_text::Violation::SensitiveData);
8807 }
8808 let has_language = violations
8812 .iter()
8813 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8814 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8815 self.state.event(
8816 "github-text",
8817 if has_language {
8818 format!("description rejected: {violations:?}; requesting one rewrite")
8819 } else {
8820 format!("description rejected: {violations:?}; asking the owner")
8821 },
8822 );
8823 let mut rewritten = false;
8824 let mut rejected_rewrite = None;
8825 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8826 {
8827 let output = text;
8828 if let Some(title) = summary_title(&output) {
8829 let raw_body = summary_without_title(&output);
8832 let clean_title = crate::scrub::scrub(&title, &identity);
8833 let mut body = crate::scrub::scrub(&raw_body, &identity);
8834 let redacted = clean_title != title || body != raw_body;
8835 let title = clean_title;
8836 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8837 if !complete {
8838 self.state.event(
8839 "github-text",
8840 "description rewrite omitted required findings",
8841 );
8842 }
8843 if !body.trim().is_empty() {
8844 body.push_str("\n\n");
8845 body.push_str(&public_pr_evidence(&self.state, head));
8846 }
8847 let marker = format!("magi:run/{}", self.state.id);
8848 if !body.trim().is_empty() && !body.contains(&marker) {
8849 body.push_str(&format!("\n\n{marker}"));
8850 }
8851 let rewrite = PrMessage {
8852 title: title.clone(),
8853 body: body.clone(),
8854 };
8855 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8856 let mut failed = crate::github_text::check_for(
8857 &self.state.config.graph,
8858 &title,
8859 &body,
8860 redecision,
8861 );
8862 if (!crate::github_text::shareable(&title)
8863 || !crate::github_text::shareable(&body))
8864 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8865 {
8866 failed.push(crate::github_text::Violation::SensitiveData);
8867 }
8868 if complete
8869 && !title.trim().is_empty()
8870 && !body.trim().is_empty()
8871 && !failed.is_empty()
8872 {
8873 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8874 title: title.clone(),
8875 body: body.clone(),
8876 failed: failed
8877 .iter()
8878 .map(|v| crate::github_text::category(*v).to_owned())
8879 .collect(),
8880 });
8881 }
8882 if complete
8883 && !title.trim().is_empty()
8884 && !raw_body.trim().is_empty()
8885 && failed.is_empty()
8886 {
8887 decision = redecision;
8888 pr = PrMessage { title, body };
8889 rewritten = true;
8890 self.state
8891 .event("github-text", "description rewrite passed");
8892 if redacted {
8893 self.state.event(
8894 "github-text",
8895 "sensitive data removed from description rewrite",
8896 );
8897 }
8898 }
8899 }
8900 }
8901 if !rewritten {
8902 self.state.event(
8903 "github-text",
8904 "description rewrite unavailable or rejected; asking the owner",
8905 );
8906 if let Some(w) =
8907 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8908 {
8909 self.retire_github_text_question("the withheld text changed");
8912 let record = crate::run::GithubTextAsk {
8913 rewrite: rejected_rewrite,
8914 fingerprint: raw_fp,
8915 title: w.title,
8916 body: w.body,
8917 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8918 question: None,
8919 asks: 0,
8920 resolved: false,
8921 chosen_title: None,
8922 choice: None,
8923 publication: None,
8924 };
8925 return self.settle_withheld(winner, &pr, record).await;
8926 }
8927 }
8928 }
8929 let (title, body) =
8930 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8931 Ok(Some(PrMessage { title, body }))
8932 }
8933
8934 async fn summarize_change(
8945 &mut self,
8946 winner: &Candidate,
8947 head: Option<&str>,
8948 ) -> std::result::Result<(), String> {
8949 let Some(head) = head else {
8950 return Err("the final commit could not be read".to_owned());
8951 };
8952 let remote = self.state.config.merge.remote.clone();
8953 let tracking = format!("{remote}/{}", self.state.base_branch);
8954 if !git::rev_exists(&winner.worktree, &tracking).await {
8955 return Err(format!("the remote base `{tracking}` could not be read"));
8956 }
8957 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8958 Ok(b) if !b.is_empty() => b,
8959 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8960 };
8961 let evidence = crate::pr_outcome::summary_evidence(
8962 &self.state,
8963 crate::pr_outcome::FinalHead::At(head),
8964 );
8965 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8966 if self
8967 .state
8968 .change_summary
8969 .as_ref()
8970 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8971 {
8972 return Ok(());
8973 }
8974 if self.state.change_summary.take().is_some() {
8975 self.state.event(
8976 "pr-summary",
8977 "the final diff or evidence changed; the earlier change summary was discarded",
8978 );
8979 }
8980 if let Err(e) = write_artifact(
8982 &self.state,
8983 &format!("public-description-{fingerprint}.md"),
8984 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8985 ) {
8986 tracing::warn!("could not save public description evidence: {e:#}");
8987 }
8988 let chain = match agent::pick_chain(
8989 &self.state.config.agents,
8990 self.state.config.roles.synthesizer.as_ref(),
8991 &agent::installed,
8992 "synthesizer",
8993 ) {
8994 Ok(chain) => chain,
8995 Err(e) => {
8996 tracing::warn!("no agent for the change summary: {e:#}");
8997 self.state.event("pr-summary", "no summary agent available");
8998 return Err("no agent was available to summarize it".to_owned());
8999 }
9000 };
9001 let subjects = git::subjects(&winner.worktree, &base, head)
9002 .await
9003 .unwrap_or_default();
9004 let stat = git::diff_stat(&winner.worktree, &base, head)
9005 .await
9006 .unwrap_or_default();
9007 let stat: String = stat.chars().take(6_000).collect();
9008 let mut prompt = crate::prompt::change_summary_in(
9009 &base,
9010 head,
9011 &subjects,
9012 stat.trim(),
9013 &self.state.config.graph.github_language,
9014 );
9015 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
9016 let sessions = self.state.config.graph.sessions;
9017 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
9018 let prompts = self.state.config.prompts.clone();
9019 let cache = self.state.config.cache_dir();
9020 let run = self.state.id.clone();
9021 for (i, spec) in chain.into_iter().enumerate() {
9022 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
9023 let github_spec = match agent::for_github_text(
9024 &spec,
9025 &self.state.config.graph.github_language,
9026 &winner.worktree,
9027 ) {
9028 Ok(spec) => spec,
9029 Err(e) => {
9030 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9031 continue;
9032 }
9033 };
9034 let job = SeatJob {
9035 spec: github_spec,
9036 seat,
9037 cwd: winner.worktree.clone(),
9038 prompt: prompt.clone(),
9039 timeout,
9040 allow_write: false,
9041 sessions,
9042 artifacts: agent::artifacts_dir(&self.state.dir()),
9043 stem: if i == 0 {
9044 "change-summary".to_owned()
9045 } else {
9046 format!("change-summary-{}", spec.id)
9047 },
9048 handover: None,
9049 };
9050 let ctx = WaveCtx {
9051 run: &run,
9052 node: "pr-summary",
9053 prompts: &prompts,
9054 cache: cache.as_deref(),
9055 round: None,
9056 carry_seats: false,
9057 pauses: Some((&self.pause, &self.interrupt)),
9058 };
9059 let (seat, outcome) =
9060 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9061 self.state.seats.insert(seat.key.clone(), seat);
9062 let text = match outcome {
9063 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9064 AgentOutcome::ResourceBlocked(why) => {
9065 if self.state.resource_block.is_none() {
9068 self.state.resource_block = Some(crate::run::ResourceBlock {
9069 cause: crate::run::ResourceBlockCause::Other,
9070 node: "pr-summary".to_owned(),
9071 kind: "shared resource".to_owned(),
9072 detail: why,
9073 at: Timestamp::now(),
9074 });
9075 }
9076 return Err("a shared resource was busy".to_owned());
9077 }
9078 _ => continue,
9079 };
9080 let blind = &self.state.config.blind;
9081 let Some(title) = summary_title(&text) else {
9082 continue;
9083 };
9084 let body: String = summary_without_title(&text)
9085 .lines()
9086 .filter(|l| !l.contains("magi:run/"))
9087 .collect::<Vec<_>>()
9088 .join("\n");
9089 let body = blind::sanitize_prose(body.trim(), blind);
9090 let title = blind::sanitize_prose(&title, blind);
9091 if body.trim().is_empty() || title.trim().is_empty() {
9092 continue;
9093 }
9094 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9095 continue;
9096 }
9097 if git::rev_parse(&winner.worktree, "HEAD")
9099 .await
9100 .ok()
9101 .as_deref()
9102 != Some(head)
9103 {
9104 return Err("the branch moved while the summary was written".to_owned());
9105 }
9106 if git::merge_base(&winner.worktree, &tracking, head)
9107 .await
9108 .ok()
9109 .as_deref()
9110 != Some(base.as_str())
9111 || crate::pr_outcome::summary_evidence(
9112 &self.state,
9113 crate::pr_outcome::FinalHead::At(head),
9114 ) != evidence
9115 {
9116 return Err("the summary evidence changed while it was written".to_owned());
9117 }
9118 self.state.change_summary = Some(crate::run::ChangeSummary {
9119 head: head.to_owned(),
9120 base,
9121 evidence: fingerprint,
9122 title,
9123 body,
9124 });
9125 self.state.event(
9126 "pr-summary",
9127 format!("change summary written by {} for the final diff", spec.id),
9128 );
9129 if let Err(e) = self.state.save() {
9130 tracing::warn!("could not save the change summary: {e:#}");
9131 }
9132 return Ok(());
9133 }
9134 self.state
9135 .event("pr-summary", "no agent produced a change summary");
9136 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9137 }
9138
9139 async fn rewrite_pr_text(
9149 &mut self,
9150 winner: &Candidate,
9151 pr: &PrMessage,
9152 violations: &[crate::github_text::Violation],
9153 ) -> Option<String> {
9154 let prompt = format!(
9155 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9156 pr.title, pr.body
9157 );
9158 let sessions = self.state.config.graph.sessions;
9159 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9160 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9163 let key = format!("impl-{}", winner.label);
9164 if let Some(spec) = self
9165 .state
9166 .config
9167 .agents
9168 .iter()
9169 .find(|a| a.id == winner.agent)
9170 .cloned()
9171 && let Some(existing) = self.state.seats.get(&key)
9172 && existing.agent == spec.id
9173 && agent::has_session(spec.kind, existing, sessions)
9174 {
9175 attempts.push((spec, Some(existing.clone())));
9176 }
9177 match agent::pick_chain(
9178 &self.state.config.agents,
9179 self.state.config.roles.language_judge.as_ref(),
9180 &agent::installed,
9181 "language_judge",
9182 ) {
9183 Ok(chain) => {
9184 for spec in chain {
9185 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9186 continue;
9187 }
9188 attempts.push((spec, None));
9189 }
9190 }
9191 Err(e) if attempts.is_empty() => {
9192 tracing::warn!("no agent for the description rewrite: {e:#}");
9193 self.state
9194 .event("github-text", "no rewrite agent available");
9195 return None;
9196 }
9197 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9198 }
9199 let prompts = self.state.config.prompts.clone();
9200 let cache = self.state.config.cache_dir();
9201 let run = self.state.id.clone();
9202 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9203 let resumed = resumed_seat.is_some();
9204 let seat = resumed_seat.unwrap_or_else(|| {
9205 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9206 });
9207 let github_spec = match agent::for_github_text(
9208 &spec,
9209 &self.state.config.graph.github_language,
9210 &winner.worktree,
9211 ) {
9212 Ok(spec) => spec,
9213 Err(e) => {
9214 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9215 continue;
9216 }
9217 };
9218 let job = SeatJob {
9219 spec: github_spec,
9220 seat,
9221 cwd: winner.worktree.clone(),
9222 prompt: prompt.clone(),
9223 timeout: retry_budget(timeout_full, resumed),
9224 allow_write: false,
9225 sessions,
9226 artifacts: agent::artifacts_dir(&self.state.dir()),
9227 stem: if i == 0 {
9228 "github-text-rewrite".to_owned()
9229 } else {
9230 format!("github-text-rewrite-{}", spec.id)
9231 },
9232 handover: None,
9233 };
9234 let ctx = WaveCtx {
9235 run: &run,
9236 node: "github-text",
9237 prompts: &prompts,
9238 cache: cache.as_deref(),
9239 round: None,
9240 carry_seats: false,
9241 pauses: Some((&self.pause, &self.interrupt)),
9242 };
9243 let (seat, outcome) =
9244 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9245 self.state.seats.insert(seat.key.clone(), seat);
9246 match outcome {
9247 AgentOutcome::Ok(o)
9248 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9249 {
9250 return Some(o.text);
9251 }
9252 AgentOutcome::ResourceBlocked(_) => return None,
9253 _ => {}
9254 }
9255 }
9256 None
9257 }
9258
9259 fn retire_github_text_question(&mut self, why: &str) {
9261 let Some(id) = self
9262 .state
9263 .github_text
9264 .as_ref()
9265 .and_then(|g| g.question.clone())
9266 else {
9267 return;
9268 };
9269 let _ = ask::Questions::open().update(&id, |q| {
9270 q.abandon(why);
9271 Ok(())
9272 });
9273 }
9274
9275 async fn settle_withheld(
9279 &mut self,
9280 winner: &Candidate,
9281 pr: &PrMessage,
9282 mut g: crate::run::GithubTextAsk,
9283 ) -> Result<Option<PrMessage>> {
9284 use crate::github_text as gt;
9285 let store = ask::Questions::open();
9286 let timeout = self.state.config.graph.answer_timeout;
9287 if !g.resolved {
9288 let mut standing = g
9289 .question
9290 .as_ref()
9291 .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9292 if let Some(q) = standing.as_ref()
9293 && q.status.open()
9294 && !store.held_reply(&q.id)
9295 && gt::expired(q, timeout)
9296 {
9297 let why = format!("no answer within {}s of asking", timeout.max(1));
9298 standing = store
9299 .update(&q.id, |q| {
9300 q.abandon(&why);
9301 Ok(())
9302 })
9303 .ok()
9304 .map(|(q, ())| q);
9305 }
9306 let mut ask_again: Option<Option<Vec<String>>> = None;
9307 match standing {
9308 None if g.asks <= 1 => {
9311 g.asks = 0;
9312 ask_again = Some(None);
9313 }
9314 None => {
9315 g.resolved = true;
9316 g.chosen_title = None;
9317 g.choice = Some(GithubTextChoice::Fallback);
9318 }
9319 Some(q) if q.status.open() => {
9320 self.park_on_github_text(&g)?;
9321 return Ok(None);
9322 }
9323 Some(q) => match gt::read_reply(&q) {
9324 gt::Reply::Fallback => {
9325 g.resolved = true;
9326 g.chosen_title = None;
9327 g.choice = Some(GithubTextChoice::Fallback);
9328 }
9329 gt::Reply::Original => {
9330 g.resolved = true;
9331 g.chosen_title = None;
9332 g.choice = Some(GithubTextChoice::Original);
9333 }
9334 gt::Reply::Rewrite => {
9335 g.resolved = true;
9336 g.chosen_title = None;
9337 g.choice = Some(if g.rewrite.is_some() {
9338 GithubTextChoice::Rewrite
9339 } else {
9340 GithubTextChoice::Fallback
9341 });
9342 }
9343 gt::Reply::Title(text) => {
9344 let first = text
9345 .lines()
9346 .map(str::trim)
9347 .find(|l| !l.is_empty())
9348 .unwrap_or("");
9349 let decision = if first.is_empty() {
9350 None
9351 } else {
9352 let candidate = PrMessage {
9353 title: first.to_owned(),
9354 body: String::new(),
9355 };
9356 self.judge_pr_language(&winner.worktree, &candidate).await
9357 };
9358 match gt::vet_title(
9359 &text,
9360 decision,
9361 self.state.config.graph.github_english(),
9362 ) {
9363 Ok(title) => {
9364 g.resolved = true;
9365 g.chosen_title = Some(title);
9366 g.choice = Some(GithubTextChoice::MyTitle);
9367 }
9368 Err(cats) if g.asks < 2 => {
9369 ask_again = Some(Some(
9370 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9371 ))
9372 }
9373 Err(_) => {
9374 g.resolved = true;
9375 g.chosen_title = None;
9376 g.choice = Some(GithubTextChoice::Fallback);
9377 }
9378 }
9379 }
9380 },
9381 }
9382 if let Some(rejected) = ask_again {
9383 let w = gt::Withheld {
9386 title: g.title,
9387 body: g.body,
9388 categories: g.categories.clone(),
9389 };
9390 let artifact = match crate::run::write_artifact(
9393 &self.state,
9394 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9395 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9396 ) {
9397 Ok(p) => Some(p),
9398 Err(e) => {
9399 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9400 None
9401 }
9402 };
9403 let (detail, comparison) = gt::question_content(
9404 &w,
9405 &pr.title,
9406 &pr.body,
9407 &self.state.id,
9408 rejected.as_deref(),
9409 artifact.as_deref(),
9410 g.rewrite.as_ref(),
9411 );
9412 let mut q = ask::Question::new(
9413 self.state.id.clone(),
9414 gt::ASK_NODE.to_owned(),
9415 gt::ASK_SEAT.to_owned(),
9416 gt::question_summary(&self.state.config.graph.language, &w),
9417 detail,
9418 gt::question_choices(&w, g.rewrite.as_ref()),
9419 );
9420 q.posting_comparison = Some(comparison);
9421 q.answer_timeout = timeout;
9422 g.question = Some(q.id.clone());
9425 g.asks += 1;
9426 self.state.github_text = Some(g.clone());
9427 self.state.event(
9428 "github-text",
9429 format!("asking the owner about the withheld text ({})", q.short()),
9430 );
9431 self.state.save()?;
9432 store
9433 .put_executable(&mut q)
9434 .context("file the posting-gate question")?;
9435 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9436 tracing::warn!(
9437 "could not notify about posting-gate question {}: {e:#}",
9438 q.short()
9439 );
9440 }
9441 self.park_on_github_text(&g)?;
9442 return Ok(None);
9443 }
9444 }
9445 let resolved = g.chosen_title.clone();
9446 let original = g.choice == Some(GithubTextChoice::Original);
9447 self.state.event(
9449 "github-text",
9450 if original {
9451 format!(
9452 "posting the original text for: {}",
9453 match (g.title, g.body) {
9454 (true, true) => "title, description",
9455 (true, false) => "title",
9456 _ => "description",
9457 }
9458 )
9459 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9460 "posting the saved rewrite title and description".to_owned()
9461 } else if resolved.is_some() {
9462 "posting the owner's replacement title".to_owned()
9463 } else {
9464 "posting the neutral text for the withheld fields".to_owned()
9465 },
9466 );
9467 self.state.github_text = Some(g.clone());
9468 self.state.save()?;
9469 if g.choice == Some(GithubTextChoice::Rewrite)
9470 && let Some(r) = g.rewrite.as_ref()
9471 {
9472 return Ok(Some(PrMessage {
9473 title: r.title.clone(),
9474 body: r.body.clone(),
9475 }));
9476 }
9477 let w = gt::Withheld {
9478 title: g.title,
9479 body: g.body,
9480 categories: g.categories.clone(),
9481 };
9482 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9483 if let Some(t) = resolved {
9484 title = t;
9485 }
9486 let english = gt::LanguageDecision {
9488 title_english: true,
9489 body_english: true,
9490 body_complete: true,
9491 };
9492 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9493 if original {
9494 if g.title {
9497 title = pr.title.clone();
9498 }
9499 if g.body {
9500 body = pr.body.clone();
9501 }
9502 }
9503 Ok(Some(PrMessage { title, body }))
9504 }
9505
9506 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9507 self.state.github_text = Some(g.clone());
9508 self.state.end_pass_parked(ParkCause::Question);
9509 self.state.parked = true;
9510 self.state.event(
9511 "github-text",
9512 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9513 );
9514 self.state.save()
9515 }
9516
9517 async fn judge_pr_language(
9520 &mut self,
9521 cwd: &Path,
9522 pr: &PrMessage,
9523 ) -> Option<crate::github_text::LanguageDecision> {
9524 self.state.config.roles.language_judge.as_ref()?;
9525 if !self.state.config.graph.language_gate() {
9526 return None;
9527 }
9528 let decision =
9529 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9530 self.state.event(
9531 "github-text",
9532 if decision.is_some() {
9533 "language decided by the language judge"
9534 } else {
9535 "language judge unavailable; using built-in heuristics"
9536 },
9537 );
9538 decision
9539 }
9540
9541 fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9542 Box::pin(self.merge_inner())
9543 }
9544
9545 async fn merge_inner(&mut self) -> Result<()> {
9546 if self.resource_halt()? || self.hold_base_sync().await? {
9547 return Ok(());
9548 }
9549 if self
9564 .state
9565 .base_sync
9566 .as_ref()
9567 .is_some_and(|s| s.conflict.is_some())
9568 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9569 != Some(RunStatus::Gating)
9570 || !self.state.gate_allows_progress()
9578 {
9579 return Ok(());
9580 }
9581 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9590 return Ok(());
9591 }
9592 let Some(winner) = self.state.winner().cloned() else {
9593 return Ok(());
9594 };
9595 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9596 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9597 self.state.status = RunStatus::Blocked;
9598 self.state.event(
9599 "gate",
9600 "winner changed after gate; fresh verification required",
9601 );
9602 self.state.save()?;
9603 return Ok(());
9604 }
9605 let wait = self.repository_wait();
9606 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9607 return Ok(());
9608 }
9609 let repo = self.state.repo.clone();
9610 let base = self.state.base_branch.clone();
9611 let mode = self.state.config.merge.mode;
9612 let style = self.state.config.merge.style;
9613 let facts = if is_review_run(&self.state) {
9614 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9615 let start = review_base(
9616 &repo,
9617 &self.state.config.merge.remote,
9618 &base,
9619 &self.state.base_commit,
9620 &winner.branch,
9621 )
9622 .await;
9623 branch_facts(&repo, &start, &winner.branch).await
9624 } else {
9625 None
9626 };
9627 let leaked = if is_review_run(&self.state) {
9630 leaked_subjects(&self.state, &winner.branch).await
9631 } else {
9632 Some(Vec::new())
9633 };
9634 let Some(pr) = self
9638 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9639 .await?
9640 else {
9641 return Ok(());
9642 };
9643 let message = pr.commit_message();
9644
9645 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9646 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9647 return Ok(());
9648 }
9649 let outcome = match mode {
9650 MergeMode::None => MergeOutcome {
9651 mode,
9652 ok: true,
9653 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9654 empty: false,
9655 },
9656 MergeMode::Pr | MergeMode::Local
9657 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9658 {
9659 MergeOutcome {
9660 mode,
9661 ok: false,
9662 detail: empty_candidate_detail(&self.state, &base),
9663 empty: true,
9664 }
9665 }
9666 MergeMode::Local => {
9667 let on = git::current_branch(&repo).await?;
9668 if on.as_deref() != Some(base.as_str()) {
9669 MergeOutcome {
9670 mode,
9671 ok: false,
9672 detail: format!(
9673 "{} has {} checked out, not the base branch {base}",
9674 repo.display(),
9675 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9676 ),
9677 empty: false,
9678 }
9679 } else if !git::is_clean(&repo).await? {
9680 MergeOutcome {
9681 mode,
9682 ok: false,
9683 detail: format!("{} is dirty; refusing to merge", repo.display()),
9684 empty: false,
9685 }
9686 } else {
9687 let out = match style {
9688 MergeStyle::Merge => {
9689 git::merge_no_ff(&repo, &winner.branch, &message).await?
9690 }
9691 MergeStyle::Squash => {
9692 git::merge_squash(&repo, &winner.branch, &message).await?
9693 }
9694 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9695 };
9696 MergeOutcome {
9697 mode,
9698 ok: out.ok(),
9699 detail: if out.ok() { out.stdout } else { out.stderr },
9700 empty: false,
9701 }
9702 }
9703 }
9704 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9705 self.state.pr_publication_pending = true;
9706 self.state.save()?;
9707 let out =
9708 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9709 self.state.pr_publication_pending = out.is_err();
9710 MergeOutcome {
9711 mode,
9712 ok: out.is_ok(),
9713 detail: match out {
9714 Ok(url) => url,
9715 Err(e) => format!("{e:#}"),
9716 },
9717 empty: false,
9718 }
9719 }
9720 MergeMode::Pr => {
9721 let remote = self.state.config.merge.remote.clone();
9722 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9723 let pushed = if self.state.foreign_checkout.is_some() {
9724 let refspec = format!("{delivery_head}:refs/heads/{public}");
9725 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9726 } else {
9727 git::push(&winner.worktree, &remote, &winner.branch).await?
9728 };
9729 if !pushed.ok() {
9730 MergeOutcome {
9731 mode,
9732 ok: false,
9733 detail: pushed.stderr,
9734 empty: false,
9735 }
9736 } else {
9737 if self.state.foreign_checkout.is_some() {
9738 self.state.pr_pushed_head = Some(delivery_head.clone());
9739 }
9740 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9746 let out = match pr_merge_plan(found) {
9747 PrPlan::Create => {
9748 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9749 .await
9750 }
9751 PrPlan::Adopt { url, title } => {
9752 self.state
9753 .event("merge", format!("Pr: adopted open pull request {url}"));
9754 if self.state.explicit_github_text_selection() {
9755 land::set_pr_text_verbatim(
9758 &winner.worktree,
9759 &url,
9760 &pr.title,
9761 &pr.body,
9762 )
9763 .await
9764 .map(|()| url)
9765 } else {
9766 if title != pr.title
9767 && (!is_review_run(&self.state)
9768 || leaked
9769 .as_deref()
9770 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9771 && let Err(e) =
9772 if self.state.github_text.as_ref().is_some_and(|g| {
9773 g.resolved
9774 && g.title
9775 && g.choice == Some(GithubTextChoice::Original)
9776 }) {
9777 land::set_pr_title_verbatim(
9779 &winner.worktree,
9780 &url,
9781 &pr.title,
9782 )
9783 .await
9784 } else {
9785 land::set_pr_title(
9786 &mut self.state,
9787 &winner.worktree,
9788 &url,
9789 &pr.title,
9790 )
9791 .await
9792 }
9793 {
9794 tracing::warn!("could not refresh title of {url}: {e:#}");
9795 self.state
9796 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9797 }
9798 Ok(url)
9799 }
9800 }
9801 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9802 };
9803 match out {
9804 Ok(url) => MergeOutcome {
9805 mode,
9806 ok: true,
9807 detail: url,
9808 empty: false,
9809 },
9810 Err(e) => MergeOutcome {
9811 mode,
9812 ok: false,
9813 detail: e.to_string(),
9814 empty: false,
9815 },
9816 }
9817 }
9818 }
9819 };
9820
9821 self.state.status = match (mode, outcome.ok) {
9822 (MergeMode::None, _) => RunStatus::Ready,
9823 (_, true) if outcome.empty => RunStatus::Ready,
9824 (MergeMode::Pr, true) => RunStatus::Ready,
9825 (_, true) => RunStatus::Merged,
9826 (_, false) => RunStatus::Blocked,
9827 };
9828 self.state.event(
9829 "merge",
9830 format!(
9831 "{:?}: {}",
9832 mode,
9833 outcome.detail.lines().next().unwrap_or("")
9834 ),
9835 );
9836 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9837 self.state.record_github_text_publication(&outcome.detail);
9838 }
9839 self.state.merge = Some(outcome);
9840 self.state.save()?;
9841
9842 if self.state.config.graph.land
9848 && mode == MergeMode::Pr
9849 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9850 {
9851 self.run_land().await?;
9852 }
9853 self.settle_questions();
9858 Ok(())
9859 }
9860
9861 fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9872 Box::pin(self.run_land_inner())
9873 }
9874
9875 async fn run_land_inner(&mut self) -> Result<()> {
9876 crate::pr_head::ensure_owner(&self.state)?;
9877 let url = self
9878 .state
9879 .pr
9880 .as_ref()
9881 .map(|pr| pr.url.clone())
9882 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9883 .unwrap_or_default();
9884 let url = url.lines().next().unwrap_or("").trim().to_owned();
9885 if !url.starts_with("http") {
9886 return Ok(());
9887 }
9888 loop {
9891 match land::land(&mut self.state, &url).await {
9892 Ok(_)
9893 if self
9894 .state
9895 .merge_repairs
9896 .last()
9897 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9898 {
9899 if self.repair_merge_review().await? {
9900 continue;
9901 }
9902 }
9903 Ok(_) if self.state.resource_block.is_some() => {
9904 return self.stop_for_resource();
9905 }
9906 Ok(pr) if self.state.parked => {
9907 let _ = pr;
9911 }
9912 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9913 self.state.status = RunStatus::Superseded;
9914 self.state.save()?;
9915 }
9916 Ok(pr) => {
9917 self.state.status = match pr.state {
9918 land::PrLifecycle::Merged => RunStatus::Merged,
9919 _ => RunStatus::Blocked,
9920 };
9921 if bump::should_release_bump(self.state.status)
9928 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9929 {
9930 self.state
9936 .event("bump", format!("release bump skipped: {e:#}"));
9937 }
9938 if self.state.status == RunStatus::Merged {
9941 crate::followup::after_merge(&mut self.state, &pr.url).await;
9942 }
9943 self.state.save()?;
9944 }
9945 Err(e) => {
9946 if e.is::<crate::repo_lock::ResourceBlocked>() {
9947 return Err(e);
9948 }
9949 self.state.status = RunStatus::Blocked;
9950 self.state.event("land", format!("gave up: {e}"));
9951 self.state.save()?;
9952 }
9953 }
9954 break;
9955 }
9956 Ok(())
9957 }
9958
9959 async fn repair_merge_review(&mut self) -> Result<bool> {
9962 use crate::run::MergeRepairPhase as Phase;
9963 crate::pr_head::ensure_owner(&self.state)?;
9964 let i = self.state.merge_repairs.len() - 1;
9965 let request = self.state.merge_repairs[i].clone();
9966 if request.phase == Phase::Fixing {
9967 let restored = match self.state.winner().cloned() {
9968 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9969 None => Err(anyhow::anyhow!("repair has no winning branch")),
9970 };
9971 let why = match restored {
9972 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9973 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9974 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9975 };
9976 self.hold_merge_repair(i, why)?;
9977 return Ok(false);
9978 }
9979 if request.phase == Phase::Reviewing && self.state.gate_ran {
9984 let winner = self
9985 .state
9986 .winner()
9987 .context("repair has no winning branch")?;
9988 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9989 if self.state.gate_head.as_deref() != Some(head.as_str())
9990 || !git::is_clean(&winner.worktree).await?
9991 {
9992 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9993 return Ok(false);
9994 }
9995 }
9996 let failure = if i >= self.state.config.graph.land_rounds
9997 || self.state.config.graph.review_rounds == 0
9998 {
9999 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
10000 } else {
10001 None
10002 };
10003 if let Some(why) = failure {
10004 self.hold_merge_repair(i, why)?;
10005 return Ok(false);
10006 }
10007 if request.phase == Phase::Requested {
10008 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
10009 Ok(pr) => pr,
10010 Err(e) => {
10011 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
10012 return Ok(false);
10013 }
10014 };
10015 self.state.merge_repairs[i].phase = Phase::Fixing;
10016 self.state.land_approval = None;
10017 self.state.land_armed_head = None;
10018 self.state.save()?;
10019 let evidence = serde_json::to_string_pretty(&request)?;
10022 let reason = format!(
10023 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
10024 );
10025 let budget = self.state.config.graph.land_rounds;
10026 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10027 match result {
10028 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10029 Ok(land::Fixed::Deferred) => {
10030 let worktree = self
10034 .state
10035 .winner()
10036 .context("repair has no winning branch")?
10037 .worktree
10038 .clone();
10039 self.graft_restore(&worktree, &request.head).await?;
10040 self.state.merge_repairs[i].phase = Phase::Requested;
10041 self.stop_for_resource()?;
10042 return Ok(false);
10043 }
10044 Ok(land::Fixed::Failed(why)) => {
10045 self.hold_merge_repair(
10046 i,
10047 format!(
10048 "repair failed: {why}; inspect the branch and run an explicit review"
10049 ),
10050 )?;
10051 return Ok(false);
10052 }
10053 Err(e) => {
10054 self.hold_merge_repair(
10055 i,
10056 format!(
10057 "repair failed: {e:#}; inspect the branch and run an explicit review"
10058 ),
10059 )?;
10060 return Ok(false);
10061 }
10062 }
10063 self.begin_merge_repair_review(i)?;
10067 }
10068 self.review_loop().await?;
10069 self.gate().await?;
10070 if self.resource_halt()? || self.state.parked {
10071 return Ok(false);
10072 }
10073 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10074 != Some(RunStatus::Gating)
10075 || !self.state.gate_allows_progress()
10076 {
10077 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10078 return Ok(false);
10079 }
10080 let winner = self
10083 .state
10084 .winner()
10085 .cloned()
10086 .context("repair has no winning branch")?;
10087 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10088 if self.state.gate_head.as_deref() != Some(after.as_str())
10089 || !git::is_clean(&winner.worktree).await?
10090 {
10091 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10092 return Ok(false);
10093 }
10094 let expected = self
10095 .state
10096 .pr_pushed_head
10097 .as_deref()
10098 .unwrap_or(&request.head)
10099 .to_owned();
10100 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10101 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10104 return Ok(false);
10105 }
10106 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10107 let refspec = format!("{after}:refs/heads/{public}");
10108 let push = git::git_raw(
10109 &winner.worktree,
10110 &["push", &lease, &self.state.config.merge.remote, &refspec],
10111 )
10112 .await;
10113 match push {
10114 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10115 Ok(out) => {
10116 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10117 return Ok(false);
10118 }
10119 Err(e) => {
10120 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10121 return Ok(false);
10122 }
10123 }
10124 self.state.merge_repairs[i].phase = Phase::Complete;
10125 self.state.status = RunStatus::Landing;
10126 self.state.event(
10127 "repair",
10128 "new review and gate completed; fresh owner merge decision required",
10129 );
10130 self.state.save()?;
10131 Ok(true)
10132 }
10133
10134 fn merge_repair_artifacts(&self) -> PathBuf {
10135 let artifacts = agent::artifacts_dir(&self.state.dir());
10136 match self
10137 .state
10138 .merge_repairs
10139 .last()
10140 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10141 {
10142 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10143 None => artifacts,
10144 }
10145 }
10146
10147 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10148 if let Some(e2e) = self.state.head_e2e.take() {
10149 self.state
10150 .verification_history
10151 .push(crate::run::VerificationEvidence {
10152 superseded_by: None,
10153 archived_at: Some(Timestamp::now()),
10154 fixes_spent: Some(self.state.gate_fixes.len()),
10155 gate_head: None,
10156 gate_ran: false,
10157 gate: Vec::new(),
10158 e2e: Some(e2e),
10159 });
10160 }
10161 self.state.reviews.clear();
10162 self.state.gate.clear();
10163 self.state.gate_ran = false;
10164 self.state.gate_head = None;
10165 self.state.gate_fixes.clear();
10166 self.state.gate_repair_e2e = None;
10167 self.state.pre_gate.clear();
10168 self.state.pre_gate_commit = None;
10169 self.state.contested_handoff = None;
10170 self.state.land_reruns = None;
10171 self.state.land_approval = None;
10172 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10173 self.state.status = RunStatus::Reviewing;
10174 self.state.save()
10175 }
10176
10177 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10178 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10179 self.state.merge_repairs[index].failure = Some(why.clone());
10180 self.state.status = RunStatus::Blocked;
10181 self.state.event("repair", why);
10182 self.state.save()
10183 }
10184
10185 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10189 if let Some(existing) = self.state.seats.get(key)
10190 && existing.agent == agent
10191 {
10192 return existing.clone();
10193 }
10194 let fresh = if self.state.seats.contains_key(key) {
10197 handover_seat(key, agent, self.state.next_seat_seed())
10198 } else {
10199 SeatState::new(key, agent, self.state.seed)
10200 };
10201 self.state.seats.insert(key.to_owned(), fresh.clone());
10202 fresh
10203 }
10204
10205 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10211 match self.state.seats.get(key) {
10212 Some(s) if s.agent != spec.id => {
10213 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10214 }
10215 _ => spec,
10216 }
10217 }
10218
10219 fn view(&self, c: &Candidate) -> CandidateView {
10221 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10222 .unwrap_or_default();
10223 let (patch, _) = blind::sanitize_patch(
10224 &format!("candidate {} patch", c.label),
10225 &raw,
10226 &self.state.config.blind,
10227 );
10228 CandidateView {
10229 label: c.label,
10230 branch: c.branch.clone(),
10231 summary: c.summary.clone(),
10232 stat: c.stat.clone(),
10233 patch,
10234 }
10235 }
10236
10237 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10239 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10240 prompt::judge(
10241 "(see above)",
10242 &views,
10243 self.roles.judges.len(),
10244 base_short,
10245 "en",
10246 )
10247 }
10248
10249 fn vote_prompt_full(
10255 &self,
10256 j: usize,
10257 labels: &[char],
10258 language: &str,
10259 candidates: &[Candidate],
10260 base_short: &str,
10261 ) -> String {
10262 let mut text = format!(
10263 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10264 prompt::final_vote(labels, language),
10265 self.state.instruction,
10266 self.candidate_block(candidates, base_short)
10267 );
10268 if let Some(own) = self
10269 .state
10270 .judgements
10271 .get(j)
10272 .filter(|r| !r.ranking.is_empty())
10273 {
10274 let reasons = own
10275 .reasons
10276 .iter()
10277 .map(|(k, v)| format!("- {k}: {v}"))
10278 .collect::<Vec<_>>()
10279 .join("\n");
10280 text.push_str(&format!(
10281 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10282 own.ranking.iter().collect::<String>(),
10283 if reasons.is_empty() {
10284 ""
10285 } else {
10286 ", because:\n"
10287 }
10288 ));
10289 }
10290 if !self.state.deliberation.is_empty() {
10291 text.push_str("\n# What was argued before this vote\n");
10292 for t in self.transcript(&[], j) {
10293 text.push_str(&format!(
10294 "\n## {}{}\n\n{}\n",
10295 t.who,
10296 if t.is_self { " (you)" } else { "" },
10297 t.body.trim()
10298 ));
10299 }
10300 }
10301 text
10302 }
10303
10304 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10311 let mut turns = Vec::new();
10312 for j in &self.state.judgements {
10313 if j.ranking.is_empty() {
10314 continue;
10315 }
10316 let reasons = j
10317 .reasons
10318 .iter()
10319 .map(|(k, v)| format!("- {k}: {v}"))
10320 .collect::<Vec<_>>()
10321 .join("\n");
10322 turns.push(Turn {
10323 who: format!("Judge {} (opening ranking)", j.judge),
10324 is_self: j.judge == self_idx + 1,
10325 body: format!(
10326 "Ranked {}{}{reasons}",
10327 j.ranking.iter().collect::<String>(),
10328 if reasons.is_empty() {
10329 ""
10330 } else {
10331 ", because:\n"
10332 }
10333 ),
10334 });
10335 }
10336 for t in self
10337 .state
10338 .deliberation
10339 .iter()
10340 .flat_map(|r| r.turns.iter())
10341 .chain(current)
10342 {
10343 turns.push(Turn {
10344 who: format!("Judge {}", t.judge),
10345 is_self: t.judge == self_idx + 1,
10346 body: t.body.clone(),
10347 });
10348 }
10349 turns
10350 }
10351}
10352
10353fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10355 agent::has_session(spec.kind, seat, sessions)
10356}
10357
10358fn next_untried_in_roster<'a>(
10379 roster: &'a [AgentSpec],
10380 start: usize,
10381 tried: &BTreeSet<String>,
10382) -> Option<&'a AgentSpec> {
10383 roster
10384 .get(start + 1..)?
10385 .iter()
10386 .find(|s| !tried.contains(&s.id))
10387}
10388
10389fn pick_successor<'a>(
10410 roster: &'a [AgentSpec],
10411 start: usize,
10412 tried: &BTreeSet<String>,
10413 carried: Option<&BTreeSet<String>>,
10414 others: &BTreeSet<String>,
10415) -> Option<&'a AgentSpec> {
10416 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10417 roster
10418 .get(start + 1..)
10419 .and_then(|tail| {
10420 tail.iter()
10421 .filter(free)
10422 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10423 })
10424 .or_else(|| {
10425 carried
10426 .is_some()
10427 .then(|| roster.iter().find(free))
10428 .flatten()
10429 })
10430 .or_else(|| match carried {
10431 Some(c) => next_for_seat(roster, start, tried, c),
10432 None => next_untried_in_roster(roster, start, tried),
10433 })
10434}
10435
10436fn next_for_seat<'a>(
10446 roster: &'a [AgentSpec],
10447 start: usize,
10448 round_tried: &BTreeSet<String>,
10449 carried_failed: &BTreeSet<String>,
10450) -> Option<&'a AgentSpec> {
10451 roster
10452 .get(start + 1..)?
10453 .iter()
10454 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10455 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10456}
10457
10458fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10464 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10465 return spec;
10466 };
10467 let ok = |id: &str| !h.failed.contains(id);
10468 if let Some(last) = h.last_ok.as_deref()
10469 && ok(last)
10470 && let Some(s) = roster.iter().find(|s| s.id == last)
10471 {
10472 return s.clone();
10473 }
10474 if ok(&spec.id) {
10475 return spec;
10476 }
10477 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10478 roster
10479 .get(from + 1..)
10480 .into_iter()
10481 .flatten()
10482 .chain(roster.iter())
10483 .find(|s| ok(&s.id))
10484 .cloned()
10485 .unwrap_or(spec)
10486}
10487
10488pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10492 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10493}
10494
10495const TIMED_OUT: &str = "timed out";
10498
10499impl FailClass {
10500 fn of(out: &AgentOutcome) -> Option<Self> {
10502 match out {
10503 AgentOutcome::Ok(_) => None,
10504 AgentOutcome::Quota(_) => Some(Self::Quota),
10505 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10506 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10507 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10508 AgentOutcome::ResourceBlocked(_) => None,
10511 }
10512 }
10513
10514 fn stem_word(&self) -> &'static str {
10516 match self {
10517 Self::Quota => "quota",
10518 _ => "handover",
10519 }
10520 }
10521}
10522
10523fn failure_signature(msg: &str) -> String {
10527 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10528 if let Some(code) = line
10531 .strip_prefix("process exited with no usable output (exit code ")
10532 .and_then(|s| s.strip_suffix(')'))
10533 .and_then(|s| s.parse::<i32>().ok())
10534 {
10535 line = format!("exited with some({code}) and no usable output");
10536 } else if matches!(
10537 line.as_str(),
10538 "process exited with no usable output (no exit code reported)"
10539 | "process exited with no usable output (exit code unavailable)"
10540 ) {
10541 line = "exited with none and no usable output".to_owned();
10542 }
10543 let mut out = Vec::new();
10544 for word in line.split_whitespace() {
10545 if word.contains('/') || word.contains('\\') {
10546 out.push("<path>".to_owned());
10547 continue;
10548 }
10549 let mut w = String::new();
10550 let mut in_digits = false;
10551 for c in word.chars() {
10552 if c.is_ascii_digit() {
10553 if !in_digits {
10554 w.push('#');
10555 }
10556 in_digits = true;
10557 } else {
10558 in_digits = false;
10559 w.push(c);
10560 }
10561 }
10562 out.push(w);
10563 }
10564 out.join(" ").chars().take(120).collect()
10565}
10566
10567fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10574 match cur {
10575 FailClass::Quota | FailClass::Timeout => true,
10576 FailClass::Other(_) => prev != Some(cur),
10577 }
10578}
10579
10580fn fail_reason(out: &AgentOutcome) -> String {
10582 match out {
10583 AgentOutcome::Ok(_) => String::new(),
10584 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10585 AgentOutcome::Dropped(o) => format!(
10586 "the CLI dropped the stream ({})",
10587 o.dropped
10588 .as_ref()
10589 .map(|d| d.why.as_str())
10590 .unwrap_or("it ended without delivering its answer")
10591 ),
10592 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10593 e.lines().next().unwrap_or("").chars().take(160).collect()
10594 }
10595 }
10596}
10597
10598pub(crate) fn record_handover(
10601 state: &mut RunState,
10602 node: &str,
10603 seat: &str,
10604 from: &str,
10605 to: &str,
10606 class: &FailClass,
10607 reason: &str,
10608) {
10609 let message = if *class == FailClass::Quota {
10610 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10611 } else {
10612 format!("{seat}: handed over {from} -> {to} ({reason})")
10613 };
10614 state.event(node, message);
10615 state.handovers.push(Handover {
10616 at: Timestamp::now(),
10617 node: node.to_owned(),
10618 seat: seat.to_owned(),
10619 from: from.to_owned(),
10620 to: to.to_owned(),
10621 reason: reason.to_owned(),
10622 });
10623}
10624
10625fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10639 commands.iter().any(|c| c.exit_code.is_none())
10640}
10641
10642fn verified_noop_claim(
10655 usable: bool,
10656 commands: &[agent::CommandEvidence],
10657 text: &str,
10658) -> Option<String> {
10659 (usable && !has_unconfirmed_command(commands))
10660 .then(|| verdict::verified_noop(text))
10661 .flatten()
10662}
10663
10664fn short(commit: &str) -> String {
10665 commit.chars().take(7).collect()
10666}
10667
10668fn make_executable(path: &Path) -> Result<()> {
10669 #[cfg(unix)]
10670 {
10671 use std::os::unix::fs::PermissionsExt as _;
10672 let mut perms = std::fs::metadata(path)?.permissions();
10673 perms.set_mode(0o755);
10674 std::fs::set_permissions(path, perms)?;
10675 }
10676 #[cfg(not(unix))]
10677 {
10678 let _ = path;
10679 }
10680 Ok(())
10681}
10682
10683struct WaveCtx<'a> {
10690 run: &'a str,
10693 node: &'a str,
10695 prompts: &'a Prompts,
10696 cache: Option<&'a Path>,
10698 round: Option<usize>,
10701 carry_seats: bool,
10705 pauses: Option<(&'a Pause, &'a Pause)>,
10708}
10709
10710fn repository_wait(
10712 state: &RunState,
10713 pauses: Option<(&Pause, &Pause)>,
10714) -> crate::repo_lock::WaitPolicy {
10715 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10716 let inherited = crate::repo_lock::WaitPolicy::current();
10717 crate::repo_lock::WaitPolicy::new(
10718 Duration::from_secs(state.config.graph.timeout_resource_wait),
10719 move || {
10720 inherited.cancelled()
10721 || pauses
10722 .as_ref()
10723 .is_some_and(|(a, b)| a.parked() || b.parked())
10724 },
10725 )
10726}
10727
10728fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10729 if error.is::<crate::repo_lock::ResourceBlocked>() {
10730 state.resource_block = Some(crate::run::ResourceBlock {
10731 cause: crate::run::ResourceBlockCause::Other,
10732 node: "repository".into(),
10733 kind: if error.to_string().contains("disk pressure") {
10734 "disk pressure"
10735 } else if error.to_string().contains("disk admission") {
10736 "disk admission"
10737 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10738 "incremental cache admission"
10739 } else {
10740 "repository coordination"
10741 }
10742 .into(),
10743 detail: error.to_string(),
10744 at: Timestamp::now(),
10745 });
10746 state.event("resource", error.to_string());
10747 }
10748}
10749
10750async fn coordinate_operation(
10751 state: &mut RunState,
10752 cwd: &Path,
10753 wait: &crate::repo_lock::WaitPolicy,
10754) -> Result<Option<crate::repo_lock::SeatLease>> {
10755 let result =
10756 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10757 .await;
10758 if let Err(e) = &result {
10759 record_repository_block(state, e);
10760 }
10761 result
10762}
10763
10764async fn run_one(
10766 job: SeatJob,
10767 sem: Arc<Semaphore>,
10768 ctx: &WaveCtx<'_>,
10769 state: &mut RunState,
10770 attempt: usize,
10771) -> (SeatState, AgentOutcome) {
10772 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10773 .await
10774 .pop()
10775 .expect("one job in, one result out");
10776 (seat, out)
10777}
10778
10779async fn wave(
10785 jobs: Vec<SeatJob>,
10786 sem: Arc<Semaphore>,
10787 ctx: &WaveCtx<'_>,
10788 state: &mut RunState,
10789 attempt: usize,
10790) -> Vec<(usize, SeatState, AgentOutcome)> {
10791 let WaveCtx {
10792 run,
10793 node,
10794 prompts,
10795 cache,
10796 round,
10797 carry_seats: _,
10798 pauses,
10799 } = *ctx;
10800 for job in &jobs {
10801 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10802 }
10803 if let Err(e) = state.save() {
10804 tracing::warn!("could not persist in-progress seats: {e:#}");
10809 }
10810 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10826 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10830 let cache_guard = if let Some(cache_dir) = cache {
10831 if jobs_had_a_writer {
10832 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10833 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10834 match acquire_cache_lease(
10835 state,
10836 cache_dir,
10837 &owner,
10838 resource_limit,
10839 node,
10840 node,
10841 &cancel,
10842 )
10843 .await
10844 {
10845 Ok(Some(g)) => Some(g),
10846 Ok(None) => {
10849 if let Some(block) = &mut state.resource_block {
10850 block.cause = crate::run::ResourceBlockCause::Cancelled {
10851 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10852 };
10853 }
10854 return block_wave(
10855 jobs,
10856 state,
10857 node,
10858 attempt,
10859 "parked while waiting for the shared build cache",
10860 );
10861 }
10862 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10865 Err(e) => {
10866 let why = format!("{e:#}");
10870 state.event(
10871 "resource",
10872 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10873 );
10874 return block_wave(jobs, state, node, attempt, &why);
10875 }
10876 }
10877 } else {
10878 None
10879 }
10880 } else {
10881 None
10882 };
10883 state.seats_spawning(node, attempt);
10886 if let Err(e) = state.save() {
10887 tracing::warn!("could not persist the start of a wave: {e:#}");
10888 }
10889 let slot_limit = resource_limit
10893 + jobs
10894 .iter()
10895 .map(|j| j.timeout)
10896 .max()
10897 .unwrap_or_default()
10898 .saturating_mul(jobs.len() as u32);
10899 let mut set = tokio::task::JoinSet::new();
10900 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10903 let overlay = prompts.overlay(node);
10904 let repository_wait = repository_wait(state, pauses);
10905 for (i, mut job) in jobs.into_iter().enumerate() {
10906 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10907 if cache.is_some() {
10908 job.prompt.push('\n');
10909 job.prompt
10910 .push_str(&prompt::build_cache_note(node, job.allow_write));
10911 }
10912 let sem = Arc::clone(&sem);
10913 let run = run.to_owned();
10914 let node = node.to_owned();
10915 let attachments = if node == "implement" {
10918 state.attachments.clone()
10919 } else {
10920 Vec::new()
10921 };
10922 let cache = cache
10933 .filter(|_| job.allow_write && cache_guard.is_some())
10934 .map(Path::to_path_buf);
10935 let spawned_tx = spawned_tx.clone();
10936 let repository_wait = repository_wait.clone();
10937 let coordination = state.config.coordination.clone();
10938 let fetched_base = state.fetched_base.clone();
10939 set.spawn(async move {
10940 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10944 Ok(p) => p,
10945 Err(_) => {
10946 return (
10947 i,
10948 job.seat,
10949 AgentOutcome::ResourceBlocked(format!(
10950 "no parallel slot became free within {}s",
10951 slot_limit.as_secs()
10952 )),
10953 false,
10954 );
10955 }
10956 };
10957 spawned_tx.send(job.seat.key.clone()).ok();
10958 let mut seat = job.seat;
10959 let out = repository_wait
10960 .scope(agent::invoke(
10961 &job.spec,
10962 &mut seat,
10963 &Invocation {
10964 coordination: coordination.as_ref(),
10965 fetched_base: fetched_base.as_ref(),
10966 cwd: &job.cwd,
10967 prompt: &job.prompt,
10968 timeout: job.timeout,
10969 allow_write: job.allow_write,
10970 sessions: job.sessions,
10971 artifacts: &job.artifacts,
10972 stem: &job.stem,
10973 run: &run,
10974 node: &node,
10975 cache_dir: cache.as_deref(),
10976 attachments: &attachments,
10977 writable: &[],
10978 },
10979 ))
10980 .await;
10981 let interrupted = out
10982 .as_ref()
10983 .err()
10984 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10985 let out = match out {
10986 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10987 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10988 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10996 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10997 Ok(o) => AgentOutcome::Failed(format!(
10998 "process exited with no usable output ({})",
10999 crate::notices::exit_status(o.exit_code)
11000 )),
11001 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
11002 AgentOutcome::ResourceBlocked(e.to_string())
11003 }
11004 Err(e) => AgentOutcome::Failed(e.to_string()),
11005 };
11006 (i, seat, out, interrupted)
11007 });
11008 }
11009 drop(spawned_tx);
11010 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
11011 loop {
11012 let joined = tokio::select! {
11013 Some(key) = spawned_rx.recv() => {
11014 state.seat_spawned(&key);
11016 if let Err(e) = state.save() {
11017 tracing::warn!("could not persist a seat's start: {e:#}");
11018 }
11019 continue;
11020 }
11021 joined = set.join_next() => joined,
11022 };
11023 let Some(joined) = joined else {
11024 break;
11025 };
11026 let (i, seat, out, interrupted) = match joined {
11027 Ok(v) => v,
11028 Err(e) => {
11032 tracing::error!("agent task panicked: {e}");
11033 continue;
11034 }
11035 };
11036 state.seat_finished(&seat.key);
11037 if interrupted
11038 && node == "implement"
11039 && let Some(label) = state
11040 .candidates
11041 .iter()
11042 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11043 .map(|candidate| candidate.label)
11044 {
11045 state.candidate_preservation.insert(
11046 label,
11047 "interrupted implementation awaiting persistence".into(),
11048 );
11049 state.interrupted_implementations.insert(label);
11050 }
11051 if let AgentOutcome::ResourceBlocked(why) = &out
11052 && state.resource_block.is_none()
11053 {
11054 state.resource_block = Some(crate::run::ResourceBlock {
11055 cause: crate::run::ResourceBlockCause::Other,
11056 node: node.to_owned(),
11057 kind: if why.contains("disk pressure") {
11058 "disk pressure"
11059 } else if why.contains("disk admission") {
11060 "disk admission"
11061 } else if why.contains(crate::incremental::BLOCKED) {
11062 "incremental cache admission"
11063 } else if why.contains("repository coordination") {
11064 "repository coordination"
11065 } else {
11066 "parallel slot"
11067 }
11068 .to_owned(),
11069 detail: why.clone(),
11070 at: Timestamp::now(),
11071 });
11072 let activity = if interrupted {
11073 "agent execution interrupted"
11074 } else {
11075 "no agent was started"
11076 };
11077 state.event("resource", format!("{node}: {activity} - {why}"));
11078 }
11079 record_jobs(state, node, round, &seat.key, &out);
11080 if let Err(e) = state.save() {
11081 tracing::warn!("could not persist a seat's completion: {e:#}");
11082 }
11083 if collected.len() <= i {
11084 collected.resize_with(i + 1, || None);
11085 }
11086 collected[i] = Some((i, seat, out));
11087 }
11088 if state
11094 .active
11095 .values()
11096 .any(|a| a.node == node && a.attempt == attempt)
11097 {
11098 state
11099 .active
11100 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11101 if let Err(e) = state.save() {
11102 tracing::warn!("could not persist the end of a wave: {e:#}");
11103 }
11104 }
11105 if let Some(cache_dir) = cache
11112 && jobs_had_a_writer
11113 {
11114 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11115 }
11116 if let Some(guard) = cache_guard {
11117 guard.release();
11118 }
11119 collected.into_iter().flatten().collect()
11120}
11121
11122fn record_jobs(
11133 state: &mut RunState,
11134 node: &str,
11135 round: Option<usize>,
11136 seat: &str,
11137 out: &AgentOutcome,
11138) {
11139 let commands: &[agent::CommandEvidence] = match out {
11140 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11141 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11142 };
11143 let checked_at = Timestamp::now();
11144 for c in commands {
11145 state.jobs.push(JobRecord {
11146 node: node.to_owned(),
11147 round,
11148 seat: seat.to_owned(),
11149 id: c.id.clone(),
11150 description: c.description.clone(),
11151 checked_at,
11152 status: match c.exit_code {
11153 Some(0) => JobStatus::Completed,
11154 Some(_) => JobStatus::Failed,
11155 None => JobStatus::Unknown,
11156 },
11157 exit_code: c.exit_code,
11158 result_summary: c.result_summary.clone(),
11159 source: c.source.clone(),
11160 });
11161 }
11162}
11163
11164fn round_is_clean(
11185 blocking: usize,
11186 e2e_ok: bool,
11187 answered: usize,
11188 expected: usize,
11189 quota_missing: usize,
11190 policy: IncompleteReviewPolicy,
11191) -> bool {
11192 if blocking != 0 || !e2e_ok {
11193 return false;
11194 }
11195 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11196 return true;
11197 }
11198 answered > 0 && expected - answered <= quota_missing
11199}
11200
11201pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11225 if max_rounds == 0
11226 || reviews.last().is_some_and(|r| {
11227 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11228 })
11229 {
11230 return Some(RunStatus::Gating);
11231 }
11232 let last = reviews.last()?;
11233 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11234 if reviews.len() < max_rounds && !stagnant {
11235 return None;
11236 }
11237 if last.incomplete() && last.blocking == 0 {
11238 return Some(RunStatus::Blocked);
11239 }
11240 if last.e2e_status() == E2eStatus::ResourceBlocked
11241 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11242 {
11243 return None;
11244 }
11245 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11246 RunStatus::Gating
11247 } else {
11248 RunStatus::Blocked
11249 })
11250}
11251
11252pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11267 if nudged {
11268 (full / 4).max(Duration::from_secs(120)).min(full)
11269 } else {
11270 full
11271 }
11272}
11273
11274#[allow(clippy::too_many_arguments)]
11303async fn ask_json_wave<T>(
11304 jobs: Vec<SeatJob>,
11305 sem: Arc<Semaphore>,
11306 retries: usize,
11307 roster: &[AgentSpec],
11308 ctx: &WaveCtx<'_>,
11309 losses: &mut Vec<QuotaLoss>,
11310 state: &mut RunState,
11311 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11312) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11313where
11314 T: serde::de::DeserializeOwned + Send + 'static,
11315{
11316 ask_wave_with(
11317 jobs,
11318 sem,
11319 retries,
11320 roster,
11321 ctx,
11322 losses,
11323 state,
11324 &|text: &str| {
11325 let v = verdict::extract_json::<T>(text)?;
11326 validate(&v)?;
11327 Ok(v)
11328 },
11329 )
11330 .await
11331}
11332
11333#[allow(clippy::too_many_arguments)]
11342async fn ask_wave_with<T>(
11343 jobs: Vec<SeatJob>,
11344 sem: Arc<Semaphore>,
11345 retries: usize,
11346 roster: &[AgentSpec],
11347 ctx: &WaveCtx<'_>,
11348 losses: &mut Vec<QuotaLoss>,
11349 state: &mut RunState,
11350 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11351) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11352where
11353 T: Send + 'static,
11354{
11355 let n = jobs.len();
11356 let originals: Vec<SeatJob> = jobs;
11357 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11358 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11359 let mut nudges: Vec<usize> = vec![0; n];
11366 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11370 let mut tried: Vec<BTreeSet<String>> = specs
11371 .iter()
11372 .map(|s| BTreeSet::from([s.id.clone()]))
11373 .collect();
11374 let starts: Vec<usize> = specs
11375 .iter()
11376 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11377 .collect();
11378 let carry = ctx.carry_seats && !roster.is_empty();
11379 let carried: Vec<BTreeSet<String>> = originals
11382 .iter()
11383 .map(|j| {
11384 state
11385 .seat_history
11386 .get(&j.seat.key)
11387 .filter(|_| carry)
11388 .map(|h| h.failed.clone())
11389 .unwrap_or_default()
11390 })
11391 .collect();
11392 let mut prev: Vec<Option<FailClass>> = originals
11393 .iter()
11394 .map(|j| {
11395 state
11396 .seat_history
11397 .get(&j.seat.key)
11398 .filter(|_| carry)
11399 .and_then(|h| h.last_fail.clone())
11400 })
11401 .collect();
11402 let next_agent =
11403 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11404 let others: BTreeSet<String> = specs
11405 .iter()
11406 .enumerate()
11407 .filter(|(j, _)| *j != i)
11408 .map(|(_, s)| s.id.clone())
11409 .collect();
11410 pick_successor(
11411 roster,
11412 starts[i],
11413 tried,
11414 carry.then(|| &carried[i]),
11415 &others,
11416 )
11417 .cloned()
11418 };
11419 let mut fresh: Vec<Option<String>> = vec![None; n];
11420 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11421 let mut pending: Vec<usize> = (0..n).collect();
11422
11423 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11428 for round in 0..max_rounds {
11429 if pending.is_empty() {
11430 break;
11431 }
11432 let mut batch = Vec::with_capacity(pending.len());
11433 let mut renudged: Vec<&str> = Vec::new();
11434 for &i in &pending {
11435 let src = &originals[i];
11436 let full: &str = match &src.handover {
11441 Some(h) if specs[i].id != src.spec.id => h,
11442 _ => &src.prompt,
11443 };
11444 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11447 (
11448 full.to_owned(),
11449 src.timeout,
11450 format!("{}-{word}-{}", src.stem, specs[i].id),
11451 )
11452 } else if nudges[i] == 0 {
11453 (full.to_owned(), src.timeout, src.stem.clone())
11454 } else {
11455 renudged.push(src.seat.key.as_str());
11456 let why = done[i]
11457 .as_ref()
11458 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11459 .unwrap_or_else(|| "no parsable answer".to_owned());
11460 let nudge = prompt::nudge(&why);
11461 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11462 let prompt = if nudged {
11463 nudge
11464 } else {
11465 format!("{full}\n\n---\n\n{nudge}")
11466 };
11467 (
11468 prompt,
11469 retry_budget(src.timeout, nudged),
11470 format!("{}-retry{}", src.stem, nudges[i]),
11471 )
11472 };
11473 batch.push(SeatJob {
11474 spec: specs[i].clone(),
11475 seat: seats[i].clone(),
11476 cwd: src.cwd.clone(),
11477 prompt,
11478 timeout,
11479 allow_write: src.allow_write,
11480 sessions: src.sessions,
11481 artifacts: src.artifacts.clone(),
11482 stem,
11483 handover: None,
11484 });
11485 }
11486
11487 if !renudged.is_empty() {
11488 state.event(
11489 ctx.node,
11490 format!("retry {round}: re-asking {}", renudged.join(", ")),
11491 );
11492 }
11493 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11494 let mut still = Vec::new();
11495 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11496 seats[i] = seat;
11497 if let AgentOutcome::ResourceBlocked(why) = &out {
11500 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11501 continue;
11502 }
11503 let class = FailClass::of(&out);
11504 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11507 && nudges[i] < retries
11508 && !roster.is_empty();
11509 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11510 let next = should_hand_over(prev[i].as_ref(), &cur)
11511 .then(|| next_agent(i, &tried[i], &specs))
11512 .flatten();
11513 if carry {
11514 let h = state
11515 .seat_history
11516 .entry(originals[i].seat.key.clone())
11517 .or_default();
11518 h.failed.insert(specs[i].id.clone());
11519 h.last_fail = Some(cur.clone());
11520 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11521 h.last_ok = None;
11522 }
11523 if let Err(e) = state.save() {
11526 tracing::warn!("could not persist a seat's failure history: {e:#}");
11527 }
11528 }
11529 if let Some(next) = next {
11530 record_handover(
11531 state,
11532 ctx.node,
11533 &originals[i].seat.key,
11534 &specs[i].id,
11535 &next.id,
11536 &cur,
11537 &fail_reason(&out),
11538 );
11539 tried[i].insert(next.id.clone());
11540 prev[i] = Some(cur.clone());
11541 seats[i] =
11542 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11543 specs[i] = next;
11544 fresh[i] = Some(cur.stem_word().to_owned());
11545 nudges[i] = 0;
11546 done[i] = Some(Err(anyhow::anyhow!(
11547 "handed over after: {}",
11548 fail_reason(&out)
11549 )));
11550 still.push(i);
11551 continue;
11552 }
11553 }
11554 if carry
11555 && !nudge_first
11556 && let Some(cur) = class.clone()
11557 {
11558 let h = state
11560 .seat_history
11561 .entry(originals[i].seat.key.clone())
11562 .or_default();
11563 h.failed.insert(specs[i].id.clone());
11564 h.last_fail = Some(cur);
11565 }
11566 let parsed = match out {
11567 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11568 AgentOutcome::Quota(o) => {
11569 last_quota[i] = o.quota.clone();
11570 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11571 }
11572 AgentOutcome::Dropped(o) => {
11576 let why = o
11577 .dropped
11578 .as_ref()
11579 .map(|d| d.why.as_str())
11580 .unwrap_or("the CLI ended the stream without delivering its answer");
11581 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11582 }
11583 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11584 Err(anyhow::anyhow!(e))
11585 }
11586 };
11587 let quota = class == Some(FailClass::Quota);
11588 let failed = parsed.is_err();
11589 if !failed && matches!(ctx.node, "judge" | "vote") {
11590 state
11593 .quota
11594 .retain(|loss| loss.seat != originals[i].seat.key);
11595 }
11596 done[i] = Some(parsed);
11597 if carry && !failed {
11598 let h = state
11599 .seat_history
11600 .entry(originals[i].seat.key.clone())
11601 .or_default();
11602 h.failed.remove(&specs[i].id);
11603 h.last_ok = Some(specs[i].id.clone());
11604 h.last_fail = None;
11605 }
11606 let agent_failure = class.is_some()
11614 && !nudge_first
11615 && !roster.is_empty()
11616 && next_agent(i, &tried[i], &specs).is_some();
11617 if failed && !quota && !agent_failure && nudges[i] < retries {
11618 nudges[i] += 1;
11619 still.push(i);
11620 }
11621 }
11622 pending = still;
11623 }
11624
11625 for (i, q) in last_quota.into_iter().enumerate() {
11628 if let Some(quota) = q {
11629 losses.push(QuotaLoss::from_report(
11630 originals[i].seat.key.clone(),
11631 ctx.node,
11632 Some("a),
11633 ));
11634 }
11635 }
11636
11637 if state.status != RunStatus::Stalled
11640 && ctx.node != "deliberate"
11641 && n > 0
11642 && losses.len() == n
11643 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11644 {
11645 block_machine_outage(state, ctx.node, losses);
11646 state.quota.append(losses);
11647 }
11648
11649 seats
11650 .into_iter()
11651 .zip(done)
11652 .zip(nudges)
11653 .map(|((seat, res), attempts)| {
11654 (
11655 seat,
11656 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11657 attempts,
11658 )
11659 })
11660 .collect()
11661}
11662
11663pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11666 let mut auth: Vec<_> = losses
11667 .iter()
11668 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11669 .map(|loss| loss.agent.clone())
11670 .collect();
11671 auth.sort();
11672 auth.dedup();
11673 let (kind, detail) = if auth.is_empty() {
11674 (
11675 "agent usage limit",
11676 "All seats reached their agent usage limits; resume after usage is available"
11677 .to_owned(),
11678 )
11679 } else {
11680 (
11681 "agent authentication",
11682 format!(
11683 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11684 auth.join(", ")
11685 ),
11686 )
11687 };
11688 state.resource_block = Some(crate::run::ResourceBlock {
11689 node: node.to_owned(),
11690 kind: kind.to_owned(),
11691 cause: crate::run::ResourceBlockCause::Other,
11692 detail: detail.clone(),
11693 at: Timestamp::now(),
11694 });
11695 state.event(node, detail);
11696}
11697
11698fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11700 results
11701 .iter()
11702 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11703}
11704
11705fn block_wave(
11709 jobs: Vec<SeatJob>,
11710 state: &mut RunState,
11711 node: &str,
11712 attempt: usize,
11713 why: &str,
11714) -> Vec<(usize, SeatState, AgentOutcome)> {
11715 state.resource_wait = None;
11716 state
11717 .active
11718 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11719 if let Err(e) = state.save() {
11720 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11721 }
11722 jobs.into_iter()
11723 .enumerate()
11724 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11725 .collect()
11726}
11727
11728async fn acquire_cache_lease(
11741 state: &mut RunState,
11742 cache_dir: &Path,
11743 owner: &crate::cache::Owner,
11744 budget: Duration,
11745 node: &str,
11746 context: &str,
11747 cancel: &(dyn Fn() -> bool + Sync),
11748) -> Result<Option<crate::cache::Guard>> {
11749 let home = crate::run::home();
11750 let started = Instant::now();
11751 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11752 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11753 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11754 Err(e) => {
11755 state.event(
11756 "verify",
11757 format!("{context}: could not check the shared build cache: {e:#}"),
11758 );
11759 if let Err(e2) = state.save() {
11760 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11761 }
11762 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11763 }
11764 };
11765 state.event(
11766 "verify",
11767 format!(
11768 "{context}: waiting for the shared build cache at {} ({})",
11769 cache_dir.display(),
11770 busy.describe()
11771 ),
11772 );
11773 state.resource_wait = Some(crate::run::ResourceWait {
11776 node: context.to_owned(),
11777 kind: "shared build cache".to_owned(),
11778 holder: busy.describe(),
11779 since: Timestamp::now(),
11780 limit_secs: budget.as_secs(),
11781 });
11782 if let Err(e) = state.save() {
11783 tracing::warn!("could not persist a cache wait: {e:#}");
11784 }
11785 let remaining = budget.saturating_sub(started.elapsed());
11786 let waited = crate::cache::wait_for_unless(
11787 &home,
11788 cache_dir,
11789 owner,
11790 remaining,
11791 Duration::from_secs(5),
11792 cancel,
11793 )
11794 .await;
11795 state.resource_wait = None;
11796 let cause = match &waited {
11797 Ok(Some(_)) => None,
11798 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11799 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11800 Some(crate::run::ResourceBlockCause::Timeout {
11801 waited_secs: started.elapsed().as_secs(),
11802 limit_secs: budget.as_secs(),
11803 })
11804 }
11805 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11806 };
11807 if let Some(cause) = cause {
11808 let detail = match &waited {
11809 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11810 Err(error) => format!("{error:#}"),
11811 };
11812 state.resource_block = Some(crate::run::ResourceBlock {
11813 cause,
11814 node: node.to_owned(),
11815 kind: "shared build cache".into(),
11816 detail,
11817 at: Timestamp::now(),
11818 });
11819 state.event(
11820 "resource",
11821 state.resource_block.as_ref().unwrap().description(),
11822 );
11823 }
11824 if let Err(error) = state.save() {
11825 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11826 }
11827 waited
11828}
11829
11830#[derive(Debug)]
11835struct CacheCheckError(String);
11836
11837impl std::fmt::Display for CacheCheckError {
11838 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11839 f.write_str(&self.0)
11840 }
11841}
11842
11843impl std::error::Error for CacheCheckError {}
11844
11845fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11847 state
11848 .resource_block
11849 .get_or_insert_with(|| crate::run::ResourceBlock {
11850 cause: crate::run::ResourceBlockCause::Other,
11851 node: node.to_owned(),
11852 kind: "shared build cache".into(),
11853 detail: format!("{error:#}"),
11854 at: Timestamp::now(),
11855 });
11856 if let Err(error) = state.save() {
11857 tracing::warn!("could not persist cache resource block: {error:#}");
11858 }
11859}
11860
11861#[allow(clippy::too_many_arguments)]
11882async fn with_cache_lease<'s, F, Fut>(
11883 state: &'s mut RunState,
11884 cache_dir: Option<&Path>,
11885 node: &str,
11886 seat: &str,
11887 worktree: &Path,
11888 head: &str,
11889 budget: Duration,
11890 context: &str,
11891 wait: &crate::repo_lock::WaitPolicy,
11892 body: F,
11893) -> Result<(Vec<CommandOutcome>, bool)>
11894where
11895 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11896 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11897{
11898 let started = Instant::now();
11901 let home = crate::run::home();
11902 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11903 let guard = if let Some(cache_dir) = cache_dir {
11904 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11905 match acquire_cache_lease(
11906 state,
11907 cache_dir,
11908 &owner,
11909 resource_limit,
11910 node,
11911 context,
11912 &|| wait.cancelled(),
11913 )
11914 .await
11915 .and_then(|g| {
11916 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11917 }) {
11918 Ok(g) => Some(g),
11919 Err(e) => {
11920 record_cache_error(state, node, &e);
11921 return Ok((
11922 vec![CommandOutcome {
11923 command: "(waiting for the shared build cache)".into(),
11924 code: None,
11925 output_tail: e.to_string(),
11926 duration_ms: started.elapsed().as_millis() as u64,
11927 resource_blocked: true,
11928 }],
11929 false,
11930 ));
11931 }
11932 }
11933 } else {
11934 None
11935 };
11936 let _coordination = coordinate_operation(state, worktree, wait).await?;
11937 if let Some(cache_dir) = cache_dir {
11938 let identity = crate::cache::Identity::new(worktree, head);
11939 if let Err(e) = crate::cache::ensure_fresh(
11940 &home,
11941 cache_dir,
11942 &identity,
11943 &state.config.verify.cargo_manifests(),
11944 ) {
11945 record_cache_error(state, node, &e);
11946 state.event(
11947 "verify",
11948 format!(
11949 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11950 worktree.display(),
11951 short(head)
11952 ),
11953 );
11954 if let Some(guard) = guard {
11955 guard.release();
11956 }
11957 return Ok((
11958 vec![CommandOutcome {
11959 command: "(confirming the shared build cache is fresh)".into(),
11960 code: None,
11961 output_tail: e.to_string(),
11962 duration_ms: started.elapsed().as_millis() as u64,
11963 resource_blocked: true,
11964 }],
11965 false,
11966 ));
11967 }
11968 }
11969 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11970 if !timed_out_pids.is_empty() {
11971 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11972 }
11973 drop(_coordination);
11976 if let Some(guard) = guard {
11977 guard.release();
11978 }
11979 Ok((outcomes, retried))
11980}
11981
11982async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11994 wait_for_pids_with(
11995 pids,
11996 crate::proc::pid_alive,
11997 LEASE_RELEASE_POLL,
11998 LEASE_RELEASE_MAX_WAIT,
11999 )
12000 .await;
12001}
12002
12003async fn wait_for_pids_with<F: Fn(u32) -> bool>(
12009 pids: &[u32],
12010 alive: F,
12011 poll: Duration,
12012 max_wait: Duration,
12013) {
12014 let deadline = Instant::now() + max_wait;
12015 loop {
12016 if pids.iter().all(|&pid| !alive(pid)) {
12017 return;
12018 }
12019 if Instant::now() >= deadline {
12020 return;
12021 }
12022 tokio::time::sleep(poll).await;
12023 }
12024}
12025
12026fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12033 outcomes.iter().any(|o| o.resource_blocked)
12034}
12035
12036pub(crate) enum GateFix {
12038 Retry,
12040 Stop,
12043 Defer,
12046}
12047
12048pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12056 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12057 red.peek().is_some()
12058 && red.all(|o| {
12059 !o.resource_blocked
12060 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12061 && !o.output_tail.trim().is_empty()
12062 })
12063}
12064
12065fn e2e_outcome_label(o: &CommandOutcome) -> String {
12069 if o.ok() {
12070 return "pass".to_owned();
12071 }
12072 let reason = if o.build_failed() {
12073 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12074 } else {
12075 format!("FAIL ({:?})", o.code)
12076 };
12077 format!(
12078 "{reason}\n{}",
12079 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12080 )
12081}
12082
12083async fn run_e2e_with_retry(
12091 state: &mut RunState,
12092 shell: &[String],
12093 commands: &[String],
12094 worktree: &Path,
12095 timeout: Duration,
12096 context: &str,
12097) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12098 let (mut e2e, mut timed_out_pids) = run_commands(
12099 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12100 )
12101 .await;
12102 for o in &e2e {
12103 state.event(
12104 "verify",
12105 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12106 );
12107 }
12108 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12112 if verify_retried {
12113 state.event(
12114 "verify",
12115 format!(
12116 "{context}: verify could not build/link, not a test result — retrying once \
12117 before concluding"
12118 ),
12119 );
12120 let retried = run_commands(
12121 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12122 )
12123 .await;
12124 e2e = retried.0;
12125 timed_out_pids.extend(retried.1);
12128 for o in &e2e {
12129 state.event(
12130 "verify",
12131 format!(
12132 "{context}: retry `{}` -> {}",
12133 o.command,
12134 e2e_outcome_label(o)
12135 ),
12136 );
12137 }
12138 }
12139 (e2e, verify_retried, timed_out_pids)
12140}
12141
12142#[allow(clippy::too_many_arguments)]
12158async fn run_commands(
12159 state: &mut RunState,
12160 node: &str,
12161 task: &str,
12162 attempt: usize,
12163 shell: &[String],
12164 commands: &[String],
12165 cwd: &Path,
12166 timeout: Duration,
12167) -> (Vec<CommandOutcome>, Vec<u32>) {
12168 if commands.is_empty() {
12169 return (Vec::new(), Vec::new());
12174 }
12175 let mut out = Vec::new();
12176 let mut timed_out_pids = Vec::new();
12177 let total = commands.len();
12178 for (idx, command) in commands.iter().enumerate() {
12179 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12180 if let Err(e) = state.save() {
12181 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12182 }
12183 let started = Instant::now();
12184 let mut cmd = tokio::process::Command::new(&shell[0]);
12185 cmd.quiet();
12186 for key in crate::worker::INVOCATION_ENV {
12190 cmd.env_remove(key);
12191 }
12192 cmd.args(&shell[1..])
12193 .arg(command)
12194 .current_dir(cwd)
12195 .stdin(std::process::Stdio::null())
12196 .stdout(std::process::Stdio::piped())
12197 .stderr(std::process::Stdio::piped())
12198 .kill_on_drop(true);
12199 let mut outputs = Vec::new();
12200 let protected = (|| -> Result<()> {
12201 crate::incremental::configure(&mut cmd, cwd)?;
12202 let mut paths = vec![cwd.to_owned()];
12203 if let Some(cache) = state.config.cache_dir() {
12204 paths.push(cache);
12205 }
12206 outputs.extend(crate::retention::OutputLease::admit(
12207 &crate::run::home(),
12208 &paths,
12209 state.config.disk.min_free_bytes,
12210 )?);
12211 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12212 outputs.push(guard);
12213 }
12214 if let Some(cache) = state.config.cache_dir() {
12215 outputs.push(crate::retention::OutputLease::cache(
12216 &crate::run::home(),
12217 &cache,
12218 false,
12219 )?);
12220 }
12221 Ok(())
12222 })();
12223 if let Err(e) = protected {
12224 let detail = format!("execution admission unavailable: {e:#}");
12225 state.resource_block = Some(crate::run::ResourceBlock {
12226 cause: crate::run::ResourceBlockCause::Other,
12227 node: node.to_owned(),
12228 kind: if detail.contains("disk admission") {
12229 "disk admission"
12230 } else {
12231 "output ownership"
12232 }
12233 .to_owned(),
12234 detail: detail.clone(),
12235 at: Timestamp::now(),
12236 });
12237 out.push(CommandOutcome {
12238 command: command.clone(),
12239 code: None,
12240 output_tail: detail,
12241 duration_ms: 0,
12242 resource_blocked: true,
12243 });
12244 break;
12245 }
12246 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12247 let (code, body) = match spawned {
12248 Ok(mut running) => {
12249 let pid = running.child().id();
12250 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12251 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12252 let mut paths = vec![cwd.to_owned()];
12253 if let Some(cache) = state.config.cache_dir() {
12254 paths.push(cache);
12255 }
12256 let mut pressure = None;
12257 let status = tokio::select! {
12258 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12259 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12260 pressure = Some(reason); None
12261 }
12262 };
12263 let (code, failure) = match status {
12264 Some(Ok(Ok(status))) => (
12265 status.code(),
12266 status
12267 .code()
12268 .is_none()
12269 .then(|| format!("command terminated: {status}")),
12270 ),
12271 Some(Ok(Err(error))) => {
12272 (None, Some(format!("failed waiting for command: {error}")))
12273 }
12274 Some(Err(_)) => {
12275 if !running.coordinated()
12279 && let Some(pid) = pid
12280 {
12281 timed_out_pids.push(pid);
12282 }
12283 (
12284 None,
12285 Some(format!("timed out after {}s", timeout.as_secs())),
12286 )
12287 }
12288 None => (None, None),
12289 };
12290 running.finish().await;
12291 if let Some(reason) = pressure {
12292 state.resource_block = Some(crate::run::ResourceBlock {
12293 cause: crate::run::ResourceBlockCause::Other,
12294 node: node.to_owned(),
12295 kind: "disk pressure".to_owned(),
12296 detail: reason.clone(),
12297 at: Timestamp::now(),
12298 });
12299 out.push(CommandOutcome {
12300 command: command.clone(),
12301 code: None,
12302 output_tail: reason,
12303 duration_ms: started.elapsed().as_millis() as u64,
12304 resource_blocked: true,
12305 });
12306 break;
12307 }
12308 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12309 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12310 if let Some(failure) = failure {
12311 body.push('\n');
12312 body.push_str(&failure);
12313 }
12314 (code, body)
12315 }
12316 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12317 };
12318 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12319 if !body.is_empty() {
12320 match crate::run::write_verification_artifact(state, task, &body) {
12324 Ok(name) if summary != body => {
12325 summary = format!("[full output: artifacts/{name}]\n{summary}");
12326 }
12327 Ok(_) => {}
12328 Err(e) => {
12329 let warning = format!("could not save the full {task} output: {e:#}");
12330 tracing::warn!("{warning}");
12331 state.event(node, warning);
12332 if summary != body {
12333 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12334 }
12335 }
12336 }
12337 }
12338 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12339 out.push(CommandOutcome {
12340 command: command.clone(),
12341 code,
12342 output_tail: summary,
12343 duration_ms: started.elapsed().as_millis() as u64,
12344 resource_blocked,
12345 });
12346 if resource_blocked {
12347 state.resource_block = Some(crate::run::ResourceBlock {
12348 cause: crate::run::ResourceBlockCause::Other,
12349 node: node.to_owned(),
12350 kind: "incremental cache admission".into(),
12351 detail: body,
12352 at: Timestamp::now(),
12353 });
12354 break;
12355 }
12356 }
12357 state.task_finished(task);
12358 if let Err(e) = state.save() {
12359 tracing::warn!("could not persist the end of {task}: {e:#}");
12360 }
12361 (out, timed_out_pids)
12362}
12363
12364fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12376 let repo = repo.display();
12377 match style {
12378 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12379 MergeStyle::Squash => {
12380 let subject = message
12383 .lines()
12384 .next()
12385 .unwrap_or(branch)
12386 .replace(['\\', '"', '$', '`'], "");
12387 format!(
12388 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12389 )
12390 }
12391 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12392 }
12393}
12394
12395const PR_TITLE_MAX: usize = 240;
12406
12407fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12418 let line = queue::first_line(text)?;
12419 let chars: Vec<char> = line.chars().collect();
12420 let (kept, mark) = if chars.len() <= max {
12421 (line.to_owned(), "")
12422 } else if let Some(end) = sentence_end(&chars, max) {
12423 (chars[..end].iter().collect::<String>(), "")
12424 } else {
12425 let room = max.saturating_sub(3);
12426 let cut = if chars[room].is_whitespace() {
12429 room
12430 } else {
12431 chars[..room]
12432 .iter()
12433 .rposition(|c| c.is_whitespace())
12434 .unwrap_or(room)
12435 };
12436 let head: String = chars[..cut].iter().collect();
12437 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12438 (head.to_owned(), "...")
12439 };
12440 if kept.is_empty() {
12441 return None;
12442 }
12443 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12444 return None;
12445 }
12446 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12447 return None;
12448 }
12449 Some(format!("{kept}{mark}"))
12450}
12451
12452fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12456 const MIN: usize = 20;
12457 for i in MIN..max.min(chars.len()) {
12458 if !matches!(chars[i], '.' | '!' | '?') {
12459 continue;
12460 }
12461 let Some(&next) = chars.get(i + 1) else {
12462 continue;
12463 };
12464 if !next.is_whitespace() {
12465 continue;
12466 }
12467 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12468 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12469 continue;
12470 }
12471 let word: String = chars[..i]
12472 .iter()
12473 .rev()
12474 .take_while(|c| !c.is_whitespace())
12475 .collect::<Vec<_>>()
12476 .into_iter()
12477 .rev()
12478 .collect();
12479 let word = word.to_ascii_lowercase();
12480 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12481 continue;
12482 }
12483 let end = chars[..i]
12484 .iter()
12485 .rposition(|c| !c.is_whitespace())
12486 .map_or(i, |p| p + 1);
12487 return Some(end);
12488 }
12489 None
12490}
12491
12492struct PrMessage {
12497 title: String,
12498 body: String,
12499}
12500
12501impl PrMessage {
12502 fn commit_message(&self) -> String {
12506 format!("{}\n\n{}", self.title, self.body)
12507 }
12508}
12509
12510fn title_marker(line: &str) -> Option<&str> {
12512 let line = line.trim();
12513 let head = line.get(..6)?;
12514 head.eq_ignore_ascii_case("title:")
12515 .then(|| line[6..].trim())
12516}
12517
12518fn summary_title(summary: &str) -> Option<String> {
12523 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12524 let raw = title_marker(first)?;
12525 if raw.is_empty() {
12526 return None;
12527 }
12528 let title = queue::title_from(raw, PR_TITLE_MAX);
12529 let lower = title.to_ascii_lowercase();
12530 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12531 return None;
12532 }
12533 Some(title)
12534}
12535
12536const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12538
12539pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12541
12542pub(crate) fn is_review_run(state: &RunState) -> bool {
12546 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12547}
12548
12549fn review_title(state: &RunState) -> Option<String> {
12554 english_subject(
12555 state.reviewed_commits.as_ref()?.first()?,
12556 state.config.graph.github_english(),
12557 )
12558}
12559
12560fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12563 let raw = raw.trim();
12564 if raw.is_empty() {
12565 return None;
12566 }
12567 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12568 return None;
12569 }
12570 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12571 return None;
12572 }
12573 let title = queue::title_from(raw, PR_TITLE_MAX);
12574 let lower = title.to_ascii_lowercase();
12575 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12576 return None;
12577 }
12578 Some(title)
12579}
12580
12581#[derive(Debug, Clone, PartialEq, Eq)]
12584struct BranchFacts {
12585 commits: Vec<(String, String)>,
12587 stat: String,
12589}
12590
12591const STAT_FILE_LINES: usize = 25;
12593const COMMITS_MAX_CHARS: usize = 20_000;
12595
12596async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12599 let commits = git::commit_log(repo, base, branch).await.ok()?;
12600 if commits.is_empty() {
12601 return None;
12602 }
12603 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12604 let lines: Vec<&str> = stat.lines().collect();
12605 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12606 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12607 let more = format!(" ... {omitted} more file(s)");
12608 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12609 kept.push(&more);
12610 kept.push(lines[lines.len() - 1]);
12611 kept.join("\n")
12612 } else {
12613 lines.join("\n")
12614 };
12615 Some(BranchFacts { commits, stat })
12616}
12617
12618fn markdown_safe(text: &str) -> String {
12621 text.replace("</details>", "</details>")
12622 .replace("\x60\x60\x60", "~~~")
12623}
12624
12625fn neutral_title(state: &RunState, winner: char) -> String {
12626 format!(
12627 "chore: land candidate {} of run {}",
12628 winner.to_ascii_uppercase(),
12629 state.id
12630 )
12631}
12632
12633pub(crate) async fn repair_landing_gate(
12636 state: &mut RunState,
12637 winner: &Candidate,
12638 outcomes: &[CommandOutcome],
12639) -> Result<GateFix> {
12640 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12641 return Ok(GateFix::Stop);
12642 }
12643 let mut runner = Runner {
12644 roles: state.config.resolve_roles()?,
12645 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12646 state: state.clone(),
12647 pause: Pause::new(),
12648 interrupt: Pause::new(),
12649 };
12650 let result = runner.gate_fix_round(winner, outcomes).await;
12651 *state = runner.state;
12652 result
12653}
12654
12655pub(crate) async fn verify_current_head(
12658 state: &mut RunState,
12659 winner: &Candidate,
12660 wait: &crate::repo_lock::WaitPolicy,
12661) -> Result<bool> {
12662 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12663 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12665 state
12666 .verification_history
12667 .push(crate::run::VerificationEvidence {
12668 superseded_by: Some(head.clone()),
12669 archived_at: Some(Timestamp::now()),
12670 fixes_spent: Some(state.gate_fixes.len()),
12671 gate_head: None,
12672 gate_ran: false,
12673 gate: Vec::new(),
12674 e2e: state.head_e2e.take(),
12675 });
12676 }
12677 if let Some(e) = &state.head_e2e
12678 && e.head == head
12679 {
12680 if e.passed {
12681 return Ok(true);
12682 }
12683 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12684 state.status = RunStatus::Blocked;
12685 state.save()?;
12686 return Ok(false);
12687 }
12688 } else if state.reviews.last().is_some_and(|r| {
12689 r.verified_head.as_deref() == Some(head.as_str())
12690 && (r.e2e_status() == E2eStatus::Passed
12691 || (r.e2e_status() == E2eStatus::NotConfigured
12692 && state.config.verify.e2e.is_empty()))
12693 }) || state
12694 .gate_repair_e2e
12695 .as_ref()
12696 .is_some_and(|e| e.head == head && e.passed)
12697 {
12698 return Ok(true);
12699 }
12700 if let Some(e) = &state.gate_repair_e2e
12701 && e.head == head
12702 && !e.passed
12703 && !e.outcomes.is_empty()
12704 && !verify_inconclusive(&e.outcomes)
12705 {
12706 state.status = RunStatus::Blocked;
12707 state.save()?;
12708 return Ok(false);
12709 }
12710 state.head_e2e = Some(crate::run::GateRepairE2e {
12711 head: head.clone(),
12712 passed: false,
12713 outcomes: Vec::new(),
12714 });
12715 state.save()?;
12716 let commands = state.config.verify.e2e.clone();
12717 let shell = state.config.shell();
12718 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12719 let cache_dir = state.config.cache_dir();
12720 let outcomes = if commands.is_empty() {
12721 Vec::new()
12722 } else {
12723 let (outcomes, _) = with_cache_lease(
12724 state,
12725 cache_dir.as_deref(),
12726 "e2e",
12727 "e2e",
12728 &winner.worktree,
12729 &head,
12730 timeout,
12731 "current head e2e",
12732 wait,
12733 |state, budget| async move {
12734 run_e2e_with_retry(
12735 state,
12736 &shell,
12737 &commands,
12738 &winner.worktree,
12739 budget,
12740 "current head e2e",
12741 )
12742 .await
12743 },
12744 )
12745 .await?;
12746 outcomes
12747 };
12748 let unresolved = verify_inconclusive(&outcomes);
12749 let passed = !unresolved
12750 && outcomes.iter().all(CommandOutcome::ok)
12751 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12752 if !outcomes.is_empty() && !unresolved {
12753 state.record_base_progress(&head);
12754 }
12755 state.head_e2e = Some(crate::run::GateRepairE2e {
12756 head: head.clone(),
12757 passed,
12758 outcomes,
12759 });
12760 if !passed && !unresolved {
12761 state.status = RunStatus::Blocked;
12762 state.event(
12763 "e2e",
12764 format!("verify.e2e failed on head {head}; nothing published"),
12765 );
12766 }
12767 state.save()?;
12768 Ok(passed)
12769}
12770
12771pub(crate) async fn check_gate_repair_e2e(
12774 state: &mut RunState,
12775 winner: &Candidate,
12776 wait: &crate::repo_lock::WaitPolicy,
12777) -> Result<GateFix> {
12778 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12779 return Ok(GateFix::Retry);
12780 }
12781 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12782 if state
12783 .gate_repair_e2e
12784 .as_ref()
12785 .is_some_and(|r| r.head == head && r.passed)
12786 {
12787 return Ok(GateFix::Retry);
12788 }
12789 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12790 head: head.clone(),
12791 passed: false,
12792 outcomes: Vec::new(),
12793 });
12794 state.save()?;
12795 let commands = state.config.verify.e2e.clone();
12796 let outcomes = if commands.is_empty() {
12797 Vec::new()
12798 } else {
12799 let shell = state.config.shell();
12800 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12801 let cache_dir = state.config.cache_dir();
12802 let (outcomes, _) = with_cache_lease(
12803 state,
12804 cache_dir.as_deref(),
12805 "e2e",
12806 "e2e",
12807 &winner.worktree,
12808 &head,
12809 timeout,
12810 "gate repair e2e",
12811 wait,
12812 |state, budget| async move {
12813 run_e2e_with_retry(
12814 state,
12815 &shell,
12816 &commands,
12817 &winner.worktree,
12818 budget,
12819 "gate repair e2e",
12820 )
12821 .await
12822 },
12823 )
12824 .await?;
12825 outcomes
12826 };
12827 let deferred = verify_inconclusive(&outcomes);
12828 let passed = outcomes.iter().all(CommandOutcome::ok)
12829 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12830 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12831 head,
12832 passed,
12833 outcomes,
12834 });
12835 state.save()?;
12836 if deferred {
12837 Ok(GateFix::Defer)
12838 } else if !passed {
12839 state.event(
12840 "gate",
12841 "verify.e2e failed after gate repair; nothing published",
12842 );
12843 Ok(GateFix::Stop)
12844 } else {
12845 Ok(GateFix::Retry)
12846 }
12847}
12848
12849pub(crate) async fn run_landing_gate(
12851 state: &mut RunState,
12852 winner: &Candidate,
12853 wait: &crate::repo_lock::WaitPolicy,
12854) -> Result<Vec<CommandOutcome>> {
12855 let shell = state.config.shell();
12856 let gate_commands = state.config.verify.gate.clone();
12857 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12858 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12859 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12860 Vec::new()
12861 } else {
12862 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12863 let cache_dir = state.config.cache_dir();
12864 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12865 let (outcomes, _) = with_cache_lease(
12866 state,
12867 cache_dir.as_deref(),
12868 "gate",
12869 "gate",
12870 &winner.worktree,
12871 &head,
12872 timeout,
12873 "final gate",
12874 wait,
12875 |state, budget| {
12876 let shell = shell.clone();
12877 let gate_commands = gate_commands.clone();
12878 let winner = winner.clone();
12879 async move {
12880 Runner::run_pre_gate(state, &winner).await;
12881 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12883 let (outcomes, timed_out_pids) = run_commands(
12884 state,
12885 "gate",
12886 "gate",
12887 0,
12888 &shell,
12889 &gate_commands,
12890 &winner.worktree,
12891 budget,
12892 )
12893 .await;
12894 (outcomes, false, timed_out_pids)
12895 }
12896 },
12897 )
12898 .await?;
12899 outcomes
12900 };
12901 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12902 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12903 let mut outcomes = outcomes;
12904 outcomes.push(CommandOutcome {
12905 command: "gate head stability".into(), code: None,
12906 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12907 duration_ms: 0, resource_blocked: false,
12908 });
12909 return Ok(outcomes);
12910 }
12911 if outcomes.is_empty() {
12912 state.event("gate", "no gate commands configured; not tested");
12917 }
12918 for o in &outcomes {
12919 state.event(
12920 "gate",
12921 format!(
12922 "`{}` -> {}",
12923 o.command,
12924 if o.ok() {
12925 "pass".to_owned()
12926 } else {
12927 format!(
12928 "FAIL ({:?})\n{}",
12929 o.code,
12930 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12931 )
12932 }
12933 ),
12934 );
12935 }
12936 Ok(outcomes)
12937}
12938
12939pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12945 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12946 ""
12947 } else {
12948 pr_title
12949 }
12950}
12951
12952pub fn landing_subject_source(state: &RunState) -> String {
12956 if is_review_run(state) {
12957 let winner = state.candidates.first().map_or('A', |c| c.label);
12958 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12959 }
12960 state.instruction.clone()
12961}
12962
12963fn summary_without_title(summary: &str) -> String {
12966 let mut lines = summary.trim().lines().peekable();
12967 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12968 lines.next();
12969 }
12970 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12971}
12972
12973#[cfg(test)]
12985fn pr_message(state: &RunState, winner: char) -> PrMessage {
12986 pr_message_with(state, winner, None)
12987}
12988
12989#[cfg(test)]
12992fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12993 let raw = pr_message_raw(
12994 state,
12995 winner,
12996 facts,
12997 crate::pr_outcome::FinalHead::Unreported,
12998 ChangeNote::NotAsked,
12999 );
13000 let id = crate::scrub::Identity::current();
13001 PrMessage {
13002 title: crate::scrub::scrub(&raw.title, &id),
13003 body: crate::scrub::scrub(&raw.body, &id),
13004 }
13005}
13006
13007#[derive(Debug, Clone, Copy)]
13009enum ChangeNote<'a> {
13010 NotAsked,
13012 Summary(&'a crate::run::ChangeSummary),
13014 Unavailable(&'a str),
13016}
13017
13018fn pr_message_raw(
13019 state: &RunState,
13020 winner: char,
13021 facts: Option<&BranchFacts>,
13022 head: crate::pr_outcome::FinalHead<'_>,
13023 change: ChangeNote<'_>,
13024) -> PrMessage {
13025 if !matches!(change, ChangeNote::NotAsked) {
13026 let (title, mut body) = match change {
13027 ChangeNote::Summary(c) => (
13028 c.title.clone(),
13029 format!("## Summary\n\n{}\n\n", c.body.trim()),
13030 ),
13031 ChangeNote::Unavailable(why) => (
13032 neutral_title(state, winner),
13033 format!(
13034 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13035 ),
13036 ),
13037 ChangeNote::NotAsked => unreachable!(),
13038 };
13039 body.push_str(&public_pr_evidence(state, head));
13040 body.push_str(&format!(
13041 "\n---\nmagi:run/{} magi:candidate-{}\n",
13042 state.id,
13043 winner.to_ascii_lowercase()
13044 ));
13045 return PrMessage { title, body };
13046 }
13047 pr_preview(state, winner, facts, head)
13048}
13049
13050fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13053 let mut body = crate::pr_outcome::review_rounds(state);
13054 body.push_str(&crate::pr_outcome::final_verification(state, head));
13055 let open = state.last_round_findings();
13056 if !open.is_empty() {
13057 body.push_str("## Open review findings\n\n");
13058 for f in open {
13059 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13060 }
13061 body.push('\n');
13062 }
13063 if let Some(c) = state.change_summary.as_ref()
13064 && head == crate::pr_outcome::FinalHead::At(&c.head)
13065 {
13066 body.push_str(&format!(
13067 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13068 c.base, c.head
13069 ));
13070 }
13071 body
13072}
13073
13074fn pr_preview(
13076 state: &RunState,
13077 winner: char,
13078 facts: Option<&BranchFacts>,
13079 head: crate::pr_outcome::FinalHead<'_>,
13080) -> PrMessage {
13081 let summary = state
13082 .candidates
13083 .iter()
13084 .find(|c| c.label == winner)
13085 .map(|c| c.summary.as_str())
13086 .unwrap_or_default();
13087 let review = is_review_run(state);
13090 let title = if review {
13091 facts
13092 .and_then(|f| {
13093 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13094 })
13095 .or_else(|| review_title(state))
13096 .or_else(|| {
13097 state
13098 .candidates
13099 .iter()
13100 .find(|c| c.label == winner)
13101 .filter(|c| !c.branch.starts_with("magi/"))
13102 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13103 })
13104 .unwrap_or_else(|| neutral_title(state, winner))
13105 } else {
13106 summary_title(summary).unwrap_or_else(|| {
13107 english_title(
13108 &state.instruction,
13109 PR_TITLE_MAX,
13110 state.config.graph.github_english(),
13111 )
13112 .unwrap_or_else(|| neutral_title(state, winner))
13113 })
13114 };
13115
13116 let mut body = String::new();
13117 let what = summary_without_title(summary);
13118 if !what.is_empty() {
13119 body.push_str("## Summary\n\n");
13120 body.push_str(&what);
13121 body.push_str("\n\n");
13122 }
13123
13124 body.push_str(&crate::pr_outcome::review_rounds(state));
13126 body.push_str(&crate::pr_outcome::final_verification(state, head));
13127 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13128
13129 let open = state.open_findings();
13130 if !open.is_empty() {
13131 body.push_str("## Open review findings\n\n");
13132 for f in &open {
13133 body.push_str(&format!(
13134 "- `{}` [{:?}] {}\n",
13135 f.id, f.severity, "Context retained in local review records."
13136 ));
13137 }
13138 body.push('\n');
13139 }
13140
13141 if let Some(fix) = fix
13142 && !fix.rejected.is_empty()
13143 {
13144 body.push_str("## Declined by the fixer\n\n");
13145 for r in &fix.rejected {
13146 body.push_str(&format!(
13147 "- `{}`: {}\n",
13148 r.id, "Reason retained in local review records."
13149 ));
13150 }
13151 body.push('\n');
13152 }
13153
13154 if review {
13155 body.push_str("## Commits under review\n\n");
13157 if let Some(facts) = facts {
13158 let mut left = COMMITS_MAX_CHARS;
13159 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13160 let mut entry = format!("- {}\n", markdown_safe(subject));
13161 for l in markdown_safe(text).lines() {
13162 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13163 }
13164 if left == 0 {
13165 body.push_str(&format!(
13166 "- ... {} more commit(s)\n",
13167 facts.commits.len() - i
13168 ));
13169 break;
13170 }
13171 if entry.len() > left {
13172 let mut end = left;
13175 while !entry.is_char_boundary(end) {
13176 end -= 1;
13177 }
13178 entry.truncate(end);
13179 entry.push_str("\n ... (truncated)\n");
13180 left = 0;
13181 } else {
13182 left -= entry.len();
13183 }
13184 body.push_str(&entry);
13185 }
13186 if !facts.stat.trim().is_empty() {
13187 body.push_str(&format!(
13188 "\n## Diff stat\n\n```\n{}\n```\n",
13189 markdown_safe(facts.stat.trim())
13190 ));
13191 }
13192 } else {
13193 match &state.reviewed_commits {
13194 Some(subjects) => {
13195 for s in subjects {
13196 body.push_str(&format!("- {}\n", s.trim()));
13197 }
13198 }
13199 None => {
13200 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13203 body.push_str(rest.trim());
13204 body.push('\n');
13205 }
13206 }
13207 }
13208 } else {
13209 let task = state.instruction.trim();
13210 let task = if task.is_empty() {
13211 "(empty task)"
13212 } else {
13213 task
13214 };
13215 body.push_str(&format!(
13216 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13217 task.replace("</details>", "</details>")
13218 ));
13219 }
13220
13221 body.push_str(&format!(
13222 "\n---\nmagi:run/{} magi:candidate-{}\n",
13223 state.id,
13224 winner.to_ascii_lowercase()
13225 ));
13226
13227 PrMessage { title, body }
13228}
13229
13230fn seeded_instruction(state: &RunState) -> String {
13233 let mut inputs = Vec::new();
13234 if let Some(start) = &state.sources.start_from {
13235 inputs.push(format!(
13236 "Start from `{}` at `{}` (including ancestry).",
13237 start.token, start.sha
13238 ));
13239 }
13240 for pick in &state.sources.pick_commits {
13241 inputs.push(format!(
13242 "Pick `{}` at `{}` (single commit).",
13243 pick.token, pick.sha
13244 ));
13245 }
13246 if inputs.is_empty() {
13247 state.instruction.clone()
13248 } else {
13249 format!(
13250 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13251 state.instruction,
13252 inputs.join("\n")
13253 )
13254 }
13255}
13256
13257async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13261 let base = &state.base_branch;
13262 let mut against = base.clone();
13263 if mode == MergeMode::Pr {
13264 let remote = &state.config.merge.remote;
13268 let tracking = format!("{remote}/{base}");
13269 let fetched = git::fetch(repo, remote, base).await;
13270 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13271 against = tracking;
13272 } else {
13273 return false;
13274 }
13275 }
13276 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13277}
13278
13279fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13282 let mut detail = format!(
13283 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13284 nothing to open a pull request for"
13285 );
13286 detail.push_str(
13287 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13288 );
13289 detail
13290}
13291
13292#[derive(Debug, PartialEq, Eq)]
13295enum PrPlan {
13296 Create,
13297 Adopt { url: String, title: String },
13298 Stop(String),
13299}
13300
13301fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13304 match found {
13305 Ok(land::OpenPr::None) => PrPlan::Create,
13306 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13307 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13308 "several open pull requests exist for this branch, not picking one: {}",
13309 urls.join(" ")
13310 )),
13311 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13312 }
13313}
13314
13315pub(crate) async fn gh_pr_create(
13317 cwd: &Path,
13318 base: &str,
13319 head: &str,
13320 title: &str,
13321 body: &str,
13322) -> Result<String> {
13323 let out = tokio::process::Command::new("gh")
13324 .args([
13325 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13326 ])
13327 .current_dir(cwd)
13328 .quiet()
13329 .stdin(std::process::Stdio::null())
13330 .output()
13331 .await
13332 .context("spawn gh")?;
13333 if out.status.success() {
13334 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13335 } else {
13336 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13337 }
13338}
13339
13340async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13342 match std::fs::symlink_metadata(path) {
13343 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13344 git::worktree_remove_clean(repo, path).await?;
13347 return Ok(());
13348 }
13349 Err(e) => return Err(e.into()),
13350 Ok(_) => {}
13351 }
13352 if !git::is_clean(path).await? {
13353 bail!(
13354 "worktree has uncommitted work; refusing removal: {}",
13355 path.display()
13356 );
13357 }
13358 if !git::worktree_remove_clean(repo, path).await? {
13359 bail!("worktree removal refused: {}", path.display());
13360 }
13361 Ok(())
13362}
13363
13364async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13367 if !crate::retention::read_seat(path) || !path.exists() {
13368 return remove_preserved_worktree(&state.repo, path).await;
13369 }
13370 let owned = std::fs::symlink_metadata(path)?.is_dir()
13371 && path.parent().map(Path::canonicalize).transpose()?
13372 == Some(state.worktree_root().canonicalize()?)
13373 && crate::retention::worktrees(state)
13374 .await?
13375 .contains(&path.canonicalize()?);
13376 let common = crate::repo_lock::common_dir(&state.repo).await?;
13377 if !owned
13378 || common.is_none()
13379 || crate::repo_lock::common_dir(path).await? != common
13380 || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13381 .await?
13382 .ok()
13383 || state.candidates.iter().any(|c| c.worktree == path)
13384 {
13385 return remove_preserved_worktree(&state.repo, path).await;
13386 }
13387 let changes = git::status_porcelain(path).await?;
13388 if !git::worktree_remove(&state.repo, path).await? {
13389 bail!(
13390 "read-only seat worktree removal refused: {}",
13391 path.display()
13392 );
13393 }
13394 if !changes.is_empty() {
13395 let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13396 let paths: String = paths.chars().take(1024).collect();
13397 state.event(
13398 "fold",
13399 format!(
13400 "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13401 path.file_name().unwrap_or_default().to_string_lossy(),
13402 changes.lines().count()
13403 ),
13404 );
13405 }
13406 Ok(())
13407}
13408
13409pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13418 fold_run_with_policy(state, drop_winner, home, false).await
13419}
13420
13421pub async fn fold_run_overriding_recovery_retention(
13423 state: &mut RunState,
13424 drop_winner: bool,
13425 home: &Path,
13426) -> Result<Vec<String>> {
13427 fold_run_with_policy(state, drop_winner, home, true).await
13428}
13429
13430async fn fold_run_with_policy(
13431 state: &mut RunState,
13432 drop_winner: bool,
13433 home: &Path,
13434 override_recovery_retention: bool,
13435) -> Result<Vec<String>> {
13436 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state)
13437 && !override_recovery_retention
13438 {
13439 bail!("refusing fold: {reason}");
13440 }
13441 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13442 bail!("candidate preservation incomplete; refusing cleanup");
13443 }
13444 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13448 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13449 if override_recovery_retention {
13451 *state = RunState::load_under(&state.id, home)?;
13452 anyhow::ensure!(
13453 state.status.done() && !state.status.resumable(),
13454 "recovery retention override requires a finished, non-resumable run"
13455 );
13456 let claimed = crate::daemon::is_working_on_other(home, &state.id, Timestamp::now())?;
13457 anyhow::ensure!(
13458 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
13459 "run is live or its ownership is unknown"
13460 );
13461 anyhow::ensure!(
13462 state.candidate_preservation.is_empty() && state.interrupted_implementations.is_empty(),
13463 "candidate preservation incomplete; refusing cleanup"
13464 );
13465 }
13466 let retention_reason = crate::pr_recovery::source_retention_reason(home, state);
13467 if let Some(reason) = &retention_reason
13468 && !override_recovery_retention
13469 {
13470 bail!("refusing fold: {reason}");
13471 }
13472 let _driver = crate::driver::maintenance(home, &state.id)?;
13473 let mut output_guards = Vec::new();
13474 for entry in std::fs::read_dir(state.worktree_root())
13475 .into_iter()
13476 .flatten()
13477 .flatten()
13478 {
13479 if entry.path().join(".git").is_file() {
13480 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13481 }
13482 }
13483 let repo = state.repo.clone();
13484 let wait = repository_wait(state, None);
13487 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13488 Ok(lease) => lease,
13489 Err(e) => {
13490 state.save_under(home)?;
13491 return Err(e);
13492 }
13493 };
13494 let _source_outputs = if retention_reason.is_some()
13497 || (state.status == RunStatus::Merged && state.merge.as_ref().is_some_and(|merge| merge.ok))
13498 {
13499 verify_merged_fold_sources(state, drop_winner, home).await?
13500 } else {
13501 Vec::new()
13502 };
13503 if let Some(reason) = retention_reason {
13504 state.event(
13505 "fold",
13506 format!(
13507 "operator CLI --override-recovery-retention authorized deletion for run {}: bypassed {reason}",
13508 state.id
13509 ),
13510 );
13511 state.save_under(home)?;
13512 }
13513 let root = state.worktree_root();
13514 let winner = state.tally.as_ref().map(|t| t.winner);
13515 let mut removed = Vec::new();
13516
13517 for i in 0..state.candidates.len() {
13518 let c = state.candidates[i].clone();
13519 let is_winner = Some(c.label) == winner;
13520 if is_winner && !drop_winner {
13521 continue;
13522 }
13523 let existed = c.worktree.exists();
13524 remove_preserved_worktree(&repo, &c.worktree).await?;
13527 if existed {
13528 removed.push(c.worktree.to_string_lossy().into_owned());
13529 }
13530 let handed_over = state.released_branches.contains(&c.branch);
13533 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13534 if !git::branch_delete(&repo, &c.branch).await? {
13535 bail!("candidate branch removal refused");
13536 }
13537 removed.push(c.branch.clone());
13538 }
13539 state.candidates[i].folded = true;
13540 }
13541
13542 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13543 let path = name.path();
13544 let keep = !drop_winner
13545 && winner.is_some_and(|w| {
13546 path.file_name()
13547 .is_some_and(|n| n == format!("cand-{w}").as_str())
13548 });
13549 if keep {
13550 continue;
13551 }
13552 remove_read_seat_worktree(state, &path).await?;
13553 removed.push(path.to_string_lossy().into_owned());
13554 }
13555
13556 remove_if_empty(&root);
13565
13566 if state.enabled_worktree_config && drop_winner {
13567 git::release_worktree_config(&repo).await.ok();
13571 state.enabled_worktree_config = false;
13572 }
13573 state.save_under(home)?;
13574 Ok(removed)
13575}
13576
13577async fn verify_merged_fold_sources(
13578 state: &RunState,
13579 drop_winner: bool,
13580 home: &Path,
13581) -> Result<Vec<std::fs::File>> {
13582 let winner = state.tally.as_ref().map(|t| t.winner);
13583 let mut heads = BTreeSet::new();
13584 for candidate in &state.candidates {
13585 if Some(candidate.label) == winner && !drop_winner {
13586 continue;
13587 }
13588 anyhow::ensure!(
13589 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13590 "candidate recovery source is outside the inventoried run root: {}",
13591 candidate.worktree.display()
13592 );
13593 match std::fs::symlink_metadata(&candidate.worktree) {
13594 Ok(_) => {}
13595 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13596 Err(error) => return Err(error.into()),
13597 }
13598 if !state.released_branches.contains(&candidate.branch)
13599 && git::branch_exists(&state.repo, &candidate.branch).await?
13600 {
13601 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13602 }
13603 }
13604 let mut cargo_guards = Vec::new();
13605 let entries = match std::fs::read_dir(state.worktree_root()) {
13606 Ok(entries) => Some(entries),
13607 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13608 Err(error) => return Err(error.into()),
13609 };
13610 for entry in entries.into_iter().flatten() {
13611 let path = entry?.path();
13612 let metadata = std::fs::symlink_metadata(&path)?;
13613 anyhow::ensure!(
13614 metadata.is_dir() && !metadata.file_type().is_symlink(),
13615 "non-directory or symlink recovery source retained: {}",
13616 path.display()
13617 );
13618 if !drop_winner
13619 && winner.is_some_and(|w| {
13620 path.file_name()
13621 .is_some_and(|name| name == format!("cand-{w}").as_str())
13622 })
13623 {
13624 continue;
13625 }
13626 anyhow::ensure!(
13627 path.join(".git").is_file(),
13628 "unregistered recovery source retained: {}",
13629 path.display()
13630 );
13631 anyhow::ensure!(
13632 git::git(
13633 &path,
13634 &[
13635 "status",
13636 "--porcelain",
13637 "--untracked-files=all",
13638 "--ignored",
13639 "--",
13640 ".",
13641 ":(exclude)target"
13642 ]
13643 )
13644 .await?
13645 .is_empty(),
13646 "source changes or ignored recovery files retained: {}",
13647 path.display()
13648 );
13649 match std::fs::symlink_metadata(path.join("target")) {
13650 Ok(_) => {
13651 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13652 }
13653 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13654 Err(error) => return Err(error.into()),
13655 }
13656 for checkpoint in [
13657 "rebase-merge",
13658 "rebase-apply",
13659 "sequencer",
13660 "CHERRY_PICK_HEAD",
13661 "MERGE_HEAD",
13662 "REBASE_HEAD",
13663 "REVERT_HEAD",
13664 "BISECT_LOG",
13665 "AUTO_MERGE",
13666 "MERGE_MSG",
13667 ] {
13668 let location = git::git(
13669 &path,
13670 &[
13671 "rev-parse",
13672 "--path-format=absolute",
13673 "--git-path",
13674 checkpoint,
13675 ],
13676 )
13677 .await?;
13678 anyhow::ensure!(
13679 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13680 if e.kind() == std::io::ErrorKind::NotFound),
13681 "Git recovery checkpoint retained: {checkpoint}"
13682 );
13683 }
13684 heads.insert(git::rev_parse(&path, "HEAD").await?);
13685 }
13686 crate::fold_evidence::preserve(state, home, &heads).await?;
13687
13688 Ok(cargo_guards)
13689}
13690
13691fn remove_if_empty(dir: &Path) {
13702 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13703 std::fs::remove_dir(dir).ok();
13704 }
13705}
13706
13707pub fn worst_open(state: &RunState) -> Option<Severity> {
13709 state
13710 .reviews
13711 .last()?
13712 .reviews
13713 .iter()
13714 .flat_map(|r| r.findings.iter())
13715 .map(|f| f.severity)
13716 .max()
13717}
13718
13719async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13723 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13724 git::acquire_worktree_config(repo).await?;
13725 state.enabled_worktree_config = true;
13726 }
13727 Ok(())
13728}
13729
13730#[cfg(test)]
13731mod tests {
13732 #[tokio::test]
13733 async fn launcher_base_fetch_requires_remote_success() {
13734 let tmp = tempfile::tempdir().unwrap();
13735 let repo = tmp.path().join("repo");
13736 let remote = tmp.path().join("remote");
13737 std::fs::create_dir(&repo).unwrap();
13738 std::fs::create_dir(&remote).unwrap();
13739 git::git(&remote, &["init", "--bare", "-b", "main"])
13740 .await
13741 .unwrap();
13742 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13743 git::git(
13744 &repo,
13745 &[
13746 "-c",
13747 "user.name=Fixture",
13748 "-c",
13749 "user.email=fixture@example.invalid",
13750 "commit",
13751 "--allow-empty",
13752 "-m",
13753 "fixture",
13754 ],
13755 )
13756 .await
13757 .unwrap();
13758 git::git(
13759 &repo,
13760 &["remote", "add", "origin", remote.to_str().unwrap()],
13761 )
13762 .await
13763 .unwrap();
13764 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13765 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13766 assert_eq!(
13767 resolve_base(&repo, "main", "origin", None)
13768 .await
13769 .unwrap()
13770 .commit,
13771 head
13772 );
13773 #[cfg(unix)]
13774 {
13775 let policy: crate::config::Coordination = toml::from_str(
13776 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13777 ).unwrap();
13778 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13779 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13780 let held = crate::repo_lock::SeatLease::operation_with(
13781 &repo,
13782 Some(&policy),
13783 &crate::repo_lock::WaitPolicy::default(),
13784 )
13785 .await
13786 .unwrap();
13787 assert!(held.is_some());
13788 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13789 tokio::pin!(fetch);
13790 assert!(
13791 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13792 .await
13793 .is_err()
13794 );
13795 assert!(
13796 !repo.join(".git/FETCH_HEAD").exists(),
13797 "no fetch before admission"
13798 );
13799 drop(held);
13800 let fetched = fetch.await.unwrap();
13801 assert_eq!(fetched.commit, head);
13802 assert_eq!(fetched.remote, "origin");
13803 assert_eq!(fetched.branch, "main");
13804 }
13805 git::git(
13807 &repo,
13808 &[
13809 "remote",
13810 "set-url",
13811 "origin",
13812 tmp.path().join("missing").to_str().unwrap(),
13813 ],
13814 )
13815 .await
13816 .unwrap();
13817 let error = resolve_base(&repo, "main", "origin", None)
13818 .await
13819 .unwrap_err()
13820 .to_string();
13821 assert!(error.contains("cannot read origin/main"));
13822 assert!(error.contains("refusing to branch off"));
13823 }
13824
13825 #[test]
13826 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13827 let leaked = vec!["chore(deps): update a crate".to_owned()];
13828 let own = "fix(daemon): apply a chosen action";
13829 assert!(should_retitle("", own, &leaked));
13830 assert!(should_retitle(
13831 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13832 own,
13833 &leaked
13834 ));
13835 assert!(should_retitle(
13836 "chore: land candidate A of run 1",
13837 own,
13838 &leaked
13839 ));
13840 assert!(should_retitle(
13841 "magi: candidate A (uncommitted work)",
13842 own,
13843 &leaked
13844 ));
13845 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13846 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13847 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13848 }
13849
13850 #[test]
13851 fn pr_merge_plan_creates_adopts_or_stops() {
13852 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13853 assert_eq!(
13854 pr_merge_plan(Ok(land::OpenPr::One {
13855 url: "u".into(),
13856 title: "t".into()
13857 })),
13858 PrPlan::Adopt {
13859 url: "u".into(),
13860 title: "t".into()
13861 }
13862 );
13863 let PrPlan::Stop(many) =
13864 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13865 else {
13866 panic!("many must stop");
13867 };
13868 assert!(many.contains('a') && many.contains('b'));
13869 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13870 panic!("a failed lookup must stop");
13871 };
13872 assert!(err.contains("bad token"));
13873 }
13874
13875 use super::*;
13876 use crate::run::GateStatus;
13877 use std::collections::BTreeMap;
13878 use std::time::Duration;
13879
13880 fn conductor() -> AgentSpec {
13881 AgentSpec {
13882 id: "conductor".to_owned(),
13883 kind: crate::config::AgentKind::Command,
13884 model: None,
13885 command: vec!["true".to_owned()],
13886 extra_args: Vec::new(),
13887 env: BTreeMap::new(),
13888 prompt_delivery: None,
13889 max_turn_secs: None,
13890 }
13891 }
13892
13893 fn spec(id: &str) -> AgentSpec {
13894 AgentSpec {
13895 id: id.to_owned(),
13896 kind: crate::config::AgentKind::Command,
13897 model: None,
13898 command: vec!["true".to_owned()],
13899 extra_args: Vec::new(),
13900 env: BTreeMap::new(),
13901 prompt_delivery: None,
13902 max_turn_secs: None,
13903 }
13904 }
13905
13906 fn ids(xs: &[&str]) -> BTreeSet<String> {
13907 xs.iter().map(|s| (*s).to_owned()).collect()
13908 }
13909
13910 #[test]
13911 fn pick_successor_skips_an_agent_another_seat_holds() {
13912 let roster = [spec("a"), spec("b"), spec("c")];
13914 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13915 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13916 }
13917
13918 #[test]
13919 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13920 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13922 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13923 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13924 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13925 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13926 }
13927
13928 #[test]
13929 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13930 let roster = [spec("a"), spec("b")];
13931 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13932 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13933 let carried = ids(&["b"]);
13934 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13935 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13936 }
13937
13938 #[test]
13939 fn pick_successor_returns_none_without_any_untried_successor() {
13940 let roster = [spec("a"), spec("b")];
13941 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13942 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13943 let carried = ids(&["a"]);
13944 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13945 }
13946
13947 #[test]
13948 fn pick_successor_rescue_avoids_another_seats_occupant() {
13949 let roster = [spec("a"), spec("b"), spec("c")];
13951 let carried = ids(&["b", "c"]);
13952 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13953 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13954 }
13955
13956 #[test]
13957 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13958 let roster = [spec("a"), spec("b"), spec("c")];
13959 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13960 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13961 }
13962
13963 #[test]
13964 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13965 let roster = [spec("a"), spec("b"), spec("c")];
13966 let failed = ids(&["a", "b", "c"]);
13967 let mut tried = ids(&["b"]);
13969 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13970 assert_eq!(first.id, "a");
13971 tried.insert(first.id.clone());
13972 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13973 assert_eq!(second.id, "c");
13974 tried.insert(second.id.clone());
13975 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13976 }
13977
13978 #[test]
13979 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13980 let roster = [spec("a"), spec("b")];
13981 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13982 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13983 }
13984
13985 #[test]
13986 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13987 let roster = [spec("a"), spec("b"), spec("c")];
13988 let h = SeatHistory {
13989 failed: ids(&["a"]),
13990 last_ok: Some("b".to_owned()),
13991 last_fail: None,
13992 };
13993 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13994 let gone = SeatHistory {
13996 last_ok: Some("zzz".to_owned()),
13997 ..h.clone()
13998 };
13999 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
14000 let failed = SeatHistory {
14001 failed: ids(&["a", "b"]),
14002 last_ok: Some("b".to_owned()),
14003 last_fail: None,
14004 };
14005 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
14006 }
14007
14008 #[test]
14009 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
14010 let first = SeatState::new("review-1", "alpha", 7);
14011 let next = handover_seat("review-1", "gamma", 7);
14012 assert_ne!(first.claude_session, next.claude_session);
14013 }
14014
14015 #[test]
14016 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
14017 let mut state = state_with_summary("x", "y");
14018 let a = handover_seat("review-1", "beta", state.next_seat_seed());
14019 let b = handover_seat("review-1", "beta", state.next_seat_seed());
14020 assert_ne!(a.claude_session, b.claude_session);
14021 }
14022
14023 #[test]
14024 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
14025 let roster = [spec("a"), spec("b")];
14026 let h = SeatHistory {
14027 failed: ids(&["a", "b"]),
14028 ..SeatHistory::default()
14029 };
14030 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
14031 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
14032 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
14033 }
14034
14035 #[test]
14042 fn failure_signature_ignores_numbers_and_paths() {
14043 assert_eq!(
14044 failure_signature("exited with Some(2) and no usable output"),
14045 failure_signature("exited with Some(137) and no usable output")
14046 );
14047 assert_eq!(
14048 failure_signature("process exited with no usable output (exit code 2)"),
14049 failure_signature("process exited with no usable output (exit code 137)")
14050 );
14051 for (old, new) in [
14052 (
14053 "exited with Some(2) and no usable output",
14054 "process exited with no usable output (exit code 2)",
14055 ),
14056 (
14057 "exited with None and no usable output",
14058 "process exited with no usable output (exit code unavailable)",
14059 ),
14060 ] {
14061 let previous = FailClass::Other(failure_signature(old));
14062 let current = FailClass::Other(failure_signature(new));
14063 assert_eq!(previous, current, "resumption retains the failure class");
14064 assert!(!should_hand_over(Some(&previous), ¤t));
14065 }
14066 assert_eq!(
14067 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
14068 failure_signature("cannot open /var/x.txt: denied")
14069 );
14070 assert_ne!(failure_signature("boom"), failure_signature("bang"));
14071 }
14072
14073 #[test]
14074 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
14075 let other = FailClass::Other("x".into());
14076 assert!(should_hand_over(None, &FailClass::Quota));
14077 assert!(should_hand_over(Some(&other), &FailClass::Quota));
14078 assert!(should_hand_over(
14079 Some(&FailClass::Timeout),
14080 &FailClass::Timeout
14081 ));
14082 assert!(should_hand_over(None, &other));
14083 assert!(!should_hand_over(Some(&other), &other));
14084 assert!(should_hand_over(
14085 Some(&other),
14086 &FailClass::Other("y".into())
14087 ));
14088 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14090 assert!(should_hand_over(Some(&FailClass::Quota), &other));
14091 }
14092
14093 #[test]
14094 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14095 let a = SeatState::new("judge-1", "alpha", 7);
14096 let b = handover_seat("judge-1", "beta", 7);
14097 assert_ne!(a.claude_session, b.claude_session);
14098 assert_eq!(b.turns, 0);
14099 }
14100
14101 #[test]
14102 fn a_timeout_is_classified_apart_from_other_failures() {
14103 assert_eq!(
14104 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14105 Some(FailClass::Timeout)
14106 );
14107 assert!(matches!(
14108 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14109 Some(FailClass::Other(_))
14110 ));
14111 }
14112
14113 #[test]
14114 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14115 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14116 let tried = BTreeSet::from(["beta".to_owned()]);
14117 let next = next_untried_in_roster(&roster, 1, &tried);
14120 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14121 }
14122
14123 #[test]
14124 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14125 let roster = vec![spec("alpha"), spec("beta")];
14126 let tried = BTreeSet::from(["beta".to_owned()]);
14127 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14131 }
14132
14133 #[test]
14134 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14135 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14136 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14137 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14141 }
14142
14143 #[test]
14144 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14145 let roster = vec![spec("a"), spec("a"), spec("b")];
14146 let tried = BTreeSet::from(["a".to_owned()]);
14147 let next = next_untried_in_roster(&roster, 0, &tried);
14148 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14149 }
14150
14151 #[test]
14152 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14153 let roster = vec![spec("a"), spec("b")];
14154 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14155 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14156 }
14157
14158 #[test]
14159 fn remove_if_empty_only_ever_takes_a_bare_directory() {
14160 let dir = tempfile::tempdir().unwrap();
14161 let bay = dir.path().join("ffff");
14162
14163 remove_if_empty(&bay);
14165 assert!(!bay.exists());
14166
14167 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14170 remove_if_empty(&bay);
14171 assert!(bay.exists(), "non-empty directory must survive");
14172
14173 std::fs::remove_dir(bay.join("cand-A")).unwrap();
14175 remove_if_empty(&bay);
14176 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14177 }
14178
14179 #[test]
14188 fn a_full_panel_that_found_nothing_is_clean() {
14189 assert!(round_is_clean(
14190 0,
14191 true,
14192 2,
14193 2,
14194 0,
14195 IncompleteReviewPolicy::Block
14196 ));
14197 }
14198
14199 #[test]
14200 fn a_missing_seat_is_never_clean_under_the_default_policy() {
14201 assert!(!round_is_clean(
14202 0,
14203 true,
14204 1,
14205 2,
14206 0,
14207 IncompleteReviewPolicy::Block
14208 ));
14209 }
14210
14211 #[test]
14212 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14213 assert!(!round_is_clean(
14214 1,
14215 true,
14216 1,
14217 2,
14218 0,
14219 IncompleteReviewPolicy::Warn
14220 ));
14221 }
14222
14223 #[test]
14224 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14225 assert!(round_is_clean(
14226 0,
14227 true,
14228 1,
14229 2,
14230 0,
14231 IncompleteReviewPolicy::Warn
14232 ));
14233 }
14234
14235 #[test]
14236 fn a_full_panel_with_an_open_finding_is_not_clean() {
14237 assert!(!round_is_clean(
14238 1,
14239 true,
14240 2,
14241 2,
14242 0,
14243 IncompleteReviewPolicy::Block
14244 ));
14245 }
14246
14247 #[test]
14248 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14249 assert!(!round_is_clean(
14250 0,
14251 false,
14252 2,
14253 2,
14254 0,
14255 IncompleteReviewPolicy::Block
14256 ));
14257 }
14258
14259 #[test]
14266 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14267 assert!(round_is_clean(
14270 0,
14271 true,
14272 1,
14273 2,
14274 1,
14275 IncompleteReviewPolicy::Block
14276 ));
14277 }
14278
14279 #[test]
14280 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14281 assert!(!round_is_clean(
14284 0,
14285 true,
14286 1,
14287 2,
14288 0,
14289 IncompleteReviewPolicy::Block
14290 ));
14291 }
14292
14293 #[test]
14294 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14295 assert!(!round_is_clean(
14296 1,
14297 true,
14298 1,
14299 2,
14300 1,
14301 IncompleteReviewPolicy::Block
14302 ));
14303 assert!(!round_is_clean(
14304 0,
14305 false,
14306 1,
14307 2,
14308 1,
14309 IncompleteReviewPolicy::Block
14310 ));
14311 }
14312
14313 #[test]
14314 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14315 assert!(!round_is_clean(
14319 0,
14320 true,
14321 0,
14322 2,
14323 2,
14324 IncompleteReviewPolicy::Block
14325 ));
14326 }
14327
14328 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14329 CommandOutcome {
14330 command: "test".to_owned(),
14331 code,
14332 output_tail: String::new(),
14333 duration_ms: 0,
14334 resource_blocked,
14335 }
14336 }
14337
14338 #[cfg(unix)]
14339 #[tokio::test]
14340 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14341 ask_test_home();
14342 let temp = tempfile::tempdir().unwrap();
14343 let mut runner = runner_at(RunStatus::Gating);
14344 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14345 let commands = vec![
14346 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14347 "touch must-not-run".to_owned(),
14348 ];
14349 let (outcomes, retried, pids) = run_e2e_with_retry(
14350 &mut runner.state,
14351 &shell,
14352 &commands,
14353 temp.path(),
14354 Duration::from_secs(5),
14355 "fixture",
14356 )
14357 .await;
14358 assert_eq!(outcomes.len(), 1);
14359 assert!(outcomes[0].resource_blocked);
14360 assert!(!outcomes[0].build_failed());
14361 assert!(!retried);
14362 assert!(pids.is_empty());
14363 assert!(!temp.path().join("must-not-run").exists());
14364 assert_eq!(
14365 runner.state.resource_block.as_ref().unwrap().kind,
14366 "incremental cache admission"
14367 );
14368 }
14369
14370 #[test]
14371 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14372 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14373 assert!(
14374 !verify_inconclusive(&[outcome(Some(1), false)]),
14375 "an ordinary failure is still evidence about the patch"
14376 );
14377 assert!(verify_inconclusive(&[outcome(None, true)]));
14378 assert!(
14379 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14380 "one inconclusive outcome taints the whole batch"
14381 );
14382 assert!(!verify_inconclusive(&[]));
14383 }
14384
14385 #[tokio::test]
14386 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14387 let calls = std::sync::atomic::AtomicUsize::new(0);
14391 let started = Instant::now();
14392 wait_for_pids_with(
14393 &[123],
14394 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14395 Duration::from_millis(5),
14396 Duration::from_secs(5),
14397 )
14398 .await;
14399 assert!(
14400 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14401 "must keep checking rather than deciding on the first answer"
14402 );
14403 assert!(
14404 started.elapsed() < Duration::from_secs(1),
14405 "must return the moment it is confirmed dead, not wait out the ceiling"
14406 );
14407 }
14408
14409 #[tokio::test]
14410 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14411 let started = Instant::now();
14412 wait_for_pids_with(
14413 &[123],
14414 |_| true, Duration::from_millis(5),
14416 Duration::from_millis(30),
14417 )
14418 .await;
14419 let elapsed = started.elapsed();
14420 assert!(
14421 elapsed >= Duration::from_millis(30),
14422 "must not give up before its own ceiling: {elapsed:?}"
14423 );
14424 assert!(
14425 elapsed < Duration::from_secs(1),
14426 "must not wait past its own ceiling either: {elapsed:?}"
14427 );
14428 }
14429
14430 #[tokio::test]
14431 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14432 let started = Instant::now();
14433 wait_for_pids_with(
14434 &[],
14435 |_| true,
14436 Duration::from_secs(5),
14437 Duration::from_secs(5),
14438 )
14439 .await;
14440 assert!(
14441 started.elapsed() < Duration::from_millis(200),
14442 "an empty pid list has nothing to confirm"
14443 );
14444 }
14445
14446 fn review_round(
14452 clean: bool,
14453 blocking: usize,
14454 answered: usize,
14455 expected: usize,
14456 progressed: bool,
14457 e2e_ok: bool,
14458 ) -> ReviewRound {
14459 ReviewRound {
14460 round: 1,
14461 head: "h".to_owned(),
14462 verified_head: None,
14463 verified_at: None,
14464 reviews: Vec::new(),
14465 e2e: vec![CommandOutcome {
14466 command: "test".to_owned(),
14467 code: Some(if e2e_ok { 0 } else { 1 }),
14468 output_tail: String::new(),
14469 duration_ms: 0,
14470 resource_blocked: false,
14471 }],
14472 verify_retried: false,
14473 e2e_deferred: false,
14474 e2e_defer_reason: None,
14475 fix: None,
14476 blocking,
14477 answered,
14478 expected,
14479 clean,
14480 progressed,
14481 vote_split: false,
14482 reconsideration: Vec::new(),
14483 verdict: None,
14484 }
14485 }
14486
14487 #[test]
14488 fn review_conclusion_is_none_when_nothing_has_run() {
14489 assert_eq!(review_conclusion(&[], 3), None);
14490 }
14491
14492 #[test]
14493 fn review_conclusion_is_none_while_rounds_remain() {
14494 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14495 assert_eq!(review_conclusion(&rounds, 3), None);
14496 }
14497
14498 #[test]
14499 fn review_conclusion_is_gating_once_a_round_is_clean() {
14500 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14501 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14502 }
14503
14504 #[test]
14505 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14506 let rounds = vec![
14507 review_round(false, 1, 2, 2, true, true),
14508 review_round(false, 1, 2, 2, true, true),
14509 ];
14510 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14511 }
14512
14513 #[test]
14514 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14515 let rounds = vec![
14516 review_round(false, 1, 2, 2, true, true),
14517 review_round(false, 1, 2, 2, true, false),
14518 ];
14519 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14520 }
14521
14522 #[test]
14523 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14524 let mut blocked = review_round(false, 1, 2, 2, true, false);
14531 blocked.e2e[0].resource_blocked = true;
14532 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14533 assert_eq!(review_conclusion(&rounds, 2), None);
14534 }
14535
14536 #[test]
14537 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14538 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14540 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14541 }
14542
14543 #[test]
14544 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14545 let rounds = vec![
14546 review_round(false, 1, 2, 2, false, true),
14547 review_round(false, 1, 2, 2, false, true),
14548 ];
14549 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14550 }
14551
14552 fn secs(n: u64) -> Duration {
14553 Duration::from_secs(n)
14554 }
14555
14556 #[tokio::test]
14559 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14560 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14561 if std::env::var_os(CHILD).is_none() {
14562 let output = std::process::Command::new(std::env::current_exe().unwrap())
14566 .args([
14567 "--exact",
14568 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14569 "--nocapture",
14570 ])
14571 .env(CHILD, "1")
14572 .env_remove("MAGI_WORKER_HOME")
14573 .env_remove("MAGI_WORKER_ID")
14574 .output()
14575 .unwrap();
14576 assert!(
14577 output.status.success()
14578 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14579 "{}\n{}",
14580 String::from_utf8_lossy(&output.stdout),
14581 String::from_utf8_lossy(&output.stderr)
14582 );
14583 return;
14584 }
14585 ask_test_home();
14586 let tmp = tempfile::tempdir().unwrap();
14587 let repo = tmp.path().join("repo");
14588 std::fs::create_dir_all(&repo).unwrap();
14589 init_repo(&repo);
14590 let g = |args: &[&str]| {
14591 let out = std::process::Command::new("git")
14592 .args(args)
14593 .current_dir(&repo)
14594 .quiet()
14595 .output()
14596 .unwrap();
14597 assert!(
14598 out.status.success(),
14599 "{}",
14600 String::from_utf8_lossy(&out.stderr)
14601 );
14602 };
14603 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14604 g(&["checkout", "-b", "historical"]);
14605 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14606 g(&["add", "."]);
14607 g(&["commit", "-m", "old alternative"]);
14608 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14609 g(&["checkout", "-b", "magi/task-selected", &base]);
14610 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14611 g(&["add", "."]);
14612 g(&["commit", "-m", "measurement infrastructure"]);
14613 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14614 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14615 g(&["add", "."]);
14616 g(&["commit", "-m", "dependent implementation"]);
14617 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14618 g(&["checkout", "main"]);
14619 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14620 g(&["add", "."]);
14621 g(&["commit", "-m", "unrelated base change"]);
14622 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14623 let origin = tmp.path().join("origin.git");
14624 g(&[
14625 "clone",
14626 "--bare",
14627 repo.to_str().unwrap(),
14628 origin.to_str().unwrap(),
14629 ]);
14630 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14631 g(&["fetch", "origin"]);
14632 let selection = crate::pr_recovery::Selection {
14633 url: "https://example.test/project/pull/1".into(),
14634 branch: "magi/task-selected".into(),
14635 head: head.clone(),
14636 source_head: None,
14637 source_run: None,
14638 source_proof: None,
14639 disposition: Some(crate::pr_recovery::Disposition::Review),
14640 instruction: "preserve measured behavior".into(),
14641 approved_at: jiff::Timestamp::now(),
14642 };
14643 let mut template = runner_at(RunStatus::Reviewing);
14644 template.state.repo = repo.clone();
14645 template.state.base_commit = current.clone();
14646 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14647 template.state.origin = Some(crate::run::Origin::queue("selected"));
14648 template.state.pr_head = Some(selection.branch.clone());
14649 template.state.pr_recovery = Some(selection.clone());
14650 let mut runner =
14651 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14652 .await
14653 .unwrap();
14654 let wt = runner.state.winner().unwrap().worktree.clone();
14655 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14656 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14657 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14658 assert!(runner.state.seeds.is_empty());
14659 assert_eq!(runner.state.instruction, "preserve measured behavior");
14660 runner.sync_to_base().await.unwrap();
14661 assert_eq!(
14662 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14663 "prerequisite"
14664 );
14665 assert_eq!(
14666 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14667 "measured implementation"
14668 );
14669 assert_eq!(
14670 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14671 "current main change"
14672 );
14673 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14674 assert_eq!(
14675 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14676 head
14677 );
14678
14679 let mut partial = runner_at(RunStatus::Reviewing);
14681 partial.state.repo = repo.clone();
14682 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14683 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14684 run: Some("exit 1".into()),
14685 ..Default::default()
14686 }];
14687 let selection = runner.state.pr_recovery.clone().unwrap();
14688 partial.state.pr_recovery = Some(selection.clone());
14689 partial.state.pr_head = Some(selection.branch.clone());
14690 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14691 let id = partial.state.id.clone();
14692 let error = match Runner::open_selected(
14693 &repo,
14694 partial.state,
14695 partial.roles,
14696 1,
14697 runner.state.base_commit.clone(),
14698 selection,
14699 )
14700 .await
14701 {
14702 Ok(_) => panic!("setup must fail"),
14703 Err(e) => e,
14704 };
14705 assert_eq!(
14706 error
14707 .downcast_ref::<crate::run::StartupFailure>()
14708 .unwrap()
14709 .run,
14710 id
14711 );
14712 let failed = RunState::load(&id).unwrap();
14713 assert_eq!(failed.status, RunStatus::Failed);
14714 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14715 assert!(Runner::resume(&id).is_err());
14716 runner.state.status = RunStatus::Ready;
14717 runner.state.driver_exited = true;
14718 runner.state.save().unwrap();
14719 let mut retry = runner_at(RunStatus::Reviewing);
14720 retry.state.repo = repo.clone();
14721 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14722 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14723 reselected.approved_at = jiff::Timestamp::now();
14724 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14725 retry.state.pr_head = Some(reselected.branch.clone());
14726 retry.state.pr_recovery = Some(reselected.clone());
14727 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14728 .await
14729 .unwrap();
14730 let restarted = Runner::open_selected(
14731 &repo,
14732 retry.state,
14733 retry.roles,
14734 1,
14735 runner.state.base_commit.clone(),
14736 reselected,
14737 )
14738 .await
14739 .unwrap();
14740 assert_eq!(
14741 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14742 .await
14743 .unwrap(),
14744 head
14745 );
14746 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14747 assert_eq!(
14748 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14749 head
14750 );
14751 }
14752
14753 #[tokio::test]
14754 #[cfg(unix)]
14755 async fn fallback_and_interrupted_restoration_require_coordination() {
14756 use std::os::fd::AsRawFd as _;
14757 ask_test_home();
14758 let tmp = tempfile::tempdir().unwrap();
14759 let mut runner = registered_runner(tmp.path()).await;
14760 runner.state.config.graph.timeout_resource_wait = 1;
14761 let winner = runner.state.candidates[0].clone();
14762 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14763 let leftover = winner.worktree.join("leftover");
14764 std::fs::write(&leftover, "untrusted edits").unwrap();
14765 let next = tmp.path().join("next-fixer");
14766 let mut failed = spec("first");
14767 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14768 let mut successor = spec("second");
14769 successor.command = vec![
14770 "sh".into(),
14771 "-c".into(),
14772 format!("touch '{}'; echo done", next.display()),
14773 ];
14774 let attempts = vec![
14775 (failed.clone(), "first".into()),
14776 (successor.clone(), "second".into()),
14777 ];
14778 let lock_path = runner.state.repo.join(".git/updater.lock");
14779 let external = std::fs::File::create(&lock_path).unwrap();
14780 assert_eq!(
14781 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14782 0
14783 );
14784 let artifacts = tmp.path().join("artifacts");
14785 let build = |spec, seat| SeatJob {
14786 spec,
14787 seat,
14788 cwd: winner.worktree.clone(),
14789 prompt: "fixture".into(),
14790 timeout: Duration::from_secs(10),
14791 allow_write: true,
14792 sessions: false,
14793 artifacts: artifacts.clone(),
14794 stem: "fix".into(),
14795 handover: None,
14796 };
14797 let (_, _, out) = runner
14798 .ask_fixer_among(
14799 attempts.clone(),
14800 "graft",
14801 None,
14802 Some((&winner.worktree, &head)),
14803 build,
14804 )
14805 .await;
14806 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14807 assert!(leftover.exists(), "contention must prevent clean");
14808 assert!(!next.exists());
14809 assert!(runner.state.handovers.is_empty());
14810 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14812 from_run: "earlier".into(),
14813 generation: 1,
14814 mode: crate::recovery::Mode::VerifyRepair,
14815 reasons: Vec::new(),
14816 reviewed_head: None,
14817 rounds: Vec::new(),
14818 open_findings: Vec::new(),
14819 evidence: None,
14820 criteria_inherited: true,
14821 repair: crate::recovery::Repair::Started {
14822 before: head.clone(),
14823 },
14824 });
14825 assert!(
14826 runner
14827 .recovery_repair(&winner)
14828 .await
14829 .unwrap_err()
14830 .is::<crate::repo_lock::ResourceBlocked>()
14831 );
14832 assert!(matches!(
14833 runner.state.recovery.as_ref().unwrap().repair,
14834 crate::recovery::Repair::Started { .. }
14835 ));
14836 assert!(leftover.exists());
14837 drop(external);
14838 runner.recovery_repair(&winner).await.unwrap();
14839 assert!(!leftover.exists());
14840 assert!(matches!(
14841 runner.state.recovery.as_ref().unwrap().repair,
14842 crate::recovery::Repair::Done { .. }
14843 ));
14844 std::fs::remove_file(&lock_path).unwrap();
14846 std::fs::create_dir(&lock_path).unwrap();
14847 std::fs::write(&leftover, "untrusted edits").unwrap();
14848 let (_, _, out) = runner
14849 .ask_fixer_among(
14850 attempts,
14851 "graft",
14852 None,
14853 Some((&winner.worktree, &head)),
14854 build,
14855 )
14856 .await;
14857 assert!(matches!(out, AgentOutcome::Failed(_)));
14858 assert!(leftover.exists());
14859 assert!(!next.exists());
14860 assert_eq!(
14861 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14862 head
14863 );
14864 }
14865
14866 #[cfg(unix)]
14867 async fn registered_runner(root: &Path) -> Runner {
14868 repository_runner(root, crate::config::test_coordination()).await
14869 }
14870
14871 async fn repository_runner(
14872 root: &Path,
14873 coordination: Option<crate::config::Coordination>,
14874 ) -> Runner {
14875 ask_test_home();
14876 let repo = root.join("repo");
14877 std::fs::create_dir_all(&repo).unwrap();
14878 init_repo(&repo);
14879 if coordination.is_some() {
14880 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14881 }
14882 let mut runner = runner_at(RunStatus::Prep);
14883 runner.state.repo = repo.clone();
14884 runner.state.config.coordination = coordination;
14885 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14886 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14887 runner.state.config.graph.advise = false;
14888 runner.roles.implementers = vec![spec("implement")];
14889 runner.roles.judges = vec![spec("judge")];
14890 runner.prep().await.unwrap();
14891 runner
14892 }
14893
14894 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14895
14896 #[tokio::test]
14897 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14898 let tmp = tempfile::tempdir().unwrap();
14899 let mut runner = repository_runner(tmp.path(), None).await;
14900 let prompts = runner.state.config.prompts.clone();
14901 let run = runner.state.id.clone();
14902 let ctx = WaveCtx {
14903 carry_seats: false,
14904 pauses: None,
14905 run: &run,
14906 node: "advise",
14907 prompts: &prompts,
14908 cache: None,
14909 round: None,
14910 };
14911 let jobs: Vec<_> = ["alpha", "beta"]
14912 .into_iter()
14913 .map(|id| SeatJob {
14914 spec: scripted(id, AUTH_OUTAGE),
14915 seat: SeatState::new(id, id, 7),
14916 prompt: "propose".into(),
14917 cwd: runner.state.candidates[0].worktree.clone(),
14918 timeout: Duration::from_secs(10),
14919 allow_write: false,
14920 sessions: false,
14921 artifacts: agent::artifacts_dir(&runner.state.dir()),
14922 stem: id.into(),
14923 handover: None,
14924 })
14925 .collect();
14926 let mut losses = Vec::new();
14927 let results = ask_wave_with(
14928 jobs.clone(),
14929 runner.sem.clone(),
14930 2,
14931 &[],
14932 &ctx,
14933 &mut losses,
14934 &mut runner.state,
14935 &|_: &str| Ok(()),
14936 )
14937 .await;
14938 assert!(
14939 results
14940 .iter()
14941 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14942 );
14943 assert_eq!(runner.state.quota.len(), 2);
14944 assert!(
14945 runner
14946 .state
14947 .quota
14948 .iter()
14949 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14950 );
14951 assert!(!runner.state.advise_attempted);
14952 runner.stop_for_resource().unwrap();
14953 assert_eq!(runner.state.status, RunStatus::Blocked);
14954 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14955 assert_eq!(
14956 runner.state.resource_block.as_ref().unwrap().kind,
14957 "agent authentication"
14958 );
14959 runner.state.resource_block = None;
14962 runner.state.quota.clear();
14963 let mut mixed = jobs;
14964 mixed[1].spec.command = vec!["false".into()];
14965 let results = ask_wave_with(
14966 mixed,
14967 runner.sem.clone(),
14968 0,
14969 &[],
14970 &ctx,
14971 &mut losses,
14972 &mut runner.state,
14973 &|_: &str| Ok(()),
14974 )
14975 .await;
14976 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14977 assert!(runner.state.resource_block.is_none());
14978 assert_eq!(losses.len(), 1);
14979 }
14980
14981 #[tokio::test]
14982 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14983 let tmp = tempfile::tempdir().unwrap();
14984 let mut runner = repository_runner(tmp.path(), None).await;
14985 let implementer = scripted("implement", AUTH_OUTAGE);
14986 runner.roles.implementers = vec![implementer.clone()];
14987 runner.state.config.agents = vec![implementer];
14988 runner.implement().await.unwrap();
14989 assert_eq!(runner.state.status, RunStatus::Blocked);
14990 assert_eq!(runner.state.quota.len(), 1);
14991 assert_eq!(
14992 runner.state.quota[0].cause,
14993 agent::QuotaCause::Authentication
14994 );
14995 let candidate = &runner.state.candidates[0];
14996 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14997 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14998 }
14999
15000 #[tokio::test]
15001 #[cfg(unix)]
15002 async fn parked_implementer_persists_completed_work() {
15003 for registered in [false, true] {
15004 let tmp = tempfile::tempdir().unwrap();
15005 let mut runner = registered_runner(tmp.path()).await;
15006 if !registered {
15007 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
15008 }
15009 let marker = tmp.path().join("started");
15010 let release = tmp.path().join("release");
15011 let mut implementer = spec("implement");
15012 implementer.command = vec![
15013 "sh".into(),
15014 "-c".into(),
15015 format!(
15016 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
15017 marker.display(),
15018 release.display()
15019 ),
15020 ];
15021 runner.roles.implementers = vec![implementer.clone()];
15022 runner.state.config.agents = vec![implementer];
15023 let pause = runner.pause.clone();
15024 let (result, ()) = tokio::join!(runner.implement(), async {
15025 tokio::time::timeout(Duration::from_secs(10), async {
15026 while !marker.exists() {
15027 tokio::time::sleep(Duration::from_millis(10)).await;
15028 }
15029 })
15030 .await
15031 .unwrap();
15032 pause.park();
15033 std::fs::write(&release, "go").unwrap();
15034 });
15035 result.unwrap();
15036 let candidate = &runner.state.candidates[0];
15037 assert!(candidate.commits > 0, "{candidate:?}");
15038 assert!(candidate.summary.contains("saved-result"));
15039 assert_eq!(candidate.files, 1);
15040 let before = candidate.commits;
15041 runner.implement().await.unwrap();
15042 assert_eq!(runner.state.candidates[0].commits, before);
15043 }
15044 }
15045
15046 #[tokio::test]
15047 #[cfg(unix)]
15048 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
15049 let tmp = tempfile::tempdir().unwrap();
15050 let mut runner = registered_runner(tmp.path()).await;
15051 let winner = runner.state.candidates[0].clone();
15052 let cache = tmp.path().join("cache");
15053 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
15054 runner.state.config.graph.timeout_resource_wait = 0;
15055 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
15056 let crate::cache::AcquireOutcome::Acquired(held) =
15057 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
15058 else {
15059 panic!("cache must be free")
15060 };
15061 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15062 from_run: "earlier".into(),
15063 generation: 1,
15064 mode: crate::recovery::Mode::VerifyRepair,
15065 reasons: Vec::new(),
15066 reviewed_head: None,
15067 rounds: Vec::new(),
15068 open_findings: Vec::new(),
15069 evidence: None,
15070 criteria_inherited: true,
15071 repair: crate::recovery::Repair::NotStarted,
15072 });
15073 runner.recovery_repair(&winner).await.unwrap();
15074 assert!(matches!(
15075 runner.state.recovery.as_ref().unwrap().repair,
15076 crate::recovery::Repair::WaitingForRestore { .. }
15077 ));
15078 drop(held);
15079 runner.state.resource_block = None;
15080 runner.state.config.graph.timeout_resource_wait = 1;
15081 runner.recovery_repair(&winner).await.unwrap();
15082 assert!(matches!(
15083 runner.state.recovery.as_ref().unwrap().repair,
15084 crate::recovery::Repair::Done { .. }
15085 ));
15086 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15087 }
15088
15089 #[tokio::test]
15090 #[cfg(unix)]
15091 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15092 use std::os::fd::AsRawFd as _;
15093 for graft in [false, true] {
15094 let tmp = tempfile::tempdir().unwrap();
15095 let mut runner = registered_runner(tmp.path()).await;
15096 runner.state.config.graph.timeout_resource_wait = 1;
15097 let winner = runner.state.candidates[0].clone();
15098 let marker = tmp.path().join("fixer-started");
15099 let mut fixer = spec("fixer");
15100 fixer.command = vec![
15101 "sh".into(),
15102 "-c".into(),
15103 format!("echo started >> '{}'; echo done", marker.display()),
15104 ];
15105 runner.roles.fixer = Some(vec![fixer]);
15106 if graft {
15107 let mut loser = winner.clone();
15108 loser.label = 'B';
15109 loser.index = 1;
15110 runner.state.candidates.push(loser);
15111 runner.state.config.graph.graft_runner_up = true;
15112 runner.state.tally = Some(Tally {
15113 first_choice: BTreeMap::new(),
15114 borda: BTreeMap::new(),
15115 winner: winner.label,
15116 rankings: 1,
15117 unanimous_initial: true,
15118 deliberated: false,
15119 changed_votes: 0,
15120 unanimous_final: true,
15121 tie_break: None,
15122 judges: 1,
15123 present: 1,
15124 quorum: 1,
15125 met_quorum: true,
15126 uncontested: None,
15127 });
15128 runner.state.judgements.push(crate::run::Judgement {
15129 judge: 1,
15130 seat: "judge-1".into(),
15131 agent: "judge".into(),
15132 ranking: vec![winner.label, 'B'],
15133 reasons: BTreeMap::new(),
15134 confidence: None,
15135 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15136 order: vec![0, 1],
15137 failed: None,
15138 duration_ms: 0,
15139 });
15140 } else {
15141 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15142 from_run: "earlier".into(),
15143 generation: 1,
15144 mode: crate::recovery::Mode::VerifyRepair,
15145 reasons: Vec::new(),
15146 reviewed_head: None,
15147 rounds: Vec::new(),
15148 open_findings: Vec::new(),
15149 evidence: None,
15150 criteria_inherited: true,
15151 repair: crate::recovery::Repair::NotStarted,
15152 });
15153 }
15154 let reader = crate::repo_lock::SeatLease::for_checkout(
15155 &winner.worktree,
15156 false,
15157 crate::config::test_coordination().as_ref(),
15158 )
15159 .await
15160 .unwrap();
15161 let pause = runner.pause.clone();
15162 let (result, ()) = tokio::join!(
15163 async {
15164 if graft {
15165 runner.graft().await
15166 } else {
15167 runner.recovery_repair(&winner).await
15168 }
15169 },
15170 async {
15171 tokio::time::sleep(Duration::from_millis(100)).await;
15172 pause.park();
15173 }
15174 );
15175 result.unwrap();
15176 assert!(!marker.exists(), "a blocked seat must not spawn");
15177 runner.state = RunState::load(&runner.state.id).unwrap();
15179 if graft {
15180 assert_eq!(
15181 runner.state.graft.as_ref().unwrap().outcome,
15182 GraftOutcome::WaitingForRestore
15183 );
15184 } else {
15185 assert!(matches!(
15186 runner.state.recovery.as_ref().unwrap().repair,
15187 crate::recovery::Repair::WaitingForRestore { .. }
15188 ));
15189 }
15190 drop(reader);
15191 runner.pause = Pause::new();
15192 runner.state.resource_block = None;
15193 let external =
15195 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15196 assert_eq!(
15197 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15198 0
15199 );
15200 let leftover = winner.worktree.join("leftover");
15201 std::fs::write(&leftover, "earlier fallback edits").unwrap();
15202 let result = if graft {
15203 runner.graft().await
15204 } else {
15205 runner.recovery_repair(&winner).await
15206 };
15207 assert!(
15208 result
15209 .unwrap_err()
15210 .is::<crate::repo_lock::ResourceBlocked>()
15211 );
15212 assert!(leftover.exists());
15213 assert!(!marker.exists());
15214 drop(external);
15215 runner.state.resource_block = None;
15216 let result = if graft {
15217 runner.graft().await
15218 } else {
15219 runner.recovery_repair(&winner).await
15220 };
15221 result.unwrap();
15222 assert!(!leftover.exists(), "restore must precede retry");
15223 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15224 if graft {
15225 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15226 } else {
15227 assert!(matches!(
15228 runner.state.recovery.as_ref().unwrap().repair,
15229 crate::recovery::Repair::Done { .. }
15230 ));
15231 }
15232 }
15233 }
15234
15235 #[tokio::test]
15236 #[cfg(unix)]
15237 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15238 use std::os::fd::AsRawFd as _;
15239 let tmp = tempfile::tempdir().unwrap();
15240 let mut runner = registered_runner(tmp.path()).await;
15241 runner.state.config.graph.timeout_resource_wait = 1;
15242 let winner = runner.state.candidates[0].clone();
15243 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15244 runner.state.tally = Some(Tally {
15245 first_choice: BTreeMap::new(),
15246 borda: BTreeMap::new(),
15247 winner: winner.label,
15248 rankings: 0,
15249 unanimous_initial: false,
15250 deliberated: false,
15251 changed_votes: 0,
15252 unanimous_final: false,
15253 tie_break: None,
15254 judges: 0,
15255 present: 0,
15256 quorum: 0,
15257 met_quorum: true,
15258 uncontested: Some("fixture".into()),
15259 });
15260 runner.state.graft = Some(GraftRecord {
15261 pre: pre.clone(),
15262 post: None,
15263 lines: None,
15264 limit: None,
15265 agent: None,
15266 outcome: GraftOutcome::Running,
15267 requested: Vec::new(),
15268 adoption: None,
15269 adoption_error: None,
15270 });
15271 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15272 git::commit_all(&winner.worktree, "unfinished graft")
15273 .await
15274 .unwrap();
15275 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15276 let leftover = winner.worktree.join("leftover");
15277 std::fs::write(&leftover, "untrusted edits").unwrap();
15278 let path = runner.state.repo.join(".git/updater.lock");
15279 let external = std::fs::File::create(&path).unwrap();
15280 assert_eq!(
15281 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15282 0
15283 );
15284 assert!(
15285 runner
15286 .graft()
15287 .await
15288 .unwrap_err()
15289 .is::<crate::repo_lock::ResourceBlocked>()
15290 );
15291 assert!(matches!(
15292 runner.state.graft.as_ref().unwrap().outcome,
15293 GraftOutcome::Running
15294 ));
15295 assert_eq!(
15296 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15297 unfinished
15298 );
15299 assert!(leftover.exists());
15300 drop(external);
15301 std::fs::remove_file(&path).unwrap();
15302 std::fs::create_dir(&path).unwrap();
15303 assert!(runner.graft().await.is_err());
15304 assert!(matches!(
15305 runner.state.graft.as_ref().unwrap().outcome,
15306 GraftOutcome::Running
15307 ));
15308 assert_eq!(
15309 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15310 unfinished
15311 );
15312 assert!(leftover.exists());
15313 std::fs::remove_dir(&path).unwrap();
15314 runner.graft().await.unwrap();
15315 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15316 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15317 assert!(!leftover.exists());
15318 }
15319
15320 #[tokio::test]
15321 #[cfg(unix)]
15322 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15323 use std::os::fd::AsRawFd as _;
15324 let tmp = tempfile::tempdir().unwrap();
15325 let mut runner = registered_runner(tmp.path()).await;
15326 runner.state.prep_complete = Some(false);
15327 runner.state.config.graph.timeout_resource_wait = 1;
15328 runner.state.save().unwrap();
15329 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15330 assert_eq!(
15331 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15332 0
15333 );
15334 runner.execute().await.unwrap();
15335 let disk = RunState::load(&runner.state.id).unwrap();
15336 assert!(disk.driver_exited);
15337 assert_eq!(disk.driver_pid, None);
15338 assert_eq!(runner.state.status, RunStatus::Blocked);
15339 assert_eq!(
15340 runner.state.resource_block.as_ref().unwrap().kind,
15341 "repository coordination"
15342 );
15343 assert!(runner.state.prep_failure.is_none());
15344 assert!(runner.state.seats.is_empty());
15345 runner.pause.park();
15346 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15349 .await
15350 .unwrap()
15351 .unwrap();
15352 assert!(runner.state.parked);
15353 assert!(runner.state.prep_failure.is_none());
15354 let disk = RunState::load(&runner.state.id).unwrap();
15355 assert!(disk.driver_exited);
15356 assert_eq!(disk.driver_pid, None);
15357 }
15358
15359 #[tokio::test]
15360 #[cfg(unix)]
15361 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15362 use std::os::fd::AsRawFd as _;
15363 let tmp = tempfile::tempdir().unwrap();
15364 let mut runner = registered_runner(tmp.path()).await;
15365 runner.state.prep_complete = Some(false);
15366 runner.state.config.graph.timeout_resource_wait = 30;
15367 runner.state.save().unwrap();
15368 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15369 assert_eq!(
15370 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15371 0
15372 );
15373 assert!(
15374 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15375 .await
15376 .is_err()
15377 );
15378 let disk = RunState::load(&runner.state.id).unwrap();
15379 assert!(disk.driver_exited);
15380 assert_eq!(disk.driver_pid, None);
15381 assert!(disk.status.resumable());
15382 assert!(runner.state.driver_exited);
15383 assert_eq!(runner.state.driver_pid, None);
15384 runner.state.save().unwrap();
15385 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15386 }
15387
15388 #[tokio::test]
15389 #[cfg(unix)]
15390 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15391 let tmp = tempfile::tempdir().unwrap();
15392 let mut runner = registered_runner(tmp.path()).await;
15393 let winner = runner.state.candidates[0].clone();
15394 let cache = tmp.path().join("cache");
15395 let started = tmp.path().join("pre-gate-started");
15396 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15397 runner.state.config.verify.gate =
15398 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15399 let home = crate::run::home();
15400 let owner =
15401 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15402 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15403 crate::cache::AcquireOutcome::Acquired(g) => g,
15404 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15405 };
15406 let mut gate = Box::pin(runner.run_gate(&winner));
15407 assert!(
15408 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15409 .await
15410 .is_err()
15411 );
15412 assert!(!started.exists(), "pre-gate must wait for the cache too");
15413 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15417 let editor = crate::repo_lock::SeatLease::checkout_with(
15418 &winner.worktree,
15419 true,
15420 crate::config::test_coordination().as_ref(),
15421 &wait,
15422 )
15423 .await
15424 .unwrap();
15425 drop(editor);
15426 guard.release();
15427 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15428 .await
15429 .unwrap()
15430 .unwrap();
15431 drop(gate);
15432 assert!(outcomes.iter().all(CommandOutcome::ok));
15433 assert!(started.exists());
15434 }
15435
15436 #[tokio::test]
15437 #[cfg(unix)]
15438 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15439 use std::os::fd::AsRawFd as _;
15440
15441 ask_test_home();
15442 let tmp = tempfile::tempdir().unwrap();
15443 let repo = tmp.path().join("repo");
15444 std::fs::create_dir_all(&repo).unwrap();
15445 init_repo(&repo);
15446 let common = repo.join(".git");
15447 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15448 let mut runner = runner_at(RunStatus::Prep);
15449 runner.state.repo = repo.clone();
15450 runner.state.config.coordination = crate::config::test_coordination();
15451 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15452 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15453 runner.state.config.graph.advise = false;
15454 runner.roles.implementers = vec![spec("implement")];
15455 runner.roles.judges = vec![spec("judge")];
15456 runner.prep().await.unwrap();
15457 let winner = runner.state.candidates[0].clone();
15458 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15461 let completed = tmp.path().join("seat-completed");
15462 runner.roles.implementers[0].command = vec![
15463 "sh".into(),
15464 "-c".into(),
15465 format!("touch '{}'; echo completed", completed.display()),
15466 ];
15467 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15468 assert_eq!(
15469 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15470 0
15471 );
15472 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15473 let mut implement = Box::pin(runner.implement());
15474 tokio::select! {
15475 result = &mut implement => panic!("rescue must wait: {result:?}"),
15476 result = tokio::time::timeout(Duration::from_secs(10), async {
15477 while !completed.exists() {
15478 tokio::time::sleep(Duration::from_millis(20)).await;
15479 }
15480 }) => result.unwrap(),
15481 }
15482 assert!(
15483 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15484 .await
15485 .is_err()
15486 );
15487 assert_eq!(
15488 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15489 before
15490 );
15491 assert_eq!(
15492 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15493 0
15494 );
15495 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15496 .await
15497 .unwrap()
15498 .unwrap();
15499 drop(implement);
15500 assert_ne!(
15501 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15502 before
15503 );
15504 assert!(git::is_clean(&winner.worktree).await.unwrap());
15505
15506 let started = tmp.path().join("pre-gate-started");
15507 let proceed = tmp.path().join("pre-gate-proceed");
15508 let checked = tmp.path().join("gate-ran");
15509 runner.state.config.verify.pre_gate = vec![format!(
15510 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15511 started.display(),
15512 proceed.display()
15513 )];
15514 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15515 assert_eq!(
15516 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15517 0
15518 );
15519 let mut gate = Box::pin(runner.run_gate(&winner));
15520 assert!(
15521 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15522 .await
15523 .is_err()
15524 );
15525 assert!(!started.exists());
15526 assert!(!checked.exists());
15527 assert!(git::is_clean(&winner.worktree).await.unwrap());
15528 assert_eq!(
15529 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15530 0
15531 );
15532 tokio::select! {
15533 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15534 result = tokio::time::timeout(Duration::from_secs(10), async {
15535 while !started.exists() {
15536 tokio::time::sleep(Duration::from_millis(20)).await;
15537 }
15538 }) => result.unwrap(),
15539 }
15540 assert_eq!(
15541 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15542 -1,
15543 "mechanical edits must hold the real lock"
15544 );
15545 std::fs::write(proceed, "ready").unwrap();
15546 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15547 .await
15548 .unwrap()
15549 .unwrap();
15550 drop(gate);
15551 assert!(outcomes.iter().all(CommandOutcome::ok));
15552 assert!(checked.exists());
15553 assert!(runner.state.pre_gate_commit.is_some());
15554 assert!(git::is_clean(&winner.worktree).await.unwrap());
15555 assert_eq!(
15556 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15557 0,
15558 "gate completion must release coordination"
15559 );
15560 drop(outsider);
15561 std::fs::remove_file(common.join("updater.lock")).unwrap();
15562 std::fs::create_dir(common.join("updater.lock")).unwrap();
15563 std::fs::remove_file(&started).unwrap();
15564 std::fs::remove_file(&checked).unwrap();
15565 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15566 let error = runner.run_gate(&winner).await.unwrap_err();
15567 assert!(
15568 error
15569 .to_string()
15570 .contains("acquire mandatory repository lock")
15571 );
15572 assert!(
15573 !started.exists(),
15574 "failed acquisition must prevent pre-gate"
15575 );
15576 assert!(!checked.exists(), "failed acquisition must prevent gate");
15577 assert!(git::is_clean(&winner.worktree).await.unwrap());
15578 assert_eq!(
15579 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15580 head
15581 );
15582 }
15583
15584 #[tokio::test]
15585 #[cfg(unix)]
15586 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15587 use std::os::fd::AsRawFd as _;
15588 use std::time::Duration;
15589
15590 ask_test_home();
15591 let tmp = tempfile::tempdir().unwrap();
15592 let repo = tmp.path().join("repo");
15593 std::fs::create_dir_all(&repo).unwrap();
15594 init_repo(&repo);
15595 let common = repo.join(".git");
15596 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15597 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15598 assert_eq!(
15599 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15600 0
15601 );
15602 let mut runner = runner_at(RunStatus::Prep);
15603 runner.state.repo = repo.clone();
15604 runner.state.config.coordination = crate::config::test_coordination();
15605 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15606 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15607 runner.state.config.graph.advise = true;
15608 runner.state.config.graph.advisors = 1;
15609 runner.roles.implementers = vec![spec("implement")];
15610 runner.roles.judges = vec![spec("judge")];
15611 let root = runner.state.worktree_root();
15612 let started = tmp.path().join("setup-started");
15613 let proceed = tmp.path().join("setup-proceed");
15614 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15615 run: Some(format!(
15616 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15617 started.display(),
15618 proceed.display()
15619 )),
15620 ..Default::default()
15621 }];
15622 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15623 let config = git::git(&repo, &["config", "--local", "--list"])
15624 .await
15625 .unwrap();
15626 let mut prep = Box::pin(runner.prep());
15627 assert!(
15628 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15629 .await
15630 .is_err()
15631 );
15632 assert!(
15633 !root.exists(),
15634 "contention must prevent even checkout creation"
15635 );
15636 assert!(!started.exists(), "contention must prevent setup");
15637 assert_eq!(
15638 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15639 branches
15640 );
15641 assert_eq!(
15642 git::git(&repo, &["config", "--local", "--list"])
15643 .await
15644 .unwrap(),
15645 config
15646 );
15647 drop(lock);
15648 tokio::select! {
15649 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15650 result = tokio::time::timeout(Duration::from_secs(10), async {
15651 while !started.exists() {
15652 tokio::time::sleep(Duration::from_millis(20)).await;
15653 }
15654 }) => result.unwrap(),
15655 }
15656 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15657 assert_eq!(
15658 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15659 -1
15660 );
15661 assert!(
15662 tokio::time::timeout(
15663 Duration::from_millis(100),
15664 crate::repo_lock::SeatLease::for_checkout(
15665 &repo,
15666 true,
15667 crate::config::test_coordination().as_ref()
15668 )
15669 )
15670 .await
15671 .is_err(),
15672 "preparation must also exclude local editors"
15673 );
15674 std::fs::write(proceed, "ready").unwrap();
15675 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15676 .await
15677 .unwrap()
15678 .unwrap();
15679 drop(prep);
15680 assert_eq!(runner.state.prep_complete, Some(true));
15681 assert!(root.join("judge-1").exists());
15682 assert!(root.join("advisor-1").exists());
15683 assert_eq!(
15684 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15685 0
15686 );
15687 }
15688
15689 #[tokio::test]
15690 #[cfg(unix)]
15691 async fn preparation_lock_failure_prevents_repository_changes() {
15692 ask_test_home();
15693 let tmp = tempfile::tempdir().unwrap();
15694 let repo = tmp.path().join("repo");
15695 std::fs::create_dir_all(&repo).unwrap();
15696 init_repo(&repo);
15697 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15698 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15701 let mut runner = runner_at(RunStatus::Prep);
15702 runner.state.repo = repo.clone();
15703 runner.state.config.coordination = crate::config::test_coordination();
15704 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15705 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15706 runner.roles.implementers = vec![spec("implement")];
15707 let error = runner.prep().await.unwrap_err();
15708 assert!(
15709 error
15710 .to_string()
15711 .contains("acquire mandatory repository lock")
15712 );
15713 assert!(!runner.state.worktree_root().exists());
15714 assert!(runner.state.candidates.is_empty());
15715 assert!(!repo.join(".git/config.worktree").exists());
15716 assert_eq!(
15717 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15718 "* main"
15719 );
15720 }
15721
15722 #[tokio::test]
15723 async fn preparation_records_creation_intent_before_git_can_fail() {
15724 ask_test_home();
15725 let tmp = tempfile::tempdir().unwrap();
15726 let repo = tmp.path().join("repo");
15727 std::fs::create_dir_all(&repo).unwrap();
15728 init_repo(&repo);
15729 let mut runner = runner_at(RunStatus::Prep);
15730 runner.state.repo = repo.clone();
15731 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15732 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15733 runner.state.config.blind.commit_msg_hook = false;
15734 runner.roles.implementers = vec![spec("one")];
15735 let label = blind::assign_labels(1, runner.state.seed)[0];
15736 let branch = runner.state.branch_for(label);
15737 git::git(&repo, &["branch", &branch]).await.unwrap();
15740 assert!(runner.prep().await.is_err());
15741 let saved = RunState::load(&runner.state.id).unwrap();
15742 assert!(saved.candidates.is_empty());
15743 let event = saved
15744 .events
15745 .iter()
15746 .find(|e| e.node == "prep-intent")
15747 .unwrap();
15748 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15749 assert_eq!(intent["branch"], branch);
15750 assert_eq!(intent["start"], runner.state.base_commit);
15751 assert_eq!(
15752 intent["worktree"],
15753 runner
15754 .state
15755 .worktree_root()
15756 .join(format!("cand-{label}"))
15757 .to_string_lossy()
15758 .as_ref()
15759 );
15760 assert_eq!(intent["picks"], serde_json::json!([]));
15761 }
15762
15763 #[tokio::test]
15764 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15765 ask_test_home();
15766 let tmp = tempfile::tempdir().unwrap();
15767 let repo = tmp.path().join("repo");
15768 std::fs::create_dir_all(&repo).unwrap();
15769 init_repo(&repo);
15770 let mut runner = runner_at(RunStatus::Prep);
15771 runner.state.repo = repo.clone();
15772 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15773 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15774 runner.state.sources.pick_commits.push(refs::Input {
15775 token: "missing".into(),
15776 sha: "0".repeat(40),
15777 provenance: refs::Provenance::ExplicitCli,
15778 });
15779 assert!(runner.prep().await.is_err());
15780 assert!(runner.state.candidates.is_empty());
15781 assert!(!runner.state.worktree_root().exists());
15782 }
15783
15784 #[tokio::test]
15785 async fn prose_and_legacy_inference_never_import_into_candidates() {
15786 ask_test_home();
15787 let tmp = tempfile::tempdir().unwrap();
15788 let repo = tmp.path().join("repo");
15789 std::fs::create_dir_all(&repo).unwrap();
15790 init_repo(&repo);
15791 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15792 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15793 .await
15794 .unwrap();
15795 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15796 git::git(&repo, &["add", "."]).await.unwrap();
15797 git::git(&repo, &["commit", "-m", "historical implementation"])
15798 .await
15799 .unwrap();
15800 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15801 git::git(&repo, &["checkout", "main"]).await.unwrap();
15802 for instruction in [
15803 format!("Example: magi/history/A at {source}"),
15804 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15805 format!("Never import magi/history/A or {source}"),
15806 format!("Conversation answer: I mentioned {source} only as background"),
15807 ] {
15808 let mut runner = runner_at(RunStatus::Prep);
15809 runner.state.repo = repo.clone();
15810 runner.state.base_commit = base.clone();
15811 runner.state.instruction = instruction.clone();
15812 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15813 runner.state.config.blind.commit_msg_hook = false;
15814 runner.state.seeds = vec![refs::Seed {
15815 token: "magi/history/A".into(),
15816 sha: source.clone(),
15817 kind: refs::SeedKind::Unmerged,
15818 branch: true,
15819 detail: String::new(),
15820 }];
15821 runner.roles.implementers = vec![spec("one")];
15822 assert_eq!(seeded_instruction(&runner.state), instruction);
15823 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15825 legacy.as_object_mut().unwrap().remove("sources");
15826 runner.state = serde_json::from_value(legacy).unwrap();
15827 assert_eq!(runner.state.sources, refs::Sources::default());
15828 runner.state.save().unwrap();
15829 runner.state = RunState::load(&runner.state.id).unwrap();
15830 runner.prep().await.unwrap();
15831 for candidate in &runner.state.candidates {
15832 assert_eq!(
15833 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15834 base
15835 );
15836 assert!(!candidate.worktree.join("unwanted").exists());
15837 }
15838 }
15839 }
15840
15841 #[tokio::test]
15842 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15843 ask_test_home();
15844 let tmp = tempfile::tempdir().unwrap();
15845 let repo = tmp.path().join("repo");
15846 std::fs::create_dir_all(&repo).unwrap();
15847 init_repo(&repo);
15848 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15849 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15850 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15851 git::git(&repo, &["add", "."]).await.unwrap();
15852 git::git(&repo, &["commit", "-m", "required seed"])
15853 .await
15854 .unwrap();
15855 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15856 git::git(&repo, &["checkout", "main"]).await.unwrap();
15857 let mut runner = runner_at(RunStatus::Prep);
15858 runner.state.repo = repo.clone();
15859 runner.state.base_commit = base.clone();
15860 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15861 runner.state.config.graph.advise = false;
15862 runner.state.config.blind.commit_msg_hook = false;
15863 runner.state.sources =
15864 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15865 .await
15866 .unwrap();
15867 runner.roles.implementers = vec![spec("one"), spec("two")];
15868 runner.roles.judges = vec![spec("judge")];
15869 let label = blind::assign_labels(2, runner.state.seed)[0];
15870 let branch = runner.state.branch_for(label);
15871 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15872 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15873 .await
15874 .unwrap();
15875 runner.state.candidates.push(Candidate {
15876 index: 0,
15877 label,
15878 agent: "one".into(),
15879 branch,
15880 worktree,
15881 summary: String::new(),
15882 stat: String::new(),
15883 files: 0,
15884 commits: 0,
15885 empty: false,
15886 failed: None,
15887 verified_noop: None,
15888 duration_ms: 0,
15889 folded: false,
15890 });
15891 runner.state.save().unwrap();
15892 runner.state = RunState::load(&runner.state.id).unwrap();
15894 assert_eq!(runner.state.prep_complete, Some(false));
15895 runner.prep().await.unwrap();
15896 assert_eq!(runner.state.prep_complete, Some(true));
15897 assert_eq!(runner.state.candidates.len(), 2);
15898 for candidate in &runner.state.candidates {
15899 assert_eq!(
15900 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15901 git::tree_of(&repo, &seed).await.unwrap()
15902 );
15903 assert_eq!(
15904 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15905 "required commit"
15906 );
15907 }
15908 assert!(runner.state.worktree_root().join("judge-1").exists());
15909 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15910 runner.prep().await.unwrap();
15911 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15912
15913 runner.state.prep_complete = Some(false);
15914 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15915 run: Some(format!(
15916 "test -f '{}'",
15917 tmp.path().join("dependency-ready").display()
15918 )),
15919 ..Default::default()
15920 }];
15921 assert!(runner.prep().await.is_err());
15922 assert!(runner.state.prep_failure.is_some());
15923 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15924 runner.state = RunState::load(&runner.state.id).unwrap();
15925 assert!(
15926 runner.prep().await.is_err(),
15927 "correcting the cause alone must not auto-retry"
15928 );
15929 let failed_at = runner.state.events.last().unwrap().at;
15930 assert!(
15931 runner.retry_preparation(failed_at).is_err(),
15932 "old approvals cannot unlock the hold"
15933 );
15934 let mut task = crate::queue::Task::new(
15935 "retry".into(),
15936 "retry setup".into(),
15937 repo.clone(),
15938 crate::queue::Source::Human,
15939 );
15940 task.start(runner.state.id.clone());
15941 task.hold_machine(Some("setup failed".into()));
15942 task.release_operator("cli", None);
15943 runner
15944 .retry_preparation(task.resumption.as_ref().unwrap().at)
15945 .unwrap();
15946 runner.prep().await.unwrap();
15947 assert!(runner.state.prep_failure.is_none());
15948 assert_eq!(runner.state.prep_complete, Some(true));
15949 assert_eq!(runner.state.candidates.len(), 2);
15950 assert!(runner.state.worktree_root().join("judge-1").exists());
15951 for candidate in &runner.state.candidates {
15952 assert_eq!(
15953 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15954 git::tree_of(&repo, &seed).await.unwrap()
15955 );
15956 }
15957 }
15958
15959 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15960 let mut heads = Vec::new();
15961 for candidate in candidates {
15962 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15963 }
15964 heads
15965 }
15966
15967 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15968 ask_test_home();
15969 let tmp = tempfile::tempdir().unwrap();
15970 let repo = tmp.path().join("repo");
15971 std::fs::create_dir(&repo).unwrap();
15972 init_repo(&repo);
15973 let mut runner = runner_at(RunStatus::Reviewing);
15974 runner.state.repo = repo.clone();
15975 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15976 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15977 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15978 let branch = format!("candidate-{label}");
15979 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15980 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15981 .await
15982 .unwrap();
15983 if label == 'B' {
15984 std::fs::write(worktree.join("change"), "winner").unwrap();
15985 git::rescue_commit(&worktree, "candidate change")
15986 .await
15987 .unwrap();
15988 }
15989 runner.state.candidates.push(Candidate {
15990 index,
15991 label,
15992 agent: "alpha".into(),
15993 branch,
15994 worktree,
15995 summary: "completed reply".into(),
15996 stat: String::new(),
15997 files: usize::from(label == 'B'),
15998 commits: usize::from(label == 'B'),
15999 empty: label == 'A',
16000 failed: None,
16001 verified_noop: None,
16002 duration_ms: 1,
16003 folded: false,
16004 });
16005 }
16006 runner.state.judge_skipped = true;
16007 runner.state.tally = Some(Tally {
16008 first_choice: BTreeMap::new(),
16009 borda: BTreeMap::new(),
16010 winner: 'B',
16011 rankings: 0,
16012 unanimous_initial: false,
16013 deliberated: false,
16014 changed_votes: 0,
16015 unanimous_final: false,
16016 tie_break: None,
16017 judges: 0,
16018 present: 0,
16019 quorum: 0,
16020 met_quorum: true,
16021 uncontested: Some("only B produced a change".into()),
16022 });
16023 runner.state.save().unwrap();
16024 (tmp, runner)
16025 }
16026
16027 #[tokio::test]
16028 async fn fold_losers_persists_discard_before_later_teardown_failure() {
16029 let (_tmp, mut runner) = preservation_resume_fixture().await;
16030 runner.state.candidates[0].folded = true;
16031 runner.state.config.graph.advise = true;
16032 runner.state.config.graph.advisors = 2;
16033 let repo = runner.state.repo.clone();
16034 let root = runner.state.worktree_root();
16035 let discarded = root.join("advisor-1");
16036 git::worktree_add_detached(&repo, &discarded, "main")
16037 .await
16038 .unwrap();
16039 std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
16040 let protected = root.join("advisor-2");
16041 git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
16042 .await
16043 .unwrap();
16044 std::fs::write(protected.join("keep"), "uncommitted").unwrap();
16045 runner.state.save().unwrap();
16046
16047 assert!(runner.fold_losers().await.is_err());
16048 assert!(!discarded.exists());
16049 assert!(protected.join("keep").exists());
16050 let saved = RunState::load(&runner.state.id).unwrap();
16051 assert!(saved.events.iter().any(|event| {
16052 event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
16053 }));
16054 }
16055
16056 #[tokio::test]
16057 async fn read_seat_teardown_discards_only_owned_detached_seats() {
16058 let (tmp, mut runner) = preservation_resume_fixture().await;
16059 let repo = runner.state.repo.clone();
16060 let root = runner.state.worktree_root();
16061 for name in ["advisor-3", "judge-1", "review-2"] {
16062 let path = root.join(name);
16063 git::worktree_add_detached(&repo, &path, "main")
16064 .await
16065 .unwrap();
16066 std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
16067 std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
16068 remove_read_seat_worktree(&mut runner.state, &path)
16069 .await
16070 .unwrap();
16071 assert!(!path.exists());
16072 }
16073 let outside = tmp.path().join("advisor-1");
16074 let unknown = root.join("unknown");
16075 let attached = root.join("advisor-4");
16076 git::worktree_add_detached(&repo, &outside, "main")
16077 .await
16078 .unwrap();
16079 git::worktree_add_detached(&repo, &unknown, "main")
16080 .await
16081 .unwrap();
16082 git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16083 .await
16084 .unwrap();
16085 let foreign = tmp.path().join("foreign");
16086 std::fs::create_dir(&foreign).unwrap();
16087 init_repo(&foreign);
16088 let wrong_repo = root.join("judge-2");
16089 git::worktree_add_detached(&foreign, &wrong_repo, "main")
16090 .await
16091 .unwrap();
16092 let candidate = runner.state.candidates[0].worktree.clone();
16093 for path in [outside, unknown, attached, wrong_repo, candidate] {
16094 std::fs::write(path.join("uncommitted"), "keep").unwrap();
16095 assert!(
16096 remove_read_seat_worktree(&mut runner.state, &path)
16097 .await
16098 .is_err()
16099 );
16100 assert_eq!(
16101 std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16102 "keep"
16103 );
16104 }
16105 }
16106
16107 #[tokio::test]
16108 #[cfg(unix)]
16109 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16110 for (solo, failed_checkpoint) in
16111 [(true, false), (false, false), (true, true), (false, true)]
16112 {
16113 let (_tmp, mut runner) = preservation_resume_fixture().await;
16114 if solo {
16115 runner.state.candidates.retain(|c| c.label == 'A');
16116 }
16117 runner.state.candidates[0].empty = false;
16118 runner.state.candidates[0].summary.clear();
16119 runner.state.tally = None;
16120 runner.state.judge_skipped = false;
16121 runner.state.status = RunStatus::Implementing;
16122 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16123 let mut implementer = spec("alpha");
16124 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16125 runner.roles.implementers = vec![implementer];
16126 let patch = runner.state.dir().join("artifacts/cand-A.patch");
16127 if failed_checkpoint {
16128 std::fs::create_dir_all(&patch).unwrap();
16129 }
16130 runner.state.save().unwrap();
16131 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16132 drop(pressure);
16133 runner.state = RunState::load(&runner.state.id).unwrap();
16134 assert_eq!(runner.state.status, RunStatus::Blocked);
16135 assert!(runner.state.interrupted_implementations.contains(&'A'));
16136 assert_eq!(runner.state.candidates[0].commits, 0);
16137 assert!(!runner.state.candidates[0].empty);
16138 assert!(runner.state.candidates[0].failed.is_none());
16139 if failed_checkpoint {
16140 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16141 std::fs::remove_dir(patch).unwrap();
16142 }
16143 runner.state.resource_block = None;
16144 runner.state.config.disk.min_free_bytes = 0;
16145 runner.roles.implementers[0].command = vec![
16146 "sh".into(),
16147 "-c".into(),
16148 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16149 ];
16150 runner.implement().await.unwrap();
16151 assert!(runner.state.interrupted_implementations.is_empty());
16152 assert!(runner.state.candidate_preservation.is_empty());
16153 assert_eq!(runner.state.candidates[0].commits, 1);
16154 assert!(!runner.state.candidates[0].empty);
16155 assert!(
16156 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16157 .unwrap()
16158 .contains("retried empty interrupted seat")
16159 );
16160 if !solo {
16161 assert_eq!(runner.state.candidates[1].commits, 1);
16162 }
16163 }
16164 }
16165
16166 #[tokio::test]
16167 #[cfg(unix)]
16168 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16169 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16170 let (_tmp, mut runner) = preservation_resume_fixture().await;
16171 runner.state.candidates.retain(|c| c.label == 'A');
16172 runner.state.candidates[0].empty = false;
16173 runner.state.candidates[0].summary.clear();
16174 runner.state.tally = None;
16175 runner.state.judge_skipped = false;
16176 runner.state.status = RunStatus::Implementing;
16177 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16180 let wt = runner.state.candidates[0].worktree.clone();
16181 std::fs::write(
16182 wt.join("interrupted-source"),
16183 "retain implementation work\n",
16184 )
16185 .unwrap();
16186 if committed {
16187 git::rescue_commit(&wt, "implementation checkpoint")
16188 .await
16189 .unwrap();
16190 }
16191 let lock = git::git(
16192 &wt,
16193 &[
16194 "rev-parse",
16195 "--path-format=absolute",
16196 "--git-path",
16197 "index.lock",
16198 ],
16199 )
16200 .await
16201 .unwrap();
16202 if block_rescue {
16203 std::fs::write(&lock, "fixture-owned lock").unwrap();
16204 }
16205 let mut implementer = spec("alpha");
16206 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16207 runner.roles.implementers = vec![implementer];
16208 runner.state.save().unwrap();
16209 let result = runner.implement().await;
16210 drop(pressure);
16211 assert_eq!(result.is_err(), block_rescue);
16212 assert_eq!(
16213 runner.state.resource_block.as_ref().unwrap().kind,
16214 "disk pressure"
16215 );
16216 assert!(
16217 runner
16218 .state
16219 .events
16220 .iter()
16221 .any(|e| e.message.contains("agent execution interrupted"))
16222 );
16223 assert!(
16224 runner
16225 .state
16226 .dir()
16227 .join("artifacts/candidate-A-recovery/staged.patch")
16228 .exists()
16229 );
16230 if block_rescue {
16231 let saved = RunState::load(&runner.state.id).unwrap();
16232 assert!(saved.candidate_preservation.contains_key(&'A'));
16233 assert!(!saved.candidates[0].empty);
16234 assert_eq!(
16235 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16236 "retain implementation work\n"
16237 );
16238 std::fs::remove_file(lock).unwrap();
16239 runner.state = saved;
16240 } else {
16241 assert_eq!(runner.state.status, RunStatus::Blocked);
16242 assert_eq!(runner.state.candidates[0].commits, 1);
16243 }
16244 runner.roles.implementers.clear();
16247 runner.state.config.disk.min_free_bytes = 0;
16248 runner.state.resource_block = None;
16249 runner.implement().await.unwrap();
16250 assert!(runner.state.candidate_preservation.is_empty());
16251 assert_eq!(runner.state.candidates[0].commits, 1);
16252 assert!(!runner.state.candidates[0].empty);
16253 assert!(git::is_clean(&wt).await.unwrap());
16254 assert!(
16255 crate::run::read_artifact(&runner.state, "cand-A.patch")
16256 .unwrap()
16257 .contains("retain implementation work")
16258 );
16259 }
16260 }
16261
16262 #[tokio::test]
16263 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16264 let (_tmp, mut runner) = preservation_resume_fixture().await;
16265 runner.implement().await.unwrap();
16267 assert!(runner.state.candidate_preservation.is_empty());
16268 assert!(
16269 !runner
16270 .state
16271 .dir()
16272 .join("artifacts/candidate-A-recovery")
16273 .exists()
16274 );
16275 runner.fold_losers().await.unwrap();
16276 assert!(!runner.state.candidates[0].worktree.exists());
16277 for schema in [14, crate::run::SCHEMA] {
16278 runner.state.schema = schema;
16279 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16280 runner.state.save().unwrap();
16281 runner.state = RunState::load(&runner.state.id).unwrap();
16282 runner.implement().await.unwrap();
16283 assert!(runner.state.candidate_preservation.is_empty());
16284 assert!(runner.state.candidates[0].folded);
16285 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16286 }
16287 }
16288
16289 #[tokio::test]
16290 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16291 let (_tmp, mut runner) = preservation_resume_fixture().await;
16292 runner.state.candidates.retain(|c| c.label == 'A');
16293 runner.state.tally = None;
16294 runner.state.judge_skipped = false;
16295 let candidate = &mut runner.state.candidates[0];
16296 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16297 candidate.failed = Some("deployment validation unavailable".into());
16298 std::fs::write(
16299 candidate.worktree.join("repair"),
16300 "future launcher policy\n",
16301 )
16302 .unwrap();
16303 runner
16304 .state
16305 .candidate_preservation
16306 .insert('A', "awaiting persistence".into());
16307 runner.persist_candidates().await.unwrap();
16308 let candidate = &runner.state.candidates[0];
16309 assert!(!candidate.empty);
16310 assert_eq!(candidate.commits, 1);
16311 assert_eq!(candidate.files, 1);
16312 assert!(candidate.failed.is_none());
16313 assert!(candidate.summary.contains("EPERM"));
16314 assert!(
16315 crate::run::read_artifact(&runner.state, "cand-A.patch")
16316 .unwrap()
16317 .contains("future launcher policy")
16318 );
16319 assert!(runner.state.candidate_preservation.is_empty());
16320 }
16321
16322 #[tokio::test]
16323 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16324 let (_tmp, mut runner) = preservation_resume_fixture().await;
16325 let wt = runner.state.candidates[0].worktree.clone();
16326 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16327 runner.state.schema = 14;
16328 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16329 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16330 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16331 for _ in 0..2 {
16332 assert!(runner.implement().await.is_err());
16333 assert_eq!(runner.state.status, RunStatus::Blocked);
16334 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16335 assert!(
16336 runner.state.candidates[0].empty,
16337 "old verdict must remain unusable"
16338 );
16339 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16340 assert!(runner.state.judge_skipped);
16341 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16342 assert!(
16343 crate::run::read_artifact(&runner.state, "cand-A.patch")
16344 .unwrap()
16345 .contains("owned edits")
16346 );
16347 assert!(wt.exists());
16348 assert_eq!(
16349 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16350 .await
16351 .unwrap(),
16352 1
16353 );
16354 runner.state = RunState::load(&runner.state.id).unwrap();
16355 }
16356 }
16357
16358 #[tokio::test]
16359 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16360 let (_tmp, mut runner) = preservation_resume_fixture().await;
16361 let wt = runner.state.candidates[0].worktree.clone();
16362 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16363 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16364 let index_lock = PathBuf::from(
16365 git::git(
16366 &wt,
16367 &[
16368 "rev-parse",
16369 "--path-format=absolute",
16370 "--git-path",
16371 "index.lock",
16372 ],
16373 )
16374 .await
16375 .unwrap(),
16376 );
16377 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16378 runner.state.schema = 14;
16379 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16380 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16381 assert!(runner.implement().await.is_err());
16382 assert!(runner.after_implement().is_err());
16383 assert!(runner.fold_losers().await.is_err());
16384 assert_eq!(
16385 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16386 "failed seat edits"
16387 );
16388 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16389 runner.state = RunState::load(&runner.state.id).unwrap();
16390 std::fs::remove_file(index_lock).unwrap();
16391 assert!(runner.implement().await.is_err());
16393 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16394 assert!(runner.state.candidates[0].empty);
16395 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16396 assert!(
16397 crate::run::read_artifact(&runner.state, "cand-A.patch")
16398 .unwrap()
16399 .contains("failed seat edits")
16400 );
16401 assert!(wt.exists());
16402 }
16403
16404 #[tokio::test]
16405 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16406 let (_tmp, mut runner) = preservation_resume_fixture().await;
16407 let repo = runner.state.repo.clone();
16408 let common = git::git(
16409 &repo,
16410 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16411 )
16412 .await
16413 .unwrap();
16414 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16415 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16416 assert!(runner.fold_losers().await.is_err());
16417 assert!(!runner.state.candidates[0].worktree.exists());
16418 assert!(!runner.state.candidates[0].folded);
16419 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16420 runner.state = RunState::load(&runner.state.id).unwrap();
16421 std::fs::remove_file(branch_lock).unwrap();
16422 let judge = runner.state.worktree_root().join("judge-1");
16423 git::worktree_add_detached(&repo, &judge, "main")
16424 .await
16425 .unwrap();
16426 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16427 git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16428 .await
16429 .unwrap();
16430 runner.roles.judges = vec![spec("judge")];
16431 assert!(runner.fold_losers().await.is_err());
16432 runner.state = RunState::load(&runner.state.id).unwrap();
16433 assert!(
16434 runner.state.candidates[0].folded,
16435 "candidate progress saved before judge failure"
16436 );
16437 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16438 assert!(judge.join("uncommitted").exists());
16439 git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16440 .await
16441 .unwrap();
16442 runner.fold_losers().await.unwrap();
16443 assert!(!judge.exists());
16444 runner.fold_losers().await.unwrap();
16445 }
16446
16447 #[tokio::test]
16448 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16449 let (_tmp, mut runner) = preservation_resume_fixture().await;
16450 runner.state.candidates.retain(|c| c.label == 'B');
16451 runner.state.tally = None;
16452 runner.state.judge_skipped = false;
16453 runner.state.prep_complete = Some(true);
16454 runner.state.config.graph.advise = false;
16455 runner.state.config.graph.selected_implementer = Some("alpha".into());
16456 let wt = runner.state.candidates[0].worktree.clone();
16457 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16458 runner
16459 .state
16460 .candidate_preservation
16461 .insert('B', "awaiting persistence".into());
16462 let index_lock = git::git(
16463 &wt,
16464 &[
16465 "rev-parse",
16466 "--path-format=absolute",
16467 "--git-path",
16468 "index.lock",
16469 ],
16470 )
16471 .await
16472 .unwrap();
16473 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16474 runner.state.save().unwrap();
16475 assert!(runner.execute_graph().await.is_err());
16476 assert_eq!(runner.state.status, RunStatus::Blocked);
16477 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16478 std::fs::remove_file(index_lock).unwrap();
16479 for _ in 0..2 {
16482 runner.state = RunState::load(&runner.state.id).unwrap();
16483 runner.pause.park();
16484 runner.execute_graph().await.unwrap();
16485 assert_eq!(runner.state.status, RunStatus::Implementing);
16486 assert!(runner.state.candidate_preservation.is_empty());
16487 assert_eq!(runner.state.candidates[0].commits, 2);
16488 assert!(runner.state.tally.is_none());
16489 }
16490 runner.implement().await.unwrap();
16491 assert_eq!(runner.state.status, RunStatus::Judging);
16492 runner.state.status = RunStatus::Blocked;
16494 runner.state.candidates[0].failed = Some("implementation failed".into());
16495 runner.state.save().unwrap();
16496 let error = runner.execute_graph().await.unwrap_err();
16497 assert!(
16498 error
16499 .to_string()
16500 .contains("selected implementation is held")
16501 );
16502 }
16503
16504 #[tokio::test]
16505 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16506 let (_tmp, mut runner) = preservation_resume_fixture().await;
16507 let repo = runner.state.repo.clone();
16508 let candidate = runner.state.candidates[0].clone();
16509 let registered = git::worktree_holding(&repo, &candidate.branch)
16512 .await
16513 .unwrap()
16514 .unwrap();
16515 assert_eq!(
16516 registered.canonicalize().unwrap(),
16517 candidate.worktree.canonicalize().unwrap()
16518 );
16519 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16520 assert_eq!(
16521 git::worktree_holding(&repo, &candidate.branch)
16522 .await
16523 .unwrap(),
16524 Some(registered)
16525 );
16526 assert!(
16527 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16528 "the stale registration initially keeps the branch checked out"
16529 );
16530 runner.fold_losers().await.unwrap();
16531 assert!(runner.state.candidates[0].folded);
16532 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16533 assert_eq!(
16534 git::worktree_holding(&repo, &candidate.branch)
16535 .await
16536 .unwrap(),
16537 None
16538 );
16539 runner.state = RunState::load(&runner.state.id).unwrap();
16540 runner.fold_losers().await.unwrap();
16541 }
16542
16543 #[tokio::test]
16544 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16545 let (_tmp, mut runner) = preservation_resume_fixture().await;
16546 let repo = runner.state.repo.clone();
16547 let candidate = runner.state.candidates[0].clone();
16548 let winner = runner.state.candidates[1].clone();
16549 let registered = git::worktree_holding(&repo, &candidate.branch)
16550 .await
16551 .unwrap()
16552 .unwrap();
16553 assert_eq!(
16554 registered.canonicalize().unwrap(),
16555 candidate.worktree.canonicalize().unwrap()
16556 );
16557 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16558 assert_eq!(
16559 git::worktree_holding(&repo, &candidate.branch)
16560 .await
16561 .unwrap(),
16562 Some(registered)
16563 );
16564 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16565
16566 for _ in 0..2 {
16567 fold_run(&mut runner.state, false, &crate::run::home())
16568 .await
16569 .unwrap();
16570 runner.state = RunState::load(&runner.state.id).unwrap();
16571 assert!(runner.state.candidates[0].folded);
16572 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16573 assert_eq!(
16574 git::worktree_holding(&repo, &candidate.branch)
16575 .await
16576 .unwrap(),
16577 None
16578 );
16579 assert!(!runner.state.candidates[1].folded);
16580 assert!(winner.worktree.exists());
16581 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16582 }
16583 }
16584 #[tokio::test]
16585 async fn fold_recovery_override_archives_sources_and_keeps_safety_checks() {
16586 for sidecar in [false, true] {
16587 let tmp = tempfile::tempdir().unwrap();
16588 let home = tmp.path().join("home");
16589 let repo = tmp.path().join("repo");
16590 std::fs::create_dir(&repo).unwrap();
16591 init_repo(&repo);
16592 let mut state = runner_at(RunStatus::Superseded).state;
16593 state.repo = repo.clone();
16594 state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16595 state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16596 state.driver_exited = true;
16597 state.reserve_under(&home).unwrap();
16598 git::git(
16599 &repo,
16600 &["update-ref", "refs/remotes/origin/main", &state.base_commit],
16601 )
16602 .await
16603 .unwrap();
16604 let wt = state.worktree_root().join("cand-A");
16605 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16606 .await
16607 .unwrap();
16608 std::fs::write(wt.join("independent"), "retained implementation\n").unwrap();
16609 git::commit_all(&wt, "fixture: independent source")
16610 .await
16611 .unwrap();
16612 let head = git::rev_parse(&wt, "HEAD").await.unwrap();
16613 state.candidates.push(Candidate {
16614 index: 0,
16615 label: 'A',
16616 agent: "fixture".into(),
16617 branch: "candidate".into(),
16618 worktree: wt.clone(),
16619 summary: String::new(),
16620 stat: String::new(),
16621 files: 0,
16622 commits: 0,
16623 empty: false,
16624 failed: None,
16625 verified_noop: None,
16626 duration_ms: 0,
16627 folded: false,
16628 });
16629 let mut task = crate::queue::Task::new(
16630 "task".into(),
16631 "task".into(),
16632 repo.clone(),
16633 crate::queue::Source::Human,
16634 );
16635 task.runs.push(state.id.clone());
16636 crate::queue::Queue::at(home.join("queue"))
16637 .put(&mut task)
16638 .unwrap();
16639 if sidecar {
16640 crate::pr_recovery::retain_sources(&task, &home).unwrap();
16641 }
16642 state.save_under(&home).unwrap();
16643 let reason = crate::pr_recovery::source_retention_reason(&home, &state).unwrap();
16644 assert!(
16645 fold_run(&mut state, true, &home)
16646 .await
16647 .unwrap_err()
16648 .to_string()
16649 .contains(&reason)
16650 );
16651 let mut disk = state.config.disk.clone();
16652 disk.fold_grace_secs = 0;
16653 assert_eq!(
16654 crate::clean::fold_due(
16655 &home.join("runs"),
16656 &home,
16657 &state.worktree_root(),
16658 &disk,
16659 Timestamp::now() + std::time::Duration::from_secs(10),
16660 )
16661 .await
16662 .unwrap(),
16663 (0, 0)
16664 );
16665 assert!(wt.exists());
16666 state.status = RunStatus::Blocked;
16667 state.save_under(&home).unwrap();
16668 assert!(
16669 fold_run_overriding_recovery_retention(&mut state, true, &home)
16670 .await
16671 .is_err()
16672 );
16673 state.status = RunStatus::Superseded;
16674 state.driver_exited = false;
16675 state.driver_claimed = None;
16676 state.save_under(&home).unwrap();
16677 assert!(
16678 fold_run_overriding_recovery_retention(&mut state, true, &home)
16679 .await
16680 .is_err()
16681 );
16682 state.driver_pid = Some(std::process::id());
16683 state.driver_started_at = crate::proc::process_started_at(std::process::id());
16684 state.save_under(&home).unwrap();
16685 assert!(
16686 fold_run_overriding_recovery_retention(&mut state, true, &home)
16687 .await
16688 .is_err()
16689 );
16690 state.driver_exited = true;
16691 state.driver_pid = None;
16692 state.driver_started_at = None;
16693 state.save_under(&home).unwrap();
16694 let lease = crate::driver::maintenance(&home, &state.id).unwrap();
16695 assert!(
16696 fold_run_overriding_recovery_retention(&mut state, true, &home)
16697 .await
16698 .is_err()
16699 );
16700 drop(lease);
16701 #[cfg(unix)]
16702 {
16703 let output = crate::retention::OutputLease::take(&wt, false).unwrap();
16704 assert!(
16705 fold_run_overriding_recovery_retention(&mut state, true, &home)
16706 .await
16707 .is_err()
16708 );
16709 drop(output);
16710 }
16711 let owner = crate::pr_head::owner_lock(&home, &state.id).unwrap();
16712 assert!(
16713 fold_run_overriding_recovery_retention(&mut state, true, &home)
16714 .await
16715 .is_err()
16716 );
16717 drop(owner);
16718 let checkpoint = git::git(
16719 &wt,
16720 &[
16721 "rev-parse",
16722 "--path-format=absolute",
16723 "--git-path",
16724 "CHERRY_PICK_HEAD",
16725 ],
16726 )
16727 .await
16728 .unwrap();
16729 std::fs::write(&checkpoint, &head).unwrap();
16730 assert!(
16731 fold_run_overriding_recovery_retention(&mut state, true, &home)
16732 .await
16733 .is_err()
16734 );
16735 std::fs::remove_file(checkpoint).unwrap();
16736 std::fs::write(wt.join("dirty"), "keep").unwrap();
16737 assert!(
16738 fold_run_overriding_recovery_retention(&mut state, true, &home)
16739 .await
16740 .is_err()
16741 );
16742 assert!(wt.join("dirty").exists());
16743 std::fs::remove_file(wt.join("dirty")).unwrap();
16744 let evidence = home
16745 .join("runs")
16746 .join(&state.id)
16747 .join("artifacts/fold-evidence");
16748 std::fs::create_dir_all(evidence.parent().unwrap()).unwrap();
16749 std::fs::write(&evidence, "blocked archive").unwrap();
16750 assert!(
16751 fold_run_overriding_recovery_retention(&mut state, true, &home)
16752 .await
16753 .is_err()
16754 );
16755 assert!(wt.exists());
16756 assert!(
16757 !RunState::load_under(&state.id, &home)
16758 .unwrap()
16759 .events
16760 .iter()
16761 .any(|event| event.message.contains("--override-recovery-retention"))
16762 );
16763 std::fs::remove_file(&evidence).unwrap();
16764 fold_run_overriding_recovery_retention(&mut state, true, &home)
16765 .await
16766 .unwrap();
16767 assert!(!wt.exists());
16768 assert!(!git::branch_exists(&repo, "candidate").await.unwrap());
16769 assert!(
16770 git::worktree_holding(&repo, "candidate")
16771 .await
16772 .unwrap()
16773 .is_none()
16774 );
16775 assert!(
16776 std::fs::read_to_string(evidence.join(format!("{head}.patch")))
16777 .unwrap()
16778 .contains("retained implementation")
16779 );
16780 assert!(evidence.join(format!("{head}.index")).is_file());
16781 let saved = RunState::load_under(&state.id, &home).unwrap();
16782 assert_eq!(saved.status, RunStatus::Superseded);
16783 assert!(saved.candidates[0].folded);
16784 assert!(saved.events.iter().any(|event| {
16785 event.message.contains("--override-recovery-retention")
16786 && event.message.contains(&state.id)
16787 && event.message.contains(&reason)
16788 }));
16789 assert_eq!(
16790 crate::pr_recovery::explicit_sources_retained(&home, &state.id),
16791 sidecar
16792 );
16793 }
16794 }
16795
16796 #[tokio::test]
16797 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16798 ask_test_home();
16799 let tmp = tempfile::tempdir().unwrap();
16800 let repo = tmp.path().join("repo");
16801 std::fs::create_dir(&repo).unwrap();
16802 init_repo(&repo);
16803 let mut runner = runner_at(RunStatus::Implementing);
16804 runner.state.repo = repo.clone();
16805 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16806 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16807 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16808 let branch = format!("candidate-{label}");
16809 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16810 git::worktree_add_branch(&repo, &wt, &branch, "main")
16811 .await
16812 .unwrap();
16813 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16814 git::git(&wt, &["add", "README.md"]).await.unwrap();
16815 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16816 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16817 runner.state.candidates.push(Candidate {
16818 index,
16819 label,
16820 agent: "alpha".to_owned(),
16821 branch,
16822 worktree: wt,
16823 summary: "completed implementation".to_owned(),
16824 stat: String::new(),
16825 files: 0,
16826 commits: 0,
16827 empty: false,
16828 failed: None,
16829 verified_noop: None,
16830 duration_ms: 42,
16831 folded: false,
16832 });
16833 runner
16834 .state
16835 .candidate_preservation
16836 .insert(label, "awaiting persistence".to_owned());
16837 }
16838 let wt = runner.state.candidates[0].worktree.clone();
16839 let index_lock = git::git(
16840 &wt,
16841 &[
16842 "rev-parse",
16843 "--path-format=absolute",
16844 "--git-path",
16845 "index.lock",
16846 ],
16847 )
16848 .await
16849 .unwrap();
16850 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16851 runner.state.save().unwrap();
16852 assert!(runner.persist_candidates().await.is_err());
16853 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16854 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16855 assert_eq!(
16856 runner.state.candidates[1].commits, 1,
16857 "other results are still collected"
16858 );
16859 assert!(
16860 !runner.state.candidates[0].empty,
16861 "failed rescue is not no change"
16862 );
16863 assert_eq!(
16864 runner.state.candidates[0].summary,
16865 "completed implementation"
16866 );
16867 assert!(runner.after_implement().is_err());
16868 let home = tmp.path().join("fold-home");
16869 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16870 assert!(wt.join("new-file").exists());
16871 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16872 assert!(recovery.join("staged.patch").exists());
16873 assert!(recovery.join("unstaged.patch").exists());
16874 assert_eq!(
16875 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16876 "untracked edits\n"
16877 );
16878 runner.state.begin_pass();
16879 runner.state.end_pass_parked(ParkCause::Upgrade {
16880 from: "1.0".into(),
16881 to: Some("1.1".into()),
16882 });
16883 runner.state.save().unwrap();
16884 runner.state = RunState::load(&runner.state.id).unwrap();
16885 assert!(matches!(
16886 runner.state.passes.last().unwrap().end,
16887 Some(crate::run::PassEnd::Parked {
16888 cause: ParkCause::Upgrade { .. }
16889 })
16890 ));
16891 std::fs::remove_file(index_lock).unwrap();
16892 runner.implement().await.unwrap();
16894 assert!(runner.state.candidate_preservation.is_empty());
16895 assert!(
16896 runner
16897 .state
16898 .candidates
16899 .iter()
16900 .all(|c| c.commits == 1 && !c.empty)
16901 );
16902 assert!(git::is_clean(&wt).await.unwrap());
16903 assert!(
16904 crate::run::read_artifact(&runner.state, "cand-A.patch")
16905 .unwrap()
16906 .contains("later tracked edits")
16907 );
16908 }
16909
16910 #[tokio::test]
16911 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16912 ask_test_home();
16913 let tmp = tempfile::tempdir().unwrap();
16914 let repo = tmp.path().join("repo");
16915 std::fs::create_dir(&repo).unwrap();
16916 init_repo(&repo);
16917 let wt = tmp.path().join("candidate");
16918 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16919 .await
16920 .unwrap();
16921 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16922 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16923 assert!(wt.join("uncommitted").exists());
16924 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16925 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16926 let mut runner = runner_at(RunStatus::Implementing);
16927 runner.state.repo = repo.clone();
16928 runner.state.base_commit = "missing-base".to_owned();
16929 runner.state.candidates.push(Candidate {
16930 index: 0,
16931 label: 'A',
16932 agent: "alpha".into(),
16933 branch: "candidate".into(),
16934 worktree: wt.clone(),
16935 summary: "completed".into(),
16936 stat: String::new(),
16937 files: 0,
16938 commits: 0,
16939 empty: false,
16940 failed: None,
16941 verified_noop: None,
16942 duration_ms: 1,
16943 folded: false,
16944 });
16945 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16946 assert!(
16947 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16948 .await
16949 .is_err()
16950 );
16951 assert!(!runner.state.candidates[0].folded);
16952 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16953 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16954 runner
16955 .state
16956 .candidate_preservation
16957 .insert('A', "pending".into());
16958 assert!(runner.persist_candidates().await.is_err());
16959 assert!(!runner.state.candidates[0].empty);
16960 assert!(!runner.state.candidates[0].folded);
16961 assert!(wt.exists());
16962 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16963 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16964 std::fs::create_dir_all(&patch_path).unwrap();
16965 assert!(
16966 runner.persist_candidates().await.is_err(),
16967 "artifact failure stays pending"
16968 );
16969 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16970 assert!(!runner.state.candidates[0].empty);
16971 std::fs::remove_dir(patch_path).unwrap();
16972 runner.persist_candidates().await.unwrap();
16973 assert!(
16974 runner.state.candidates[0].empty,
16975 "only successful inspection confirms no change"
16976 );
16977 }
16978
16979 fn init_repo(dir: &Path) {
16980 let run = |args: &[&str]| {
16981 let out = std::process::Command::new("git")
16982 .args(args)
16983 .current_dir(dir)
16984 .quiet()
16985 .output()
16986 .expect("spawn git");
16987 assert!(
16988 out.status.success(),
16989 "git {args:?} failed: {}",
16990 String::from_utf8_lossy(&out.stderr)
16991 );
16992 };
16993 run(&["init", "-b", "main"]);
16994 run(&["config", "user.name", "magi test"]);
16995 run(&["config", "user.email", "magi@example.com"]);
16996 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16997 run(&["add", "-A"]);
16998 run(&["commit", "-m", "init"]);
16999 }
17000
17001 fn ask_test_home() {
17009 crate::run::pin_test_home();
17010 }
17011
17012 #[test]
17013 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
17014 ask_test_home();
17015 let mut runner = runner_at(RunStatus::Prep);
17016 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17017 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
17020 let error = guard
17021 .finish(
17022 &mut runner.state,
17023 Err(
17024 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
17025 .into(),
17026 ),
17027 )
17028 .unwrap_err();
17029 assert_eq!(
17030 error.downcast_ref::<std::io::Error>().unwrap().kind(),
17031 std::io::ErrorKind::PermissionDenied
17032 );
17033 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
17034 let message = format!("{error:#}");
17035 assert!(message.contains("original refusal"));
17036 assert!(!message.contains("could not save terminal"));
17037 assert_eq!(
17038 error.chain().count(),
17039 2,
17040 "only startup attribution wraps the original error"
17041 );
17042 let saved = RunState::load(&runner.state.id).unwrap();
17043 assert_eq!(saved.status, RunStatus::Blocked);
17044 assert!(!saved.startup.unwrap().complete);
17045 }
17046
17047 #[test]
17048 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
17049 ask_test_home();
17050 let mut runner = runner_at(RunStatus::Prep);
17051 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17052 let id = runner.state.id.clone();
17053 drop(guard);
17054 let saved = RunState::load(&id).unwrap();
17055 assert_eq!(saved.status, RunStatus::Blocked);
17056 assert!(!saved.startup.as_ref().unwrap().complete);
17057 assert!(
17058 saved
17059 .startup
17060 .unwrap()
17061 .failure
17062 .unwrap()
17063 .contains("interrupted")
17064 );
17065 assert!(Runner::resume(&id).is_err());
17066 }
17067
17068 #[test]
17069 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
17070 ask_test_home();
17071 let mut runner = runner_at(RunStatus::Prep);
17072 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17073 let error = guard
17074 .finish(
17075 &mut runner.state,
17076 Err(
17077 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
17078 .into(),
17079 ),
17080 )
17081 .unwrap_err();
17082 assert_eq!(
17083 error.downcast_ref::<std::io::Error>().unwrap().kind(),
17084 std::io::ErrorKind::PermissionDenied
17085 );
17086 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
17087 let mut task = crate::queue::Task::new(
17088 "review".into(),
17089 "criteria".into(),
17090 runner.state.repo.clone(),
17091 crate::queue::Source::Human,
17092 );
17093 task.record_startup_failure(failure.clone());
17094 task.record_startup_failure(failure.clone());
17095 assert_eq!(task.startup_failures.len(), 1);
17096 assert!(task.runs.is_empty());
17097 assert_eq!(task.attempts, 0);
17098 }
17099
17100 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
17101 ask_test_home();
17102 let tmp = tempfile::tempdir().unwrap();
17103 let repo = tmp.path().join("repo");
17104 std::fs::create_dir(&repo).unwrap();
17105 init_repo(&repo);
17106 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17107 let origin = tmp.path().join("origin.git");
17108 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
17109 .await
17110 .unwrap();
17111 git::git(
17112 &repo,
17113 &["remote", "add", "origin", origin.to_str().unwrap()],
17114 )
17115 .await
17116 .unwrap();
17117 git::git(&repo, &["checkout", "-b", "fix/retained"])
17118 .await
17119 .unwrap();
17120 std::fs::write(repo.join("retained"), "retained change").unwrap();
17121 git::git(&repo, &["add", "."]).await.unwrap();
17122 git::git(&repo, &["commit", "-m", "retained implementation"])
17123 .await
17124 .unwrap();
17125 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
17126 git::git(&repo, &["checkout", "main"]).await.unwrap();
17127 std::fs::write(repo.join("base-change"), "new base").unwrap();
17128 git::git(&repo, &["add", "."]).await.unwrap();
17129 git::git(&repo, &["commit", "-m", "base moved"])
17130 .await
17131 .unwrap();
17132 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17133 git::git(&repo, &["checkout", "fix/retained"])
17134 .await
17135 .unwrap();
17136 let mut runner = runner_at(RunStatus::Voting);
17137 runner.state.repo = repo.clone();
17138 runner.state.base_commit = base;
17139 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17140 runner.state.config.disk.min_free_bytes = 0;
17141 runner.state.config.graph.review_rounds = 1;
17142 runner.state.config.verify.e2e = vec!["true".into()];
17143 runner.state.config.verify.gate = vec!["true".into()];
17144 runner.state.candidates = vec![Candidate {
17145 index: 0,
17146 label: 'A',
17147 agent: "fixture".into(),
17148 branch: "fix/retained".into(),
17149 worktree: repo.clone(),
17150 summary: String::new(),
17151 stat: String::new(),
17152 files: 1,
17153 commits: 1,
17154 empty: false,
17155 failed: None,
17156 verified_noop: None,
17157 duration_ms: 0,
17158 folded: false,
17159 }];
17160 runner.state.tally = Some(Tally {
17161 first_choice: BTreeMap::new(),
17162 borda: BTreeMap::new(),
17163 winner: 'A',
17164 rankings: 0,
17165 unanimous_initial: true,
17166 deliberated: false,
17167 changed_votes: 0,
17168 unanimous_final: true,
17169 tie_break: None,
17170 judges: 0,
17171 present: 0,
17172 quorum: 0,
17173 met_quorum: true,
17174 uncontested: Some("review-only".into()),
17175 });
17176 let mut task = crate::queue::Task::new(
17177 "fixture".into(),
17178 "fixture".into(),
17179 repo,
17180 crate::queue::Source::Human,
17181 );
17182 task.link_run(&runner.state.id);
17183 crate::queue::Queue::open().put(&mut task).unwrap();
17184 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
17185 runner.state.base_sync = Some(BaseSync {
17186 tip: String::new(),
17187 behind: 1,
17188 attempts: BASE_SYNC_ROUNDS,
17189 conflict: None,
17190 already_in: None,
17191 });
17192 runner.sync_to_base().await.unwrap();
17193 assert_eq!(runner.state.status, RunStatus::Blocked);
17194 (tmp, runner, head, task.id)
17195 }
17196
17197 #[tokio::test]
17198 async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
17199 let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
17200 runner.state.status = RunStatus::Voting;
17201 runner.state.base_sync = None;
17202 runner.state.base_sync_recovery = Default::default();
17203 let scratch = runner.state.dir().join("base-sync");
17204 std::fs::create_dir_all(&scratch).unwrap();
17205 std::fs::write(scratch.join("obstruction"), "keep").unwrap();
17206 runner.sync_to_base().await.unwrap();
17207 assert!(runner.state.resource_block.is_some());
17208 assert!(!runner.state.base_sync_blocked());
17209 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
17210 assert_eq!(runner.state.base_episode_used(), 0);
17211 assert!(runner.state.base_sync_recovery.operations.is_empty());
17212 assert!(runner.state.rebase_fixes.is_empty());
17213 assert_eq!(
17214 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17215 head
17216 );
17217 std::fs::remove_dir_all(&scratch).unwrap();
17218 runner.state.resource_block = None;
17219 runner.sync_to_base().await.unwrap();
17220 assert!(!runner.state.base_sync_blocked());
17221 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17222 assert_eq!(runner.state.base_episode_used(), 1);
17223 assert!(runner.state.base_sync_recovery.operations[0].finished);
17224 assert!(runner.state.rebase_fixes.is_empty());
17225 assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
17226 assert!(git::is_clean(&runner.state.repo).await.unwrap());
17227 }
17228
17229 #[cfg(unix)]
17230 #[tokio::test]
17231 async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
17232 for routed in [false, true] {
17233 let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
17234 let home = crate::run::home();
17235 let queue = crate::queue::Queue::open();
17236 let task = queue.get(&owner).unwrap();
17237 runner.state.task_instruction = Some(task.instruction.clone());
17238 runner.state.driver_exited = true;
17239 runner.state.save().unwrap();
17240 let questions = crate::ask::Questions::at(home.join("questions"));
17241 let mut q = crate::ask::Question::new(
17242 runner.state.id.clone(),
17243 "base-recovery".into(),
17244 "owner".into(),
17245 "Recover bounded base movement?".into(),
17246 String::new(),
17247 vec!["recover".into()],
17248 );
17249 q.actions.insert(
17250 "recover".into(),
17251 crate::ask::ChoiceAction::RecoverBase {
17252 run: runner.state.id.clone(),
17253 stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
17254 episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
17255 },
17256 );
17257 q.routed_task = Some(owner.clone());
17258 q.answer(crate::ask::Answer::Choice("recover".into()))
17259 .unwrap();
17260 questions.put(&mut q).unwrap();
17261 crate::base_recovery_control::recover(
17262 &queue,
17263 &home,
17264 crate::base_recovery_control::Request {
17265 run: runner.state.id.clone(),
17266 head: head.clone(),
17267 actor: "decisions".into(),
17268 request: q.id.clone(),
17269 decision: Some((owner.clone(), q.answered_at.unwrap())),
17270 expected_stop: Some((
17271 runner.state.base_sync_recovery.stop.clone().unwrap(),
17272 runner.state.base_sync_recovery.episodes.last().unwrap().id,
17273 )),
17274 },
17275 )
17276 .await
17277 .unwrap();
17278 if routed {
17279 questions
17280 .update(&q.id, |q| {
17281 q.routed_task = Some("another-owner".into());
17282 Ok(())
17283 })
17284 .unwrap();
17285 } else {
17286 let mut task = queue.get(&owner).unwrap();
17287 task.hold_manual(Some("new owner decision".into()));
17288 task.status = crate::queue::TaskStatus::Queued;
17290 queue.put(&mut task).unwrap();
17291 }
17292 runner.state = RunState::load(&runner.state.id).unwrap();
17293 runner.sync_to_base().await.unwrap();
17294 let approval = runner
17295 .state
17296 .base_sync_recovery
17297 .authorizations
17298 .last()
17299 .unwrap();
17300 assert!(approval.consumed);
17301 assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17302 "routing"
17303 } else {
17304 "owner decision"
17305 }));
17306 assert_eq!(
17307 runner.state.base_sync.as_ref().unwrap().attempts,
17308 BASE_SYNC_ROUNDS
17309 );
17310 assert_eq!(
17311 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17312 head
17313 );
17314 }
17315 }
17316
17317 #[cfg(unix)]
17318 #[tokio::test]
17319 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17320 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17321 if std::env::var_os(CHILD).is_none() {
17322 let home = tempfile::tempdir().unwrap();
17323 let output = std::process::Command::new(std::env::current_exe().unwrap())
17324 .args([
17325 "--exact",
17326 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17327 "--nocapture",
17328 ])
17329 .env(CHILD, "1")
17330 .env("MAGI_HOME", home.path())
17331 .env("MAGI_WORKER_HOME", home.path())
17332 .env("MAGI_WORKER_ID", "abcdef")
17333 .output()
17334 .unwrap();
17335 assert!(
17336 output.status.success()
17337 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17338 "{}\n{}",
17339 String::from_utf8_lossy(&output.stdout),
17340 String::from_utf8_lossy(&output.stderr)
17341 );
17342 return;
17343 }
17344 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17345 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17346 let home = crate::run::home();
17347 let queue = crate::queue::Queue::open();
17348 let mut owner = queue.get(&task).unwrap();
17349 owner.release_operator("test-release", None);
17350 owner.start(runner.state.id.clone());
17351 queue.put(&mut owner).unwrap();
17352 runner.state.driver_exited = true;
17353 runner.state.task_instruction = Some(owner.instruction.clone());
17354 runner.state.save().unwrap();
17355 let _claim = queue.claim(&task).unwrap();
17356 let reserve = |id: &str| {
17357 let directory = home.join("workers/jobs").join(id);
17358 std::fs::create_dir_all(&directory).unwrap();
17359 let job = crate::worker::Job {
17360 id: id.into(),
17361 sequence: 1,
17362 kind: "task".into(),
17363 key: format!("task:{task}"),
17364 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17365 generation: "fixture-generation".into(),
17366 version: None,
17367 status: crate::worker::Status::Running,
17368 result: None,
17369 pid: Some(std::process::id()),
17370 created_at: 0,
17371 finished_at: None,
17372 };
17373 std::fs::write(
17374 directory.join("job.json"),
17375 serde_json::to_vec(&job).unwrap(),
17376 )
17377 .unwrap();
17378 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17379 };
17380 let _own = reserve("abcdef");
17381 let other = reserve("fedcba");
17382 assert!(
17383 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17384 .await
17385 .is_err()
17386 );
17387 assert!(
17388 RunState::load(&runner.state.id)
17389 .unwrap()
17390 .base_sync_recovery
17391 .authorizations
17392 .is_empty()
17393 );
17394 drop(other);
17395 assert!(crate::daemon::is_working_on(
17396 &home,
17397 &runner.state.id,
17398 Timestamp::now()
17399 ));
17400 assert!(
17401 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17402 );
17403 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17404 .await
17405 .unwrap();
17406 assert!(
17407 RunState::load(&runner.state.id)
17408 .unwrap()
17409 .base_recovery_pending()
17410 );
17411 }
17412
17413 #[cfg(unix)]
17414 #[tokio::test]
17415 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17416 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17417 let home = crate::run::home();
17418 runner.state.driver_pid = Some(std::process::id());
17419 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17420 runner.state.driver_exited = false;
17421 runner.state.begin_pass();
17422 runner.state.save().unwrap();
17423 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17424 driver.record(&runner.state).unwrap();
17425 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17426 drop(driver);
17428 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17429 assert_eq!(
17430 runner.state.liveness_under(&home, false),
17431 crate::run::Liveness::Dead
17432 );
17433 let queue = crate::queue::Queue::open();
17434 let _claim = queue.claim(&task).unwrap();
17435 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17436 .await
17437 .unwrap();
17438 let disk = RunState::load(&runner.state.id).unwrap();
17439 assert!(!disk.driver_exited);
17440 assert!(disk.base_recovery_pending());
17441 }
17442
17443 #[cfg(unix)]
17444 #[tokio::test]
17445 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17446 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17447 if std::env::var_os(CHILD).is_none() {
17448 use std::os::unix::fs::PermissionsExt;
17449 let tmp = tempfile::tempdir().unwrap();
17450 let gh = tmp.path().join("gh");
17451 std::fs::write(&gh, r#"#!/bin/sh
17452case "$1 $2" in
17453 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17454 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17455 *) exit 2 ;;
17456esac
17457"#).unwrap();
17458 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17459 let mut paths = vec![tmp.path().to_path_buf()];
17460 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17461 let output = std::process::Command::new(std::env::current_exe().unwrap())
17462 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17463 .env(CHILD, "1")
17464 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17465 .env("PATH", std::env::join_paths(paths).unwrap())
17466 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17467 .output().unwrap();
17468 assert!(
17469 output.status.success()
17470 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17471 "{}\n{}",
17472 String::from_utf8_lossy(&output.stdout),
17473 String::from_utf8_lossy(&output.stderr)
17474 );
17475 return;
17476 }
17477 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17478 let repo = runner.state.repo.clone();
17479 git::git(&repo, &["checkout", "main"]).await.unwrap();
17480 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17481 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17482 git::git(&repo, &["checkout", "fix/retained"])
17483 .await
17484 .unwrap();
17485 runner.state.pr = Some(crate::run::PrRecord {
17486 url: "https://example.test/project/pull/1".into(),
17487 number: 1,
17488 state: "open".into(),
17489 checks: "unknown".into(),
17490 round: 0,
17491 rounds: 1,
17492 red_at_merge: Vec::new(),
17493 });
17494 runner.sync_to_base().await.unwrap();
17495 assert_eq!(runner.state.status, RunStatus::Blocked);
17496 assert_eq!(
17497 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17498 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17499 );
17500 assert!(runner.state.base_sync_closure_pending());
17501 assert!(crate::handover::can_resume(&runner.state));
17502 let queue = crate::queue::Queue::open();
17503 assert!(!crate::base_sync_recovery::held_task(
17504 &queue,
17505 &queue.get(&task).unwrap()
17506 ));
17507 assert!(runner.state.authorize_base_sync(&task).is_err());
17508 runner.state.base_sync_recovery.stop = None;
17510 runner.state.status = RunStatus::Voting;
17511 runner.state.gate_ran = true;
17512 runner.state.config.graph.review_rounds = 0;
17513 runner.state.save().unwrap();
17514 assert!(crate::handover::can_resume(&runner.state));
17515 runner.state = RunState::load(&runner.state.id).unwrap();
17516 runner.execute_graph().await.unwrap();
17517 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17518 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17519 assert_eq!(
17520 runner.state.base_sync.as_ref().unwrap().attempts,
17521 BASE_SYNC_ROUNDS
17522 );
17523 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17524 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17525 assert!(runner.state.reviews.is_empty());
17526 assert!(runner.state.gate.is_empty());
17527 }
17528
17529 #[cfg(unix)]
17530 #[tokio::test]
17531 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17532 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17533 runner.state.driver_exited = true;
17534 runner.state.save().unwrap();
17535 let home = crate::run::home();
17536 let queue = crate::queue::Queue::open();
17537 let _claim = queue.claim(&task).unwrap();
17538 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17539 assert!(
17540 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17541 .await
17542 .is_err()
17543 );
17544 drop(lease);
17545 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17546 assert!(
17547 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17548 .await
17549 .is_err()
17550 );
17551 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17552 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17553 .await
17554 .unwrap();
17555 let disk = RunState::load(&runner.state.id).unwrap();
17556 assert!(disk.base_recovery_pending());
17557 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17558 assert!(
17559 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17560 .await
17561 .is_err()
17562 );
17563 assert_eq!(
17564 RunState::load(&runner.state.id)
17565 .unwrap()
17566 .base_sync_recovery
17567 .authorizations
17568 .len(),
17569 1
17570 );
17571 let mut owner = queue.get(&task).unwrap();
17572 owner.runs.push("20260101-000000-ffff".into());
17573 queue.put(&mut owner).unwrap();
17574 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17575 .await
17576 .unwrap_err();
17577 assert!(error.to_string().contains("run task ownership changed"));
17578 }
17579
17580 #[cfg(unix)]
17581 #[tokio::test]
17582 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17583 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17584 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17585 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17586 legacy["status"] = serde_json::json!("voting");
17587 runner.state = serde_json::from_value(legacy).unwrap();
17588 runner.state.save().unwrap();
17589 for _ in 0..3 {
17590 runner.execute_graph().await.unwrap();
17591 assert_eq!(runner.state.status, RunStatus::Blocked);
17592 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17593 assert!(!crate::handover::can_resume(&runner.state));
17594 assert_eq!(
17595 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17596 head
17597 );
17598 assert!(runner.state.reviews.is_empty());
17599 assert!(!runner.state.gate_ran);
17600 }
17601 let notice = crate::notices::run_ended(&runner.state).unwrap();
17602 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17603 runner.state.begin_pass();
17604 runner.state.status = RunStatus::Voting;
17605 runner.deliberate().await.unwrap();
17606 runner.vote().await.unwrap();
17607 runner.finish_after_tally().await.unwrap();
17608 assert_eq!(runner.state.status, RunStatus::Blocked);
17609 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17610 assert_eq!(
17611 crate::notices::run_ended(&runner.state).unwrap().since,
17612 notice.since
17613 );
17614 }
17615
17616 #[cfg(unix)]
17617 #[tokio::test]
17618 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17619 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17620 for detail in [
17621 "fix/retained conflicts with origin/main and did not rebase: conflict",
17622 "rebased fix/retained locally but origin refused the push",
17623 "unknown failure",
17624 ] {
17625 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17626 runner.state.record_base_sync_stop(&head);
17627 assert!(runner.state.authorize_base_sync(&task).is_err());
17628 runner.state.status = RunStatus::Voting;
17629 runner.execute_graph().await.unwrap();
17630 assert_eq!(runner.state.status, RunStatus::Blocked);
17631 assert!(!crate::handover::can_resume(&runner.state));
17632 assert_eq!(
17633 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17634 head
17635 );
17636 }
17637 }
17638
17639 #[cfg(unix)]
17640 #[tokio::test]
17641 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17642 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17643 runner.state.authorize_base_sync(&task).unwrap();
17644 assert!(runner.state.consume_base_recovery(&head));
17645 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17646 runner.state.reserve_base_operation(&head, &tip);
17647 runner.state.save().unwrap();
17648 runner.state = RunState::load(&runner.state.id).unwrap();
17649 assert!(!runner.state.consume_base_recovery(&head));
17650 runner.sync_to_base().await.unwrap();
17651 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17652 assert_eq!(runner.state.base_episode_used(), 1);
17653 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17654 assert!(
17655 runner
17656 .state
17657 .base_sync_recovery
17658 .operations
17659 .last()
17660 .unwrap()
17661 .finished
17662 );
17663 assert!(!runner.state.base_sync_blocked());
17664 runner.sync_to_base().await.unwrap();
17665 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17666 }
17667
17668 #[cfg(unix)]
17669 #[tokio::test]
17670 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17671 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17672 let original = runner
17674 .state
17675 .base_sync
17676 .as_ref()
17677 .unwrap()
17678 .conflict
17679 .clone()
17680 .unwrap();
17681 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17682 original
17683 .replace("synchronization(s)", "rebase(s)")
17684 .replace("synchronizing", "rebasing"),
17685 );
17686 runner.state.base_sync_recovery = Default::default();
17687 runner.state.record_base_sync_stop(&head);
17688 assert_eq!(
17689 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17690 crate::base_sync_recovery::StopKind::Exhausted
17691 );
17692 runner.state.authorize_base_sync(&task).unwrap();
17693 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17694 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17695 .as_object_mut()
17696 .unwrap();
17697 auth.remove("interval");
17698 auth.remove("scope");
17699 runner.state = serde_json::from_value(legacy).unwrap();
17700 assert!(runner.state.consume_base_recovery(&head));
17701 assert_eq!(runner.state.base_episode_used(), 3);
17702 runner.sync_to_base().await.unwrap();
17703 assert_eq!(runner.state.base_episode_used(), 4);
17704 runner.state.resource_block = Some(crate::run::ResourceBlock {
17705 node: "gate".into(),
17706 kind: "cache".into(),
17707 detail: "legacy wait".into(),
17708 at: Timestamp::now(),
17709 cause: crate::run::ResourceBlockCause::Unrecorded,
17710 });
17711 runner.state.save().unwrap();
17712 assert!(
17713 runner
17714 .state
17715 .base_sync_recovery
17716 .episodes
17717 .last()
17718 .unwrap()
17719 .boundary
17720 .is_none()
17721 );
17722 runner.state.resource_block = None;
17723 advance_base_fixture(&runner).await;
17724 runner.sync_to_base().await.unwrap();
17725 assert!(runner.state.base_sync_blocked());
17726 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17727 }
17728
17729 #[cfg(unix)]
17730 async fn advance_base_fixture(runner: &Runner) {
17731 let repo = &runner.state.repo;
17732 for args in [
17733 vec!["checkout", "main"],
17734 vec!["commit", "--allow-empty", "-m", "base advanced"],
17735 vec!["push", "origin", "main"],
17736 vec!["checkout", "fix/retained"],
17737 ] {
17738 git::git(repo, &args).await.unwrap();
17739 }
17740 }
17741
17742 #[cfg(unix)]
17743 fn fresh_base_interval_fixture(runner: &mut Runner) {
17744 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17745 runner.state.base_sync.as_mut().unwrap().conflict = None;
17746 runner.state.base_sync_recovery = Default::default();
17747 runner.state.status = RunStatus::Voting;
17748 }
17749
17750 #[cfg(unix)]
17751 #[tokio::test]
17752 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17753 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17754 fresh_base_interval_fixture(&mut runner);
17755 runner.state.begin_pass();
17756 runner.sync_to_base().await.unwrap();
17757 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17758 runner.state.resource_block = Some(crate::run::ResourceBlock {
17759 node: "gate".into(),
17760 kind: "build cache".into(),
17761 detail: "busy".into(),
17762 at: Timestamp::now(),
17763 cause: crate::run::ResourceBlockCause::Timeout {
17764 waited_secs: 1,
17765 limit_secs: 1,
17766 },
17767 });
17768 runner.state.save().unwrap();
17769 let queue = crate::queue::Queue::open();
17770 let mut owner = queue.get(&task).unwrap();
17771 runner.state.task_instruction = Some(owner.instruction.clone());
17772 owner.release_operator("fixture-owner", None);
17773 queue.put(&mut owner).unwrap();
17774 let approval = owner.resumption.clone().unwrap();
17775 runner.state.resource_block = None;
17776 runner.state.save().unwrap();
17777 runner.state = RunState::load(&runner.state.id).unwrap();
17778 advance_base_fixture(&runner).await;
17779 runner.sync_to_base().await.unwrap();
17780 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17781 assert_eq!(runner.state.base_episode_used(), 1);
17782 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17783 assert!(
17784 runner.state.base_sync_recovery.episodes[0]
17785 .boundary
17786 .as_ref()
17787 .unwrap()
17788 .consumed
17789 );
17790 assert!(
17791 !runner
17792 .state
17793 .renew_base_episode(&task, &approval, &head, "base")
17794 );
17795 for _ in 0..3 {
17796 owner.release_operator("fixture-owner", None);
17797 queue.put(&mut owner).unwrap();
17798 runner.state = RunState::load(&runner.state.id).unwrap();
17799 runner.state.begin_pass();
17800 runner.sync_to_base().await.unwrap();
17801 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17802 assert_eq!(runner.state.base_episode_used(), 1);
17803 }
17804 runner.state.resource_block = Some(crate::run::ResourceBlock {
17805 node: "gate".into(),
17806 kind: "disk pressure".into(),
17807 detail: "free space required".into(),
17808 at: Timestamp::now(),
17809 cause: crate::run::ResourceBlockCause::Other,
17810 });
17811 runner.state.end_pass_parked(ParkCause::Interrupt {
17812 reason: "disk pressure".into(),
17813 });
17814 runner.state.save().unwrap();
17815 owner.release_operator("fixture-owner-after-disk", None);
17816 queue.put(&mut owner).unwrap();
17817 runner.state.resource_block = None;
17818 advance_base_fixture(&runner).await;
17819 runner.sync_to_base().await.unwrap();
17820 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17821 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17822 assert_eq!(runner.state.base_episode_used(), 1);
17823 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17824 }
17825
17826 #[cfg(unix)]
17827 #[tokio::test]
17828 async fn base_recovery_continuous_advancement_stops_at_four() {
17829 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17830 fresh_base_interval_fixture(&mut runner);
17831 for spent in 1..=BASE_SYNC_ROUNDS {
17832 runner.sync_to_base().await.unwrap();
17833 assert!(!runner.state.base_sync_blocked());
17834 assert_eq!(runner.state.base_episode_used(), spent);
17835 advance_base_fixture(&runner).await;
17836 }
17837 runner.sync_to_base().await.unwrap();
17838 assert_eq!(
17839 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17840 crate::base_sync_recovery::StopKind::Exhausted
17841 );
17842 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17843 assert!(stop.unverified_sync);
17844 let action = runner.state.base_sync_next_action();
17845 assert!(action.contains("never reached its gate"));
17846 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17847 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17848 runner.state = RunState::load(&runner.state.id).unwrap();
17849 runner.sync_to_base().await.unwrap();
17850 assert_eq!(runner.state.base_episode_used(), 4);
17851 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17852 }
17853
17854 #[cfg(unix)]
17855 #[tokio::test]
17856 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17857 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17858 runner.state.ensure_base_episode();
17859 runner.state.record_base_progress(&head);
17861 runner.state.base_sync_recovery.stop = None;
17862 runner.state.record_base_sync_stop(&head);
17863 assert!(
17864 !runner
17865 .state
17866 .base_sync_recovery
17867 .stop
17868 .as_ref()
17869 .unwrap()
17870 .unverified_sync
17871 );
17872
17873 runner.state.finish_base_operation(Some(&head));
17874 runner.state.gate_ran = true;
17876 runner.state.gate_head = Some("older-head".into());
17877 runner.state.base_sync_recovery.stop = None;
17878 runner.state.record_base_sync_stop(&head);
17879 runner.state.save().unwrap();
17880 runner.state = RunState::load(&runner.state.id).unwrap();
17881 assert!(
17882 runner
17883 .state
17884 .base_sync_recovery
17885 .stop
17886 .as_ref()
17887 .unwrap()
17888 .unverified_sync
17889 );
17890
17891 runner.state.gate_head = Some(head.clone());
17892 runner.state.base_sync_recovery.stop = None;
17893 runner.state.record_base_sync_stop(&head);
17894 assert!(
17895 !runner
17896 .state
17897 .base_sync_recovery
17898 .stop
17899 .as_ref()
17900 .unwrap()
17901 .unverified_sync
17902 );
17903 assert!(
17904 !runner
17905 .state
17906 .base_sync_next_action()
17907 .contains("never reached its gate")
17908 );
17909
17910 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17912 legacy["base_sync_recovery"]["stop"]
17913 .as_object_mut()
17914 .unwrap()
17915 .remove("unverified_sync");
17916 legacy["base_sync_recovery"]["episodes"][0]
17917 .as_object_mut()
17918 .unwrap()
17919 .remove("last_sync_head");
17920 runner.state = serde_json::from_value(legacy).unwrap();
17921 assert!(
17922 !runner
17923 .state
17924 .base_sync_recovery
17925 .stop
17926 .as_ref()
17927 .unwrap()
17928 .unverified_sync
17929 );
17930 assert!(
17931 runner.state.base_sync_recovery.episodes[0]
17932 .last_sync_head
17933 .is_none()
17934 );
17935 }
17936
17937 #[cfg(unix)]
17938 #[tokio::test]
17939 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17940 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17941 runner.state.authorize_base_sync(&task).unwrap();
17942 assert!(
17943 runner
17944 .state
17945 .base_sync_next_action()
17946 .contains("Approval recorded")
17947 );
17948 assert!(
17949 runner
17950 .state
17951 .base_sync_next_action()
17952 .contains("4/4 in episode 0, 4 lifetime")
17953 );
17954 assert!(runner.state.consume_base_recovery(&head));
17955 assert!(
17956 runner
17957 .state
17958 .base_sync_next_action()
17959 .contains("0/4 in episode 1, 4 lifetime")
17960 );
17961 }
17962
17963 #[cfg(unix)]
17964 #[tokio::test]
17965 async fn base_recovery_changed_head_invalidates_without_spending() {
17966 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17967 runner.state.authorize_base_sync(&task).unwrap();
17968 git::git(
17969 &runner.state.repo,
17970 &["commit", "--allow-empty", "-m", "external head change"],
17971 )
17972 .await
17973 .unwrap();
17974 runner.sync_to_base().await.unwrap();
17975 let a = runner
17976 .state
17977 .base_sync_recovery
17978 .authorizations
17979 .last()
17980 .unwrap();
17981 assert!(a.consumed);
17982 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17983 runner.state = RunState::load(&runner.state.id).unwrap();
17984 assert!(
17985 runner
17986 .state
17987 .base_sync_next_action()
17988 .contains("Approval invalidated")
17989 );
17990 assert!(!runner.state.base_recovery_pending());
17991 assert!(runner.state.base_sync_blocked());
17992 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17993 }
17994
17995 #[cfg(unix)]
17996 #[tokio::test]
17997 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17998 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17999 runner.state.driver_exited = true;
18000 runner.state.save().unwrap();
18001 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18002 .await
18003 .unwrap();
18004 runner.state = RunState::load(&runner.state.id).unwrap();
18005 advance_base_fixture(&runner).await;
18006 runner.sync_to_base().await.unwrap();
18007 let a = runner
18008 .state
18009 .base_sync_recovery
18010 .authorizations
18011 .last()
18012 .unwrap();
18013 assert!(
18014 a.consumed
18015 && a.invalidated
18016 .as_deref()
18017 .unwrap()
18018 .contains("fetched base changed")
18019 );
18020 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18021 assert_eq!(
18022 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18023 head
18024 );
18025 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18026 .await
18027 .unwrap();
18028 runner.state = RunState::load(&runner.state.id).unwrap();
18029 runner.sync_to_base().await.unwrap();
18030 assert!(!runner.state.base_sync_blocked());
18031 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18032 }
18033
18034 #[cfg(unix)]
18035 #[tokio::test]
18036 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
18037 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18038 runner.state.authorize_base_sync(&task).unwrap();
18039 assert!(runner.state.consume_base_recovery(&head));
18040 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18041 runner.state.reserve_base_operation(&head, &tip);
18042 runner.state.save().unwrap();
18043 let scratch = runner.state.dir().join("base-sync");
18044 assert_eq!(
18045 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18046 .await
18047 .unwrap(),
18048 crate::rebase::Rebased::Applied
18049 );
18050 runner.state = RunState::load(&runner.state.id).unwrap();
18051 runner.sync_to_base().await.unwrap();
18052 assert!(
18053 runner
18054 .state
18055 .base_sync_recovery
18056 .operations
18057 .last()
18058 .unwrap()
18059 .finished
18060 );
18061 assert_eq!(runner.state.base_episode_used(), 1);
18062 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18063 assert!(!runner.state.base_sync_blocked());
18064 }
18065
18066 #[cfg(unix)]
18067 #[tokio::test]
18068 async fn base_recovery_crash_preserves_untracked_files() {
18069 for ignored in [false, true] {
18070 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18071 runner.state.authorize_base_sync(&task).unwrap();
18072 assert!(runner.state.consume_base_recovery(&head));
18073 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18074 runner.state.reserve_base_operation(&head, &tip);
18075 runner.state.save().unwrap();
18076 let scratch = runner.state.dir().join("base-sync");
18077 assert_eq!(
18078 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18079 .await
18080 .unwrap(),
18081 crate::rebase::Rebased::Applied
18082 );
18083 let repo = &runner.state.repo;
18084 let result = git::rev_parse(repo, "HEAD").await.unwrap();
18085 assert_ne!(result, head);
18086 if ignored {
18087 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
18088 .await
18089 .unwrap();
18090 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
18091 }
18092 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
18093 assert!(
18094 git::git_raw(repo, &["diff", "--quiet", &head])
18095 .await
18096 .unwrap()
18097 .ok()
18098 );
18099 runner.state = RunState::load(&runner.state.id).unwrap();
18101 runner.sync_to_base().await.unwrap();
18102 assert!(runner.state.base_sync_blocked());
18103 assert!(!runner.state.base_sync_recovery.operations[0].finished);
18104 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18105 assert_eq!(
18106 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18107 result
18108 );
18109 assert_eq!(
18110 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
18111 "retained work\n"
18112 );
18113 assert!(
18114 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
18115 .await
18116 .unwrap()
18117 .ok()
18118 );
18119 }
18120 }
18121
18122 #[cfg(unix)]
18123 #[tokio::test]
18124 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
18125 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18126 runner.state.authorize_base_sync(&task).unwrap();
18127 assert!(runner.state.consume_base_recovery(&head));
18128 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18129 runner.state.reserve_base_operation(&head, &tip);
18130 runner.state.save().unwrap();
18131 let scratch = runner.state.dir().join("base-sync");
18132 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18133 .await
18134 .unwrap();
18135 git::sync_to_head(&runner.state.repo).await.unwrap();
18136 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
18137 git::git(&runner.state.repo, &["add", "foreign"])
18138 .await
18139 .unwrap();
18140 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
18141 .await
18142 .unwrap();
18143 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18144 runner.state = RunState::load(&runner.state.id).unwrap();
18145 runner.sync_to_base().await.unwrap();
18146 assert!(runner.state.base_sync_blocked());
18147 assert!(
18148 !runner
18149 .state
18150 .base_sync_recovery
18151 .operations
18152 .last()
18153 .unwrap()
18154 .finished
18155 );
18156 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18157 assert_eq!(
18158 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18159 foreign_head
18160 );
18161 }
18162
18163 #[cfg(unix)]
18164 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
18165 {
18166 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18167 fresh_base_interval_fixture(&mut runner);
18168 let repo = &runner.state.repo;
18169 git::git(repo, &["checkout", "main"]).await.unwrap();
18170 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
18171 git::git(repo, &["add", "retained"]).await.unwrap();
18172 git::git(repo, &["commit", "-m", "upstream retained change"])
18173 .await
18174 .unwrap();
18175 git::git(repo, &["push", "origin", "main"]).await.unwrap();
18176 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
18177 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
18178 let scratch = runner.state.dir().join("base-sync");
18179 assert!(matches!(
18180 git::rebase_start(repo, &scratch, "fix/retained", &tip)
18181 .await
18182 .unwrap(),
18183 git::RebaseStart::Conflicted(_)
18184 ));
18185 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
18186 agent: "fixture".into(),
18187 paths: vec!["retained".into()],
18188 from: Some(head.clone()),
18189 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
18190 index_outside: None,
18191 finished: false,
18192 error: None,
18193 launcher: None,
18194 infra: None,
18195 outage: None,
18196 });
18197 (tmp, runner, head, tip, scratch)
18198 }
18199
18200 #[cfg(unix)]
18201 #[tokio::test]
18202 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
18203 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18204 std::fs::write(
18207 scratch.join("retained"),
18208 "upstream and retained resolution\n",
18209 )
18210 .unwrap();
18211 runner.state.resource_block = Some(crate::run::ResourceBlock {
18212 node: "rebase".into(),
18213 kind: "repository coordination".into(),
18214 detail: "busy".into(),
18215 at: Timestamp::now(),
18216 cause: crate::run::ResourceBlockCause::Other,
18217 });
18218 runner.state.save().unwrap();
18219 let newer = git::git(
18222 &runner.state.repo,
18223 &[
18224 "commit-tree",
18225 "main^{tree}",
18226 "-p",
18227 "main",
18228 "-m",
18229 "base advanced during wait",
18230 ],
18231 )
18232 .await
18233 .unwrap();
18234 git::git(
18235 &runner.state.repo,
18236 &[
18237 "push",
18238 "origin",
18239 &format!("{}:refs/heads/main", newer.trim()),
18240 ],
18241 )
18242 .await
18243 .unwrap();
18244 runner.state = RunState::load(&runner.state.id).unwrap();
18245 runner.state.resource_block = None;
18246 runner.sync_to_base().await.unwrap();
18247 assert!(!runner.state.base_sync_blocked());
18248 let op = &runner.state.base_sync_recovery.operations[0];
18249 assert_eq!(op.head, head);
18250 assert_eq!(op.tip, tip);
18251 assert!(op.finished);
18252 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18253 assert_eq!(runner.state.rebase_fixes.len(), 1);
18254 let latest = git::rev_parse(&runner.state.repo, "origin/main")
18255 .await
18256 .unwrap();
18257 assert_ne!(latest, tip);
18258 runner.state = RunState::load(&runner.state.id).unwrap();
18259 runner.sync_to_base().await.unwrap();
18260 assert!(!runner.state.base_sync_blocked());
18261 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
18262 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
18263 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18264 }
18265
18266 #[cfg(unix)]
18267 #[tokio::test]
18268 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
18269 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18270 runner.state.reserve_base_operation(&head, &tip);
18271 runner.state.save().unwrap();
18272 std::fs::write(
18273 scratch.join("retained"),
18274 "upstream and retained resolution\n",
18275 )
18276 .unwrap();
18277 git::git(&scratch, &["add", "retained"]).await.unwrap();
18278 git::git(
18279 &scratch,
18280 &["-c", "core.editor=true", "rebase", "--continue"],
18281 )
18282 .await
18283 .unwrap();
18284 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18285 assert!(
18286 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18287 .await
18288 .unwrap()
18289 .is_empty()
18290 );
18291 assert_eq!(
18292 crate::rebase::finish(
18293 &mut runner.state,
18294 &scratch,
18295 "fix/retained",
18296 &head,
18297 &tip,
18298 &["retained".into()],
18299 ""
18300 )
18301 .await
18302 .unwrap(),
18303 crate::rebase::Rebased::Applied
18304 );
18305 assert!(!scratch.exists());
18306 runner.state = RunState::load(&runner.state.id).unwrap();
18308 let op = &runner.state.base_sync_recovery.operations[0];
18309 assert!(!op.finished);
18310 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18311 runner.sync_to_base().await.unwrap();
18312 assert!(!runner.state.base_sync_blocked());
18313 assert!(runner.state.base_sync_recovery.operations[0].finished);
18314 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18315 assert_eq!(runner.state.rebase_fixes.len(), 1);
18316 let winner = runner.state.winner().unwrap().clone();
18317 let wait = runner.repository_wait();
18318 assert!(
18319 verify_current_head(&mut runner.state, &winner, &wait)
18320 .await
18321 .unwrap()
18322 );
18323 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18324 }
18325
18326 #[cfg(unix)]
18327 #[tokio::test]
18328 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18329 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18330 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18331 runner.state.reserve_base_operation(&head, &tip);
18332 runner.state.save().unwrap();
18333 std::fs::write(
18334 scratch.join("retained"),
18335 "upstream and retained resolution\n",
18336 )
18337 .unwrap();
18338 git::git(&scratch, &["add", "retained"]).await.unwrap();
18339 git::git(
18340 &scratch,
18341 &["-c", "core.editor=true", "rebase", "--continue"],
18342 )
18343 .await
18344 .unwrap();
18345 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18346 assert!(!git::rebase_in_progress(&scratch).await);
18347 assert!(
18348 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18349 .await
18350 .unwrap()
18351 .is_empty()
18352 );
18353 match damage {
18354 Some("record") => {
18355 let path =
18356 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18357 .await
18358 .unwrap();
18359 std::fs::write(scratch.join(path), "").unwrap();
18360 }
18361 Some("foreign") => {
18362 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18363 git::git(&scratch, &["add", "foreign"]).await.unwrap();
18364 git::git(&scratch, &["commit", "-m", "foreign work"])
18365 .await
18366 .unwrap();
18367 }
18368 Some("dirty") => {
18369 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18370 }
18371 _ => {}
18372 }
18373 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18374 runner.state = RunState::load(&runner.state.id).unwrap();
18376 assert!(
18377 runner.state.base_sync_recovery.operations[0]
18378 .validated_head
18379 .is_none()
18380 );
18381 runner.sync_to_base().await.unwrap();
18382 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18383 assert_eq!(runner.state.base_episode_used(), 1);
18384 assert_eq!(runner.state.rebase_fixes.len(), 1);
18385 assert_eq!(
18386 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18387 retained_head
18388 );
18389 if damage.is_some() {
18390 assert!(runner.state.base_sync_blocked());
18391 assert!(scratch.exists());
18392 assert!(!runner.state.base_sync_recovery.operations[0].finished);
18393 assert!(
18394 runner.state.base_sync_recovery.operations[0]
18395 .validated_head
18396 .is_none()
18397 );
18398 } else {
18399 assert!(!runner.state.base_sync_blocked());
18400 assert!(!scratch.exists());
18401 assert!(runner.state.base_sync_recovery.operations[0].finished);
18402 assert_eq!(
18403 runner.state.base_sync_recovery.operations[0]
18404 .validated_head
18405 .as_deref(),
18406 Some(result.as_str())
18407 );
18408 let winner = runner.state.winner().unwrap().clone();
18409 let wait = runner.repository_wait();
18410 assert!(
18411 verify_current_head(&mut runner.state, &winner, &wait)
18412 .await
18413 .unwrap()
18414 );
18415 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18416 runner.state = RunState::load(&runner.state.id).unwrap();
18417 runner.sync_to_base().await.unwrap();
18418 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18419 }
18420 }
18421 }
18422
18423 #[cfg(unix)]
18424 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18425 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18426 runner.state.prep_complete = Some(true);
18428 runner.state.base_sync = None;
18429 runner.state.base_sync_recovery = Default::default();
18430 runner.state.config.graph.gate_fix_rounds = 1;
18431 let mut review = review_round(true, 0, 1, 1, false, true);
18432 review.head = head.clone();
18433 review.verified_head = Some(head.clone());
18434 runner.state.reviews = vec![review];
18435 runner.state.gate = vec![CommandOutcome {
18436 command: "old gate".into(),
18437 code: Some(1),
18438 output_tail: "old failure".into(),
18439 duration_ms: 0,
18440 resource_blocked: false,
18441 }];
18442 runner.state.gate_head = Some(head.clone());
18443 runner.state.gate_ran = true;
18444 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18445 agent: "fixture".into(),
18446 failed: runner.state.gate.clone(),
18447 notes: "spent".into(),
18448 committed: false,
18449 error: None,
18450 }];
18451 runner.state.land_approval = Some(crate::run::LandApproval {
18452 question: "old approval".into(),
18453 head: head.clone(),
18454 });
18455 runner.state.status = RunStatus::Blocked;
18456 (tmp, runner, head)
18457 }
18458
18459 #[cfg(unix)]
18460 #[tokio::test]
18461 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18462 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18463 let (tmp, mut runner, old) = resumed_gate_fixture().await;
18464 let phases = tmp.path().join("phases");
18465 runner.state.config.verify.e2e = vec![format!(
18466 "echo e2e >> '{}'; {}",
18467 phases.display(),
18468 if e2e_ok {
18469 "true"
18470 } else {
18471 "echo new-e2e-failure; false"
18472 }
18473 )];
18474 runner.state.config.verify.gate = vec![format!(
18475 "echo gate >> '{}'; {}",
18476 phases.display(),
18477 if gate_ok {
18478 "true"
18479 } else {
18480 "echo new-gate-failure; false"
18481 }
18482 )];
18483 runner.state.save().unwrap();
18484 runner.state = RunState::load(&runner.state.id).unwrap();
18485 runner.sync_to_base().await.unwrap();
18486 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18487 assert_ne!(head, old);
18488 runner.state = RunState::load(&runner.state.id).unwrap();
18490 runner.execute_graph().await.unwrap();
18491 assert_eq!(runner.state.gate_fixes.len(), 1);
18492 assert_eq!(runner.state.reviews.len(), 1);
18493 assert_eq!(runner.state.reviews[0].head, old);
18494 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18495 assert_eq!(runner.state.verification_history.len(), 1);
18496 let history = &runner.state.verification_history[0];
18497 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18498 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18499 assert!(history.archived_at.is_some());
18500 assert_eq!(history.fixes_spent, Some(1));
18501 assert_eq!(history.gate[0].output_tail, "old failure");
18502 let archive_events: Vec<_> = runner
18503 .state
18504 .events
18505 .iter()
18506 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18507 .collect();
18508 assert_eq!(archive_events.len(), 1);
18509 assert!(archive_events[0].message.contains(&short(&old)));
18510 assert!(archive_events[0].message.contains(&short(&head)));
18511 assert!(
18512 archive_events[0]
18513 .message
18514 .contains("1 gate-fix round(s) already spent stay spent")
18515 );
18516 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18517 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18518 if e2e_ok {
18519 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18520 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18521 } else {
18522 assert!(!runner.state.gate_ran);
18523 }
18524 assert_eq!(
18525 runner.state.status,
18526 if e2e_ok && gate_ok {
18527 RunStatus::Ready
18528 } else {
18529 RunStatus::Blocked
18530 }
18531 );
18532 if !e2e_ok || !gate_ok {
18533 let cause = runner.state.verification_blocking_cause().unwrap();
18534 assert!(cause.contains(&head));
18535 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18536 runner.state.event("land", "rebased successfully");
18538 let notice = crate::notices::run_ended(&runner.state).unwrap();
18539 assert!(!format!("{notice:?}").contains("rebased successfully"));
18540 assert!(runner.state.merge.is_none());
18541 }
18542 let before = std::fs::read_to_string(&phases).unwrap();
18543 runner.state.save().unwrap();
18544 runner.state = RunState::load(&runner.state.id).unwrap();
18545 runner.review_loop().await.unwrap();
18546 runner.gate().await.unwrap();
18547 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18548 assert_eq!(runner.state.verification_history.len(), 1);
18549 let history = &runner.state.verification_history[0];
18550 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18551 assert!(history.archived_at.is_some());
18552 assert_eq!(history.fixes_spent, Some(1));
18553 }
18554 }
18555
18556 #[cfg(unix)]
18557 #[tokio::test]
18558 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18559 let (tmp, mut runner, head) = resumed_gate_fixture().await;
18560 let marker = tmp.path().join("gate-ran");
18561 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18562 runner.review_loop().await.unwrap();
18563 runner.gate().await.unwrap();
18564 assert_eq!(runner.state.status, RunStatus::Blocked);
18565 assert!(!marker.exists());
18566 assert!(runner.state.verification_history.is_empty());
18567 runner.state.gate_head = None;
18568 runner.gate().await.unwrap();
18569 assert!(marker.exists());
18570 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18571 assert_eq!(runner.state.verification_history.len(), 1);
18572 assert_eq!(runner.state.gate_fixes.len(), 1);
18573 }
18574
18575 #[cfg(unix)]
18576 #[tokio::test]
18577 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18578 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18579 runner.sync_to_base().await.unwrap();
18580 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18581 runner.state.reconcile_verification_head(&head);
18582 for outcomes in [
18584 Vec::new(),
18585 vec![CommandOutcome {
18586 command: "true".into(),
18587 code: None,
18588 output_tail: "cache unavailable".into(),
18589 duration_ms: 0,
18590 resource_blocked: true,
18591 }],
18592 ] {
18593 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18594 head: head.clone(),
18595 passed: false,
18596 outcomes,
18597 });
18598 runner.state.gate_ran = false;
18599 runner.state.gate.clear();
18600 runner.state.save().unwrap();
18601 runner.state = RunState::load(&runner.state.id).unwrap();
18602 runner.merge().await.unwrap();
18603 assert!(runner.state.merge.is_none());
18604 runner.review_loop().await.unwrap();
18605 runner.gate().await.unwrap();
18606 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18607 assert!(runner.state.gate_allows_progress());
18608 assert_eq!(runner.state.gate_fixes.len(), 1);
18609 assert!(runner.state.merge.is_none());
18610 }
18611 git::git(
18613 &runner.state.repo,
18614 &["commit", "--allow-empty", "-m", "external update"],
18615 )
18616 .await
18617 .unwrap();
18618 runner.merge().await.unwrap();
18619 assert!(runner.state.merge.is_none());
18620 assert_eq!(runner.state.status, RunStatus::Blocked);
18621 }
18622
18623 #[cfg(unix)]
18624 #[tokio::test]
18625 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18626 for waiting_on_gate in [false, true] {
18627 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18628 runner.sync_to_base().await.unwrap();
18629 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18630 runner.state.reconcile_verification_head(&head);
18631 if waiting_on_gate {
18632 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18633 head: head.clone(),
18634 passed: true,
18635 outcomes: Vec::new(),
18636 });
18637 }
18638 let cache = tmp.path().join("cache");
18639 runner.state.config.verify.e2e =
18640 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18641 runner.state.config.verify.gate = vec!["true".into()];
18642 runner.state.config.graph.timeout_resource_wait = 1;
18643 let owner =
18644 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18645 let guard =
18646 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18647 crate::cache::AcquireOutcome::Acquired(g) => g,
18648 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18649 };
18650 runner.review_loop().await.unwrap();
18651 runner.gate().await.unwrap();
18652 assert!(!runner.state.gate_ran);
18653 assert_eq!(runner.state.gate_fixes.len(), 1);
18654 assert!(runner.state.resource_block.is_some());
18655 runner.merge().await.unwrap();
18656 assert!(runner.state.merge.is_none());
18657 drop(guard);
18658 runner.state = RunState::load(&runner.state.id).unwrap();
18659 runner.state.resource_block = None;
18661 runner.review_loop().await.unwrap();
18662 runner.gate().await.unwrap();
18663 assert!(runner.state.gate_allows_progress());
18664 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18665 assert_eq!(runner.state.gate_fixes.len(), 1);
18666 }
18667 }
18668
18669 #[cfg(unix)]
18670 #[tokio::test]
18671 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18672 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18673 runner.sync_to_base().await.unwrap();
18674 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18675 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18676 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18678 runner.review_loop().await.unwrap();
18679 runner.gate().await.unwrap();
18680 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18681 assert_ne!(before, after);
18682 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18683 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18684 assert!(!runner.state.gate_allows_progress());
18685 runner.merge().await.unwrap();
18686 assert!(runner.state.merge.is_none());
18687 assert!(
18688 runner
18689 .state
18690 .verification_history
18691 .iter()
18692 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18693 );
18694
18695 runner.state.reviews[0].clean = false;
18697 runner.state.reviews[0].answered = 0;
18698 runner.state.config.verify.e2e = vec!["true".into()];
18699 runner.review_loop().await.unwrap();
18700 runner.gate().await.unwrap();
18701 runner.merge().await.unwrap();
18702 assert_eq!(runner.state.status, RunStatus::Blocked);
18703 assert!(runner.state.merge.is_none());
18704 assert!(
18705 runner
18706 .state
18707 .verification_blocking_cause()
18708 .unwrap()
18709 .contains("review blocked")
18710 );
18711 }
18712
18713 #[cfg(unix)]
18714 #[tokio::test]
18715 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18716 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18717 runner.state.gate_head = None;
18718 runner.state.config.verify.gate =
18719 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18720 runner.review_loop().await.unwrap();
18721 runner.gate().await.unwrap();
18722 assert_ne!(
18723 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18724 old
18725 );
18726 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18727 assert!(!runner.state.gate_allows_progress());
18728 assert_eq!(runner.state.status, RunStatus::Blocked);
18729 runner.merge().await.unwrap();
18730 assert!(runner.state.merge.is_none());
18731 }
18732
18733 #[cfg(unix)]
18734 #[tokio::test]
18735 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18736 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18737 let mut old_review = review_round(true, 0, 1, 1, false, true);
18738 old_review.head = head.clone();
18739 old_review.verified_head = Some(head.clone());
18740 runner.state.reviews = vec![old_review];
18741 runner.state.gate = vec![CommandOutcome {
18742 command: "old gate".into(),
18743 code: None,
18744 output_tail: "timed out after 2400s".into(),
18745 duration_ms: 2_400_000,
18746 resource_blocked: false,
18747 }];
18748 runner.state.gate_ran = true;
18749 runner.state.gate_head = Some(head.clone());
18750 runner.state.land_approval = Some(crate::run::LandApproval {
18751 question: "old-head-approval".into(),
18752 head: head.clone(),
18753 });
18754 runner.state.authorize_base_sync(&task).unwrap();
18755 runner.sync_to_base().await.unwrap();
18756 assert!(!runner.state.base_sync_blocked());
18757 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18758 assert_ne!(head, fresh);
18759 assert_eq!(runner.state.reviews.len(), 1);
18760 assert!(!runner.state.gate_ran);
18761 let saved = &runner.state.base_sync_recovery.authorizations[0];
18762 assert_eq!(saved.reviews[0].head, head);
18763 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18764 assert!(saved.gate_ran);
18765 assert!(!saved.gate[0].ok());
18766 runner.review_loop().await.unwrap();
18767 runner.gate().await.unwrap();
18768 assert_eq!(runner.state.reviews[0].head, head);
18769 assert_eq!(
18770 runner.state.reviews[0].verified_head.as_deref(),
18771 Some(head.as_str())
18772 );
18773 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18774 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18775 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18776 assert!(runner.state.gate_ran);
18777 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18778 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18779 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18780 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18781 assert_eq!(
18782 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18783 "retained change"
18784 );
18785 }
18786
18787 #[cfg(unix)]
18788 #[tokio::test]
18789 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18790 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18791 let queue = crate::queue::Queue::open();
18792 let mut owner = queue.get(&task).unwrap();
18793 owner.release_operator("test-resume", None);
18794 queue.put(&mut owner).unwrap();
18795 let generation = owner.resumption.as_ref().unwrap().generation;
18796 runner.state.task_instruction = Some(owner.instruction.clone());
18797 runner.state.driver_exited = true;
18798 runner.state.save().unwrap();
18799 let home = crate::run::home();
18800 let _claim = queue.claim(&task).unwrap();
18801 owner.resumption.as_mut().unwrap().at =
18802 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18803 queue.put(&mut owner).unwrap();
18804 assert!(
18805 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18806 .await
18807 .is_err(),
18808 "an approval from before the stop grants nothing"
18809 );
18810 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18811 queue.put(&mut owner).unwrap();
18812 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18813 .await
18814 .unwrap();
18815 assert!(
18816 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18817 .await
18818 .is_err()
18819 );
18820 runner.state = RunState::load(&runner.state.id).unwrap();
18821 assert_eq!(
18822 runner.state.base_sync_recovery.authorizations[0].generation,
18823 Some(generation)
18824 );
18825 runner.sync_to_base().await.unwrap();
18826 let repo = runner.state.repo.clone();
18827 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18828 git::git(&repo, &["checkout", "main"]).await.unwrap();
18829 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18830 git::git(&repo, &["add", "."]).await.unwrap();
18831 git::git(&repo, &["commit", "-m", "move base again"])
18832 .await
18833 .unwrap();
18834 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18835 git::git(&repo, &["checkout", "fix/retained"])
18836 .await
18837 .unwrap();
18838 runner.sync_to_base().await.unwrap();
18839 assert!(!runner.state.base_sync_blocked());
18840 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18841 assert_eq!(runner.state.base_episode_used(), 2);
18842 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18843 assert_ne!(fresh, head);
18844 assert!(!runner.state.base_recovery_pending());
18845 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18847 queue.put(&mut owner).unwrap();
18848 assert!(
18849 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18850 .await
18851 .is_err()
18852 );
18853 runner.state = RunState::load(&runner.state.id).unwrap();
18854 runner.sync_to_base().await.unwrap();
18855 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18856 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18857 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18858 }
18859
18860 #[cfg(unix)]
18861 #[tokio::test]
18862 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18863 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18864 runner.state.base_sync = None;
18865 runner.state.status = RunStatus::Blocked;
18866 let mut second = runner.state.candidates[0].clone();
18867 second.label = 'B';
18868 runner.state.candidates.push(second);
18869 assert!(runner.state.votes.is_empty());
18870 assert!(runner.state.deliberation.is_empty());
18871 runner.deliberate().await.unwrap();
18872 runner.vote().await.unwrap();
18873 assert_eq!(runner.state.status, RunStatus::Blocked);
18874 assert!(runner.state.votes.is_empty());
18875 assert!(runner.state.deliberation.is_empty());
18876 }
18877
18878 fn runner_at(status: RunStatus) -> Runner {
18881 let mut state = RunState::new(
18882 PathBuf::from("/nonexistent/repo"),
18883 "main".to_owned(),
18884 "deadbeef".to_owned(),
18885 "task".to_owned(),
18886 Config::default(),
18887 );
18888 state.config.disk.min_free_bytes = 0;
18890 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18893 state.id = format!(
18894 "20000101-000000-{:04x}",
18895 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18896 );
18897 state.status = status;
18898 Runner {
18899 state,
18900 roles: ResolvedRoles {
18901 implementers: Vec::new(),
18902 judges: Vec::new(),
18903 reviewers: Vec::new(),
18904 fixer: None,
18905 conductor: conductor(),
18906 implementer_roster: Vec::new(),
18907 judge_roster: Vec::new(),
18908 reviewer_roster: Vec::new(),
18909 },
18910 sem: Arc::new(Semaphore::new(1)),
18911 pause: Pause::new(),
18912 interrupt: Pause::new(),
18913 }
18914 }
18915
18916 #[tokio::test]
18917 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18918 crate::run::pin_test_home();
18919 let temp = tempfile::tempdir().unwrap();
18920 let root = temp.path().canonicalize().unwrap();
18921 let repo = root.join("repo");
18922 std::fs::create_dir(&repo).unwrap();
18923 init_repo(&repo);
18924 git::git(&repo, &["config", "core.autocrlf", "false"])
18925 .await
18926 .unwrap();
18927 let mut runner = runner_at(RunStatus::Blocked);
18928 runner.state.repo = repo.clone();
18929 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18930 runner.state.config.agents = vec![conductor()];
18931 runner.state.config.graph.selected_implementer = Some("conductor".into());
18932 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18933 let (shell, command, setup_output) = if cfg!(windows) {
18934 (
18935 vec!["cmd".into(), "/C".into()],
18936 "echo ready>seat-ready",
18937 "ready\r\n",
18938 )
18939 } else {
18940 (
18941 vec!["sh".into(), "-c".into()],
18942 "printf ready > seat-ready",
18943 "ready",
18944 )
18945 };
18946 runner.state.config.verify.shell = Some(shell);
18947 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18948 run: Some(command.into()),
18949 ..Default::default()
18950 }];
18951 runner.state.save().unwrap();
18952 let path = runner.state.worktree_root().join("judge-1");
18953 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18954 .await
18955 .unwrap();
18956 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18957 git::git(&path, &["add", "."]).await.unwrap();
18958 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18959 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18960 .await
18961 .unwrap();
18962 git::git(&path, &["checkout", "--detach", &head])
18963 .await
18964 .unwrap();
18965 let evidence = crate::fold_evidence::snapshot(
18966 &runner.state,
18967 &crate::run::home().canonicalize().unwrap(),
18968 &head,
18969 64 * 1024 * 1024,
18970 )
18971 .await
18972 .unwrap();
18973 let mut contents = BTreeMap::new();
18974 contents.insert(
18975 repo.join("logs/HEAD"),
18976 format!("{head} {head} fixture\n").into_bytes(),
18977 );
18978 let record = serde_json::json!({
18979 "version":1, "run":runner.state.id, "source":path, "head":head,
18980 "evidence":evidence, "admin":{}, "admin_contents":contents,
18981 "history":{head.clone():evidence}, "phase":"evicted"
18982 });
18983 std::fs::write(
18984 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18985 serde_json::to_vec(&record).unwrap(),
18986 )
18987 .unwrap();
18988 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18989 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18990 .await
18991 .unwrap();
18992 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18993 assert!(
18994 !git::git_raw(&repo, &["cat-file", "-e", &head])
18995 .await
18996 .unwrap()
18997 .ok()
18998 );
18999 let mut resumed = Runner::resume(&runner.state.id).unwrap();
19000 let error = resumed.execute().await.unwrap_err();
19001 assert!(
19002 error
19003 .to_string()
19004 .contains("selected implementation is held"),
19005 "{error:#}"
19006 );
19007 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
19008 assert_eq!(
19009 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
19010 setup_output
19011 );
19012 assert_eq!(
19013 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
19014 "unrelated seat"
19015 );
19016 let error = resumed.execute().await.unwrap_err();
19017 assert!(
19018 error
19019 .to_string()
19020 .contains("selected implementation is held"),
19021 "{error:#}"
19022 );
19023 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
19024 git::git(&path, &["add", "."]).await.unwrap();
19025 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
19026 let next = git::git(
19027 &path,
19028 &[
19029 "commit-tree",
19030 &next_tree,
19031 "-p",
19032 &head,
19033 "-m",
19034 "next review checkpoint",
19035 ],
19036 )
19037 .await
19038 .unwrap();
19039 git::git(&path, &["checkout", "--detach", &next])
19040 .await
19041 .unwrap();
19042 resumed.state.reviews.push(
19043 serde_json::from_value(serde_json::json!({
19044 "round":1, "head":next, "reviews":[]
19045 }))
19046 .unwrap(),
19047 );
19048 resumed.state.save().unwrap();
19049 let error = resumed.execute().await.unwrap_err();
19050 assert!(
19051 error
19052 .to_string()
19053 .contains("selected implementation is held"),
19054 "{error:#}"
19055 );
19056 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
19057 }
19058
19059 #[tokio::test]
19060 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
19061 crate::run::pin_test_home();
19062 let mut runner = runner_at(RunStatus::Gating);
19063 runner.state.save().unwrap();
19064 let id = runner.state.id.clone();
19065 let passes = runner.state.passes.len();
19066 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19067 let mut claiming = Box::pin(runner.claim_driver());
19068 std::future::poll_fn(|cx| {
19069 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
19070 std::task::Poll::Ready(())
19071 })
19072 .await;
19073 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
19074 drop(owner);
19075 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
19076 .await
19077 .unwrap()
19078 .unwrap();
19079 assert_eq!(execution.runner.state.passes.len(), passes + 1);
19080 assert!(execution.runner.state.resource_block.is_none());
19081 assert_eq!(
19082 execution.runner.state.liveness(false),
19083 crate::run::Liveness::Live
19084 );
19085 drop(execution);
19086 assert!(RunState::load(&id).unwrap().driver_exited);
19087 }
19088
19089 #[tokio::test]
19090 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
19091 crate::run::pin_test_home();
19092 let mut runner = runner_at(RunStatus::Gating);
19093 runner.state.save().unwrap();
19094 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
19097 assert!(runner.claim_driver().await.is_err());
19098 assert!(runner.state.driver_exited);
19099 assert_eq!(runner.state.driver_pid, None);
19100 let disk = RunState::load(&runner.state.id).unwrap();
19101 assert!(disk.driver_exited);
19102 assert_eq!(disk.driver_pid, None);
19103 assert_eq!(disk.status, RunStatus::Gating);
19104 }
19105
19106 #[tokio::test]
19107 async fn an_early_graph_error_releases_its_driver() {
19108 crate::run::pin_test_home();
19109 let mut runner = runner_at(RunStatus::Blocked);
19110 runner.state.config.graph.selected_implementer = Some("selected".into());
19111 runner.state.save().unwrap();
19112 assert!(runner.execute().await.is_err());
19113 let disk = RunState::load(&runner.state.id).unwrap();
19114 assert!(disk.driver_exited);
19115 assert_eq!(disk.driver_pid, None);
19116 assert_eq!(disk.status, RunStatus::Blocked);
19117 }
19118
19119 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
19120 crate::run::MergeRepair {
19121 question: "repair-question".into(),
19122 head: "old-head".into(),
19123 pr: "https://github.com/o/r/pull/42".into(),
19124 reviews: vec![review_round(false, 1, 1, 1, false, true)],
19125 gate: vec![outcome(Some(0), false)],
19126 gate_head: Some("old-head".into()),
19127 gate_fixes: Vec::new(),
19128 pre_gate: Vec::new(),
19129 phase,
19130 failure: None,
19131 }
19132 }
19133
19134 #[tokio::test]
19135 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
19136 crate::run::pin_test_home();
19137 let mut runner = runner_at(RunStatus::Landing);
19138 runner.state.merge_repairs.push(merge_repair_request(
19139 crate::run::MergeRepairPhase::Requested,
19140 ));
19141 runner.state.save().unwrap();
19142 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
19143 assert!(
19144 runner
19145 .execute()
19146 .await
19147 .unwrap_err()
19148 .to_string()
19149 .contains("already running")
19150 );
19151 assert_eq!(
19152 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19153 crate::run::MergeRepairPhase::Requested
19154 );
19155 drop(claim);
19156 let mut disk = runner.state.clone();
19157 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
19158 disk.save().unwrap();
19159 assert!(
19160 runner
19161 .execute()
19162 .await
19163 .unwrap_err()
19164 .to_string()
19165 .contains("decision changed")
19166 );
19167 assert_eq!(
19168 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19169 crate::run::MergeRepairPhase::Complete
19170 );
19171 }
19172
19173 #[test]
19174 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
19175 crate::run::pin_test_home();
19176 let mut runner = runner_at(RunStatus::Landing);
19177 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
19178 runner.state.reviews = request.reviews.clone();
19179 runner.state.gate = request.gate.clone();
19180 runner.state.gate_ran = true;
19181 runner.state.land_approval = Some(crate::run::LandApproval {
19182 question: "old".into(),
19183 head: "old-head".into(),
19184 });
19185 runner.state.merge_repairs.push(request);
19186 runner.begin_merge_repair_review(0).unwrap();
19187 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
19188 assert!(!runner.state.gate_status().ok());
19189 assert!(runner.state.land_approval.is_none());
19190 let disk = RunState::load(&runner.state.id).unwrap();
19191 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
19192 assert!(disk.merge_repairs[0].gate[0].ok());
19193 assert_eq!(
19194 disk.merge_repairs[0].phase,
19195 crate::run::MergeRepairPhase::Reviewing
19196 );
19197 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
19198 assert!(disk.reviews.is_empty());
19199 }
19200
19201 #[tokio::test]
19202 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
19203 crate::run::pin_test_home();
19204 for phase in [
19205 crate::run::MergeRepairPhase::Fixing,
19206 crate::run::MergeRepairPhase::Requested,
19207 ] {
19208 let mut runner = runner_at(RunStatus::Landing);
19209 runner.state.config.graph.land_rounds = 0;
19210 runner.state.merge_repairs.push(merge_repair_request(phase));
19211 runner.state.save().unwrap();
19212 assert!(!runner.repair_merge_review().await.unwrap());
19213 assert_eq!(runner.state.status, RunStatus::Blocked);
19214 assert_eq!(
19215 runner.state.merge_repairs[0].phase,
19216 crate::run::MergeRepairPhase::Failed
19217 );
19218 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
19219 assert!(
19220 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
19221 "interrupted"
19222 } else {
19223 "budget exhausted"
19224 })
19225 );
19226 assert!(runner.state.merge.is_none());
19227 }
19228 }
19229
19230 #[test]
19234 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
19235 crate::run::pin_test_home();
19236 let mut runner = runner_at(RunStatus::Implementing);
19237 let interrupt = Pause::new();
19238 runner.watch_interrupt(interrupt.clone());
19239
19240 interrupt.park_because("task a1b2 asked to run first");
19241
19242 assert!(runner.park_here().expect("park_here"));
19243 assert!(runner.state.parked);
19244 let last = runner.state.events.last().expect("a park event");
19245 assert_eq!(last.node, "park");
19246 assert!(
19247 last.message.contains("task a1b2 asked to run first"),
19248 "expected the interrupt reason in {:?}",
19249 last.message
19250 );
19251 }
19252
19253 #[test]
19256 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
19257 crate::run::pin_test_home();
19258 let mut runner = runner_at(RunStatus::Implementing);
19259 runner.state.begin_pass();
19260 let shutdown = Pause::new();
19261 runner.on_pause(shutdown.clone());
19262 let upgrade = ParkCause::Upgrade {
19263 from: "0.121.0".to_owned(),
19264 to: Some("0.121.1".to_owned()),
19265 };
19266 shutdown.park_for(upgrade.clone());
19267 shutdown.park_for(ParkCause::Operator);
19268
19269 assert!(runner.park_here().expect("park_here"));
19270 assert_eq!(
19271 runner.state.passes[0].end,
19272 Some(crate::run::PassEnd::Parked { cause: upgrade })
19273 );
19274 let last = runner.state.events.last().expect("a park event");
19275 assert!(last.message.contains("0.121.0"), "{}", last.message);
19276 }
19277
19278 #[tokio::test]
19279 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
19280 crate::run::pin_test_home();
19281 let mut runner = runner_at(RunStatus::Reviewing);
19282 runner.state.save().unwrap();
19283 let mut execution = runner.claim_driver().await.expect("first execution");
19284 assert_eq!(execution.runner.state.passes.len(), 1);
19285 execution
19286 .driver
19287 .finish(&mut execution.runner.state)
19288 .await
19289 .unwrap();
19290 }
19291
19292 #[tokio::test]
19293 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19294 crate::run::pin_test_home();
19295 let mut runner = runner_at(RunStatus::Reviewing);
19296 runner.state.config.agents = vec![conductor()];
19297 runner.state.config.graph.timeout_resource_wait = 0;
19298 runner.state.save().unwrap();
19299 let id = runner.state.id.clone();
19300 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19301 assert!(runner.claim_driver().await.is_err());
19302 drop(owner);
19303 drop(runner);
19304 let mut resumed = Runner::resume(&id).unwrap();
19305 assert_eq!(resumed.state.driver_claimed, Some(false));
19306 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19307 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19308 let mut execution = resumed
19309 .claim_driver()
19310 .await
19311 .expect("retry initial claim after reload");
19312 assert_eq!(execution.runner.state.driver_claimed, Some(true));
19313 assert_eq!(execution.runner.state.passes.len(), 1);
19314 execution
19315 .driver
19316 .finish(&mut execution.runner.state)
19317 .await
19318 .unwrap();
19319 }
19320
19321 #[tokio::test]
19322 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19323 crate::run::pin_test_home();
19324 let mut runner = runner_at(RunStatus::Reviewing);
19325 runner.state.config.agents = vec![conductor()];
19326 runner.state.save().unwrap();
19327 let id = runner.state.id.clone();
19328 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19329 let mut resumed = Runner::resume(&id).unwrap();
19330 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19331 assert!(resumed.claim_driver().await.is_err());
19332 assert!(RunState::load(&id).unwrap().passes.is_empty());
19333 drop(lease);
19334 let mut execution = resumed.claim_driver().await.unwrap();
19335 execution
19336 .driver
19337 .finish(&mut execution.runner.state)
19338 .await
19339 .unwrap();
19340 }
19341
19342 #[tokio::test]
19343 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19344 crate::run::pin_test_home();
19345 let mut runner = runner_at(RunStatus::Reviewing);
19346 let state = &mut runner.state;
19347 state.config.agents = vec![conductor()];
19348 state.driver_claimed = None;
19349 state.driver_pid = Some(std::process::id());
19350 state.driver_started_at = None;
19351 state.begin_pass();
19352 state.save().unwrap();
19353 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19354 std::fs::write(
19355 state.dir().join("driver.json"),
19356 serde_json::to_vec(&serde_json::json!({
19357 "pid": state.driver_pid, "started_at": null,
19358 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19359 }))
19360 .unwrap(),
19361 )
19362 .unwrap();
19363 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19364 drop(lease);
19365 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19366 let mut resumed = Runner::resume(&state.id).unwrap();
19367 assert!(!resumed.state.driver_exited);
19368 let mut execution = resumed
19369 .claim_driver()
19370 .await
19371 .expect("lease proves previous execution ended");
19372 assert_eq!(execution.runner.state.passes.len(), 2);
19373 execution
19374 .driver
19375 .finish(&mut execution.runner.state)
19376 .await
19377 .unwrap();
19378 }
19379
19380 #[tokio::test]
19381 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19382 crate::run::pin_test_home();
19383 let mut runner = runner_at(RunStatus::Reviewing);
19384 let mut json = serde_json::to_value(&runner.state).unwrap();
19385 json.as_object_mut().unwrap().remove("driver_claimed");
19386 runner.state = serde_json::from_value(json).unwrap();
19387 assert_eq!(runner.state.driver_claimed, None);
19388 runner.state.save().unwrap();
19389 let mut execution = runner
19390 .claim_driver()
19391 .await
19392 .expect("the execution lease, not a missing marker, excludes a second driver");
19393 assert_eq!(execution.runner.state.passes.len(), 1);
19394 execution
19395 .driver
19396 .finish(&mut execution.runner.state)
19397 .await
19398 .unwrap();
19399 }
19400
19401 #[tokio::test]
19402 async fn driver_admission_refuses_a_proven_live_driver() {
19403 crate::run::pin_test_home();
19404 let mut runner = runner_at(RunStatus::Reviewing);
19405 let pid = std::process::id();
19406 runner.state.driver_pid = Some(pid);
19407 runner.state.driver_started_at = crate::proc::process_started_at(pid);
19408 assert!(runner.state.driver_started_at.is_some());
19409 runner.state.save().unwrap();
19410 let error = runner
19411 .claim_driver()
19412 .await
19413 .err()
19414 .expect("a live driver must refuse");
19415 assert!(error.to_string().contains("live driver"), "{error:#}");
19416 let saved = RunState::load(&runner.state.id).unwrap();
19417 assert_eq!(saved.driver_pid, Some(pid));
19418 assert_eq!(saved.status, RunStatus::Reviewing);
19419 assert!(saved.passes.is_empty());
19420 }
19421
19422 #[test]
19423 fn parking_resource_halt_persists_the_pass_once() {
19424 crate::run::pin_test_home();
19425 let mut runner = runner_at(RunStatus::Reviewing);
19426 runner.state.begin_pass();
19427 runner.state.resource_block = Some(crate::run::ResourceBlock {
19428 node: "fix".into(),
19429 kind: "shared build cache".into(),
19430 detail: "cancelled".into(),
19431 at: Timestamp::now(),
19432 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19433 });
19434 let cause = ParkCause::Upgrade {
19435 from: "old".into(),
19436 to: Some("new".into()),
19437 };
19438 runner.pause.park_for(cause.clone());
19439 runner.stop_for_resource().unwrap();
19442 assert!(runner.resource_halt().unwrap());
19443 assert!(runner.park_here().unwrap());
19444 let saved = RunState::load(&runner.state.id).unwrap();
19445 assert!(saved.parked);
19446 assert_eq!(saved.status, RunStatus::Reviewing);
19447 assert!(saved.passes[0].ended_at.is_some());
19448 assert_eq!(
19449 saved.passes[0].end,
19450 Some(crate::run::PassEnd::Parked {
19451 cause: cause.clone()
19452 })
19453 );
19454 assert_eq!(
19455 saved.resource_block.unwrap().cause,
19456 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19457 );
19458 assert_eq!(
19459 saved
19460 .events
19461 .iter()
19462 .filter(|event| event.node == "park")
19463 .count(),
19464 1
19465 );
19466 }
19467
19468 #[test]
19476 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19477 crate::run::pin_test_home();
19478 let mut runner = runner_at(RunStatus::Implementing);
19479 let shutdown = Pause::new();
19480 runner.on_pause(shutdown.clone());
19481 let interrupt = Pause::new();
19482 runner.watch_interrupt(interrupt.clone());
19483
19484 assert!(!runner.park_here().expect("park_here"));
19486 assert!(!runner.state.parked);
19487
19488 interrupt.park_because("test");
19490 assert!(!shutdown.parked());
19491 assert!(runner.park_here().expect("park_here"));
19492 }
19493
19494 #[tokio::test]
19508 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19509 crate::run::pin_test_home();
19510 let mut runner = runner_at(RunStatus::Implementing);
19511 let interrupt = Pause::new();
19512 runner.watch_interrupt(interrupt.clone());
19513
19514 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19515 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19516
19517 let node = async move {
19521 started_tx.send(()).expect("send started");
19522 finish_rx.await.expect("recv finish");
19523 "node finished"
19524 };
19525
19526 let interrupter = async move {
19527 started_rx.await.expect("recv started");
19528 interrupt.park_because("higher-priority task waiting");
19530 tokio::task::yield_now().await;
19534 finish_tx.send(()).expect("send finish");
19535 };
19536
19537 let (node_result, ()) = tokio::join!(node, interrupter);
19538 assert_eq!(
19539 node_result, "node finished",
19540 "the in-flight call ran to completion"
19541 );
19542
19543 assert!(runner.park_here().expect("park_here"));
19546 assert!(runner.state.parked);
19547 }
19548
19549 #[test]
19555 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19556 crate::run::pin_test_home();
19557 let mut runner = runner_at(RunStatus::Judging);
19558 runner.state.config.agents = vec![conductor()];
19562 runner.state.candidates = vec![Candidate {
19563 index: 0,
19564 label: 'A',
19565 agent: "alpha".to_owned(),
19566 branch: "magi/x/A".to_owned(),
19567 worktree: PathBuf::from("/nonexistent/worktree"),
19568 summary: "did the thing".to_owned(),
19569 stat: "1 file changed".to_owned(),
19570 files: 1,
19571 commits: 1,
19572 empty: false,
19573 failed: None,
19574 verified_noop: None,
19575 duration_ms: 1234,
19576 folded: false,
19577 }];
19578 let run_id = runner.state.id.clone();
19579
19580 let interrupt = Pause::new();
19581 runner.watch_interrupt(interrupt.clone());
19582 interrupt.park_because("task c3d4 asked to run first");
19583 assert!(runner.park_here().expect("park_here"));
19584
19585 let resumed = Runner::resume(&run_id).expect("resume");
19586 assert_eq!(resumed.state.candidates.len(), 1);
19587 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19588 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19589 assert_eq!(resumed.state.status, runner.state.status);
19590 assert!(
19591 resumed.state.parked,
19592 "still parked until `execute` actually walks the graph again"
19593 );
19594 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19595 }
19596
19597 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19599 let mut q = ask::Question::new(
19600 run.to_owned(),
19601 "implement".to_owned(),
19602 "impl-A".to_owned(),
19603 "Which storage backend should the cache use?".to_owned(),
19604 String::new(),
19605 vec!["SQLite".to_owned(), "Redis".to_owned()],
19606 );
19607 store.put(&mut q).unwrap();
19608 q
19609 }
19610
19611 #[test]
19612 fn a_failed_runs_open_question_is_abandoned() {
19613 ask_test_home();
19614 let store = ask::Questions::open();
19615 let mut runner = runner_at(RunStatus::Failed);
19616 let run = runner.state.id.clone();
19617 let q = ask_open_question(&store, &run);
19618
19619 runner.settle_questions();
19620
19621 let back = store.get(&q.id).unwrap();
19622 assert!(
19623 !back.status.open(),
19624 "the seat that asked died with the run; nobody is left to read an answer"
19625 );
19626 assert!(
19627 back.detail.contains(&run) && back.detail.contains("failed"),
19628 "the reason names what the run became, not just that it is gone: {}",
19629 back.detail
19630 );
19631 }
19632
19633 #[test]
19634 fn a_merged_runs_open_question_is_abandoned_too() {
19635 ask_test_home();
19636 let store = ask::Questions::open();
19637 for status in [RunStatus::Merged, RunStatus::Ready] {
19640 let mut runner = runner_at(status);
19641 let run = runner.state.id.clone();
19642 let q = ask_open_question(&store, &run);
19643
19644 runner.settle_questions();
19645
19646 let back = store.get(&q.id).unwrap();
19647 assert!(
19648 !back.status.open(),
19649 "{status:?} run's question must not outlive the run"
19650 );
19651 }
19652 }
19653
19654 #[test]
19655 fn a_still_resumable_runs_open_question_is_left_alone() {
19656 ask_test_home();
19657 let store = ask::Questions::open();
19658 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19664 let mut runner = runner_at(status);
19665 let run = runner.state.id.clone();
19666 let q = ask_open_question(&store, &run);
19667
19668 runner.settle_questions();
19669
19670 let back = store.get(&q.id).unwrap();
19671 assert!(
19672 back.status.open(),
19673 "{status:?} is still alive; the question must still be waiting"
19674 );
19675 }
19676 }
19677
19678 #[test]
19679 fn settle_questions_never_touches_an_already_answered_question() {
19680 ask_test_home();
19681 let store = ask::Questions::open();
19682 let mut runner = runner_at(RunStatus::Failed);
19683 let run = runner.state.id.clone();
19684 let mut q = ask_open_question(&store, &run);
19685 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19686 .unwrap();
19687 store.put(&mut q).unwrap();
19688
19689 runner.settle_questions();
19694 runner.settle_questions();
19695
19696 let back = store.get(&q.id).unwrap();
19697 assert_eq!(
19698 back.status,
19699 ask::QuestionStatus::Answered,
19700 "a real answer is a decision on record, never overwritten by a sweep"
19701 );
19702 }
19703
19704 #[tokio::test]
19715 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19716 crate::run::pin_test_home();
19717 let tmp = tempfile::tempdir().expect("tempdir");
19718 let repo = tmp.path().join("repo");
19719 std::fs::create_dir_all(&repo).unwrap();
19720 init_repo(&repo);
19721
19722 let mut config = Config::default();
19723 config.graph.worktree_root = Some(tmp.path().join("wt"));
19724
19725 let mut state = RunState::new(
19726 repo.clone(),
19727 "main".to_owned(),
19728 "deadbeef".to_owned(),
19729 "task".to_owned(),
19730 config,
19731 );
19732 let root = state.worktree_root();
19733 let wt_a = root.join("cand-A");
19734 let wt_b = root.join("cand-B");
19735 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19736 .await
19737 .expect("worktree A");
19738 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19739 .await
19740 .expect("worktree B");
19741
19742 state.candidates = vec![
19743 Candidate {
19744 index: 0,
19745 label: 'A',
19746 agent: "alpha".to_owned(),
19747 branch: "magi/x/A".to_owned(),
19748 worktree: wt_a.clone(),
19749 summary: String::new(),
19750 stat: String::new(),
19751 files: 0,
19752 commits: 0,
19753 empty: false,
19754 failed: None,
19755 verified_noop: None,
19756 duration_ms: 0,
19757 folded: false,
19758 },
19759 Candidate {
19760 index: 1,
19761 label: 'B',
19762 agent: "beta".to_owned(),
19763 branch: "magi/x/B".to_owned(),
19764 worktree: wt_b.clone(),
19765 summary: String::new(),
19766 stat: String::new(),
19767 files: 0,
19768 commits: 0,
19769 empty: false,
19770 failed: None,
19771 verified_noop: None,
19772 duration_ms: 0,
19773 folded: false,
19774 },
19775 ];
19776 state.tally = Some(Tally {
19777 first_choice: BTreeMap::from([('A', 1)]),
19778 borda: BTreeMap::new(),
19779 winner: 'A',
19780 rankings: 1,
19781 unanimous_initial: true,
19782 deliberated: false,
19783 changed_votes: 0,
19784 unanimous_final: true,
19785 tie_break: None,
19786 judges: 1,
19787 present: 1,
19788 quorum: 1,
19789 met_quorum: true,
19790 uncontested: None,
19791 });
19792 state.status = RunStatus::Ready;
19793
19794 fold_run(&mut state, false, &crate::run::home())
19795 .await
19796 .expect("fold_run");
19797
19798 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19799 assert!(
19800 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19801 "the unmerged winner's branch survives"
19802 );
19803 assert!(
19804 !state.candidates[0].folded,
19805 "the winner is not marked folded"
19806 );
19807
19808 assert!(!wt_b.exists(), "the loser's worktree is removed");
19809 assert!(
19810 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19811 "the loser's branch is removed"
19812 );
19813 assert!(state.candidates[1].folded, "the loser is marked folded");
19814 }
19815
19816 #[tokio::test]
19817 #[cfg(unix)]
19818 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19819 use std::os::fd::AsRawFd as _;
19820 use std::time::Duration;
19821
19822 let tmp = tempfile::tempdir().unwrap();
19823 let repo = tmp.path().join("repo");
19824 std::fs::create_dir_all(&repo).unwrap();
19825 init_repo(&repo);
19826 let common = repo.join(".git");
19827 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19828 let home = tmp.path().join("home");
19829 let mut config = Config::default();
19830 config.coordination = crate::config::test_coordination();
19831 config.graph.worktree_root = Some(tmp.path().join("wt"));
19832 config.graph.timeout_resource_wait = 1;
19833 let mut state = RunState::new(
19834 repo.clone(),
19835 "main".into(),
19836 "deadbeef".into(),
19837 "task".into(),
19838 config,
19839 );
19840 let root = state.worktree_root();
19841 let candidate = root.join("cand-A");
19842 let reviewer = root.join("review-1");
19843 let branch = state.branch_for('A');
19844 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19845 .await
19846 .unwrap();
19847 git::worktree_add_detached(&repo, &reviewer, "main")
19848 .await
19849 .unwrap();
19850 git::acquire_worktree_config(&repo).await.unwrap();
19851 state.enabled_worktree_config = true;
19852 state.status = RunStatus::Ready;
19853 state.candidates.push(Candidate {
19854 index: 0,
19855 label: 'A',
19856 agent: "implement".into(),
19857 branch: branch.clone(),
19858 worktree: candidate.clone(),
19859 summary: String::new(),
19860 stat: String::new(),
19861 files: 0,
19862 commits: 0,
19863 empty: false,
19864 failed: None,
19865 verified_noop: None,
19866 duration_ms: 0,
19867 folded: false,
19868 });
19869 state.save_under(&home).unwrap();
19870 let config_before = std::fs::read(common.join("config")).unwrap();
19871 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19872 .await
19873 .unwrap();
19874 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19877 assert_eq!(
19878 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19879 0
19880 );
19881 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19882 assert!(
19883 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19884 .await
19885 .is_err()
19886 );
19887 assert!(candidate.exists());
19888 assert!(reviewer.exists());
19889 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19890 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19891 assert_eq!(
19892 git::git(&repo, &["worktree", "list", "--porcelain"])
19893 .await
19894 .unwrap(),
19895 worktrees_before
19896 );
19897 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19898 .await
19899 .unwrap()
19900 .unwrap_err();
19901 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19902 drop(fold);
19903 assert!(candidate.exists() && reviewer.exists());
19904 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19905 assert_eq!(
19906 state.resource_block.as_ref().unwrap().kind,
19907 "repository coordination"
19908 );
19909 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19910 drop(external);
19911 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19912 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19913 .await
19914 .unwrap()
19915 .unwrap();
19916 drop(fold);
19917 assert!(removed.contains(&branch));
19918 assert!(!root.exists());
19919 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19920 assert!(state.candidates[0].folded);
19921 assert!(!state.enabled_worktree_config);
19922 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19923 let enabled = git::git_raw(
19924 &repo,
19925 &["config", "--local", "--get", "extensions.worktreeConfig"],
19926 )
19927 .await
19928 .unwrap();
19929 assert!(!enabled.ok(), "the last config reference was released");
19930 }
19931
19932 #[tokio::test]
19935 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19936 let tmp = tempfile::tempdir().expect("tempdir");
19937 let repo = tmp.path().join("repo");
19938 std::fs::create_dir_all(&repo).unwrap();
19939 init_repo(&repo);
19940 let home = tmp.path().join("home");
19941
19942 let mut config = Config::default();
19943 config.graph.worktree_root = Some(tmp.path().join("wt"));
19944 let mut state = RunState::new(
19945 repo.clone(),
19946 "main".to_owned(),
19947 "deadbeef".to_owned(),
19948 "task".to_owned(),
19949 config,
19950 );
19951 git::git(&repo, &["branch", "magi/x/A", "main"])
19953 .await
19954 .expect("branch");
19955 state.candidates = vec![Candidate {
19956 index: 0,
19957 label: 'A',
19958 agent: "alpha".to_owned(),
19959 branch: "magi/x/A".to_owned(),
19960 worktree: state.worktree_root().join("cand-A"),
19961 summary: String::new(),
19962 stat: String::new(),
19963 files: 0,
19964 commits: 0,
19965 empty: false,
19966 failed: None,
19967 verified_noop: None,
19968 duration_ms: 0,
19969 folded: true,
19970 }];
19971 state.released_to = Some("20260901-000000-new1".to_owned());
19972 state.released_branches = vec!["magi/x/A".to_owned()];
19973
19974 fold_run(&mut state, true, &home).await.expect("fold_run");
19975 state.status = RunStatus::Merged;
19976 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19977 state.merge = Some(crate::run::MergeOutcome {
19978 mode: MergeMode::Pr,
19979 ok: true,
19980 detail: "confirmed merged".into(),
19981 empty: false,
19982 });
19983 fold_run(&mut state, true, &home)
19984 .await
19985 .expect("merged fold");
19986
19987 assert!(
19988 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19989 "the handed-over branch survives a fold"
19990 );
19991 }
19992
19993 #[tokio::test]
19997 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19998 let tmp = tempfile::tempdir().expect("tempdir");
19999 let repo = tmp.path().join("repo");
20000 std::fs::create_dir_all(&repo).unwrap();
20001 init_repo(&repo);
20002 let run = |args: &[&str]| {
20003 let out = std::process::Command::new("git")
20004 .quiet()
20005 .args(args)
20006 .current_dir(&repo)
20007 .output()
20008 .expect("spawn git");
20009 assert!(out.status.success(), "git {args:?}");
20010 };
20011 run(&["branch", "magi/x/A"]);
20012 run(&["checkout", "-q", "-b", "magi/x/B"]);
20013 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
20014 run(&["add", "-A"]);
20015 run(&["commit", "-q", "-m", "work"]);
20016 run(&["checkout", "-q", "main"]);
20017
20018 let probe = RunState::new(
20022 repo.clone(),
20023 "main".to_owned(),
20024 "deadbeef".to_owned(),
20025 "task".to_owned(),
20026 Config::default(),
20027 );
20028 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
20029 let bare = tmp.path().join("origin.git");
20030 let out = std::process::Command::new("git")
20031 .quiet()
20032 .args(["init", "-q", "--bare"])
20033 .arg(&bare)
20034 .output()
20035 .expect("spawn git");
20036 assert!(out.status.success(), "git init --bare");
20037 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
20038 run(&["push", "-q", "origin", "main"]);
20039
20040 let mut state = RunState::new(
20041 repo.clone(),
20042 "main".to_owned(),
20043 "deadbeef".to_owned(),
20044 "task".to_owned(),
20045 Config::default(),
20046 );
20047 state.seeds = vec![refs::Seed {
20048 token: "magi/27b2/A".to_owned(),
20049 kind: refs::SeedKind::Unresolved,
20050 sha: String::new(),
20051 branch: true,
20052 detail: "no branch or commit named magi/27b2/A".to_owned(),
20053 }];
20054
20055 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
20056 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
20057 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
20058 let detail = empty_candidate_detail(&state, "main");
20059 assert!(detail.starts_with("empty candidate"), "{detail}");
20060 assert!(!detail.contains("magi/27b2/A"), "{detail}");
20061 assert!(detail.contains("--start-from"), "{detail}");
20062 }
20063
20064 #[tokio::test]
20073 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
20074 let tmp = tempfile::tempdir().expect("tempdir");
20075 let repo = tmp.path().join("repo");
20076 std::fs::create_dir_all(&repo).unwrap();
20077 init_repo(&repo);
20078
20079 let mut config = Config::default();
20080 config.merge.mode = MergeMode::Local;
20081
20082 let mut state = RunState::new(
20083 repo.clone(),
20084 "main".to_owned(),
20085 "deadbeef".to_owned(),
20086 "task".to_owned(),
20087 config,
20088 );
20089 state.candidates = vec![Candidate {
20090 index: 0,
20091 label: 'A',
20092 agent: "alpha".to_owned(),
20093 branch: "does-not-exist".to_owned(),
20094 worktree: repo.clone(),
20095 summary: String::new(),
20096 stat: String::new(),
20097 files: 0,
20098 commits: 0,
20099 empty: false,
20100 failed: None,
20101 verified_noop: None,
20102 duration_ms: 0,
20103 folded: false,
20104 }];
20105 state.tally = Some(Tally {
20106 first_choice: BTreeMap::from([('A', 1)]),
20107 borda: BTreeMap::new(),
20108 winner: 'A',
20109 rankings: 1,
20110 unanimous_initial: true,
20111 deliberated: false,
20112 changed_votes: 0,
20113 unanimous_final: true,
20114 tie_break: None,
20115 judges: 0,
20116 present: 0,
20117 quorum: 0,
20118 met_quorum: true,
20119 uncontested: Some("only candidate A produced a change".to_owned()),
20120 });
20121 state.reviews = vec![ReviewRound {
20122 round: 1,
20123 head: "deadbeef".to_owned(),
20124 verified_head: None,
20125 verified_at: None,
20126 reviews: Vec::new(),
20127 e2e: Vec::new(),
20128 fix: None,
20129 blocking: 0,
20130 answered: 0,
20131 expected: 0,
20132 clean: true,
20133 verify_retried: false,
20134 e2e_deferred: false,
20135 e2e_defer_reason: None,
20136 progressed: false,
20137 vote_split: false,
20138 reconsideration: Vec::new(),
20139 verdict: None,
20140 }];
20141 state.gate = vec![CommandOutcome {
20142 command: "test".to_owned(),
20143 code: Some(0),
20144 output_tail: String::new(),
20145 duration_ms: 0,
20146 resource_blocked: false,
20147 }];
20148 state.gate_ran = true;
20149 state.status = RunStatus::Ready;
20154 state.merge = Some(MergeOutcome {
20155 mode: MergeMode::Local,
20156 ok: false,
20157 detail: "already concluded".to_owned(),
20158 empty: false,
20159 });
20160
20161 let mut runner = Runner {
20162 state,
20163 roles: ResolvedRoles {
20164 implementers: Vec::new(),
20165 judges: Vec::new(),
20166 reviewers: Vec::new(),
20167 fixer: None,
20168 conductor: conductor(),
20169 implementer_roster: Vec::new(),
20170 judge_roster: Vec::new(),
20171 reviewer_roster: Vec::new(),
20172 },
20173 sem: Arc::new(Semaphore::new(1)),
20174 pause: Pause::new(),
20175 interrupt: Pause::new(),
20176 };
20177
20178 runner.merge().await.expect("merge");
20179
20180 assert_eq!(
20181 runner.state.status,
20182 RunStatus::Ready,
20183 "a concluded run's status must not change on reentry"
20184 );
20185 assert_eq!(
20186 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20187 Some("already concluded"),
20188 "merge must not run again once the node already recorded an outcome"
20189 );
20190 }
20191
20192 #[tokio::test]
20201 async fn merge_refuses_a_gate_that_has_not_actually_run() {
20202 let tmp = tempfile::tempdir().expect("tempdir");
20203 let repo = tmp.path().join("repo");
20204 std::fs::create_dir_all(&repo).unwrap();
20205 init_repo(&repo);
20206
20207 let mut config = Config::default();
20208 config.merge.mode = MergeMode::Local;
20209
20210 let mut state = RunState::new(
20211 repo.clone(),
20212 "main".to_owned(),
20213 "deadbeef".to_owned(),
20214 "task".to_owned(),
20215 config,
20216 );
20217 state.candidates = vec![Candidate {
20218 index: 0,
20219 label: 'A',
20220 agent: "alpha".to_owned(),
20221 branch: "does-not-exist".to_owned(),
20222 worktree: repo.clone(),
20223 summary: String::new(),
20224 stat: String::new(),
20225 files: 0,
20226 commits: 0,
20227 empty: false,
20228 failed: None,
20229 verified_noop: None,
20230 duration_ms: 0,
20231 folded: false,
20232 }];
20233 state.tally = Some(Tally {
20234 first_choice: BTreeMap::from([('A', 1)]),
20235 borda: BTreeMap::new(),
20236 winner: 'A',
20237 rankings: 1,
20238 unanimous_initial: true,
20239 deliberated: false,
20240 changed_votes: 0,
20241 unanimous_final: true,
20242 tie_break: None,
20243 judges: 0,
20244 present: 0,
20245 quorum: 0,
20246 met_quorum: true,
20247 uncontested: Some("only candidate A produced a change".to_owned()),
20248 });
20249 state.reviews = vec![ReviewRound {
20250 round: 1,
20251 head: "deadbeef".to_owned(),
20252 verified_head: None,
20253 verified_at: None,
20254 reviews: Vec::new(),
20255 e2e: Vec::new(),
20256 fix: None,
20257 blocking: 0,
20258 answered: 0,
20259 expected: 0,
20260 clean: true,
20261 verify_retried: false,
20262 e2e_deferred: false,
20263 e2e_defer_reason: None,
20264 progressed: false,
20265 vote_split: false,
20266 reconsideration: Vec::new(),
20267 verdict: None,
20268 }];
20269 state.gate = Vec::new();
20271 state.gate_ran = false;
20272 state.status = RunStatus::Gating;
20273
20274 let mut runner = Runner {
20275 state,
20276 roles: ResolvedRoles {
20277 implementers: Vec::new(),
20278 judges: Vec::new(),
20279 reviewers: Vec::new(),
20280 fixer: None,
20281 conductor: conductor(),
20282 implementer_roster: Vec::new(),
20283 judge_roster: Vec::new(),
20284 reviewer_roster: Vec::new(),
20285 },
20286 sem: Arc::new(Semaphore::new(1)),
20287 pause: Pause::new(),
20288 interrupt: Pause::new(),
20289 };
20290
20291 runner.merge().await.expect("merge");
20292
20293 assert!(
20294 runner.state.merge.is_none(),
20295 "an empty gate must never be read as a passing one: {:?}",
20296 runner.state.merge
20297 );
20298 }
20299
20300 #[tokio::test]
20307 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20308 let tmp = tempfile::tempdir().expect("tempdir");
20309 let repo = tmp.path().join("repo");
20310 std::fs::create_dir_all(&repo).unwrap();
20311 init_repo(&repo);
20312
20313 let config = Config::default();
20315
20316 let mut state = RunState::new(
20317 repo.clone(),
20318 "main".to_owned(),
20319 "deadbeef".to_owned(),
20320 "task".to_owned(),
20321 config,
20322 );
20323 state.candidates = vec![Candidate {
20324 index: 0,
20325 label: 'A',
20326 agent: "alpha".to_owned(),
20327 branch: "does-not-exist".to_owned(),
20328 worktree: repo.clone(),
20329 summary: String::new(),
20330 stat: String::new(),
20331 files: 0,
20332 commits: 0,
20333 empty: false,
20334 failed: None,
20335 verified_noop: None,
20336 duration_ms: 0,
20337 folded: false,
20338 }];
20339 state.tally = Some(Tally {
20340 first_choice: BTreeMap::from([('A', 1)]),
20341 borda: BTreeMap::new(),
20342 winner: 'A',
20343 rankings: 1,
20344 unanimous_initial: true,
20345 deliberated: false,
20346 changed_votes: 0,
20347 unanimous_final: true,
20348 tie_break: None,
20349 judges: 0,
20350 present: 0,
20351 quorum: 0,
20352 met_quorum: true,
20353 uncontested: Some("only candidate A produced a change".to_owned()),
20354 });
20355 state.reviews = vec![ReviewRound {
20356 round: 1,
20357 head: "deadbeef".to_owned(),
20358 verified_head: None,
20359 verified_at: None,
20360 reviews: Vec::new(),
20361 e2e: Vec::new(),
20362 fix: None,
20363 blocking: 0,
20364 answered: 0,
20365 expected: 0,
20366 clean: true,
20367 verify_retried: false,
20368 e2e_deferred: false,
20369 e2e_defer_reason: None,
20370 progressed: false,
20371 vote_split: false,
20372 reconsideration: Vec::new(),
20373 verdict: None,
20374 }];
20375
20376 let mut runner = Runner {
20377 state,
20378 roles: ResolvedRoles {
20379 implementers: Vec::new(),
20380 judges: Vec::new(),
20381 reviewers: Vec::new(),
20382 fixer: None,
20383 conductor: conductor(),
20384 implementer_roster: Vec::new(),
20385 judge_roster: Vec::new(),
20386 reviewer_roster: Vec::new(),
20387 },
20388 sem: Arc::new(Semaphore::new(1)),
20389 pause: Pause::new(),
20390 interrupt: Pause::new(),
20391 };
20392
20393 runner.gate().await.expect("gate");
20394 assert!(
20395 runner.state.gate_ran,
20396 "zero configured commands is still a real attempt, not an unrun gate"
20397 );
20398 assert!(runner.state.gate.is_empty());
20399 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20400 assert_ne!(
20401 runner.state.status,
20402 RunStatus::Blocked,
20403 "a gate with nothing to check must not read as failed"
20404 );
20405
20406 runner.merge().await.expect("merge");
20407 assert_eq!(
20408 runner.state.status,
20409 RunStatus::Ready,
20410 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20411 );
20412 }
20413
20414 #[tokio::test]
20425 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20426 crate::run::pin_test_home();
20427 let home = crate::run::home();
20428
20429 let tmp = tempfile::tempdir().expect("tempdir");
20430 let repo = tmp.path().join("repo");
20431 std::fs::create_dir_all(&repo).unwrap();
20432 init_repo(&repo);
20433 let cache_dir = tmp.path().join("target");
20436
20437 let mut config = Config::default();
20438 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20439 config.graph.timeout_verify = Some(2);
20443 config.graph.timeout_resource_wait = 0;
20444
20445 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20446 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20447 .expect("no io error acquiring directly")
20448 {
20449 crate::cache::AcquireOutcome::Acquired(g) => g,
20450 crate::cache::AcquireOutcome::Busy(b) => {
20451 panic!("expected the direct acquire to win the lease first: {b:?}")
20452 }
20453 };
20454
20455 let mut state = RunState::new(
20456 repo.clone(),
20457 "main".to_owned(),
20458 "deadbeef".to_owned(),
20459 "task".to_owned(),
20460 config,
20461 );
20462 state.candidates = vec![Candidate {
20463 index: 0,
20464 label: 'A',
20465 agent: "alpha".to_owned(),
20466 branch: "does-not-exist".to_owned(),
20467 worktree: repo.clone(),
20468 summary: String::new(),
20469 stat: String::new(),
20470 files: 0,
20471 commits: 0,
20472 empty: false,
20473 failed: None,
20474 verified_noop: None,
20475 duration_ms: 0,
20476 folded: false,
20477 }];
20478 state.tally = Some(Tally {
20479 first_choice: BTreeMap::from([('A', 1)]),
20480 borda: BTreeMap::new(),
20481 winner: 'A',
20482 rankings: 1,
20483 unanimous_initial: true,
20484 deliberated: false,
20485 changed_votes: 0,
20486 unanimous_final: true,
20487 tie_break: None,
20488 judges: 0,
20489 present: 0,
20490 quorum: 0,
20491 met_quorum: true,
20492 uncontested: Some("only candidate A produced a change".to_owned()),
20493 });
20494 state.reviews = vec![ReviewRound {
20495 round: 1,
20496 head: "deadbeef".to_owned(),
20497 verified_head: None,
20498 verified_at: None,
20499 reviews: Vec::new(),
20500 e2e: Vec::new(),
20501 fix: None,
20502 blocking: 0,
20503 answered: 0,
20504 expected: 0,
20505 clean: true,
20506 verify_retried: false,
20507 e2e_deferred: false,
20508 e2e_defer_reason: None,
20509 progressed: false,
20510 vote_split: false,
20511 reconsideration: Vec::new(),
20512 verdict: None,
20513 }];
20514
20515 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20516 state.reviews[0].e2e = vec![CommandOutcome {
20517 command: state.config.verify.e2e[0].clone(),
20518 code: Some(0),
20519 output_tail: String::new(),
20520 duration_ms: 0,
20521 resource_blocked: false,
20522 }];
20523 let mut runner = Runner {
20524 state,
20525 roles: ResolvedRoles {
20526 implementers: Vec::new(),
20527 judges: Vec::new(),
20528 reviewers: Vec::new(),
20529 fixer: None,
20530 conductor: conductor(),
20531 implementer_roster: Vec::new(),
20532 judge_roster: Vec::new(),
20533 reviewer_roster: Vec::new(),
20534 },
20535 sem: Arc::new(Semaphore::new(1)),
20536 pause: Pause::new(),
20537 interrupt: Pause::new(),
20538 };
20539
20540 runner.gate().await.expect("gate");
20541 assert!(runner.state.resource_block.is_none());
20542 assert!(
20543 !runner
20544 .state
20545 .events
20546 .iter()
20547 .any(|event| { event.message.contains("waiting for the shared build cache") }),
20548 "a gate with nothing to run must never request the held cache lease"
20549 );
20550 assert!(
20551 runner.state.gate_ran,
20552 "zero commands is still a real, immediate attempt"
20553 );
20554 assert!(runner.state.gate.is_empty());
20555 assert_ne!(
20556 runner.state.status,
20557 RunStatus::Blocked,
20558 "must not read as resource-blocked on a lease it never asked for"
20559 );
20560 }
20561
20562 #[cfg(unix)]
20565 #[tokio::test]
20566 async fn verification_is_standalone_inside_a_live_worker() {
20567 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20568 if std::env::var_os(CHILD).is_none() {
20569 let home = tempfile::tempdir().unwrap();
20570 let generation = crate::worker::pin_current(home.path()).unwrap();
20571 let output = std::process::Command::new(std::env::current_exe().unwrap())
20572 .args([
20573 "--exact",
20574 "graph::tests::verification_is_standalone_inside_a_live_worker",
20575 "--nocapture",
20576 ])
20577 .env(CHILD, "1")
20578 .env("MAGI_HOME", home.path())
20579 .env("MAGI_WORKER_HOME", home.path())
20580 .env("MAGI_WORKER_ID", "abcdef")
20581 .env("MAGI_WORKER_GENERATION", &generation.id)
20582 .env("MAGI_WORKER_EXE", &generation.executable)
20583 .env("MAGI_RUN", "verification-parent")
20584 .env("MAGI_NODE", "gate")
20585 .env("MAGI_SEAT", "worker-fixture")
20586 .env("MAGI_PROCESS_OWNER", "fixture-owner")
20587 .env("MAGI_NO_AUTOUPDATE", "0")
20588 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20589 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20590 .env(
20591 "VERIFY_EXPECT_TOOLCHAIN",
20592 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20593 )
20594 .env(
20595 "VERIFY_EXPECT_CACHE",
20596 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20597 )
20598 .output()
20599 .unwrap();
20600 assert!(
20601 output.status.success()
20602 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20603 "{}\n{}",
20604 String::from_utf8_lossy(&output.stdout),
20605 String::from_utf8_lossy(&output.stderr)
20606 );
20607 return;
20608 }
20609 let home = crate::run::home();
20610 let directory = home.join("workers/jobs/abcdef");
20611 std::fs::create_dir_all(&directory).unwrap();
20612 let job = crate::worker::Job {
20613 id: "abcdef".into(),
20614 sequence: 1,
20615 kind: "task".into(),
20616 key: "task:verification".into(),
20617 payload: serde_json::json!({}),
20618 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20619 version: Some(env!("CARGO_PKG_VERSION").into()),
20620 status: crate::worker::Status::Running,
20621 result: None,
20622 pid: Some(std::process::id()),
20623 created_at: 0,
20624 finished_at: None,
20625 };
20626 std::fs::write(
20627 directory.join("job.json"),
20628 serde_json::to_vec(&job).unwrap(),
20629 )
20630 .unwrap();
20631 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20632 let assert_worker = || {
20633 assert_eq!(
20634 crate::worker::current_job(&home).unwrap().unwrap().id,
20635 job.id
20636 );
20637 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20638 assert_eq!(
20639 crate::worker::helper_executable(&home).unwrap(),
20640 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20641 );
20642 assert!(
20643 crate::updater::Checker::new(&crate::config::Update {
20644 mode: crate::config::UpdateMode::Notify,
20645 interval: None,
20646 auto_restart: false,
20647 })
20648 .is_none(),
20649 "the driver remains pinned, even while tests are standalone"
20650 );
20651 };
20652 assert_worker();
20653 let repo = home.join("repo");
20654 std::fs::create_dir_all(&repo).unwrap();
20655 init_repo(&repo);
20656 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20659 test -z "${MAGI_WORKER_GENERATION+x}" &&
20660 test -z "${MAGI_WORKER_HOME+x}" &&
20661 test -z "${MAGI_WORKER_EXE+x}" &&
20662 test "$MAGI_HOME/repo" -ef . &&
20663 test "$MAGI_RUN" = verification-parent &&
20664 test "$MAGI_NODE" = gate &&
20665 test "$MAGI_SEAT" = worker-fixture &&
20666 test -n "$MAGI_PROCESS_OWNER" &&
20667 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20668 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20669 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20670 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20671 assert!(
20673 !std::process::Command::new("sh")
20674 .args(["-c", probe])
20675 .current_dir(&repo)
20676 .status()
20677 .unwrap()
20678 .success()
20679 );
20680 let unrelated_home = tempfile::tempdir().unwrap();
20681 assert!(
20682 crate::worker::current_job(unrelated_home.path()).is_err(),
20683 "an inherited worker id cannot be resolved in a unit fixture's home"
20684 );
20685 let mut runner = runner_at(RunStatus::Gating);
20686 runner.state.repo = repo.clone();
20687 runner.state.config.disk.min_free_bytes = 0;
20688 runner.state.config.coordination = crate::config::test_coordination();
20689 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20690 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20691 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20692 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20693 let winner = Candidate {
20694 index: 0,
20695 label: 'A',
20696 agent: "fixture".into(),
20697 branch: "main".into(),
20698 worktree: repo.clone(),
20699 summary: String::new(),
20700 stat: String::new(),
20701 files: 0,
20702 commits: 0,
20703 empty: false,
20704 failed: None,
20705 verified_noop: None,
20706 duration_ms: 0,
20707 folded: false,
20708 };
20709 let (e2e, retried, pids) = run_e2e_with_retry(
20710 &mut runner.state,
20711 &["sh".into(), "-c".into()],
20712 &[format!("{probe} && printf initial")],
20713 &repo,
20714 Duration::from_secs(10),
20715 "initial",
20716 )
20717 .await;
20718 assert!(!retried && pids.is_empty());
20719 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20720 assert_eq!(e2e[0].output_tail, "initial");
20721 let gate = runner.run_gate(&winner).await.unwrap();
20722 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20723 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20726 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20727 passed: false,
20728 outcomes: Vec::new(),
20729 });
20730 runner.state.save().unwrap();
20731 let mut resumed = RunState::load(&runner.state.id).unwrap();
20732 let wait = crate::repo_lock::WaitPolicy::default();
20733 assert!(matches!(
20734 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20735 .await
20736 .unwrap(),
20737 GateFix::Retry
20738 ));
20739 let saved = RunState::load(&resumed.id).unwrap();
20740 let repair = saved.gate_repair_e2e.unwrap();
20741 assert!(repair.passed);
20742 assert_eq!(repair.outcomes.len(), 1);
20743 assert!(repair.outcomes[0].ok());
20744 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20745 .await
20746 .unwrap();
20747 assert_eq!(landing.len(), 1);
20748 assert!(landing[0].ok(), "{landing:?}");
20749 assert_eq!(
20750 std::fs::read_to_string(repo.join("phases")).unwrap(),
20751 "pre\ngate\ne2e\npre\ngate\n"
20752 );
20753 assert_worker();
20754 }
20755
20756 #[cfg(unix)]
20757 #[tokio::test]
20758 async fn verification_keeps_the_panic_and_the_full_output() {
20759 crate::run::pin_test_home();
20760 let tmp = tempfile::tempdir().unwrap();
20761 let mut config = Config::default();
20762 config.disk.min_free_bytes = 0;
20763 let mut state = RunState::new(
20764 tmp.path().to_owned(),
20765 "main".into(),
20766 "base".into(),
20767 "verification evidence".into(),
20768 config,
20769 );
20770 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20771 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20772 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20773 let mut names = Vec::new();
20774 for _ in 0..2 {
20775 let (outcomes, _) = run_commands(
20776 &mut state,
20777 "gate",
20778 "gate",
20779 0,
20780 &["sh".into(), "-c".into()],
20781 &[script.into()],
20782 tmp.path(),
20783 Duration::from_secs(60),
20784 )
20785 .await;
20786 let tail = &outcomes[0].output_tail;
20787 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20788 assert!(tail.contains("boom assertion"));
20789 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20790 let line = tail.lines().next().unwrap();
20791 names.push(line.to_owned());
20792 }
20793 assert_ne!(names[0], names[1], "the second run must not overwrite");
20794 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20795 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20796 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20797
20798 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20800 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20801 let (outcomes, _) = run_commands(
20802 &mut state,
20803 "gate",
20804 "gate",
20805 0,
20806 &["sh".into(), "-c".into()],
20807 &["echo failure-output; exit 7".into()],
20808 tmp.path(),
20809 Duration::from_secs(10),
20810 )
20811 .await;
20812 assert_eq!(outcomes[0].code, Some(7));
20813 assert!(state.events.iter().any(|e| {
20814 e.node == "gate" && e.message.contains("could not save the full gate output:")
20815 }));
20816 }
20817
20818 #[cfg(unix)]
20819 #[tokio::test]
20820 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20821 crate::run::pin_test_home();
20822 let tmp = tempfile::tempdir().unwrap();
20823 let mut config = Config::default();
20824 config.disk.min_free_bytes = 0;
20825 let mut state = RunState::new(
20826 tmp.path().to_owned(),
20827 "main".into(),
20828 "base".into(),
20829 "verification evidence".into(),
20830 config,
20831 );
20832 let (outcomes, pids) = run_commands(
20833 &mut state,
20834 "gate",
20835 "gate",
20836 1,
20837 &["sh".into(), "-c".into()],
20838 &[
20839 "printf before-timeout; exec sleep 30".into(),
20840 "kill -TERM $$".into(),
20841 "exit 7".into(),
20842 "printf success".into(),
20843 ],
20844 tmp.path(),
20845 Duration::from_secs(1),
20846 )
20847 .await;
20848 wait_for_timed_out_children_to_die(&pids).await;
20849 assert_eq!(
20850 pids.len(),
20851 1,
20852 "unleased timeout retains its death-check PID"
20853 );
20854 assert_eq!(outcomes.len(), 4);
20855 assert_eq!(outcomes[0].code, None);
20856 assert!(outcomes[0].output_tail.contains("before-timeout"));
20857 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20858 assert_eq!(outcomes[1].code, None);
20859 assert!(outcomes[1].output_tail.contains("command terminated:"));
20860 assert!(!outcomes[1].output_tail.contains("timed out"));
20861 assert_eq!(outcomes[2].code, Some(7));
20862 assert!(!outcomes[2].output_tail.contains("timed out"));
20863 assert_eq!(outcomes[3].code, Some(0));
20864 assert_eq!(outcomes[3].output_tail, "success");
20865 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20866 }
20867
20868 #[tokio::test]
20878 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20879 crate::run::pin_test_home();
20880
20881 let tmp = tempfile::tempdir().expect("tempdir");
20882 let repo = tmp.path().join("repo");
20883 std::fs::create_dir_all(&repo).unwrap();
20884 init_repo(&repo);
20885
20886 let mut config = Config::default();
20887 config.disk.min_free_bytes = 0;
20888 config.verify.gate = vec![
20889 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20890 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20891 .to_owned(),
20892 ];
20893
20894 let mut state = RunState::new(
20895 repo.clone(),
20896 "main".to_owned(),
20897 "deadbeef".to_owned(),
20898 "task".to_owned(),
20899 config,
20900 );
20901 let run_id = state.id.clone();
20902 state.candidates = vec![Candidate {
20903 index: 0,
20904 label: 'A',
20905 agent: "alpha".to_owned(),
20906 branch: "does-not-exist".to_owned(),
20907 worktree: repo.clone(),
20908 summary: String::new(),
20909 stat: String::new(),
20910 files: 0,
20911 commits: 0,
20912 empty: false,
20913 failed: None,
20914 verified_noop: None,
20915 duration_ms: 0,
20916 folded: false,
20917 }];
20918 state.tally = Some(Tally {
20919 first_choice: BTreeMap::from([('A', 1)]),
20920 borda: BTreeMap::new(),
20921 winner: 'A',
20922 rankings: 1,
20923 unanimous_initial: true,
20924 deliberated: false,
20925 changed_votes: 0,
20926 unanimous_final: true,
20927 tie_break: None,
20928 judges: 0,
20929 present: 0,
20930 quorum: 0,
20931 met_quorum: true,
20932 uncontested: Some("only candidate A produced a change".to_owned()),
20933 });
20934 state.reviews = vec![ReviewRound {
20935 round: 1,
20936 head: "deadbeef".to_owned(),
20937 verified_head: None,
20938 verified_at: None,
20939 reviews: Vec::new(),
20940 e2e: Vec::new(),
20941 fix: None,
20942 blocking: 0,
20943 answered: 0,
20944 expected: 0,
20945 clean: true,
20946 verify_retried: false,
20947 e2e_deferred: false,
20948 e2e_defer_reason: None,
20949 progressed: false,
20950 vote_split: false,
20951 reconsideration: Vec::new(),
20952 verdict: None,
20953 }];
20954
20955 let mut runner = Runner {
20956 state,
20957 roles: ResolvedRoles {
20958 implementers: Vec::new(),
20959 judges: Vec::new(),
20960 reviewers: Vec::new(),
20961 fixer: None,
20962 conductor: conductor(),
20963 implementer_roster: Vec::new(),
20964 judge_roster: Vec::new(),
20965 reviewer_roster: Vec::new(),
20966 },
20967 sem: Arc::new(Semaphore::new(1)),
20968 pause: Pause::new(),
20969 interrupt: Pause::new(),
20970 };
20971
20972 let started_marker = repo.join("started.marker");
20973 let release_marker = repo.join("release.marker");
20974 let poller = tokio::spawn(async move {
20975 for _ in 0..100 {
20980 if started_marker.exists()
20981 && let Ok(s) = crate::run::RunState::load(&run_id)
20982 && let Some(a) = s.active.get("gate")
20983 {
20984 std::fs::write(&release_marker, b"go").expect("release marker");
20985 return Some(a.clone());
20986 }
20987 tokio::time::sleep(Duration::from_millis(50)).await;
20988 }
20989 None
20990 });
20991
20992 runner.gate().await.expect("gate");
20993 let captured = poller.await.expect("poller task");
20994 let captured = captured.expect(
20995 "the poller never saw a `gate` task entry in run.json while the command was \
20996 still blocked on its own release marker",
20997 );
20998
20999 assert_eq!(captured.task.as_deref(), Some("gate"));
21000 assert_eq!(captured.node, "gate");
21001 assert_eq!(captured.index, Some(1));
21002 assert_eq!(captured.total, Some(1));
21003 assert!(
21004 captured
21005 .command
21006 .as_deref()
21007 .is_some_and(|c| c.contains("started.marker")),
21008 "{captured:?}"
21009 );
21010
21011 assert!(
21012 runner.state.active.is_empty(),
21013 "the entry must be cleared once the command actually finished: {:?}",
21014 runner.state.active
21015 );
21016 assert!(runner.state.gate_ran);
21017 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
21018 }
21019
21020 #[tokio::test]
21023 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
21024 use crate::verdict::{Finding, ReviewVote, Severity};
21025 crate::run::pin_test_home();
21026 let tmp = tempfile::tempdir().expect("tempdir");
21027 let repo = tmp.path().join("repo");
21028 std::fs::create_dir_all(&repo).unwrap();
21029 init_repo(&repo);
21030
21031 for (severity, vote, expect) in [
21032 (Severity::Major, ReviewVote::Reject, true),
21033 (Severity::Minor, ReviewVote::Reject, false),
21034 (Severity::Major, ReviewVote::Approve, false),
21035 ] {
21036 let mut round = review_round(false, 1, 1, 1, false, true);
21037 round.reviews = vec![ReviewRecord {
21038 reviewer: 1,
21039 agent: "alpha".to_owned(),
21040 summary: String::new(),
21041 findings: vec![Finding {
21042 id: "R1-1-1".to_owned(),
21043 severity,
21044 file: None,
21045 line: None,
21046 title: "t".to_owned(),
21047 detail: String::new(),
21048 }],
21049 vote: Some(vote),
21050 failed: None,
21051 duration_ms: 0,
21052 attempts: 0,
21053 }];
21054 let mut state = RunState::new(
21055 repo.clone(),
21056 "main".to_owned(),
21057 "deadbeef".to_owned(),
21058 "task".to_owned(),
21059 Config::default(),
21060 );
21061 state.reviews = vec![round];
21062 let mut runner = Runner {
21063 state,
21064 roles: ResolvedRoles {
21065 implementers: Vec::new(),
21066 judges: Vec::new(),
21067 reviewers: Vec::new(),
21068 fixer: None,
21069 conductor: conductor(),
21070 implementer_roster: Vec::new(),
21071 judge_roster: Vec::new(),
21072 reviewer_roster: Vec::new(),
21073 },
21074 sem: Arc::new(Semaphore::new(1)),
21075 pause: Pause::new(),
21076 interrupt: Pause::new(),
21077 };
21078 let shell = runner.state.config.shell();
21079 runner
21080 .stop_reviewing("round budget spent", &shell, &repo)
21081 .await
21082 .expect("stop_reviewing");
21083 assert_eq!(runner.state.status, RunStatus::Gating);
21084 assert_eq!(
21085 runner.state.contested_handoff.is_some(),
21086 expect,
21087 "{severity:?} + {vote:?}"
21088 );
21089 }
21090 }
21091
21092 #[tokio::test]
21105 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
21106 crate::run::pin_test_home();
21107 let home = crate::run::home();
21108
21109 let tmp = tempfile::tempdir().expect("tempdir");
21110 let repo = tmp.path().join("repo");
21111 std::fs::create_dir_all(&repo).unwrap();
21112 init_repo(&repo);
21113 let head = crate::git::rev_parse(&repo, "HEAD")
21114 .await
21115 .expect("rev-parse");
21116 let cache_dir = tmp.path().join("target");
21119
21120 let mut config = Config::default();
21121 config.verify.e2e = vec![format!(
21122 "CARGO_TARGET_DIR='{}' test -f README.md",
21123 cache_dir.display()
21124 )];
21125 config.graph.review_rounds = 1;
21126 config.graph.timeout_verify = Some(2);
21129 config.graph.timeout_resource_wait = 2;
21130
21131 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21132 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21133 .expect("no io error acquiring directly")
21134 {
21135 crate::cache::AcquireOutcome::Acquired(g) => g,
21136 crate::cache::AcquireOutcome::Busy(b) => {
21137 panic!("expected the direct acquire to win the lease first: {b:?}")
21138 }
21139 };
21140
21141 let mut state = RunState::new(
21142 repo.clone(),
21143 "main".to_owned(),
21144 head.clone(),
21145 "task".to_owned(),
21146 config,
21147 );
21148 state.candidates = vec![Candidate {
21149 index: 0,
21150 label: 'A',
21151 agent: "alpha".to_owned(),
21152 branch: "does-not-exist".to_owned(),
21153 worktree: repo.clone(),
21154 summary: String::new(),
21155 stat: String::new(),
21156 files: 0,
21157 commits: 0,
21158 empty: false,
21159 failed: None,
21160 verified_noop: None,
21161 duration_ms: 0,
21162 folded: false,
21163 }];
21164 state.tally = Some(Tally {
21165 first_choice: BTreeMap::from([('A', 1)]),
21166 borda: BTreeMap::new(),
21167 winner: 'A',
21168 rankings: 1,
21169 unanimous_initial: true,
21170 deliberated: false,
21171 changed_votes: 0,
21172 unanimous_final: true,
21173 tie_break: None,
21174 judges: 0,
21175 present: 0,
21176 quorum: 0,
21177 met_quorum: true,
21178 uncontested: Some("only candidate A produced a change".to_owned()),
21179 });
21180 state.reviews = vec![ReviewRound {
21184 round: 1,
21185 head: head.clone(),
21186 verified_head: None,
21187 verified_at: None,
21188 reviews: Vec::new(),
21189 e2e: Vec::new(),
21190 fix: None,
21191 blocking: 1,
21192 answered: 1,
21193 expected: 1,
21194 clean: false,
21195 verify_retried: false,
21196 e2e_deferred: true,
21197 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
21198 progressed: false,
21199 vote_split: false,
21200 reconsideration: Vec::new(),
21201 verdict: None,
21202 }];
21203
21204 let mut runner = Runner {
21205 state,
21206 roles: ResolvedRoles {
21207 implementers: Vec::new(),
21208 judges: Vec::new(),
21209 reviewers: Vec::new(),
21210 fixer: None,
21211 conductor: conductor(),
21212 implementer_roster: Vec::new(),
21213 judge_roster: Vec::new(),
21214 reviewer_roster: Vec::new(),
21215 },
21216 sem: Arc::new(Semaphore::new(1)),
21217 pause: Pause::new(),
21218 interrupt: Pause::new(),
21219 };
21220
21221 let shell = runner.state.config.shell();
21222 runner
21223 .stop_reviewing("round budget spent", &shell, &repo)
21224 .await
21225 .expect("stop_reviewing");
21226
21227 let last = runner.state.reviews.last().expect("round record");
21228 assert_eq!(
21229 last.e2e_status(),
21230 E2eStatus::ResourceBlocked,
21231 "the shared cache is still held; the attempt must read as blocked, not deferred or \
21232 failed: {last:?}"
21233 );
21234 assert_eq!(
21235 last.verified_head.as_deref(),
21236 Some(head.as_str()),
21237 "which commit this attempt targeted is known even though nothing finished checking \
21238 it"
21239 );
21240 let first_attempt_at = last
21241 .verified_at
21242 .expect("when this attempt ran is known too");
21243 assert_ne!(
21244 runner.state.status,
21245 RunStatus::Blocked,
21246 "contention is evidence about the machine, not the patch — it must not settle the \
21247 run as blocked: {:?}",
21248 runner.state.status
21249 );
21250 assert!(
21251 !runner
21252 .state
21253 .events
21254 .iter()
21255 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
21256 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
21257 runner.state.events
21258 );
21259
21260 runner
21265 .stop_reviewing("round budget spent", &shell, &repo)
21266 .await
21267 .expect("stop_reviewing retry");
21268 assert_eq!(
21269 runner.state.reviews.len(),
21270 1,
21271 "no new round was started: {:?}",
21272 runner.state.reviews
21273 );
21274 let last = runner.state.reviews.last().expect("round record");
21275 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
21276 assert!(
21277 last.verified_at.expect("still known") > first_attempt_at,
21278 "a second reentry must be a fresh attempt, not a stale copy of the first"
21279 );
21280 assert_ne!(runner.state.status, RunStatus::Blocked);
21281
21282 held.release();
21283 }
21284
21285 #[cfg(unix)]
21286 #[tokio::test]
21287 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
21288 ask_test_home();
21289 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21290 let tmp = tempfile::tempdir().unwrap();
21291 let repo = tmp.path().join("repo");
21292 std::fs::create_dir(&repo).unwrap();
21293 init_repo(&repo);
21294 let run_git = |args: &[&str]| {
21295 let out = std::process::Command::new("git")
21296 .args(args)
21297 .current_dir(&repo)
21298 .output()
21299 .unwrap();
21300 assert!(
21301 out.status.success(),
21302 "{args:?}: {}",
21303 String::from_utf8_lossy(&out.stderr)
21304 );
21305 };
21306 let origin = tmp.path().join("origin.git");
21307 run_git(&["init", "--bare", origin.to_str().unwrap()]);
21308 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21309 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21310 run_git(&["add", "spec.rs"]);
21311 run_git(&["commit", "-m", "initial spec"]);
21312 run_git(&["push", "origin", "main"]);
21313 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21314 run_git(&["checkout", "-b", "fix/review"]);
21315 std::fs::write(
21316 repo.join("check.rs"),
21317 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21318 )
21319 .unwrap();
21320 run_git(&["add", "check.rs"]);
21321 run_git(&["commit", "-m", "use spec"]);
21322 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21323 run_git(&["checkout", "main"]);
21324 std::fs::write(
21325 repo.join("spec.rs"),
21326 "struct AgentSpec { max_turn_secs: u64 }\n",
21327 )
21328 .unwrap();
21329 run_git(&["add", "spec.rs"]);
21330 run_git(&["commit", "-m", "extend spec"]);
21331 run_git(&["push", "origin", "main"]);
21332 run_git(&["checkout", "fix/review"]);
21333
21334 let mut runner = runner_at(RunStatus::Reviewing);
21335 runner.state.repo = repo.clone();
21336 runner.state.base_commit = base;
21337 runner.state.pr_head = Some("fix/review".into());
21338 runner.state.config.graph.review_rounds = max_rounds;
21339 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21340 runner.state.config.graph.sessions = false;
21341 runner.state.config.disk.min_free_bytes = 0;
21342 runner.state.config.verify.e2e =
21343 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21344 runner.state.candidates = vec![Candidate {
21345 index: 0,
21346 label: 'A',
21347 agent: "fixture".into(),
21348 branch: "fix/review".into(),
21349 worktree: repo.clone(),
21350 summary: String::new(),
21351 stat: String::new(),
21352 files: 1,
21353 commits: 1,
21354 empty: false,
21355 failed: None,
21356 verified_noop: None,
21357 duration_ms: 0,
21358 folded: false,
21359 }];
21360 runner.state.tally = Some(Tally {
21361 first_choice: BTreeMap::new(),
21362 borda: BTreeMap::new(),
21363 winner: 'A',
21364 rankings: 0,
21365 unanimous_initial: true,
21366 deliberated: false,
21367 changed_votes: 0,
21368 unanimous_final: true,
21369 tie_break: None,
21370 judges: 0,
21371 present: 0,
21372 quorum: 0,
21373 met_quorum: true,
21374 uncontested: Some("review-only".into()),
21375 });
21376 let mut last = review_round(false, 0, 1, 1, false, false);
21377 last.head = reviewed.clone();
21378 last.e2e_deferred = true;
21379 last.e2e[0].resource_blocked = true;
21380 last.e2e[0].output_tail = "waiting for the shared build cache".into();
21381 if max_rounds == 1 {
21382 last.e2e.clear();
21383 }
21384 runner.state.reviews = vec![last];
21385 let calls = tmp.path().join("fix-calls");
21386 let mut fixer = spec("repair");
21387 fixer
21388 .env
21389 .insert("FIX_CALLS".into(), calls.display().to_string());
21390 fixer.command = vec!["sh".into(), "-c".into(), r#"
21391 echo fix >> "$FIX_CALLS"
21392 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21393 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21394 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21395 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21396 "#.into()];
21397 let mut reviewer = spec("reviewer");
21398 reviewer.command = vec![
21399 "sh".into(),
21400 "-c".into(),
21401 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21402 .into(),
21403 ];
21404 runner.roles.fixer = Some(vec![fixer]);
21405 runner.roles.reviewers = vec![reviewer];
21406
21407 runner.sync_to_base().await.unwrap();
21408 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21409 assert_ne!(rebased, reviewed);
21410 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21411 if reentry {
21412 let shell = runner.state.config.shell();
21415 runner
21416 .stop_unresolved_review("deferred verification", &shell, &repo)
21417 .await
21418 .unwrap();
21419 assert!(runner.catchup_repairable());
21420 assert!(!calls.exists());
21421 runner.state = RunState::load(&runner.state.id).unwrap();
21422 assert!(runner.catchup_repairable());
21423 }
21424 runner.review_loop().await.unwrap();
21425 let first = &runner.state.reviews[0];
21426 assert_eq!(first.head, reviewed);
21427 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21428 assert!(
21429 first.verify_retried,
21430 "a reproduced build failure gets exactly one retry: {:?}",
21431 first.e2e
21432 );
21433 assert!(first.e2e[0].output_tail.contains("E0063"));
21434 assert!(!runner.catchup_repairable());
21435 if max_rounds == 2 {
21436 assert_eq!(
21437 runner.state.status,
21438 RunStatus::Gating,
21439 "{}",
21440 crate::report::run(&runner.state)
21441 );
21442 assert_eq!(runner.state.reviews.len(), 2);
21443 assert!(first.fix.as_ref().unwrap().committed);
21444 assert!(runner.state.reviews[1].clean);
21445 assert_eq!(
21446 runner
21447 .state
21448 .reviews
21449 .iter()
21450 .filter(|r| r.fix.is_some())
21451 .count(),
21452 1
21453 );
21454 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21455 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21456 } else {
21457 assert_eq!(runner.state.status, RunStatus::Blocked);
21458 assert!(first.fix.is_none());
21459 assert!(!calls.exists());
21460 let recovery = crate::recovery::assess(
21461 "prior",
21462 Some(&runner.state),
21463 &runner.landing_base(),
21464 crate::recovery::Relation::Diverged,
21465 );
21466 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21467 let reasons = recovery.reasons.join("\n");
21468 assert!(
21469 reasons.contains(&reviewed) && reasons.contains(&rebased),
21470 "{reasons}"
21471 );
21472 assert!(
21473 reasons.contains("base sync to")
21474 && reasons.contains("additional review is required"),
21475 "{reasons}"
21476 );
21477 }
21478 }
21479 }
21480
21481 #[tokio::test]
21493 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21494 crate::run::pin_test_home();
21495 let home = crate::run::home();
21496
21497 let tmp = tempfile::tempdir().expect("tempdir");
21498 let repo = tmp.path().join("repo");
21499 std::fs::create_dir_all(&repo).unwrap();
21500 init_repo(&repo);
21501 let head = crate::git::rev_parse(&repo, "HEAD")
21502 .await
21503 .expect("rev-parse");
21504 let cache_dir = tmp.path().join("target");
21505
21506 let mut config = Config::default();
21507 config.verify.e2e = vec![format!(
21508 "CARGO_TARGET_DIR='{}' test -f README.md",
21509 cache_dir.display()
21510 )];
21511 config.graph.review_rounds = 1;
21512 config.graph.timeout_verify = Some(2);
21513 config.graph.timeout_resource_wait = 2;
21514
21515 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21516 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21517 .expect("no io error acquiring directly")
21518 {
21519 crate::cache::AcquireOutcome::Acquired(g) => g,
21520 crate::cache::AcquireOutcome::Busy(b) => {
21521 panic!("expected the direct acquire to win the lease first: {b:?}")
21522 }
21523 };
21524
21525 let mut state = RunState::new(
21526 repo.clone(),
21527 "main".to_owned(),
21528 head.clone(),
21529 "task".to_owned(),
21530 config,
21531 );
21532 state.candidates = vec![Candidate {
21533 index: 0,
21534 label: 'A',
21535 agent: "alpha".to_owned(),
21536 branch: "does-not-exist".to_owned(),
21537 worktree: repo.clone(),
21538 summary: String::new(),
21539 stat: String::new(),
21540 files: 0,
21541 commits: 0,
21542 empty: false,
21543 failed: None,
21544 verified_noop: None,
21545 duration_ms: 0,
21546 folded: false,
21547 }];
21548 state.tally = Some(Tally {
21549 first_choice: BTreeMap::from([('A', 1)]),
21550 borda: BTreeMap::new(),
21551 winner: 'A',
21552 rankings: 1,
21553 unanimous_initial: true,
21554 deliberated: false,
21555 changed_votes: 0,
21556 unanimous_final: true,
21557 tie_break: None,
21558 judges: 0,
21559 present: 0,
21560 quorum: 0,
21561 met_quorum: true,
21562 uncontested: Some("only candidate A produced a change".to_owned()),
21563 });
21564 state.reviews = vec![ReviewRound {
21568 round: 1,
21569 head: head.clone(),
21570 verified_head: Some(head.clone()),
21571 verified_at: Some(jiff::Timestamp::now()),
21572 reviews: Vec::new(),
21573 e2e: vec![CommandOutcome {
21574 command: format!(
21575 "CARGO_TARGET_DIR='{}' test -f README.md",
21576 cache_dir.display()
21577 ),
21578 code: None,
21579 output_tail: "waiting for the shared build cache".to_owned(),
21580 duration_ms: 0,
21581 resource_blocked: true,
21582 }],
21583 fix: None,
21584 blocking: 1,
21585 answered: 1,
21586 expected: 1,
21587 clean: false,
21588 verify_retried: false,
21589 e2e_deferred: false,
21590 e2e_defer_reason: None,
21591 progressed: false,
21592 vote_split: false,
21593 reconsideration: Vec::new(),
21594 verdict: None,
21595 }];
21596
21597 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21598 let mut runner = Runner {
21599 state,
21600 roles: ResolvedRoles {
21601 implementers: Vec::new(),
21602 judges: Vec::new(),
21603 reviewers: Vec::new(),
21604 fixer: None,
21605 conductor: conductor(),
21606 implementer_roster: Vec::new(),
21607 judge_roster: Vec::new(),
21608 reviewer_roster: Vec::new(),
21609 },
21610 sem: Arc::new(Semaphore::new(1)),
21611 pause: Pause::new(),
21612 interrupt: Pause::new(),
21613 };
21614
21615 runner.review_loop().await.expect("review_loop");
21620
21621 assert_eq!(
21622 runner.state.reviews.len(),
21623 1,
21624 "no new round was started on top of the unresolved one: {:?}",
21625 runner.state.reviews
21626 );
21627 let last = &runner.state.reviews[0];
21628 assert_eq!(
21629 last.e2e_status(),
21630 E2eStatus::ResourceBlocked,
21631 "still contended: {last:?}"
21632 );
21633 assert!(
21634 last.verified_at.expect("still known") > first_attempt_at,
21635 "review_loop must have actually retried the check, not left it exactly as found"
21636 );
21637 assert_ne!(
21638 runner.state.status,
21639 RunStatus::Blocked,
21640 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21641 runner.state.status
21642 );
21643
21644 held.release();
21645 }
21646
21647 #[tokio::test]
21648 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21649 crate::run::pin_test_home();
21650 for status in [RunStatus::Blocked, RunStatus::Gating] {
21651 let mut runner = runner_at(status);
21652 runner.state.config.graph.land = true;
21653 runner.state.gate_ran = true;
21654 runner.state.merge = Some(MergeOutcome {
21655 mode: MergeMode::Pr,
21656 ok: false,
21657 detail: "the fixer produced no commit".into(),
21658 empty: false,
21659 });
21660 let url = "https://example.invalid/o/r/pull/1";
21661 runner.state.pr = Some(crate::run::PrRecord {
21662 url: url.into(),
21663 number: 1,
21664 state: "open".into(),
21665 checks: "red".into(),
21666 round: 1,
21667 rounds: 1,
21668 red_at_merge: Vec::new(),
21669 });
21670 runner.state.save().unwrap();
21671 runner.execute_graph().await.unwrap();
21675 assert_eq!(runner.state.status, RunStatus::Blocked);
21676 assert!(runner.state.land_sync_pending);
21677 assert!(runner.state.events.iter().any(|event|
21678 event.node == "land" && event.message == format!("watching {url}")));
21679 assert_eq!(
21680 runner.state.merge.as_ref().unwrap().detail,
21681 "the fixer produced no commit"
21682 );
21683 }
21684 }
21685
21686 #[tokio::test]
21687 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21688 crate::run::pin_test_home();
21689 let tmp = tempfile::tempdir().expect("tempdir");
21690 let repo = tmp.path().join("repo");
21691 std::fs::create_dir_all(&repo).unwrap();
21692 init_repo(&repo);
21693
21694 let mut config = Config::default();
21695 config.merge.mode = MergeMode::Pr;
21696 config.graph.land = true;
21697 config.graph.land_approval = false;
21698
21699 let mut state = RunState::new(
21700 repo.clone(),
21701 "main".to_owned(),
21702 "deadbeef".to_owned(),
21703 "task".to_owned(),
21704 config,
21705 );
21706 state.candidates = vec![Candidate {
21707 index: 0,
21708 label: 'A',
21709 agent: "alpha".to_owned(),
21710 branch: "does-not-exist".to_owned(),
21711 worktree: repo.clone(),
21712 summary: String::new(),
21713 stat: String::new(),
21714 files: 0,
21715 commits: 0,
21716 empty: false,
21717 failed: None,
21718 verified_noop: None,
21719 duration_ms: 0,
21720 folded: false,
21721 }];
21722 state.tally = Some(Tally {
21723 first_choice: BTreeMap::from([('A', 1)]),
21724 borda: BTreeMap::new(),
21725 winner: 'A',
21726 rankings: 1,
21727 unanimous_initial: true,
21728 deliberated: false,
21729 changed_votes: 0,
21730 unanimous_final: true,
21731 tie_break: None,
21732 judges: 0,
21733 present: 0,
21734 quorum: 0,
21735 met_quorum: true,
21736 uncontested: Some("only candidate A produced a change".to_owned()),
21737 });
21738 state.reviews = vec![ReviewRound {
21739 round: 1,
21740 head: "deadbeef".to_owned(),
21741 verified_head: None,
21742 verified_at: None,
21743 reviews: Vec::new(),
21744 e2e: Vec::new(),
21745 fix: None,
21746 blocking: 0,
21747 answered: 0,
21748 expected: 0,
21749 clean: true,
21750 verify_retried: false,
21751 e2e_deferred: false,
21752 e2e_defer_reason: None,
21753 progressed: false,
21754 vote_split: false,
21755 reconsideration: Vec::new(),
21756 verdict: None,
21757 }];
21758 state.gate = vec![CommandOutcome {
21759 command: "test".to_owned(),
21760 code: Some(0),
21761 output_tail: String::new(),
21762 duration_ms: 0,
21763 resource_blocked: false,
21764 }];
21765 state.gate_ran = true;
21766 state.status = RunStatus::Landing;
21770 state.merge = Some(MergeOutcome {
21771 mode: MergeMode::Pr,
21772 ok: true,
21773 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21774 empty: false,
21775 });
21776
21777 ask_test_home();
21781 state.save().expect("save landing run");
21784 let store = ask::Questions::open();
21785 let q = ask_open_question(&store, &state.id);
21786
21787 let mut runner = Runner {
21788 state,
21789 roles: ResolvedRoles {
21790 implementers: Vec::new(),
21791 judges: Vec::new(),
21792 reviewers: Vec::new(),
21793 fixer: None,
21794 conductor: conductor(),
21795 implementer_roster: Vec::new(),
21796 judge_roster: Vec::new(),
21797 reviewer_roster: Vec::new(),
21798 },
21799 sem: Arc::new(Semaphore::new(1)),
21800 pause: Pause::new(),
21801 interrupt: Pause::new(),
21802 };
21803
21804 runner.execute().await.expect("execute");
21809
21810 assert_eq!(
21811 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21812 Some("https://example.invalid/x/y/pull/1"),
21813 "reentry must not push again or open a second pull request over the \
21814 one `land` is already watching"
21815 );
21816 assert_ne!(
21817 runner.state.status,
21818 RunStatus::Landing,
21819 "land could not actually reach the fake pull request, so it must \
21820 have given up rather than left the run silently parked forever"
21821 );
21822 assert_eq!(runner.state.status, RunStatus::Blocked);
21826 assert!(
21827 store.get(&q.id).unwrap().status.open(),
21828 "Blocked is still alive; settle_questions must have been a no-op here"
21829 );
21830 }
21831
21832 fn state_with_round(round: ReviewRound) -> RunState {
21833 let mut s = RunState::new(
21834 PathBuf::from("/repo"),
21835 "main".to_owned(),
21836 "abc1234".to_owned(),
21837 "add retries".to_owned(),
21838 Config::default(),
21839 );
21840 s.reviews = vec![round];
21841 s
21842 }
21843
21844 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21845 crate::verdict::Finding {
21846 id: id.to_owned(),
21847 severity,
21848 file: None,
21849 line: None,
21850 title: title.to_owned(),
21851 detail: String::new(),
21852 }
21853 }
21854
21855 #[test]
21856 fn pr_body_names_open_findings_and_declined_ones() {
21857 let round = ReviewRound {
21858 round: 2,
21859 head: "deadbee".to_owned(),
21860 verified_head: None,
21861 verified_at: None,
21862 reviews: vec![ReviewRecord {
21863 attempts: 0,
21864 reviewer: 1,
21865 agent: "alpha".to_owned(),
21866 summary: String::new(),
21867 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21868 vote: None,
21869 failed: None,
21870 duration_ms: 0,
21871 }],
21872 e2e: vec![CommandOutcome {
21873 command: "cargo test".to_owned(),
21874 code: Some(0),
21875 output_tail: String::new(),
21876 duration_ms: 0,
21877 resource_blocked: false,
21878 }],
21879 verify_retried: false,
21880 e2e_deferred: false,
21881 e2e_defer_reason: None,
21882 fix: Some(FixRecord {
21883 agent: "alpha".to_owned(),
21884 addressed: Vec::new(),
21885 rejected: vec![crate::verdict::Rejection {
21886 id: "R1-1-1".to_owned(),
21887 why: "not reachable from any caller".to_owned(),
21888 }],
21889 notes: String::new(),
21890 committed: true,
21891 failed: None,
21892 duration_ms: 0,
21893 continuation: None,
21894 }),
21895 blocking: 0,
21896 answered: 1,
21897 expected: 1,
21898 clean: false,
21899 progressed: true,
21900 vote_split: false,
21901 reconsideration: Vec::new(),
21902 verdict: None,
21903 };
21904 let state = state_with_round(round);
21905 let body = pr_message(&state, 'A').body;
21906
21907 assert!(body.contains("add retries"), "the task must still be there");
21908 assert!(body.contains("R2-1-1"), "{body}");
21909 assert!(!body.contains("unused import"), "{body}");
21910 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21911 assert!(
21912 !body.contains("not reachable from any caller"),
21913 "the reason it was declined: {body}"
21914 );
21915 }
21916
21917 #[test]
21918 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21919 let round = ReviewRound {
21920 round: 1,
21921 head: "deadbee".to_owned(),
21922 verified_head: None,
21923 verified_at: None,
21924 reviews: vec![ReviewRecord {
21925 attempts: 0,
21926 reviewer: 1,
21927 agent: "alpha".to_owned(),
21928 summary: String::new(),
21929 findings: Vec::new(),
21930 vote: None,
21931 failed: None,
21932 duration_ms: 0,
21933 }],
21934 e2e: Vec::new(),
21935 verify_retried: false,
21936 e2e_deferred: false,
21937 e2e_defer_reason: None,
21938 fix: None,
21939 blocking: 0,
21940 answered: 1,
21941 expected: 1,
21942 clean: true,
21943 progressed: false,
21944 vote_split: false,
21945 reconsideration: Vec::new(),
21946 verdict: None,
21947 };
21948 let state = state_with_round(round);
21949 let body = pr_message(&state, 'A').body;
21950 assert!(!body.contains("Open review findings"), "{body}");
21951 assert!(!body.contains("Declined"), "{body}");
21952 }
21953
21954 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21955 let mut state = RunState::new(
21956 PathBuf::from("/repo"),
21957 "main".to_owned(),
21958 "abc1234".to_owned(),
21959 instruction.to_owned(),
21960 Config::default(),
21961 );
21962 state.config.disk.min_free_bytes = 0;
21964 state.candidates.push(Candidate {
21965 index: 0,
21966 label: 'A',
21967 agent: "alpha".to_owned(),
21968 branch: "magi/x/A".to_owned(),
21969 worktree: PathBuf::from("/wt"),
21970 summary: summary.to_owned(),
21971 stat: String::new(),
21972 files: 1,
21973 commits: 1,
21974 empty: false,
21975 failed: None,
21976 verified_noop: None,
21977 folded: false,
21978 duration_ms: 0,
21979 });
21980 state
21981 }
21982
21983 fn review_state(subjects: &[&str]) -> RunState {
21984 let mut state = state_with_summary(
21985 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21986 "",
21987 );
21988 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21989 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21990 state
21991 }
21992
21993 #[tokio::test]
21996 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21997 ask_test_home();
21998 let tmp = tempfile::tempdir().unwrap();
21999 let repo = tmp.path().join("repo");
22000 std::fs::create_dir_all(&repo).unwrap();
22001 init_repo(&repo);
22002 let origin = tmp.path().join("origin.git");
22003 let g = |dir: &Path, args: &[&str]| {
22004 let out = std::process::Command::new("git")
22005 .args(args)
22006 .current_dir(dir)
22007 .quiet()
22008 .output()
22009 .expect("spawn git");
22010 assert!(
22011 out.status.success(),
22012 "git {args:?}: {}",
22013 String::from_utf8_lossy(&out.stderr)
22014 );
22015 };
22016 g(
22017 tmp.path(),
22018 &[
22019 "clone",
22020 "--bare",
22021 "-q",
22022 repo.to_str().unwrap(),
22023 origin.to_str().unwrap(),
22024 ],
22025 );
22026 g(
22027 &repo,
22028 &["remote", "add", "origin", origin.to_str().unwrap()],
22029 );
22030 let c1 = git::rev_parse(&repo, "main").await.unwrap();
22031
22032 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
22034 g(&repo, &["add", "-A"]);
22035 g(
22036 &repo,
22037 &["commit", "-q", "-m", "chore(deps): update a crate"],
22038 );
22039 g(&repo, &["push", "-q", "origin", "main"]);
22040 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
22041 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22042 g(&repo, &["add", "-A"]);
22043 g(
22044 &repo,
22045 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
22046 );
22047 g(&repo, &["checkout", "-q", "main"]);
22048
22049 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
22050 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
22051 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
22053 assert_eq!(fallback, c1);
22054
22055 let mut state = review_state(&[
22056 "chore(deps): update a crate",
22057 "fix(daemon): apply a chosen action",
22058 ]);
22059 state.repo = repo.clone();
22060 state.base_branch = "main".to_owned();
22061 state.base_commit = c1;
22062 refresh_reviewed_commits(&mut state, "feat/own").await;
22063 assert_eq!(
22064 state.reviewed_commits,
22065 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
22066 );
22067 assert_eq!(
22068 review_title(&state).as_deref(),
22069 Some("fix(daemon): apply a chosen action")
22070 );
22071 assert_eq!(
22072 leaked_subjects(&state, "feat/own").await,
22073 Some(vec!["chore(deps): update a crate".to_owned()])
22074 );
22075 state.config.merge.remote = "nowhere".to_owned();
22078 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
22079 }
22080
22081 struct SummaryFixture {
22085 _tmp: tempfile::TempDir,
22086 repo: PathBuf,
22087 reply: PathBuf,
22088 runner: Runner,
22089 }
22090
22091 fn summary_fixture() -> SummaryFixture {
22092 ask_test_home();
22093 let tmp = tempfile::tempdir().unwrap();
22094 let repo = tmp.path().join("repo");
22095 std::fs::create_dir_all(&repo).unwrap();
22096 init_repo(&repo);
22097 let origin = tmp.path().join("origin.git");
22098 let g = |args: &[&str]| {
22099 let out = std::process::Command::new("git")
22100 .args(args)
22101 .current_dir(&repo)
22102 .quiet()
22103 .output()
22104 .expect("spawn git");
22105 assert!(
22106 out.status.success(),
22107 "git {args:?}: {}",
22108 String::from_utf8_lossy(&out.stderr)
22109 );
22110 };
22111 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
22112 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
22113 g(&["push", "-q", "origin", "main"]);
22114 g(&["checkout", "-q", "-b", "feat/own"]);
22115 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22116 g(&["add", "-A"]);
22117 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
22118 let reply = tmp.path().join("reply.txt");
22119 std::fs::write(
22120 &reply,
22121 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
22122 )
22123 .unwrap();
22124 let mut runner = runner_at(RunStatus::Gating);
22125 let mut state = review_state(&["feat(own): add the own file"]);
22126 state.repo = repo.clone();
22127 state.base_branch = "main".to_owned();
22128 state.candidates[0].branch = "feat/own".to_owned();
22129 state.candidates[0].worktree = repo.clone();
22130 let mut author = spec("alpha");
22131 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
22132 author
22133 .env
22134 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
22135 state.config.agents = vec![author];
22136 runner.state = state;
22137 SummaryFixture {
22138 _tmp: tmp,
22139 repo,
22140 reply,
22141 runner,
22142 }
22143 }
22144
22145 async fn posted(f: &mut SummaryFixture) -> PrMessage {
22146 let winner = f.runner.state.candidates[0].clone();
22147 f.runner
22148 .guarded_pr_message(&winner, None, true)
22149 .await
22150 .unwrap()
22151 .expect("an English description never parks")
22152 }
22153
22154 fn clean_round_with_minor_findings() -> ReviewRound {
22155 let mut round = passing_round(2, None);
22156 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
22157 round.reviews = vec![ReviewRecord {
22158 attempts: 0,
22159 reviewer: 1,
22160 agent: "alpha".into(),
22161 summary: "内部レビュー".into(),
22162 findings: vec![
22163 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
22164 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
22165 ],
22166 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
22167 failed: None,
22168 duration_ms: 0,
22169 }];
22170 round
22171 }
22172
22173 #[tokio::test]
22174 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
22175 for review_only in [false, true] {
22176 let mut f = summary_fixture();
22177 if !review_only {
22178 f.runner.state.candidates[0].agent = "alpha".into();
22179 }
22180 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
22181 assert!(f.runner.state.reviews[0].clean);
22182 assert!(f.runner.state.reviews[0].fix.is_none());
22183 assert!(f.runner.state.open_findings().is_empty());
22184 assert_eq!(
22185 crate::pr_outcome::required_finding_ids(&f.runner.state),
22186 ["R2-1-1", "R2-1-2"]
22187 );
22188 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22189 let evidence = crate::pr_outcome::summary_evidence(
22190 &f.runner.state,
22191 crate::pr_outcome::FinalHead::At(&head),
22192 );
22193 assert!(
22194 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
22195 );
22196 let omitted = posted(&mut f).await;
22198 assert!(omitted.body.contains("No description"));
22199 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
22200 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
22201 assert!(f.runner.state.change_summary.is_none());
22202 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
22203 let described = posted(&mut f).await;
22204 assert!(described.body.contains("improve documentation for callers"));
22205 assert!(described.body.contains("remaining edge case"));
22206 assert!(described.body.contains("## Open review findings"));
22207 assert!(!described.body.contains("未修正"));
22208 }
22209 }
22210
22211 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
22212 let mut f = summary_fixture();
22213 if !review_only {
22214 f.runner.state.candidates[0].agent = "alpha".into();
22215 }
22216 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22217 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
22218 fixed
22219 .fix
22220 .as_mut()
22221 .unwrap()
22222 .rejected
22223 .push(crate::verdict::Rejection {
22224 id: "R1-1-1".into(),
22225 why: "呼び出されないコードのため辞退".into(),
22226 });
22227 let mut clean = clean_round_with_minor_findings();
22228 clean.verified_head = Some(head.clone());
22229 f.runner.state.reviews = vec![fixed, clean];
22230 ok_gate(&mut f.runner.state, Some(&head));
22231 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
22232 f.runner.state.config.agents.push(scripted("beta", reply));
22233 f.runner.state.config.roles.language_judge =
22234 Some(crate::config::AgentChoice::One("beta".into()));
22235 f
22236 }
22237
22238 #[tokio::test]
22239 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
22240 for review_only in [false, true] {
22241 let mut f = rewrite_fixture_with_findings(review_only,
22242 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
22243 let m = posted(&mut f).await;
22244 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22245 let expected =
22246 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
22247 assert!(m.body.contains(&expected));
22248 assert!(m.body.contains("## Review rounds"));
22249 assert!(
22250 m.body
22251 .contains("Gate (`verify.gate`): passed on the final commit")
22252 );
22253 assert!(
22254 m.body
22255 .contains("Tests (`verify.e2e`): passed on the final commit")
22256 );
22257 let c = f.runner.state.change_summary.as_ref().unwrap();
22258 assert!(
22259 m.body
22260 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
22261 );
22262 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22263 assert!(crate::pr_outcome::covers_required_findings(
22264 &f.runner.state,
22265 &m.body
22266 ));
22267 assert!(m.body.contains("documentation remains incomplete"));
22268 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
22269 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
22270 assert!(f.runner.state.github_text.is_none());
22271 }
22272 }
22273
22274 #[tokio::test]
22275 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
22276 for review_only in [false, true] {
22277 for reply in [
22278 "TITLE: fix: public description\nA short English description.",
22279 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
22280 ] {
22281 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
22282 let winner = f.runner.state.candidates[0].clone();
22283 let posted = f
22284 .runner
22285 .guarded_pr_message(&winner, None, true)
22286 .await
22287 .unwrap();
22288 assert!(posted.is_none());
22289 assert!(f.runner.state.parked);
22290 let withheld = f.runner.state.github_text.as_ref().unwrap();
22291 assert!(
22292 withheld.rewrite.is_none(),
22293 "an incomplete rewrite must not be selectable"
22294 );
22295 let q = ask::Questions::open()
22296 .get(withheld.question.as_ref().unwrap())
22297 .unwrap();
22298 assert!(
22299 !q.choices
22300 .contains(&crate::github_text::USE_REWRITE.to_owned())
22301 );
22302 assert!(
22303 q.detail.contains("R1-1-1")
22304 && q.detail.contains("R2-1-1")
22305 && q.detail.contains("R2-1-2")
22306 );
22307 }
22308 }
22309 }
22310
22311 #[tokio::test]
22312 async fn japanese_reports_stay_local_in_both_public_run_modes() {
22313 for review_only in [false, true] {
22314 let mut f = summary_fixture();
22315 if !review_only {
22316 f.runner.state.candidates[0].agent = "alpha".into();
22317 }
22318 f.runner.state.instruction = "内部タスクの説明".into();
22319 f.runner.state.candidates[0].summary = "内部の候補報告".into();
22320 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22321 f.runner.state.reviews = vec![
22322 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22323 round_with_notes(2, Some("二回目の内部修正報告です。")),
22324 passing_round(3, Some(&head)),
22325 ];
22326 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22327 ok_gate(&mut f.runner.state, Some(&head));
22328 let captured = f.repo.join("public-prompt.txt");
22329 let author = &mut f.runner.state.config.agents[0];
22330 author.command = vec![
22331 "sh".into(),
22332 "-c".into(),
22333 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22334 ];
22335 author.env.insert(
22336 "PROMPT_FILE".into(),
22337 captured.to_string_lossy().into_owned(),
22338 );
22339 let m = posted(&mut f).await;
22340 assert!(m.body.contains("Adds `own.txt`"));
22341 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22342 assert!(m.body.contains("R1-1-1"));
22343 assert!(
22344 m.body
22345 .contains("Gate (`verify.gate`): passed on the final commit")
22346 );
22347 assert!(
22348 m.body
22349 .contains("Tests (`verify.e2e`): passed on the final commit")
22350 );
22351 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22352 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22353 let prompt = std::fs::read_to_string(captured).unwrap();
22354 assert!(prompt.contains(&format!(
22355 "git diff {} {head}",
22356 f.runner.state.change_summary.as_ref().unwrap().base
22357 )));
22358 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22359 assert_eq!(
22360 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22361 "二回目の内部修正報告です。"
22362 );
22363 }
22364 }
22365
22366 #[tokio::test]
22367 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22368 for review_only in [false, true] {
22369 let mut f = summary_fixture();
22370 if !review_only {
22371 f.runner.state.candidates[0].agent = "alpha".into();
22372 }
22373 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22374 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22375 std::fs::write(&f.reply, "no usable description").unwrap();
22376 let failed = posted(&mut f).await;
22377 assert!(
22378 failed
22379 .body
22380 .contains("No description of what this change does could be generated")
22381 );
22382 assert!(failed.body.is_ascii());
22383 assert!(f.runner.state.github_text.is_none());
22384 assert!(f.runner.state.change_summary.is_none());
22385 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22386 let evidence = crate::pr_outcome::summary_evidence(
22387 &f.runner.state,
22388 crate::pr_outcome::FinalHead::At(&head),
22389 );
22390 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22391 assert!(
22392 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22393 .exists()
22394 );
22395 f.runner.state =
22396 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22397 std::fs::write(
22398 &f.reply,
22399 "TITLE: feat(own): add the own file\nAdds the committed own file.",
22400 )
22401 .unwrap();
22402 let retried = posted(&mut f).await;
22403 assert!(retried.body.contains("Adds the committed own file."));
22404 assert_eq!(
22405 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22406 "日本語の修正報告"
22407 );
22408 }
22409 }
22410
22411 #[tokio::test]
22412 async fn same_head_verification_and_findings_invalidate_public_prose() {
22413 let mut f = summary_fixture();
22414 posted(&mut f).await;
22415 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22416 let original = f
22417 .runner
22418 .state
22419 .change_summary
22420 .as_ref()
22421 .unwrap()
22422 .evidence
22423 .clone();
22424 ok_gate(&mut f.runner.state, Some(&head));
22425 std::fs::write(
22426 &f.reply,
22427 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22428 )
22429 .unwrap();
22430 let verified = posted(&mut f).await;
22431 assert!(verified.body.contains("now verified"));
22432 assert_ne!(
22433 f.runner.state.change_summary.as_ref().unwrap().evidence,
22434 original
22435 );
22436 let mut round = round_with_notes(1, Some("内部報告"));
22437 round.reviews.push(ReviewRecord {
22438 attempts: 0,
22439 reviewer: 1,
22440 agent: "alpha".into(),
22441 summary: "内部レビュー".into(),
22442 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22443 vote: None,
22444 failed: None,
22445 duration_ms: 0,
22446 });
22447 round
22448 .fix
22449 .as_mut()
22450 .unwrap()
22451 .rejected
22452 .push(crate::verdict::Rejection {
22453 id: "R1-1-1".into(),
22454 why: "呼び出し元から到達できない".into(),
22455 });
22456 f.runner.state.reviews = vec![round];
22457 let omitted = posted(&mut f).await;
22459 assert!(omitted.body.contains("No description"));
22460 assert!(omitted.body.contains("R1-1-1"));
22461 assert!(!omitted.body.contains("到達"));
22462 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22463 let described = posted(&mut f).await;
22464 assert!(described.body.contains("unreachable from callers"));
22465 assert!(described.body.contains("R1-1-1"));
22466 assert!(!described.body.contains("到達"));
22467 let evidence = f
22468 .runner
22469 .state
22470 .change_summary
22471 .as_ref()
22472 .unwrap()
22473 .evidence
22474 .clone();
22475 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22476 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22477 assert!(posted(&mut f).await.body.contains("rationale updated"));
22478 assert_ne!(
22479 f.runner.state.change_summary.as_ref().unwrap().evidence,
22480 evidence
22481 );
22482 f.runner
22484 .state
22485 .change_summary
22486 .as_mut()
22487 .unwrap()
22488 .evidence
22489 .clear();
22490 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22491 assert!(posted(&mut f).await.body.contains("current specification"));
22492 }
22493
22494 #[tokio::test]
22495 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22496 let mut f = summary_fixture();
22497 let m = posted(&mut f).await;
22498 assert_eq!(m.title, "feat(own): add the own file");
22499 assert!(
22500 m.body
22501 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22502 "{}",
22503 m.body
22504 );
22505 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22506 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22507 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22508 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22509 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22510 let kept = f.runner.state.change_summary.as_ref().unwrap();
22511 assert_eq!(
22512 (kept.head.as_str(), kept.base.as_str()),
22513 (head.as_str(), base.as_str())
22514 );
22515 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22516 }
22517
22518 #[tokio::test]
22519 async fn review_only_summary_receives_the_configured_github_language() {
22520 let mut f = summary_fixture();
22521 f.runner.state.config.graph.github_language = "ja".into();
22522 let captured = f.repo.join("summary-prompt.txt");
22523 let author = &mut f.runner.state.config.agents[0];
22524 author.command = vec![
22525 "sh".into(),
22526 "-c".into(),
22527 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22528 ];
22529 author.env.insert(
22530 "PROMPT_FILE".into(),
22531 captured.to_string_lossy().into_owned(),
22532 );
22533 std::fs::write(
22534 &f.reply,
22535 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22536 )
22537 .unwrap();
22538 let m = posted(&mut f).await;
22539 let prompt = std::fs::read_to_string(captured).unwrap();
22540 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22541 assert!(!prompt.contains("in English"));
22542 assert_eq!(m.title, "feat(own): ファイルを追加");
22543 assert!(m.body.contains("own.txt を追加します。"));
22544 }
22545
22546 #[tokio::test]
22547 async fn review_fixes_come_after_the_whole_change_summary() {
22548 let mut f = summary_fixture();
22549 f.runner.state.reviews = vec![
22550 round_with_notes(1, Some("handled the PATH case")),
22551 round_with_notes(2, None),
22552 ];
22553 let m = posted(&mut f).await;
22554 let summary = m.body.find("## Summary").expect("summary");
22555 let fixes = m.body.find("## Review rounds").expect("review rounds");
22556 assert!(summary < fixes, "{}", m.body);
22557 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22558 assert!(m.body.contains("## Final verification"), "{}", m.body);
22559 }
22560
22561 #[tokio::test]
22562 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22563 let mut f = summary_fixture();
22564 let first = posted(&mut f).await;
22565 let first_head = f.runner.state.change_summary.clone().unwrap().head;
22566 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22568 let again = posted(&mut f).await;
22569 assert_eq!((again.title, again.body), (first.title, first.body));
22570
22571 let g = |args: &[&str]| {
22572 let out = std::process::Command::new("git")
22573 .args(args)
22574 .current_dir(&f.repo)
22575 .quiet()
22576 .output()
22577 .expect("spawn git");
22578 assert!(out.status.success(), "git {args:?}");
22579 };
22580 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22581 g(&["add", "-A"]);
22582 g(&["commit", "-q", "-m", "fix(own): handle more"]);
22583 let second = posted(&mut f).await;
22584 let kept = f.runner.state.change_summary.clone().unwrap();
22585 assert_ne!(kept.head, first_head);
22586 assert!(second.body.contains("Not read yet."), "{}", second.body);
22587 assert!(!second.body.contains("so the first summary is read"));
22588 assert_eq!(second.title, "feat(own): other");
22589 }
22590
22591 #[tokio::test]
22592 async fn an_unavailable_summary_is_stated_and_never_invented() {
22593 let mut f = summary_fixture();
22594 std::fs::write(&f.reply, "no title line here").unwrap();
22595 let m = posted(&mut f).await;
22596 assert!(
22597 m.body
22598 .contains("No description of what this change does could be generated"),
22599 "{}",
22600 m.body
22601 );
22602 assert!(m.body.contains("internal reports remain in local records"));
22603 assert!(f.runner.state.change_summary.is_none());
22604 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22605 let winner = f.runner.state.candidates[0].clone();
22607 let preview = f
22608 .runner
22609 .guarded_pr_message(&winner, None, false)
22610 .await
22611 .unwrap()
22612 .unwrap();
22613 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22614 }
22615
22616 #[test]
22617 fn pr_message_review_single_commit_uses_its_subject() {
22618 let state = review_state(&["feat(nats): per-role user"]);
22619 let m = pr_message(&state, 'A');
22620 assert_eq!(m.title, "feat(nats): per-role user");
22621 assert!(!m.body.contains("Review the work already"), "{}", m.body);
22622 assert!(m.body.contains("## Commits under review"), "{}", m.body);
22623 }
22624
22625 #[test]
22626 fn pr_message_review_multi_commit_takes_the_oldest() {
22627 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22628 let m = pr_message(&state, 'A');
22629 assert_eq!(m.title, "feat: the change");
22630 for s in ["feat: the change", "fix: typo", "fix: again"] {
22631 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22632 }
22633 }
22634
22635 #[test]
22636 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22637 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22638 let mut state = review_state(&[first, "fix: later fixup"]);
22639 state.candidates[0].branch = "機能/ログイン".to_owned();
22640 let m = pr_message(&state, 'A');
22641 assert!(
22642 m.title.starts_with("chore: land candidate A of run"),
22643 "{}",
22644 m.title
22645 );
22646 }
22647 }
22648
22649 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22650 BranchFacts {
22651 commits: commits
22652 .iter()
22653 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22654 .collect(),
22655 stat: stat.to_owned(),
22656 }
22657 }
22658
22659 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22660 let mut r = review_round(true, 0, 1, 1, true, true);
22661 r.round = round;
22662 r.fix = notes.map(|n| FixRecord {
22663 agent: "fixer".to_owned(),
22664 addressed: Vec::new(),
22665 rejected: Vec::new(),
22666 notes: n.to_owned(),
22667 committed: true,
22668 failed: None,
22669 duration_ms: 0,
22670 continuation: None,
22671 });
22672 r
22673 }
22674
22675 #[test]
22676 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22677 let state = review_state(&["ignored"]);
22678 let f = facts(
22679 &[
22680 (
22681 "fix(login): resolve PATH on macOS",
22682 "Login shells skip rc files.",
22683 ),
22684 ("fix: address review", ""),
22685 ],
22686 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22687 );
22688 let m = pr_message_with(&state, 'A', Some(&f));
22689 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22690 assert!(
22691 m.body
22692 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22693 "{}",
22694 m.body
22695 );
22696 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22697 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22698 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22699 for banned in [
22700 "Review the work already",
22701 "no task statement",
22702 "Original task",
22703 ] {
22704 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22705 }
22706 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22707 }
22708
22709 #[test]
22710 fn pr_message_review_truncates_a_huge_first_commit_body() {
22711 let state = review_state(&["ignored"]);
22712 let f = facts(
22713 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22714 "s",
22715 );
22716 let m = pr_message_with(&state, 'A', Some(&f));
22717 assert!(m.body.len() < 30_000, "{}", m.body.len());
22718 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22719 assert!(
22720 m.body.contains("- ... 1 more commit(s)"),
22721 "{}",
22722 m.body.len()
22723 );
22724 assert!(m.body.ends_with("magi:candidate-a\n"));
22725 }
22726
22727 #[test]
22728 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22729 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22730 assert_eq!(m.title, "feat: x");
22731 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22732 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22733 assert!(!m.body.contains("no task statement"), "{}", m.body);
22734 }
22735
22736 #[test]
22737 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22738 let mut state = review_state(&["日本語の件名"]);
22739 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22740 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22741 state.candidates[0].branch = "機能/ログイン".to_owned();
22742 assert!(
22743 pr_message(&state, 'A')
22744 .title
22745 .starts_with("chore: land candidate A")
22746 );
22747 }
22748
22749 #[test]
22750 fn pr_message_review_fixes_survive_a_clean_final_round() {
22751 let mut state = review_state(&["feat: x"]);
22752 state.reviews = vec![
22753 round_with_notes(1, Some("handled the PATH case")),
22754 round_with_notes(2, None),
22755 ];
22756 let body = pr_message(&state, 'A').body;
22757 assert!(
22758 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22759 "{body}"
22760 );
22761 assert!(body.contains("### Round 1"), "{body}");
22762
22763 state.reviews = vec![
22764 round_with_notes(1, Some("first fix")),
22765 round_with_notes(2, Some("")),
22766 round_with_notes(3, Some("second fix")),
22767 round_with_notes(4, None),
22768 ];
22769 let body = pr_message(&state, 'A').body;
22770 assert!(
22771 body.contains("### Round 1") && !body.contains("first fix"),
22772 "{body}"
22773 );
22774 assert!(
22775 body.contains("### Round 3") && !body.contains("second fix"),
22776 "{body}"
22777 );
22778 assert!(body.contains("### Round 2"), "{body}");
22779 }
22780
22781 const FINAL: &str = "abcdef1234567890";
22782
22783 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22784 let mut r = round_with_notes(round, None);
22785 r.verified_head = verified.map(str::to_owned);
22786 r
22787 }
22788
22789 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22790 state.gate = vec![CommandOutcome {
22791 command: "gate".to_owned(),
22792 code: Some(0),
22793 output_tail: String::new(),
22794 duration_ms: 0,
22795 resource_blocked: false,
22796 }];
22797 state.gate_ran = true;
22798 state.gate_head = head.map(str::to_owned);
22799 }
22800
22801 fn body_at(state: &RunState) -> String {
22802 pr_message_raw(
22803 state,
22804 'A',
22805 None,
22806 crate::pr_outcome::FinalHead::At(FINAL),
22807 ChangeNote::NotAsked,
22808 )
22809 .body
22810 }
22811
22812 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22813
22814 #[test]
22815 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22816 let mut state = review_state(&["feat: x"]);
22817 state.reviews = vec![
22818 round_with_notes(1, Some(SANDBOXED)),
22819 passing_round(2, Some(FINAL)),
22820 ];
22821 ok_gate(&mut state, Some(FINAL));
22822 let body = body_at(&state);
22823 assert!(!body.contains("Handled the PATH case."), "{body}");
22824 assert!(!body.contains("could not commit"), "{body}");
22825 assert!(!body.contains("could not run"), "{body}");
22826 assert!(
22827 body.contains("The fix is committed on the branch"),
22828 "{body}"
22829 );
22830 assert!(!body.contains("sandbox"), "{body}");
22831 assert!(
22832 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22833 "{body}"
22834 );
22835 assert!(
22836 body.contains("Gate (`verify.gate`): passed on the final commit"),
22837 "{body}"
22838 );
22839 assert!(
22841 state.reviews[0]
22842 .fix
22843 .as_ref()
22844 .unwrap()
22845 .notes
22846 .contains("could not commit")
22847 );
22848 }
22849
22850 #[test]
22851 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22852 let mut state = review_state(&["feat: x"]);
22853 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22854 r1.fix.as_mut().unwrap().committed = false;
22855 let mut r2 = passing_round(2, Some(FINAL));
22856 r2.e2e[0].code = Some(1);
22857 state.reviews = vec![r1, r2];
22858 let body = body_at(&state);
22859 assert!(!body.contains("could not commit"), "{body}");
22860 assert!(!body.contains("could not run"), "{body}");
22861 assert!(body.contains("This round produced no commit"), "{body}");
22862 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22863 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22864 assert!(!body.contains("passed"), "{body}");
22865 }
22866
22867 #[test]
22868 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22869 let mut state = review_state(&["feat: x"]);
22870 state.reviews = vec![
22871 round_with_notes(1, Some(SANDBOXED)),
22872 passing_round(2, Some("0123456789abcdef")),
22873 ];
22874 ok_gate(&mut state, Some("0123456789abcdef"));
22875 let body = body_at(&state);
22876 assert!(!body.contains("HTTP tests could not run"), "{body}");
22877 assert!(!body.contains("could not commit"), "{body}");
22879 assert!(
22880 body.contains("passed on an earlier commit (`0123456`)"),
22881 "{body}"
22882 );
22883 assert!(!body.contains("passed on the final commit"), "{body}");
22884
22885 ok_gate(&mut state, None);
22887 let body = body_at(&state);
22888 assert!(
22889 body.contains("passed on a commit that was not recorded"),
22890 "{body}"
22891 );
22892 }
22893
22894 #[test]
22895 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22896 let state = state_with_summary(
22897 "add retries to the client",
22898 "TITLE: feat: retries\n\nDid it.",
22899 );
22900 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22901 assert_eq!(m.title, "feat: retries");
22902 assert!(
22903 m.body.contains("<summary>Original task</summary>"),
22904 "{}",
22905 m.body
22906 );
22907 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22908 assert!(
22909 m.body
22910 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22911 "{}",
22912 m.body
22913 );
22914 }
22915
22916 #[test]
22917 fn pr_message_review_bounds_a_long_english_subject() {
22918 let long = format!("feat: {}", "word ".repeat(100));
22919 let m = pr_message(&review_state(&[&long]), 'A');
22920 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22921 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22922 }
22923
22924 #[test]
22925 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22926 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22927 let m = pr_message(&state, 'A');
22928 assert_eq!(m.title, "add retries");
22929 assert!(m.body.contains("<summary>Original task</summary>"));
22930 assert!(!m.body.contains("Commits under review"));
22931 assert_eq!(landing_subject_source(&state), state.instruction);
22932 }
22933
22934 #[test]
22935 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22936 let state = review_state(&["feat: the change", "fix: typo"]);
22937 let source = landing_subject_source(&state);
22938 assert_eq!(land::merge_subject("", &source), "feat: the change");
22939 assert_eq!(
22940 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22941 "feat: the change"
22942 );
22943 assert_eq!(
22945 land::merge_subject("feat: renamed by hand", &source),
22946 "feat: renamed by hand"
22947 );
22948 let blank = review_state(&["日本語"]);
22949 assert!(
22950 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22951 );
22952 }
22953
22954 #[test]
22955 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22956 let state = review_state(&["feat: the change"]);
22957 let source = landing_subject_source(&state);
22958 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22959 assert_eq!(
22960 land::merge_subject(landing_title(&state, old), &source),
22961 "feat: the change"
22962 );
22963 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22964 let task = state_with_summary("add retries", "");
22965 assert_eq!(landing_title(&task, old), old);
22966 }
22967
22968 #[test]
22969 fn pr_message_describes_the_change_not_the_task() {
22970 let state = state_with_summary(
22971 "今回やってほしいこと: results projector を直す",
22972 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22973 );
22974 let m = pr_message(&state, 'A');
22975 assert_eq!(m.title, "fix(web): batch the runs list reads");
22976 assert!(
22977 m.body.starts_with("## Summary\n\n- reads run.json once"),
22978 "{}",
22979 m.body
22980 );
22981 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22982 let task_at = m.body.find("今回やってほしいこと").unwrap();
22983 let details_at = m.body.find("<details>").unwrap();
22984 assert!(
22985 details_at < task_at,
22986 "the task lives inside <details>: {}",
22987 m.body
22988 );
22989 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22990 assert!(m.body.contains("magi:candidate-a"));
22991 }
22992
22993 #[test]
22994 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22995 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22996 let m = pr_message(&state, 'A');
22997 assert_eq!(m.title, "add retries");
22998 assert!(
22999 m.body.contains("## Summary\n\n- did some things"),
23000 "{}",
23001 m.body
23002 );
23003
23004 let none = RunState::new(
23005 PathBuf::from("/repo"),
23006 "main".to_owned(),
23007 "abc1234".to_owned(),
23008 "add retries".to_owned(),
23009 Config::default(),
23010 );
23011 let m = pr_message(&none, 'A');
23012 assert_eq!(m.title, "add retries");
23013 assert!(!m.body.contains("## Summary"), "{}", m.body);
23014 }
23015
23016 #[test]
23017 fn pr_message_refuses_the_candidate_commit_subject() {
23018 for bad in [
23019 "TITLE: magi: candidate A (uncommitted work)",
23020 "TITLE: chore: stuff (uncommitted work)",
23021 "TITLE: ",
23022 ] {
23023 let state = state_with_summary("add retries", bad);
23024 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
23025 }
23026 }
23027
23028 #[test]
23029 fn pr_message_bounds_a_very_long_task_and_title() {
23030 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
23031 let state = state_with_summary(&long, "- nothing");
23032 let m = pr_message(&state, 'A');
23033 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23034 assert!(!m.title.contains('\n'));
23035
23036 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
23037 let m = pr_message(&state, 'A');
23038 assert!(m.title.starts_with("feat: "));
23039 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23040 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
23041 }
23042
23043 fn long_title_of(instruction: &str) -> String {
23044 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
23045 }
23046
23047 #[test]
23048 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
23049 let first = "Make the landing path keep a readable title for long tasks";
23050 let line = format!(
23051 "{first}. {}",
23052 "And then keep going with more words ".repeat(20)
23053 );
23054 let t = long_title_of(&line);
23055 assert_eq!(t, first);
23056 assert!(!t.starts_with("chore: land"));
23057 }
23058
23059 #[test]
23060 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
23061 let line = "word ".repeat(200);
23062 let t = long_title_of(&line);
23063 assert!(t.ends_with("word..."), "{t}");
23064 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
23065 }
23066
23067 #[test]
23068 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
23069 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
23070 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
23071 }
23072 }
23073
23074 #[test]
23075 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
23076 let mut state = state_with_summary("再試行を追加する", "- nothing");
23077 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
23078 state.config.graph.github_language = "ja".into();
23079 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
23080 }
23081
23082 #[test]
23083 fn pr_message_title_limit_is_exact() {
23084 let at = "a".repeat(PR_TITLE_MAX);
23085 assert_eq!(long_title_of(&at), at);
23086 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
23087 assert!(over.ends_with("..."), "{over}");
23088 assert_eq!(over.chars().count(), PR_TITLE_MAX);
23089 }
23090
23091 #[test]
23092 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
23093 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
23094 let t = long_title_of(&line);
23095 assert!(t.ends_with("..."), "{t}");
23096 assert!(t.is_ascii(), "{t}");
23097 }
23098
23099 #[test]
23100 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
23101 let line = format!(
23102 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
23103 "plus more filler words ".repeat(20)
23104 );
23105 let t = long_title_of(&line);
23106 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
23107 }
23108
23109 #[test]
23110 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
23111 let line = format!("\n\n# {}", "title words ".repeat(40));
23112 let state = state_with_summary(&line, "- nothing");
23113 let m = pr_message(&state, 'A');
23114 assert!(m.title.starts_with("title words"), "{}", m.title);
23115 assert_eq!(
23116 land::merge_subject(&m.title, &landing_subject_source(&state)),
23117 m.title
23118 );
23119 assert_eq!(
23121 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
23122 "feat: renamed by hand"
23123 );
23124 }
23125
23126 #[tokio::test]
23127 async fn github_text_rewrite_is_bounded_and_falls_back() {
23128 crate::run::pin_test_home();
23129 for (reply, accepted) in [
23130 (
23131 "TITLE: fix: retries\nAdd retries for failed requests.",
23132 true,
23133 ),
23134 (
23135 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
23136 false,
23137 ),
23138 (
23139 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
23140 true,
23141 ),
23142 (
23143 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
23144 true,
23145 ),
23146 (
23147 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
23148 true,
23149 ),
23150 (
23151 "TITLE: fix: retries\n日本語の説明です。 token=secret",
23152 false,
23153 ),
23154 ("TITLE: fix: retries\n", false),
23155 ] {
23156 let dir = tempfile::tempdir().unwrap();
23157 let mut runner = runner_at(RunStatus::Gating);
23158 runner.state = state_with_summary(
23159 "add retries",
23160 "TITLE: fix: retries\n日本語の説明を書きます。",
23161 );
23162 runner.state.repo = dir.path().to_owned();
23163 runner.state.candidates[0].worktree = dir.path().to_owned();
23164 let mut author = spec("alpha");
23165 author.command = vec![
23166 "sh".into(),
23167 "-c".into(),
23168 "printf '%s' \"$REWRITE_REPLY\"".into(),
23169 ];
23170 author.env.insert("REWRITE_REPLY".into(), reply.into());
23171 runner.state.config.agents = vec![author];
23172 seed_public_summary(&mut runner);
23173 let winner = runner.state.candidates[0].clone();
23174 let first = runner
23175 .guarded_pr_message(&winner, None, true)
23176 .await
23177 .unwrap();
23178 let message = if accepted {
23179 first.expect("a passing rewrite never asks")
23180 } else {
23181 assert!(first.is_none());
23183 assert!(runner.state.parked);
23184 let store = ask::Questions::open();
23185 let asked: Vec<_> = store
23186 .list()
23187 .into_iter()
23188 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23189 .collect();
23190 assert_eq!(asked.len(), 1);
23191 assert!(!asked[0].detail.contains("secret"));
23192 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
23193 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
23194 assert_eq!(
23195 asked[0]
23196 .choices
23197 .contains(&crate::github_text::USE_REWRITE.to_owned()),
23198 rewrite.is_some()
23199 );
23200 store
23203 .update(&asked[0].id, |q| {
23204 q.abandon("test");
23205 Ok(())
23206 })
23207 .unwrap();
23208 let again = runner
23209 .guarded_pr_message(&winner, None, true)
23210 .await
23211 .unwrap()
23212 .expect("an abandoned question falls back");
23213 let n = store
23214 .list()
23215 .into_iter()
23216 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23217 .count();
23218 assert_eq!(n, 1, "asked once per rejected text");
23219 again
23220 };
23221 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
23222 assert!(!message.body.contains("secret"));
23223 assert!(!message.body.contains("prefix"));
23224 assert!(!message.title.contains("secret"));
23225 assert_eq!(
23226 message.body.contains("Add retries for failed requests."),
23227 accepted
23228 );
23229 assert_eq!(
23230 runner.state.seats["github-text-rewrite"].turns, 1,
23231 "only one rewrite invocation"
23232 );
23233 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
23234 if accepted {
23235 assert!(runner.state.github_text.is_none());
23236 assert!(!runner.state.parked);
23237 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
23238 assert!(message.body.contains("## Final verification"));
23239 assert!(message.body.contains("Evidence: final diff"));
23240 assert!(
23241 message
23242 .body
23243 .contains(&format!("magi:run/{}", runner.state.id))
23244 );
23245 }
23246 if !accepted {
23247 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23248 assert!(
23249 message
23250 .body
23251 .contains(&format!("magi:run/{}", runner.state.id))
23252 );
23253 }
23254 }
23255 }
23256
23257 fn scripted(id: &str, reply: &str) -> AgentSpec {
23258 let mut a = spec(id);
23259 a.command = vec![
23260 "sh".into(),
23261 "-c".into(),
23262 "printf '%s' \"$REWRITE_REPLY\"".into(),
23263 ];
23264 a.env.insert("REWRITE_REPLY".into(), reply.into());
23265 a
23266 }
23267
23268 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
23269 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
23270
23271 fn seed_public_summary(runner: &mut Runner) {
23273 let repo = &runner.state.repo;
23274 let g = |args: &[&str]| {
23275 let out = std::process::Command::new("git")
23276 .args(args)
23277 .current_dir(repo)
23278 .quiet()
23279 .output()
23280 .unwrap();
23281 assert!(out.status.success(), "git {args:?}");
23282 String::from_utf8_lossy(&out.stdout).trim().to_owned()
23283 };
23284 g(&["init", "-q", "-b", "main"]);
23285 g(&[
23286 "-c",
23287 "user.name=Test",
23288 "-c",
23289 "user.email=test@example.invalid",
23290 "commit",
23291 "-q",
23292 "--allow-empty",
23293 "-m",
23294 "feat: fixture",
23295 ]);
23296 let head = g(&["rev-parse", "HEAD"]);
23297 g(&["update-ref", "refs/remotes/origin/main", &head]);
23298 let evidence = crate::pr_outcome::summary_evidence(
23299 &runner.state,
23300 crate::pr_outcome::FinalHead::At(&head),
23301 );
23302 let generated = &runner.state.candidates[0].summary;
23303 runner.state.change_summary = Some(crate::run::ChangeSummary {
23304 base: head.clone(),
23305 head,
23306 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23307 title: summary_title(generated).unwrap(),
23308 body: summary_without_title(generated),
23309 });
23310 }
23311
23312 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23313 let mut runner = runner_at(RunStatus::Gating);
23314 runner.state = state_with_summary("add retries", JA_SUMMARY);
23315 if review_only {
23316 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23317 }
23318 runner.state.repo = dir.to_owned();
23319 runner.state.candidates[0].worktree = dir.to_owned();
23320 runner.state.config.agents = agents;
23321 seed_public_summary(&mut runner);
23322 runner
23323 }
23324
23325 fn gate_questions(runner: &Runner) -> usize {
23326 ask::Questions::open()
23327 .list()
23328 .into_iter()
23329 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23330 .count()
23331 }
23332
23333 #[tokio::test]
23334 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23335 crate::run::pin_test_home();
23336 let dir = tempfile::tempdir().unwrap();
23337 let mut runner = rewrite_runner(
23338 dir.path(),
23339 true,
23340 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23341 );
23342 runner.state.config.roles.language_judge =
23343 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23344 let winner = runner.state.candidates[0].clone();
23345 let msg = runner
23346 .guarded_pr_message(&winner, None, true)
23347 .await
23348 .unwrap()
23349 .expect("an English rewrite posts without asking");
23350 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23351 assert!(msg.body.contains("<details>kept</details>"));
23352 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23353 assert_eq!(gate_questions(&runner), 0);
23354 assert!(runner.state.github_text.is_none());
23355 assert!(!runner.state.seats.contains_key("impl-A"));
23356 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23357 }
23358
23359 #[tokio::test]
23360 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23361 crate::run::pin_test_home();
23362 let dir = tempfile::tempdir().unwrap();
23363 let mut runner = rewrite_runner(
23364 dir.path(),
23365 false,
23366 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23367 );
23368 runner.state.config.roles.language_judge =
23369 Some(crate::config::AgentChoice::One("beta".into()));
23370 runner.state.seats.insert(
23371 "impl-A".into(),
23372 SeatState::new("impl-A", "alpha", runner.state.seed),
23373 );
23374 let winner = runner.state.candidates[0].clone();
23375 let msg = runner
23376 .guarded_pr_message(&winner, None, true)
23377 .await
23378 .unwrap()
23379 .expect("rewritten");
23380 assert!(msg.body.contains("<details>kept</details>"));
23381 assert_eq!(runner.state.seats["impl-A"].turns, 0);
23382 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23383 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23384 }
23385
23386 #[tokio::test]
23387 async fn rewrite_reuses_a_resumable_implementer_session() {
23388 crate::run::pin_test_home();
23389 let dir = tempfile::tempdir().unwrap();
23390 let mut runner = rewrite_runner(
23391 dir.path(),
23392 false,
23393 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23394 );
23395 runner.state.config.roles.language_judge =
23396 Some(crate::config::AgentChoice::One("beta".into()));
23397 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23398 seat.turns = 1;
23399 runner.state.seats.insert("impl-A".into(), seat);
23400 let winner = runner.state.candidates[0].clone();
23401 runner
23402 .guarded_pr_message(&winner, None, true)
23403 .await
23404 .unwrap()
23405 .expect("rewritten");
23406 assert_eq!(runner.state.seats["impl-A"].turns, 2);
23407 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23408 }
23409
23410 #[tokio::test]
23411 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23412 crate::run::pin_test_home();
23415 let dir = tempfile::tempdir().unwrap();
23416 let mut exhausted = spec("alpha");
23417 exhausted.command = vec!["false".into()];
23418 let mut runner = rewrite_runner(
23419 dir.path(),
23420 false,
23421 vec![exhausted, scripted("beta", EN_REWRITE)],
23422 );
23423 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23425 "alpha".into(),
23426 "beta".into(),
23427 ]));
23428 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23429 seat.turns = 1;
23430 runner.state.seats.insert("impl-A".into(), seat);
23431 let winner = runner.state.candidates[0].clone();
23432 let msg = runner
23433 .guarded_pr_message(&winner, None, true)
23434 .await
23435 .unwrap()
23436 .expect("the role rewrites after the implementer fails");
23437 assert!(msg.body.contains("`src/net.rs`"));
23438 assert_eq!(
23439 runner.state.seats["impl-A"].turns, 2,
23440 "the implementer was tried once"
23441 );
23442 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23443 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23444 }
23445
23446 #[tokio::test]
23447 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23448 crate::run::pin_test_home();
23449 let dir = tempfile::tempdir().unwrap();
23450 let mut broken = spec("bad");
23451 broken.command = vec!["false".into()];
23452 let mut runner = rewrite_runner(
23453 dir.path(),
23454 true,
23455 vec![
23456 broken,
23457 scripted("noformat", "Add retries."),
23458 scripted("good", EN_REWRITE),
23459 ],
23460 );
23461 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23462 "bad".into(),
23463 "noformat".into(),
23464 "good".into(),
23465 ]));
23466 let winner = runner.state.candidates[0].clone();
23467 let msg = runner
23468 .guarded_pr_message(&winner, None, true)
23469 .await
23470 .unwrap()
23471 .expect("the third agent rewrites");
23472 assert!(msg.body.contains("`src/net.rs`"));
23473 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23474 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23475 }
23476
23477 #[tokio::test]
23478 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23479 crate::run::pin_test_home();
23480 for (agents, role, saved) in [
23482 (
23483 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23484 Some("beta"),
23485 true,
23486 ),
23487 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23488 (Vec::new(), None, false),
23489 ] {
23490 let dir = tempfile::tempdir().unwrap();
23491 let mut runner = rewrite_runner(dir.path(), true, agents);
23492 runner.state.config.roles.language_judge =
23493 role.map(|r| crate::config::AgentChoice::One(r.into()));
23494 let winner = runner.state.candidates[0].clone();
23495 let first = runner
23496 .guarded_pr_message(&winner, None, true)
23497 .await
23498 .unwrap();
23499 assert!(first.is_none(), "withheld text parks the run");
23500 assert!(runner.state.parked);
23501 assert_eq!(gate_questions(&runner), 1);
23502 let g = runner.state.github_text.as_ref().unwrap();
23503 assert_eq!(g.rewrite.is_some(), saved);
23504 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23505 }
23506 }
23507
23508 #[tokio::test]
23509 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23510 crate::run::pin_test_home();
23511 let dir = tempfile::tempdir().unwrap();
23512 let mut runner = runner_at(RunStatus::Gating);
23513 runner.state =
23514 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23515 runner.state.repo = dir.path().to_owned();
23516 runner.state.candidates[0].worktree = dir.path().to_owned();
23517 let mut author = spec("alpha");
23518 author.command = vec![
23519 "sh".into(),
23520 "-c".into(),
23521 "printf '%s' \"$REWRITE_REPLY\"".into(),
23522 ];
23523 author.env.insert(
23524 "REWRITE_REPLY".into(),
23525 format!(
23526 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23527 "English sentence. ".repeat(600)
23528 ),
23529 );
23530 runner.state.config.agents = vec![author];
23531 seed_public_summary(&mut runner);
23532 let winner = runner.state.candidates[0].clone();
23533 assert!(
23534 runner
23535 .guarded_pr_message(&winner, None, true)
23536 .await
23537 .unwrap()
23538 .is_none()
23539 );
23540 let record = runner.state.github_text.clone().unwrap();
23541 let rewrite = record.rewrite.clone().unwrap();
23542 assert_eq!(rewrite.failed, ["body-language"]);
23543 assert!(
23544 rewrite
23545 .body
23546 .ends_with(&format!("magi:run/{}", runner.state.id))
23547 );
23548 let store = ask::Questions::open();
23549 let q = store.get(record.question.as_ref().unwrap()).unwrap();
23550 assert!(
23551 q.choices
23552 .contains(&crate::github_text::USE_REWRITE.to_owned())
23553 );
23554 for label in [
23555 "ORIGINAL",
23556 "FALLBACK",
23557 "REWRITE",
23558 "Rules still failed: body-language",
23559 "truncated",
23560 ] {
23561 assert!(q.detail.contains(label), "{label}");
23562 }
23563 let artifact = std::fs::read_to_string(crate::run::artifact_path(
23564 &runner.state,
23565 &crate::github_text::artifact_name(&record.fingerprint),
23566 ))
23567 .unwrap();
23568 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23569 runner.state =
23571 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23572 store
23573 .update(&q.id, |q| {
23574 q.say("まだ日本語")?;
23575 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23576 })
23577 .unwrap();
23578 assert!(
23579 runner
23580 .guarded_pr_message(&winner, None, true)
23581 .await
23582 .unwrap()
23583 .is_none()
23584 );
23585 let retry = runner.state.github_text.as_ref().unwrap();
23586 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23587 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23588 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23589 assert!(
23590 q.posting_comparison
23591 .as_ref()
23592 .unwrap()
23593 .matches_detail(&q.detail)
23594 );
23595 store
23596 .update(&q.id, |q| {
23597 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23598 })
23599 .unwrap();
23600 let message = runner
23601 .guarded_pr_message(&winner, None, true)
23602 .await
23603 .unwrap()
23604 .unwrap();
23605 assert_eq!(message.title, rewrite.title);
23606 assert_eq!(message.body, rewrite.body);
23607 assert_eq!(
23608 runner.state.github_text.as_ref().unwrap().choice,
23609 Some(GithubTextChoice::Rewrite)
23610 );
23611 let again = runner
23612 .guarded_pr_message(&winner, None, true)
23613 .await
23614 .unwrap()
23615 .unwrap();
23616 assert_eq!(again.title, message.title);
23617 assert_eq!(again.body, message.body);
23618 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23619 assert!(
23620 !runner
23621 .state
23622 .events
23623 .iter()
23624 .any(|e| e.message.contains(&rewrite.title)
23625 || e.message.contains("English sentence."))
23626 );
23627 }
23628
23629 #[tokio::test]
23630 async fn historical_posting_reply_never_expires_into_fallback() {
23631 crate::run::pin_test_home();
23632 let dir = tempfile::tempdir().unwrap();
23633 let mut runner = runner_at(RunStatus::Gating);
23634 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23635 runner.state.repo = dir.path().to_owned();
23636 runner.state.candidates[0].worktree = dir.path().to_owned();
23637 seed_public_summary(&mut runner);
23638 let winner = runner.state.candidates[0].clone();
23639 assert!(
23640 runner
23641 .guarded_pr_message(&winner, None, true)
23642 .await
23643 .unwrap()
23644 .is_none()
23645 );
23646 let id = runner
23647 .state
23648 .github_text
23649 .as_ref()
23650 .unwrap()
23651 .question
23652 .clone()
23653 .unwrap();
23654 let store = ask::Questions::open();
23655 let mut q = store.get(&id).unwrap();
23656 q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23657 q.answer(ask::Answer::Choice(
23658 crate::github_text::POST_ORIGINAL.into(),
23659 ))
23660 .unwrap();
23661 q.answer_delivered = true;
23662 store.put(&mut q).unwrap();
23663 let source = std::fs::read(store.path_of(&id)).unwrap();
23664 let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23665 std::fs::create_dir_all(&migration).unwrap();
23666 std::fs::write(
23667 migration.join("state.json"),
23668 serde_json::to_vec(&serde_json::json!({
23669 "schema": 1, "sources": [id], "pending": [id]
23670 }))
23671 .unwrap(),
23672 )
23673 .unwrap();
23674 for _ in 0..2 {
23675 assert!(
23676 runner
23677 .guarded_pr_message(&winner, None, true)
23678 .await
23679 .unwrap()
23680 .is_none()
23681 );
23682 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23683 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23684 }
23685 let home = crate::run::home();
23686 crate::task_chat::sweep(
23687 &crate::queue::Queue::at(home.join("queue")),
23688 &crate::talk::Talks::at(home.join("talks")),
23689 &store,
23690 &home,
23691 );
23692 let outbox = crate::task_inquiry::Outbox::at(&home);
23693 outbox
23694 .update_decision(&id, |d| {
23695 d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23696 d.source_answer_at = q.answered_at;
23697 Ok(true)
23698 })
23699 .unwrap();
23700 outbox
23701 .settle(
23702 &id,
23703 "owner-chat",
23704 "Owner confirms no publication remains necessary",
23705 None,
23706 )
23707 .unwrap();
23708 assert!(
23709 runner
23710 .guarded_pr_message(&winner, None, true)
23711 .await
23712 .unwrap()
23713 .is_none()
23714 );
23715 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23716 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23717 }
23718
23719 #[tokio::test]
23720 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23721 crate::run::pin_test_home();
23722 let dir = tempfile::tempdir().unwrap();
23723 let mut runner = runner_at(RunStatus::Gating);
23724 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23725 runner.state.repo = dir.path().to_owned();
23726 runner.state.candidates[0].worktree = dir.path().to_owned();
23727 seed_public_summary(&mut runner);
23728 let winner = runner.state.candidates[0].clone();
23729 assert!(
23730 runner
23731 .guarded_pr_message(&winner, None, true)
23732 .await
23733 .unwrap()
23734 .is_none()
23735 );
23736 let record = runner.state.github_text.as_ref().unwrap();
23737 assert!(record.rewrite.is_none());
23738 let mut old = serde_json::to_value(record).unwrap();
23740 old.as_object_mut().unwrap().remove("rewrite");
23741 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23742 assert!(old.rewrite.is_none());
23743 let id = record.question.clone().unwrap();
23744 ask::Questions::open()
23745 .update(&id, |q| {
23746 q.choices.push(crate::github_text::USE_REWRITE.into());
23747 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23748 })
23749 .unwrap();
23750 let message = runner
23751 .guarded_pr_message(&winner, None, true)
23752 .await
23753 .unwrap()
23754 .unwrap();
23755 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23756 assert_eq!(
23757 runner.state.github_text.as_ref().unwrap().choice,
23758 Some(GithubTextChoice::Fallback)
23759 );
23760 }
23761
23762 #[tokio::test]
23763 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23764 crate::run::pin_test_home();
23765 let dir = tempfile::tempdir().unwrap();
23766 let mut runner = runner_at(RunStatus::Gating);
23767 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23768 runner.state.repo = dir.path().to_owned();
23769 runner.state.candidates[0].worktree = dir.path().to_owned();
23770 seed_public_summary(&mut runner);
23771 let winner = runner.state.candidates[0].clone();
23772 assert!(
23773 runner
23774 .guarded_pr_message(&winner, None, true)
23775 .await
23776 .unwrap()
23777 .is_none()
23778 );
23779 let store = ask::Questions::open();
23780 let find = |run: &str| {
23781 store
23782 .list()
23783 .into_iter()
23784 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23785 .unwrap()
23786 };
23787 let run = runner.state.id.clone();
23788 let q = find(&run);
23789 let fp = runner
23790 .state
23791 .github_text
23792 .as_ref()
23793 .unwrap()
23794 .fingerprint
23795 .clone();
23796 let name = crate::github_text::artifact_name(&fp);
23797 let path = crate::run::artifact_path(&runner.state, &name);
23798 assert!(q.detail.contains(&path.display().to_string()));
23799 assert!(q.detail.contains("title-language: title"));
23800 let saved = std::fs::read_to_string(&path).expect("artifact written");
23801 assert!(saved.contains("日本語のタイトル"));
23802 assert!(q.cwd.is_none());
23803 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23804 let say = |id: &str, text: &str, choice: &str| {
23806 store
23807 .update(id, |q| {
23808 q.thread.push(ask::Turn {
23809 who: ask::Who::Operator,
23810 body: text.to_owned(),
23811 at: jiff::Timestamp::now(),
23812 note: None,
23813 });
23814 q.answer(ask::Answer::Choice(choice.to_owned()))
23815 })
23816 .unwrap();
23817 };
23818 say(&q.id, "まだ日本語", "use my text");
23819 assert!(
23820 runner
23821 .guarded_pr_message(&winner, None, true)
23822 .await
23823 .unwrap()
23824 .is_none()
23825 );
23826 let second = find(&run);
23827 assert_ne!(second.id, q.id);
23828 assert!(second.detail.contains("did not pass"));
23829 assert!(second.detail.contains("日本語のタイトル"));
23832 assert!(!second.detail.contains("まだ日本語"));
23833 assert!(second.detail.contains(&path.display().to_string()));
23834 say(&second.id, "fix: retry failed requests", "use my text");
23835 let message = runner
23836 .guarded_pr_message(&winner, None, true)
23837 .await
23838 .unwrap()
23839 .expect("a vetted replacement is posted");
23840 assert_eq!(message.title, "fix: retry failed requests");
23841 assert_eq!(
23842 runner
23843 .state
23844 .github_text
23845 .as_ref()
23846 .unwrap()
23847 .chosen_title
23848 .as_deref(),
23849 Some("fix: retry failed requests")
23850 );
23851 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23852 let again = runner
23854 .guarded_pr_message(&winner, None, true)
23855 .await
23856 .unwrap()
23857 .unwrap();
23858 assert_eq!(again.title, message.title);
23859 assert_eq!(
23860 store
23861 .list()
23862 .into_iter()
23863 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23864 .count(),
23865 2
23866 );
23867 }
23868
23869 #[test]
23870 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23871 let mut state = state_with_summary(
23875 "add retries",
23876 "TITLE: fix(web): batch reads\n- reads run.json once",
23877 );
23878 state.config.graph.language = "ja".to_owned();
23879 let m = pr_message(&state, 'A');
23880 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23881 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23882
23883 let task = "今回やってほしいこと: results projector を直す";
23886 let mut state = state_with_summary(task, "- no title line");
23887 state.config.graph.language = "ja".to_owned();
23888 let m = pr_message(&state, 'A');
23889 assert_eq!(
23890 m.title,
23891 format!("chore: land candidate A of run {}", state.id)
23892 );
23893 assert!(
23894 m.body.contains(&format!(
23895 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23896 )),
23897 "{}",
23898 m.body
23899 );
23900 }
23901
23902 #[test]
23903 fn pr_message_scrubs_home_paths_and_addresses() {
23904 let state = state_with_summary(
23905 "fix it in /Users/someone/src/x",
23906 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23907 );
23908 let m = pr_message(&state, 'A');
23909 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23910 assert!(!m.body.contains(leak), "{}", m.body);
23911 }
23912 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23913 }
23914
23915 #[test]
23916 fn pr_message_survives_a_task_that_closes_details() {
23917 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23918 let m = pr_message(&state, 'A');
23919 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23920 }
23921
23922 #[test]
23923 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23924 let cmd = manual_merge_command(
23925 MergeStyle::Squash,
23926 Path::new("/repo"),
23927 "b",
23928 "fix: \"quoted\" $(x) `y`\n\nbody",
23929 );
23930 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23931 }
23932
23933 #[test]
23934 fn manual_merge_command_matches_the_configured_style() {
23935 let repo = Path::new("/repo");
23936 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23937
23938 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23939 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23940
23941 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23942 assert_eq!(
23943 squash,
23944 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23945 \"Merge magi run 0832 (candidate A)\""
23946 );
23947
23948 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23949 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23950 }
23951
23952 #[test]
23953 fn a_nudge_gets_a_quarter_of_the_budget() {
23954 assert_eq!(retry_budget(secs(1200), true), secs(300));
23956 assert_eq!(retry_budget(secs(3600), true), secs(900));
23957 }
23958
23959 #[test]
23960 fn a_resent_prompt_keeps_the_whole_budget() {
23961 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23964 assert_eq!(retry_budget(secs(60), false), secs(60));
23965 }
23966
23967 #[test]
23968 fn the_floor_never_exceeds_the_original_budget() {
23969 assert_eq!(retry_budget(secs(60), true), secs(60));
23973 assert_eq!(retry_budget(secs(480), true), secs(120));
23974 assert_eq!(retry_budget(secs(0), true), secs(0));
23975 }
23976
23977 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23978 agent::CommandEvidence {
23979 id: "item1".to_owned(),
23980 description: "cargo test".to_owned(),
23981 exit_code,
23982 result_summary: String::new(),
23983 source: "codex".to_owned(),
23984 }
23985 }
23986
23987 #[test]
23988 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23989 assert!(!has_unconfirmed_command(&[]));
23993 }
23994
23995 #[test]
23996 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23997 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
24001 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
24002 assert!(!has_unconfirmed_command(&[
24003 evidence(Some(0)),
24004 evidence(Some(101))
24005 ]));
24006 }
24007
24008 #[test]
24009 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
24010 assert!(has_unconfirmed_command(&[
24011 evidence(Some(0)),
24012 evidence(None)
24013 ]));
24014 }
24015
24016 #[test]
24017 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
24018 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24019 assert_eq!(
24020 verified_noop_claim(true, &[], text).as_deref(),
24021 Some("already fixed by b32cfc4, on main.")
24022 );
24023 }
24024
24025 #[test]
24026 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
24027 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24030 assert!(verified_noop_claim(false, &[], text).is_none());
24031 }
24032
24033 #[test]
24034 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
24035 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24036 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
24037 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
24039 }
24040
24041 #[test]
24042 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
24043 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
24044 }
24045
24046 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
24049 runner.state.candidates = shape
24050 .iter()
24051 .enumerate()
24052 .map(|(i, &(empty, verified))| Candidate {
24053 index: i,
24054 label: (b'A' + i as u8) as char,
24055 agent: "sonnet".to_owned(),
24056 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
24057 worktree: PathBuf::from(format!("/wt/{i}")),
24058 summary: String::new(),
24059 stat: String::new(),
24060 files: 0,
24061 commits: 0,
24062 empty,
24063 failed: None,
24064 verified_noop: verified.map(str::to_owned),
24065 duration_ms: 0,
24066 folded: false,
24067 })
24068 .collect();
24069 }
24070
24071 #[test]
24072 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
24073 ask_test_home();
24074 let mut runner = runner_at(RunStatus::Implementing);
24075 set_candidates(
24076 &mut runner,
24077 &[
24078 (true, Some("already on main at b32cfc4")),
24079 (true, Some("same fix, see the existing test")),
24080 ],
24081 );
24082
24083 runner
24084 .after_implement()
24085 .expect("a verified no-op is not an error");
24086
24087 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
24088 }
24089
24090 #[test]
24091 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
24092 ask_test_home();
24093 let mut runner = runner_at(RunStatus::Implementing);
24094 set_candidates(
24098 &mut runner,
24099 &[(true, Some("already on main at b32cfc4")), (true, None)],
24100 );
24101
24102 let err = runner
24103 .after_implement()
24104 .expect_err("an unverified empty candidate must still fail the run");
24105
24106 assert!(
24107 err.to_string().contains("no candidate produced a change"),
24108 "{err}"
24109 );
24110 assert_eq!(runner.state.status, RunStatus::Failed);
24111 }
24112
24113 #[test]
24114 fn after_implement_still_fails_an_ordinary_all_empty_run() {
24115 ask_test_home();
24116 let mut runner = runner_at(RunStatus::Implementing);
24117 set_candidates(&mut runner, &[(true, None), (true, None)]);
24118
24119 let err = runner
24120 .after_implement()
24121 .expect_err("no candidate declared anything; this is an ordinary failure");
24122
24123 assert!(
24124 err.to_string().contains("no candidate produced a change"),
24125 "{err}"
24126 );
24127 assert_eq!(runner.state.status, RunStatus::Failed);
24128 }
24129}
24130
24131#[derive(Debug)]
24134struct GraftRestoreFailed(String);
24135
24136impl std::fmt::Display for GraftRestoreFailed {
24137 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24138 write!(
24139 f,
24140 "could not restore the winner to its pre-graft commit: {}",
24141 self.0
24142 )
24143 }
24144}
24145
24146impl std::error::Error for GraftRestoreFailed {}
24147
24148#[cfg(test)]
24152mod resource_wait_tests {
24153 use super::*;
24154 use std::collections::BTreeMap;
24155
24156 fn sh(id: &str, script: &str) -> AgentSpec {
24157 AgentSpec {
24158 id: id.to_owned(),
24159 kind: crate::config::AgentKind::Command,
24160 model: None,
24161 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
24162 extra_args: Vec::new(),
24163 env: BTreeMap::new(),
24164 prompt_delivery: None,
24165 max_turn_secs: None,
24166 }
24167 }
24168
24169 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
24170 SeatJob {
24171 seat: SeatState::new("impl-A", &spec.id, 1),
24172 spec,
24173 cwd: dir.to_path_buf(),
24174 prompt: "do it".to_owned(),
24175 timeout,
24176 allow_write: true,
24177 sessions: false,
24178 artifacts: dir.join("art"),
24179 stem: "impl-A".to_owned(),
24180 handover: None,
24181 }
24182 }
24183
24184 fn state(resource_wait_secs: u64) -> RunState {
24185 let mut config = Config::default();
24186 config.graph.timeout_resource_wait = resource_wait_secs;
24187 RunState::new(
24188 PathBuf::from("/repo"),
24189 "main".to_owned(),
24190 "abc1234".to_owned(),
24191 "task".to_owned(),
24192 config,
24193 )
24194 }
24195
24196 fn hold(cache: &Path) -> crate::cache::Guard {
24197 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
24198 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
24199 crate::cache::AcquireOutcome::Acquired(g) => g,
24200 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
24201 }
24202 }
24203
24204 async fn run(
24205 jobs: Vec<SeatJob>,
24206 sem: Arc<Semaphore>,
24207 cache: Option<&Path>,
24208 st: &mut RunState,
24209 ) -> Vec<(usize, SeatState, AgentOutcome)> {
24210 let prompts = Prompts::default();
24211 let ctx = WaveCtx {
24212 run: "r",
24213 node: "implement",
24214 prompts: &prompts,
24215 cache,
24216 round: None,
24217 carry_seats: false,
24218 pauses: None,
24219 };
24220 wave(jobs, sem, &ctx, st, 0).await
24221 }
24222
24223 #[tokio::test]
24224 async fn verification_cache_wait_preserves_the_command_budget() {
24225 crate::run::pin_test_home();
24226 let tmp = tempfile::tempdir().unwrap();
24227 let cache = tmp.path().join("target");
24228 let guard = hold(&cache);
24229 let release = tokio::spawn(async move {
24230 tokio::time::sleep(Duration::from_millis(100)).await;
24231 guard.release();
24232 });
24233 let mut st = state(10);
24234 let budget = Duration::from_millis(50);
24235 let wait = repository_wait(&st, None);
24236 let (outcomes, _) = with_cache_lease(
24237 &mut st,
24238 Some(&cache),
24239 "gate",
24240 "gate",
24241 tmp.path(),
24242 "abc1234",
24243 budget,
24244 "test gate",
24245 &wait,
24246 |_, available| async move {
24247 assert_eq!(
24248 available, budget,
24249 "contention must not shorten verification"
24250 );
24251 (Vec::new(), false, Vec::new())
24252 },
24253 )
24254 .await
24255 .unwrap();
24256 release.await.unwrap();
24257 assert!(outcomes.is_empty(), "{outcomes:?}");
24258 assert!(
24259 st.events
24260 .iter()
24261 .any(|event| event.message.contains("waiting for"))
24262 );
24263 assert!(st.resource_wait.is_none());
24264 hold(&cache).release();
24266 }
24267
24268 #[tokio::test]
24269 async fn verification_cache_wait_expires_without_starting_commands() {
24270 crate::run::pin_test_home();
24271 let tmp = tempfile::tempdir().unwrap();
24272 let cache = tmp.path().join("target");
24273 let _held = hold(&cache);
24274 let mut st = state(0);
24275 let wait = repository_wait(&st, None);
24276 let (outcomes, _) = with_cache_lease(
24277 &mut st,
24278 Some(&cache),
24279 "gate",
24280 "gate",
24281 tmp.path(),
24282 "abc1234",
24283 Duration::from_secs(60),
24284 "test gate",
24285 &wait,
24286 |_, _| async { panic!("commands must not start without the cache lease") },
24287 )
24288 .await
24289 .unwrap();
24290 assert_eq!(outcomes.len(), 1);
24291 assert!(outcomes[0].resource_blocked);
24292 assert!(st.resource_wait.is_none());
24293 assert!(matches!(
24294 st.resource_block.unwrap().cause,
24295 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24296 ));
24297 }
24298
24299 #[tokio::test]
24300 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24301 crate::run::pin_test_home();
24302 let tmp = tempfile::tempdir().unwrap();
24303 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24304 let cache = tmp.path().join("cache");
24305 let mut state = state(21600);
24306 let wait = repository_wait(&state, None);
24307 let (outcomes, _) = with_cache_lease(
24308 &mut state,
24309 Some(&cache),
24310 "gate",
24311 "gate",
24312 tmp.path(),
24313 "head",
24314 Duration::from_secs(60),
24315 "gate",
24316 &wait,
24317 |_, _| async { panic!("freshness failure must not start verification") },
24318 )
24319 .await
24320 .unwrap();
24321 assert_eq!(outcomes.len(), 1);
24322 assert!(outcomes[0].resource_blocked);
24323 let block = state.resource_block.unwrap();
24324 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24325 assert!(!block.description().contains("timed out"));
24326 }
24327
24328 #[tokio::test]
24329 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24330 crate::run::pin_test_home();
24331 let tmp = tempfile::tempdir().unwrap();
24332 let cache = tmp.path().join("cache");
24333 let _held = hold(&cache);
24334 let mut state = state(21600);
24335 let pause = Pause::new();
24336 let interrupt = Pause::new();
24337 let cause = ParkCause::Operator;
24338 pause.park_for(cause.clone());
24339 let prompts = Prompts::default();
24340 let ctx = WaveCtx {
24341 run: "test",
24342 node: "fix",
24343 prompts: &prompts,
24344 cache: Some(&cache),
24345 round: None,
24346 carry_seats: false,
24347 pauses: Some((&pause, &interrupt)),
24348 };
24349 let result = wave(
24350 vec![job(
24351 sh("writer", "exit 99"),
24352 tmp.path(),
24353 Duration::from_secs(1),
24354 )],
24355 Arc::new(Semaphore::new(1)),
24356 &ctx,
24357 &mut state,
24358 0,
24359 )
24360 .await;
24361 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24362 assert_eq!(result[0].1.turns, 0);
24363 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24364 assert_eq!(
24365 state.resource_block.unwrap().cause,
24366 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24367 );
24368 assert!(state.active.is_empty());
24369 }
24370
24371 #[tokio::test]
24372 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24373 crate::run::pin_test_home();
24374 let tmp = tempfile::tempdir().unwrap();
24375 let cache = tmp.path().join("target-a");
24376 let guard = hold(&cache);
24377 tokio::spawn(async move {
24379 tokio::time::sleep(Duration::from_millis(1500)).await;
24380 guard.release();
24381 });
24382 let mut st = state(60);
24383 let results = run(
24384 vec![job(
24385 sh("alpha", "echo ok"),
24386 tmp.path(),
24387 Duration::from_secs(1),
24388 )],
24389 Arc::new(Semaphore::new(1)),
24390 Some(&cache),
24391 &mut st,
24392 )
24393 .await;
24394 assert!(
24395 matches!(results[0].2, AgentOutcome::Ok(_)),
24396 "the wait must not be charged to the agent"
24397 );
24398 assert!(st.resource_block.is_none());
24399 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24400 assert!(st.resource_wait.is_none() && st.active.is_empty());
24401 }
24402
24403 #[tokio::test]
24404 #[cfg(unix)]
24405 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24406 use std::os::fd::AsRawFd as _;
24407 crate::run::pin_test_home();
24408 let tmp = tempfile::tempdir().unwrap();
24409 crate::git::git(tmp.path(), &["init"]).await.unwrap();
24410 let common = tmp.path().join(".git");
24411 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24412 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24413 assert_eq!(
24414 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24415 0
24416 );
24417 let mut st = state(1);
24418 st.config.coordination = crate::config::test_coordination();
24419 let mut reader = job(
24420 sh("reader", "echo must-not-run"),
24421 tmp.path(),
24422 Duration::from_secs(30),
24423 );
24424 reader.allow_write = false;
24425 let results = run(
24426 vec![reader.clone()],
24427 Arc::new(Semaphore::new(1)),
24428 None,
24429 &mut st,
24430 )
24431 .await;
24432 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24433 assert_eq!(results[0].1.turns, 0);
24434 assert!(FailClass::of(&results[0].2).is_none());
24435 assert_eq!(
24436 st.resource_block.as_ref().unwrap().kind,
24437 "repository coordination"
24438 );
24439 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24440 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24441 drop(external);
24442 let editor = crate::repo_lock::SeatLease::for_checkout(
24443 tmp.path(),
24444 true,
24445 crate::config::test_coordination().as_ref(),
24446 )
24447 .await
24448 .unwrap();
24449 let pause = Pause::new();
24450 let interrupt = Pause::new();
24451 let prompts = Prompts::default();
24452 let ctx = WaveCtx {
24453 run: "r",
24454 node: "review",
24455 prompts: &prompts,
24456 cache: None,
24457 round: None,
24458 carry_seats: false,
24459 pauses: Some((&pause, &interrupt)),
24460 };
24461 let (results, ()) = tokio::join!(
24462 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24463 async {
24464 tokio::time::sleep(Duration::from_millis(50)).await;
24465 pause.park();
24466 }
24467 );
24468 assert!(
24469 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24470 );
24471 assert_eq!(results[0].1.turns, 0);
24472 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24473 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24474 drop(editor);
24475 }
24476
24477 #[tokio::test]
24478 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24479 crate::run::pin_test_home();
24480 let tmp = tempfile::tempdir().unwrap();
24481 let cache = tmp.path().join("target-b");
24482 let _held = hold(&cache);
24483 let mut st = state(1);
24484 let results = run(
24485 vec![job(
24486 sh("alpha", "echo ok"),
24487 tmp.path(),
24488 Duration::from_secs(1),
24489 )],
24490 Arc::new(Semaphore::new(1)),
24491 Some(&cache),
24492 &mut st,
24493 )
24494 .await;
24495 let (_, seat, out) = &results[0];
24496 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24497 assert!(wave_blocked(&results));
24498 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24499 assert_eq!(seat.turns, 0, "nothing was asked");
24500 assert!(
24501 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24502 "no agent was spawned"
24503 );
24504 assert!(FailClass::of(out).is_none(), "not a failure class");
24505 let block = st.resource_block.as_ref().expect("recorded for the resume");
24506 assert_eq!(block.kind, "shared build cache");
24507 assert!(matches!(
24508 block.cause,
24509 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24510 ));
24511 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24512 assert!(st.active.is_empty());
24513 }
24514
24515 #[tokio::test]
24516 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24517 crate::run::pin_test_home();
24518 let tmp = tempfile::tempdir().unwrap();
24519 let mut st = state(1);
24520 let results = run(
24521 vec![job(
24522 sh("alpha", "echo ok"),
24523 tmp.path(),
24524 Duration::from_secs(1),
24525 )],
24526 Arc::new(Semaphore::new(0)),
24527 None,
24528 &mut st,
24529 )
24530 .await;
24531 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24532 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24533 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24534 }
24535
24536 #[tokio::test]
24537 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24538 crate::run::pin_test_home();
24539 let tmp = tempfile::tempdir().unwrap();
24540 let mut st = state(60);
24541 let results = run(
24542 vec![job(
24543 sh("alpha", "sleep 30"),
24544 tmp.path(),
24545 Duration::from_secs(1),
24546 )],
24547 Arc::new(Semaphore::new(1)),
24548 None,
24549 &mut st,
24550 )
24551 .await;
24552 let out = &results[0].2;
24553 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24554 let class = FailClass::of(out).expect("a failure");
24555 assert_eq!(class, FailClass::Timeout);
24556 assert!(should_hand_over(None, &class), "existing handover policy");
24557 assert!(st.resource_block.is_none());
24558 }
24559}