1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => {
1256 worktree_setup::prepare_with(
1257 &state.config,
1258 repo,
1259 &worktree,
1260 crate::build_target::Coordination::Held(_coordination.as_ref()),
1261 )
1262 .await
1263 }
1264 Err(e) => Err(e),
1265 };
1266 if let Err(e) = setup {
1267 git::worktree_remove_clean(repo, &worktree).await.ok();
1268 return Err(e);
1269 }
1270 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1271 .await
1272 .unwrap_or(0);
1273 if commits == 0 {
1274 git::worktree_remove_clean(repo, &worktree).await.ok();
1275 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1276 }
1277 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1278 .await
1279 .map(|f| f.len())
1280 .unwrap_or(0);
1281 if files == 0
1282 && let (Ok(head_tree), Ok(base_tree)) = (
1283 git::tree_of(&worktree, "HEAD").await,
1284 git::tree_of(&worktree, &base_commit).await,
1285 )
1286 && head_tree == base_tree
1287 {
1288 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1289 git::worktree_remove_clean(repo, &worktree).await.ok();
1290 bail!(
1291 "`{branch}` at {} has a tree identical to base {}; this usually means \
1292 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1293 against `{}/{branch}`) rather than an empty change",
1294 short(&head),
1295 short(&base_commit),
1296 state.config.merge.remote
1297 );
1298 }
1299 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1300 .await
1301 .unwrap_or_default();
1302
1303 state.candidates.push(Candidate {
1304 index: 0,
1305 label: 'A',
1306 agent: EXISTING_BRANCH.to_owned(),
1309 branch: review_branch,
1310 worktree,
1311 summary: String::new(),
1312 stat,
1313 files,
1314 commits,
1315 empty: false,
1316 failed: None,
1317 verified_noop: None,
1318 duration_ms: 0,
1319 folded: false,
1320 });
1321 state.tally = Some(Tally {
1322 first_choice: BTreeMap::from([('A', 0)]),
1323 borda: BTreeMap::new(),
1324 winner: 'A',
1325 rankings: 0,
1326 unanimous_initial: false,
1327 deliberated: false,
1328 changed_votes: 0,
1329 unanimous_final: false,
1330 tie_break: None,
1331 judges: 0,
1335 present: 0,
1336 quorum: 0,
1337 met_quorum: true,
1338 uncontested: Some("review-only run: nothing competed".to_owned()),
1339 });
1340 state.status = RunStatus::Reviewing;
1341 state.prep_complete = Some(true);
1342 state.event(
1343 "start",
1344 format!(
1345 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1346 state.id
1347 ),
1348 );
1349 Ok(())
1350 }
1351
1352 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1355 Self::authorize_base_recovery_request(home, id, task, false, None).await
1356 }
1357
1358 pub(crate) async fn authorize_base_recovery_from_resumption(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 ) -> Result<()> {
1364 Self::authorize_base_recovery_request(home, id, task, true, None).await
1365 }
1366
1367 pub(crate) async fn authorize_base_recovery_control(
1368 home: &Path,
1369 task: &str,
1370 control: &mut crate::base_recovery_control::Transaction,
1371 ) -> Result<()> {
1372 let id = control.request.run.clone();
1373 Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1374 }
1375
1376 async fn authorize_base_recovery_request(
1377 home: &Path,
1378 id: &str,
1379 task: &str,
1380 resumption: bool,
1381 mut control: Option<&mut crate::base_recovery_control::Transaction>,
1382 ) -> Result<()> {
1383 let _owner = crate::pr_head::owner_lock(home, id)?;
1384 let mut state = RunState::load_under(id, home)?;
1385 crate::pr_head::ensure_owner(&state)?;
1386 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1387 anyhow::ensure!(
1388 queued.execution_runs().last().is_some_and(|run| run == id)
1389 && queued.status != crate::queue::TaskStatus::Done
1390 && !queued.fresh_start
1391 && queued.review_branch.is_none(),
1392 "run task ownership changed"
1393 );
1394 anyhow::ensure!(
1395 state
1396 .origin
1397 .as_ref()
1398 .and_then(|o| o.task.as_deref())
1399 .is_none_or(|owner| owner == task),
1400 "run belongs to another task"
1401 );
1402 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1403 anyhow::ensure!(
1404 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1405 "run is live or its ownership is unknown"
1406 );
1407 let _execution = crate::driver::maintenance(home, id)?;
1410 anyhow::ensure!(
1411 state.status.resumable()
1412 && state.released_to.is_none()
1413 && state.retired_to.is_none()
1414 && state.pr_taken_over_by.is_none(),
1415 "run ownership is final"
1416 );
1417 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1418 anyhow::ensure!(
1419 !state.pr_publication_pending
1420 && !crate::handover::landing_checkpoint(&state)
1421 && state.merge_decision.is_none(),
1422 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1423 );
1424 let winner = state.winner().context("run has no retained winner")?;
1425 anyhow::ensure!(
1426 git::is_clean(&winner.worktree).await?,
1427 "retained worktree has uncommitted changes"
1428 );
1429 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1430 anyhow::ensure!(
1431 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1432 "retained worktree does not match the winner branch"
1433 );
1434 if let Some(control) = control.as_ref() {
1435 anyhow::ensure!(
1436 head.starts_with(&control.request.head),
1437 "retained HEAD changed since the owner request"
1438 );
1439 anyhow::ensure!(
1440 state
1441 .base_sync_recovery
1442 .stop
1443 .as_ref()
1444 .is_none_or(|stop| stop.head == head),
1445 "retained HEAD differs from the recorded stop head; inspect it first"
1446 );
1447 }
1448 state.ensure_base_episode();
1449 state.record_base_sync_stop(&head);
1450 anyhow::ensure!(
1451 state
1452 .base_sync_recovery
1453 .stop
1454 .as_ref()
1455 .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1456 "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1457 );
1458 let fetched =
1459 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1460 anyhow::ensure!(
1461 fetched.ok(),
1462 "cannot authorize base recovery without a fetched base"
1463 );
1464 let approved_tip = git::rev_parse(
1465 &state.repo,
1466 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1467 )
1468 .await?;
1469 if let Some(control) = control.as_deref_mut() {
1470 let queue = crate::queue::Queue::at(home.join("queue"));
1471 control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1472 if let Some(prior) = state
1473 .base_sync_recovery
1474 .authorizations
1475 .iter()
1476 .find(|a| a.request.as_deref() == Some(&control.key()))
1477 {
1478 anyhow::ensure!(
1479 !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1480 "recovery request was consumed or invalidated; it cannot queue again"
1481 );
1482 return control.enqueue(&queue);
1483 }
1484 }
1485 if resumption {
1486 anyhow::ensure!(
1487 crate::daemon::compatible_run_instruction(&state, &queued),
1488 "resumption instruction does not match the retained run"
1489 );
1490 let approval = queued
1491 .resumption
1492 .as_ref()
1493 .context("no operator resumption")?;
1494 anyhow::ensure!(
1495 approval.instruction == queued.instruction,
1496 "resumption approval does not match the current task instruction"
1497 );
1498 state.authorize_base_sync_resumption(task, approval)?;
1499 } else {
1500 state.authorize_base_sync(task)?;
1501 }
1502 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1503 state
1504 .base_sync_recovery
1505 .authorizations
1506 .last_mut()
1507 .unwrap()
1508 .scope = Some(crate::base_sync_recovery::Scope {
1509 tip: approved_tip,
1510 instruction: queued.instruction.clone(),
1511 episode,
1512 });
1513 if let Some(control) = control.as_deref_mut() {
1514 let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1515 authorization.request = Some(control.key());
1516 authorization.generation = control.generation();
1517 authorization.decision = control.decision();
1518 authorization.question = control
1519 .request
1520 .decision
1521 .as_ref()
1522 .map(|_| control.request.request.clone());
1523 }
1524 state.event(
1525 "base-recovery",
1526 "owner authorized a bounded base synchronization interval; fresh verification required",
1527 );
1528 state.save_under(home)?;
1529 if let Some(control) = control {
1530 control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1531 }
1532 Ok(())
1533 }
1534
1535 pub fn resume(id: &str) -> Result<Self> {
1537 let state = RunState::load(id)?;
1538 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1539 if state.startup.as_ref().is_some_and(|s| !s.complete)
1540 || (state.status == RunStatus::Prep
1541 && state.instruction.trim().is_empty()
1542 && state.candidates.is_empty()
1543 && state.prep_complete == Some(false))
1544 {
1545 bail!(
1546 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1547 );
1548 }
1549 crate::pr_head::ensure_owner(&state)?;
1550 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1551 bail!(
1552 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1553 );
1554 }
1555 if let Some(to) = &state.released_to {
1556 bail!(
1557 "run {} cannot be resumed: its worktree was released to run {}",
1558 state.short(),
1559 crate::run::short_of(to)
1560 );
1561 }
1562 let roles = state.config.resolve_roles()?;
1563 if let Some(id) = &state.config.graph.selected_implementer
1564 && state.candidates.iter().any(|c| c.agent != *id)
1565 {
1566 bail!(
1567 "replacement implementer pin does not match the saved candidates; task must stay held"
1568 );
1569 }
1570 let max_parallel = state.config.graph.max_parallel.max(1);
1571 Ok(Self {
1572 state,
1573 roles,
1574 sem: Arc::new(Semaphore::new(max_parallel)),
1575 pause: Pause::new(),
1576 interrupt: Pause::new(),
1577 })
1578 }
1579
1580 pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1587 Box::pin(self.execute_inner())
1590 }
1591
1592 async fn execute_inner(&mut self) -> Result<()> {
1593 anyhow::ensure!(
1594 self.state.startup.as_ref().is_none_or(|s| s.complete),
1595 "startup incomplete; inspect retained evidence before explicit recovery"
1596 );
1597 if let Some(selection) = &self.state.pr_recovery
1598 && (selection.source_proof.is_some() || selection.source_run.is_some())
1599 {
1600 Box::pin(selection.verify_source(&self.state.repo)).await?;
1601 }
1602 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1605 || self.state.land_approval.is_some()
1606 || self.state.merge_decision.is_some()
1607 || !self.state.merge_repairs.is_empty()
1608 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1609 {
1610 let claim = FixClaim::acquire(&self.state.dir())?;
1611 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1612 let disk = RunState::load(&self.state.id)?;
1613 let checkpoint = |s: &RunState| {
1614 (
1615 s.merge_repairs.len(),
1616 s.merge_repairs
1617 .last()
1618 .map(|r| (r.question.clone(), r.phase)),
1619 s.land_approval
1620 .as_ref()
1621 .map(|a| (a.question.clone(), a.head.clone())),
1622 )
1623 };
1624 if checkpoint(&disk) != checkpoint(&self.state) {
1625 bail!(
1626 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1627 );
1628 }
1629 Some(claim)
1630 } else {
1631 None
1632 };
1633 let mut execution = self.claim_driver().await?;
1634 let runner = &mut *execution.runner;
1635 let wait = runner.repository_wait();
1636 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1639 let result = match result {
1640 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1641 record_repository_block(&mut runner.state, &e);
1642 runner.park_here()?;
1643 Ok(())
1644 }
1645 other => other,
1646 };
1647 execution.driver.finish(&mut runner.state).await?;
1648 let ended = if result.is_err() {
1649 Some({
1650 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1651 if let Err(error) = &result {
1652 notice.message = format!(
1653 "Run {} stopped with an error: {}",
1654 runner.state.id,
1655 error.to_string().lines().next().unwrap_or("unknown error")
1656 );
1657 }
1658 notice
1659 })
1660 } else {
1661 crate::notices::run_ended(&runner.state)
1662 };
1663 if let Some(notice) = ended {
1664 crate::notices::raise(notice);
1665 }
1666 result
1667 }
1668
1669 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1670 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1671 }
1672
1673 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1674 let wait = self.repository_wait();
1675 coordinate_operation(&mut self.state, cwd, &wait).await
1676 }
1677
1678 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1680 let owner = match crate::pr_head::owner_lock_wait(
1681 &crate::run::home(),
1682 &self.state.id,
1683 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1684 )
1685 .await
1686 {
1687 Ok(owner) => owner,
1688 Err(error) => {
1689 let detail = format!("worktree maintenance or ownership update: {error:#}");
1690 self.state.resource_block = Some(crate::run::ResourceBlock {
1694 cause: crate::run::ResourceBlockCause::Other,
1695 node: "start".into(),
1696 kind: "worktree maintenance".into(),
1697 detail: detail.clone(),
1698 at: Timestamp::now(),
1699 });
1700 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1701 }
1702 };
1703 let disk = RunState::load(&self.state.id)?;
1704 crate::pr_head::ensure_owner(&disk)?;
1705 let claimed =
1706 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1707 anyhow::ensure!(
1712 disk.liveness(claimed) != crate::run::Liveness::Live,
1713 "run already has a live driver; wait for its execution to end"
1714 );
1715 if let Some(to) = &disk.released_to {
1716 bail!(
1717 "run {} cannot continue: its worktree was released to run {}",
1718 self.state.short(),
1719 crate::run::short_of(to)
1720 );
1721 }
1722 anyhow::ensure!(
1723 serde_json::to_value(&disk.base_sync_recovery)?
1724 == serde_json::to_value(&self.state.base_sync_recovery)?,
1725 "base recovery changed since this run was loaded; reload the saved run"
1726 );
1727 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1728 let mut execution = DriverExecution {
1729 runner: self,
1730 driver,
1731 };
1732 let _owner = owner;
1735 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1736 .await?;
1737 let state = &mut execution.runner.state;
1738 state.parked = false;
1743 state.resource_wait = None;
1744 state.resource_block = None;
1745 state.clear_active();
1752 let pid = std::process::id();
1767 state.driver_pid = Some(pid);
1768 state.driver_started_at = crate::proc::process_started_at(pid);
1769 state.driver_exited = false;
1770 state.driver_claimed = Some(true);
1771 state.begin_pass();
1772 execution.driver.bind(state);
1773 state.save()?;
1774 execution.driver.record(state)?;
1775 Ok(execution)
1776 }
1777
1778 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1781 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1782 if let Err(error) = &result
1783 && !error.is::<crate::repo_lock::ResourceBlocked>()
1784 && self
1785 .state
1786 .repair_grants
1787 .last()
1788 .is_some_and(|g| g.conflict.is_none())
1789 {
1790 self.state.repair_grants.last_mut().unwrap().phase =
1791 crate::repair_budget::Phase::Complete;
1792 self.state.status = RunStatus::Blocked;
1793 self.state.save()?;
1794 bail!(
1795 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1796 self.state.passes.last().map_or(0, |p| p.index)
1797 );
1798 }
1799 result
1800 }
1801
1802 async fn execute_granted_repair(&mut self) -> Result<()> {
1805 use crate::repair_budget::{Kind, Phase};
1806 let index = self.state.repair_grants.len() - 1;
1807 let winner = self
1808 .state
1809 .winner()
1810 .cloned()
1811 .context("repair winner missing")?;
1812 loop {
1813 let grant = self.state.repair_grants[index].clone();
1814 match grant.phase {
1815 Phase::Pending => {
1816 if grant.spent >= grant.receipt.request.rounds {
1817 self.state.repair_grants[index].phase = Phase::Complete;
1818 self.state.status = RunStatus::Blocked;
1819 self.state.save()?;
1820 return Ok(());
1821 }
1822 self.state.repair_grants[index].spent += 1;
1825 self.state.repair_grants[index].phase = Phase::Fixing;
1826 self.state.save()?;
1827 match grant.receipt.request.kind {
1828 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1829 Kind::Gate => {
1830 let failed = self.state.gate.clone();
1831 let _ = self.gate_fix_round(&winner, &failed).await?;
1832 }
1833 Kind::Review => {
1834 let mut record = self
1835 .state
1836 .reviews
1837 .last()
1838 .cloned()
1839 .context("review failure missing")?;
1840 let round = record.round;
1841 let step = self.run_fix_round(&winner, &mut record).await?;
1842 self.state.reviews[round - 1] = record;
1843 if matches!(step, FixStep::Recorded { report_lost: true }) {
1844 self.state.repair_grants[index].phase = Phase::Complete;
1845 self.state.status = RunStatus::Blocked;
1846 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1847 self.state.save()?;
1848 return Ok(());
1849 }
1850 if matches!(step, FixStep::Resource) {
1851 self.stop_for_resource()?;
1852 return Ok(());
1853 }
1854 self.state.save()?;
1855 }
1856 }
1857 if self.resource_halt()? {
1858 return Ok(());
1859 }
1860 }
1861 Phase::Fixing => {
1862 if grant.receipt.request.kind == Kind::BaseConflict {
1863 self.repair_base_conflict(false).await?;
1864 }
1865 let _coordination = self.coordinate(&winner.worktree).await?;
1868 let rescued = git::rescue_commit(
1869 &winner.worktree,
1870 "magi: preserve interrupted granted repair",
1871 )
1872 .await?;
1873 self.state.note_withheld("repair-grant", &rescued.withheld);
1874 self.state
1877 .verification_history
1878 .push(crate::run::VerificationEvidence {
1879 superseded_by: None,
1880 archived_at: Some(Timestamp::now()),
1881 fixes_spent: Some(self.state.gate_fixes.len()),
1882 gate_head: self.state.gate_head.take(),
1883 gate_ran: std::mem::take(&mut self.state.gate_ran),
1884 gate: std::mem::take(&mut self.state.gate),
1885 e2e: self.state.head_e2e.take(),
1886 });
1887 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1888 self.state
1889 .verification_history
1890 .push(crate::run::VerificationEvidence {
1891 superseded_by: None,
1892 archived_at: Some(Timestamp::now()),
1893 fixes_spent: Some(self.state.gate_fixes.len()),
1894 gate_head: None,
1895 gate_ran: false,
1896 gate: Vec::new(),
1897 e2e: Some(e2e),
1898 });
1899 }
1900 self.state.repair_grants[index].review_start = self.state.reviews.len();
1901 self.state.repair_grants[index].head =
1902 git::rev_parse(&winner.worktree, "HEAD").await?;
1903 self.state.repair_grants[index].phase = Phase::Reviewing;
1904 if grant.receipt.request.kind == Kind::BaseConflict {
1905 self.state.base_sync_recovery.stop = None;
1906 if let Some(sync) = &mut self.state.base_sync {
1907 sync.tip = grant
1908 .receipt
1909 .request
1910 .tip
1911 .clone()
1912 .context("repair base tip missing")?;
1913 sync.conflict = None;
1914 sync.behind = 0;
1915 }
1916 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1917 }
1918 self.state.save()?;
1919 }
1920 Phase::Reviewing => {
1921 self.review_loop_pass().await?;
1922 if self.resource_halt()? {
1923 return Ok(());
1924 }
1925 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1926 let clean = self.state.reviews.len() > grant.review_start
1927 && self.state.reviews.last().is_some_and(|r| {
1928 r.head == head
1929 && r.clean
1930 && r.expected > 0
1931 && r.answered == r.expected
1932 && matches!(
1933 r.e2e_status(),
1934 E2eStatus::Passed | E2eStatus::NotConfigured
1935 )
1936 });
1937 if !clean {
1938 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1940 == Kind::Review
1941 && grant.spent < grant.receipt.request.rounds
1942 && self.state.reviews.last().is_some_and(|r| {
1943 r.fix.is_none()
1944 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1945 }) {
1946 Phase::Pending
1947 } else {
1948 Phase::Complete
1949 };
1950 self.state.status = RunStatus::Blocked;
1951 self.state.event(
1952 "repair-grant",
1953 "fresh review did not clear the repaired head; no publication",
1954 );
1955 self.state.save()?;
1956 if self.state.repair_grants[index].phase == Phase::Pending {
1957 continue;
1958 }
1959 return Ok(());
1960 }
1961 self.state.repair_grants[index].phase = Phase::Gating;
1962 self.state.save()?;
1963 }
1964 Phase::Gating => {
1965 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1966 self.state.reconcile_verification_head(&head);
1967 let wait = self.repository_wait();
1968 if self
1969 .state
1970 .reviews
1971 .last()
1972 .is_none_or(|r| r.head != head || !r.clean)
1973 {
1974 self.state.status = RunStatus::Blocked;
1975 self.state.repair_grants[index].phase = Phase::Complete;
1976 self.state.event(
1977 "repair-grant",
1978 "head changed since fresh review; repair held",
1979 );
1980 self.state.save()?;
1981 return Ok(());
1982 }
1983 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1984 return Ok(());
1985 }
1986 let outcomes = self.run_gate(&winner).await?;
1987 if verify_inconclusive(&outcomes) {
1988 return Ok(());
1989 }
1990 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1991 if after != head {
1992 self.state.repair_grants[index].review_start = self.state.reviews.len();
1993 self.state.repair_grants[index].head = after;
1994 self.state.repair_grants[index].phase = Phase::Reviewing;
1995 self.state.status = RunStatus::Blocked;
1996 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1997 self.state.save()?;
1998 return Ok(());
1999 }
2000 let passed = outcomes.iter().all(CommandOutcome::ok);
2001 if passed
2004 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
2005 {
2006 return Ok(());
2007 }
2008 self.state.repair_grants[index].succeeded = passed;
2009 self.state.gate = outcomes;
2010 self.state.gate_ran = true;
2011 self.state.status = if passed {
2012 RunStatus::Gating
2013 } else {
2014 RunStatus::Blocked
2015 };
2016 self.state.repair_grants[index].phase = if !passed
2017 && grant.receipt.request.kind == Kind::Gate
2018 && grant.spent < grant.receipt.request.rounds
2019 {
2020 Phase::Pending
2021 } else {
2022 Phase::Complete
2023 };
2024 if passed && self.state.config.merge.mode == MergeMode::Pr {
2025 self.state.pr_publication_pending = true;
2027 } else if passed {
2028 self.state.status = RunStatus::Ready;
2029 }
2030 self.state.save()?;
2031 if !passed {
2032 continue;
2033 }
2034 if self.state.config.merge.mode != MergeMode::Pr {
2038 self.state.status = RunStatus::Ready;
2039 self.state.event(
2040 "repair-grant",
2041 "repair verified locally; grant carries no merge authorization",
2042 );
2043 self.state.save()?;
2044 return Ok(());
2045 }
2046 return self.merge().await;
2047 }
2048 Phase::Complete => return Ok(()),
2049 }
2050 }
2051 }
2052
2053 async fn execute_graph(&mut self) -> Result<()> {
2054 let archive_home = crate::run::home().canonicalize()?;
2058 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2059 let repo = self.state.repo.clone();
2060 let _coordination = self.coordinate(&repo).await?;
2061 ensure_setup_config(&mut self.state, &repo).await?;
2062 crate::retention::restore_execution_seats(
2063 &archive_home,
2064 &self.state,
2065 _coordination.as_ref(),
2066 )
2067 .await?;
2068 }
2069 self.state.configure_recovery_resume()?;
2070 if self
2071 .state
2072 .repair_grants
2073 .last()
2074 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2075 {
2076 return self.execute_granted_repair().await;
2077 }
2078 if self
2079 .state
2080 .repair_grants
2081 .last()
2082 .is_some_and(|g| !g.succeeded)
2083 {
2084 self.state.status = RunStatus::Blocked;
2085 self.state.save()?;
2086 return Ok(());
2087 }
2088 self.persist_candidates().await?;
2091 if self.state.base_sync_blocked() {
2094 self.sync_to_base().await?;
2095 if !self.state.base_sync_blocked()
2096 && self.state.status != RunStatus::AlreadyInBase
2097 && !self.resource_halt()?
2098 {
2099 self.finish_after_tally().await?;
2100 }
2101 return Ok(());
2102 }
2103 if self
2104 .state
2105 .merge_decision
2106 .as_ref()
2107 .is_some_and(|decision| !decision.closed)
2108 && self.state.status != RunStatus::Landing
2109 {
2110 if crate::merge_decision::resumable_hold(&self.state) {
2111 self.state.merge = Some(MergeOutcome {
2112 mode: MergeMode::Pr,
2113 ok: false,
2114 detail: self
2115 .state
2116 .merge_decision
2117 .as_ref()
2118 .unwrap()
2119 .target
2120 .pr
2121 .clone(),
2122 empty: false,
2123 });
2124 self.run_land().await?;
2125 self.settle_questions();
2126 return Ok(());
2127 }
2128 bail!(
2129 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2130 );
2131 }
2132 if self.state.config.graph.selected_implementer.is_some()
2133 && self.state.status == RunStatus::Blocked
2134 && self.state.tally.is_none()
2135 && self.state.candidate_preservation.is_empty()
2136 && self.state.interrupted_implementations.is_empty()
2137 {
2138 bail!(
2139 "selected implementation is held; explicitly release the owning task to retry it"
2140 );
2141 }
2142
2143 if self.state.merge_repairs.last().is_some_and(|r| {
2144 matches!(
2145 r.phase,
2146 crate::run::MergeRepairPhase::Requested
2147 | crate::run::MergeRepairPhase::Fixing
2148 | crate::run::MergeRepairPhase::Reviewing
2149 )
2150 }) {
2151 if self.repair_merge_review().await? {
2152 self.run_land().await?;
2153 }
2154 self.settle_questions();
2155 return Ok(());
2156 }
2157 if self.state.pr_publication_pending {
2160 self.merge().await?;
2161 if self.resource_halt()? {
2162 return Ok(());
2163 }
2164 self.state.save()?;
2165 self.settle_questions();
2166 return Ok(());
2167 }
2168 if self.state.status == RunStatus::Stalled {
2181 if self.recover_stall().await? {
2182 self.finish_after_tally().await?;
2183 } else {
2184 self.state.save()?;
2186 }
2187 return Ok(());
2188 }
2189 if self.state.status == RunStatus::Landing
2199 || crate::handover::landing_checkpoint(&self.state)
2200 {
2201 self.state.land_sync_pending = true;
2202 self.state.save()?;
2203 self.run_land().await?;
2204 self.settle_questions();
2209 return Ok(());
2210 }
2211 if self.state.github_text.is_some()
2216 && self.state.merge.is_none()
2217 && self.state.status == RunStatus::Gating
2218 {
2219 self.merge().await?;
2220 if self.resource_halt()? {
2221 return Ok(());
2222 }
2223 self.state.save()?;
2224 return Ok(());
2225 }
2226 if let Err(e) = self.prep().await {
2227 if e.is::<crate::repo_lock::ResourceBlocked>() {
2228 return Err(e);
2229 }
2230 self.state.status = RunStatus::Blocked;
2231 self.state
2232 .prep_failure
2233 .get_or_insert_with(|| format!("{e:#}"));
2234 self.state
2235 .event("prep", format!("preparation stopped: {e:#}"));
2236 self.state.save()?;
2237 return Err(e);
2238 }
2239 if self.park_here()? {
2240 return Ok(());
2241 }
2242 self.advise().await?;
2243 if self.park_here()? {
2244 return Ok(());
2245 }
2246 self.implement().await?;
2247 if self.park_here()? {
2248 return Ok(());
2249 }
2250 if self.state.status == RunStatus::VerifiedNoop {
2254 return Ok(());
2255 }
2256 self.judge().await?;
2257 if self.park_here()? {
2258 return Ok(());
2259 }
2260 self.deliberate().await?;
2261 if self.park_here()? {
2262 return Ok(());
2263 }
2264 self.vote().await?;
2265 if self.park_here()? {
2266 return Ok(());
2267 }
2268 self.tally()?;
2269 if self.state.status == RunStatus::Stalled {
2274 self.state.save()?;
2278 return Ok(());
2279 }
2280 self.finish_after_tally().await?;
2281 Ok(())
2282 }
2283
2284 fn stop_for_resource(&mut self) -> Result<()> {
2289 if self.pause.parked() || self.interrupt.parked() {
2292 self.park_here()?;
2293 return Ok(());
2294 }
2295 if self.state.resource_block.is_some() {
2296 self.state.status = RunStatus::Blocked;
2297 }
2298 self.state.save()
2299 }
2300
2301 fn resource_halt(&mut self) -> Result<bool> {
2303 if self.state.resource_block.is_none() {
2304 return Ok(false);
2305 }
2306 self.stop_for_resource()?;
2307 Ok(true)
2308 }
2309
2310 fn park_here(&mut self) -> Result<bool> {
2317 let parking = self.pause.parked() || self.interrupt.parked();
2318 if !parking && self.state.resource_block.is_some() {
2319 self.stop_for_resource()?;
2320 return Ok(true);
2321 }
2322 if !self.pause.parked() && !self.interrupt.parked() {
2327 return Ok(false);
2328 }
2329 if self.state.parked {
2330 return Ok(true);
2331 }
2332 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2333 if let Some(block) = &mut self.state.resource_block
2334 && matches!(
2335 block.cause,
2336 crate::run::ResourceBlockCause::Cancelled { .. }
2337 )
2338 {
2339 block.cause = crate::run::ResourceBlockCause::Cancelled {
2340 park: cause.clone(),
2341 };
2342 }
2343 let why = match cause.as_ref().map(ParkCause::label) {
2344 Some(reason) => format!(
2345 "parked after `{}` ({reason}) — resume to carry on from here",
2346 self.state.status.as_str()
2347 ),
2348 None => format!(
2349 "parked after `{}` — resume to carry on from here",
2350 self.state.status.as_str()
2351 ),
2352 };
2353 self.state.event("park", why);
2354 self.state
2355 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2356 self.state.parked = true;
2357 self.state.save()?;
2358 Ok(true)
2359 }
2360
2361 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2363 if self.state.prep_failure.is_none() {
2364 return Ok(());
2365 }
2366 let failed_at = self
2367 .state
2368 .events
2369 .iter()
2370 .rev()
2371 .find(|event| event.node == "prep")
2372 .map_or(self.state.updated_at, |event| event.at);
2373 anyhow::ensure!(
2374 approval_at > failed_at,
2375 "preparation retry requires a new explicit release after the failure"
2376 );
2377 self.state.prep_failure = None;
2378 self.state.prep_complete = Some(false);
2379 self.state
2380 .event("prep-retry", "operator approved retry of preparation");
2381 Ok(())
2382 }
2383
2384 pub fn on_pause(&mut self, pause: Pause) {
2386 self.pause = pause;
2387 }
2388
2389 pub fn watch_interrupt(&mut self, pause: Pause) {
2395 self.interrupt = pause;
2396 }
2397
2398 fn settle_questions(&mut self) {
2418 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2419 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2420 }
2421 }
2422
2423 fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2426 Box::pin(self.finish_after_tally_inner())
2427 }
2428
2429 async fn finish_after_tally_inner(&mut self) -> Result<()> {
2430 self.graft().await?;
2433 if self.resource_halt()? {
2434 return Ok(());
2435 }
2436 self.fold_losers().await?;
2437 self.sync_to_base().await?;
2442 if self.hold_base_sync().await? || self.resource_halt()? {
2443 return Ok(());
2444 }
2445 if self.state.status == RunStatus::AlreadyInBase {
2446 return Ok(());
2447 }
2448 self.review_loop().await?;
2449 if self.resource_halt()? {
2450 return Ok(());
2451 }
2452 self.sync_to_base().await?;
2453 if self.hold_base_sync().await? || self.resource_halt()? {
2454 return Ok(());
2455 }
2456 if self.state.status == RunStatus::AlreadyInBase {
2457 return Ok(());
2458 }
2459 self.gate().await?;
2460 self.merge().await?;
2461 if self.resource_halt()? {
2462 return Ok(());
2463 }
2464 self.state.save()?;
2465 Ok(())
2466 }
2467
2468 async fn setup_seat_worktree(
2475 &mut self,
2476 repo: &Path,
2477 wt: &Path,
2478 in_prep: bool,
2479 coordination: Option<&crate::repo_lock::SeatLease>,
2480 ) -> Result<()> {
2481 let result = match ensure_setup_config(&mut self.state, repo).await {
2482 Ok(()) => {
2483 worktree_setup::prepare_with(
2484 &self.state.config,
2485 repo,
2486 wt,
2487 crate::build_target::Coordination::Held(coordination),
2488 )
2489 .await
2490 }
2491 Err(e) => Err(e),
2492 };
2493 if let Err(e) = result {
2494 git::worktree_remove_clean(repo, wt).await.ok();
2495 if in_prep {
2496 self.rollback_prep(repo).await;
2497 }
2498 self.state.status = RunStatus::Blocked;
2499 self.state.event(
2500 "setup",
2501 format!("worktree setup failed in {}: {e:#}", wt.display()),
2502 );
2503 self.state.save()?;
2504 return Err(e);
2505 }
2506 Ok(())
2507 }
2508
2509 async fn rollback_prep(&mut self, repo: &Path) {
2511 let root = self.state.worktree_root();
2512 for c in std::mem::take(&mut self.state.candidates) {
2513 if matches!(
2514 git::worktree_remove_clean(repo, &c.worktree).await,
2515 Ok(true)
2516 ) {
2517 git::branch_delete(repo, &c.branch).await.ok();
2518 } else {
2519 self.state.candidates.push(c);
2520 }
2521 }
2522 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2523 for j in 1..=seats {
2524 remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2525 .await
2526 .ok();
2527 }
2528 for k in 1..=self.state.config.graph.advisors {
2529 remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2530 .await
2531 .ok();
2532 }
2533 }
2534
2535 async fn prep(&mut self) -> Result<()> {
2536 if let Some(reason) = &self.state.prep_failure {
2537 bail!(
2538 "candidate preparation previously failed: {reason}; correct the source before retrying"
2539 );
2540 }
2541 if self
2542 .state
2543 .prep_complete
2544 .unwrap_or(!self.state.candidates.is_empty())
2545 {
2546 return Ok(());
2547 }
2548 let repo = self.state.repo.clone();
2549 let _coordination = self.coordinate(&repo).await?;
2552 let base = self.state.base_commit.clone();
2553 let plan = self.state.sources.plan(&repo, &base).await?;
2554 self.rollback_prep(&repo).await;
2555 if !self.state.candidates.is_empty() {
2556 bail!(
2557 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2558 );
2559 }
2560 self.state.prep_complete = Some(false);
2561 self.state.status = RunStatus::Prep;
2562 self.state.save()?;
2563 let repo = self.state.repo.clone();
2564 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2565 let root = self.state.worktree_root();
2566 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2567
2568 let hooks_dir = self.state.dir().join("hooks");
2571 if self.state.config.blind.commit_msg_hook {
2572 std::fs::create_dir_all(&hooks_dir)
2573 .with_context(|| format!("create {}", hooks_dir.display()))?;
2574 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2575 let path = hooks_dir.join("commit-msg");
2576 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2577 make_executable(&path)?;
2578 git::acquire_worktree_config(&repo).await?;
2586 self.state.enabled_worktree_config = true;
2587 }
2588
2589 ensure_setup_config(&mut self.state, &repo).await?;
2592
2593 for (index, (spec, label)) in self
2594 .roles
2595 .implementers
2596 .clone()
2597 .into_iter()
2598 .zip(labels)
2599 .enumerate()
2600 {
2601 let branch = self.state.branch_for(label);
2602 let worktree = root.join(format!("cand-{label}"));
2603 self.state.event(
2607 "prep-intent",
2608 serde_json::json!({
2609 "branch": branch,
2610 "worktree": worktree,
2611 "start": start,
2612 "picks": plan.picks,
2613 })
2614 .to_string(),
2615 );
2616 self.state.save()?;
2617 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2618 if self.state.config.blind.commit_msg_hook {
2619 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2620 }
2621 git::local_exclude(&worktree, "/.magi/").await?;
2622 if let Err(e) = worktree_setup::prepare_with(
2623 &self.state.config,
2624 &repo,
2625 &worktree,
2626 crate::build_target::Coordination::Held(_coordination.as_ref()),
2627 )
2628 .await
2629 {
2630 git::worktree_remove_clean(&repo, &worktree).await.ok();
2633 git::branch_delete(&repo, &branch).await.ok();
2634 self.rollback_prep(&repo).await;
2635 self.state.status = RunStatus::Blocked;
2636 self.state
2637 .event("prep", format!("worktree setup failed: {e:#}"));
2638 if !e.is::<crate::repo_lock::ResourceBlocked>() {
2639 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2640 }
2641 self.state.save()?;
2642 return Err(e);
2643 }
2644 self.state.candidates.push(Candidate {
2645 index,
2646 label,
2647 agent: spec.id.clone(),
2648 branch,
2649 worktree,
2650 summary: String::new(),
2651 stat: String::new(),
2652 files: 0,
2653 commits: 0,
2654 empty: false,
2655 failed: None,
2656 verified_noop: None,
2657 duration_ms: 0,
2658 folded: false,
2659 });
2660 self.state.save()?;
2661 for pick in &plan.picks {
2662 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2663 {
2664 self.state.status = RunStatus::Blocked;
2665 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2666 self.state
2667 .event("prep", format!("cannot apply referenced commit: {e}"));
2668 self.state.save()?;
2669 return Err(e);
2670 }
2671 }
2672 }
2673
2674 for j in 1..=self.roles.judges.len() {
2675 let wt = root.join(format!("judge-{j}"));
2676 if !wt.exists() {
2677 git::worktree_add_detached(&repo, &wt, &base).await?;
2678 self.setup_seat_worktree(&repo, &wt, true, _coordination.as_ref())
2679 .await?;
2680 }
2681 }
2682
2683 if self.state.config.graph.advise {
2691 for k in 1..=self.state.config.graph.advisors {
2692 let wt = root.join(format!("advisor-{k}"));
2693 if !wt.exists() {
2694 git::worktree_add_detached(&repo, &wt, &base).await?;
2695 self.setup_seat_worktree(&repo, &wt, true, _coordination.as_ref())
2696 .await?;
2697 }
2698 }
2699 }
2700
2701 let authors: Vec<&str> = self
2706 .roles
2707 .implementers
2708 .iter()
2709 .map(|a| a.id.as_str())
2710 .collect();
2711 let overlap: Vec<String> = self
2712 .roles
2713 .judges
2714 .iter()
2715 .enumerate()
2716 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2717 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2718 .collect();
2719 if !overlap.is_empty() {
2720 let note = format!(
2721 "{} also authored a candidate; blind, but the panel is less \
2722 independent than {} distinct agents would be",
2723 overlap.join(", "),
2724 self.roles.judges.len()
2725 );
2726 self.state.event("prep", note);
2727 }
2728
2729 self.state.event(
2730 "prep",
2731 format!(
2732 "{} candidates, {} judges, base {} ({})",
2733 self.state.candidates.len(),
2734 self.roles.judges.len(),
2735 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2736 self.state.base_branch
2737 ),
2738 );
2739 self.state.prep_complete = Some(true);
2740 self.state.status = RunStatus::Implementing;
2741 self.state.save()?;
2742 Ok(())
2743 }
2744
2745 async fn advise(&mut self) -> Result<()> {
2778 let implement_untouched = self
2779 .state
2780 .candidates
2781 .iter()
2782 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2783 if !self.state.config.graph.advise || self.state.advise_attempted {
2784 return Ok(());
2785 }
2786 if !implement_untouched {
2787 self.state.event(
2788 "advise",
2789 "skipping the design-deliberation stage: at least one \
2790 candidate already shows implementation progress, so this \
2791 run is past the point the stage exists to run before"
2792 .to_owned(),
2793 );
2794 self.state.advise_attempted = true;
2795 self.state.save()?;
2796 return Ok(());
2797 }
2798 let run_id = self.state.id.clone();
2799 let prompts = self.state.config.prompts.clone();
2800 let instruction = self.state.instruction.clone();
2801 let language = self.state.config.graph.language.clone();
2802 let root = self.state.worktree_root();
2803 let n = self.state.config.graph.advisors;
2804 let where_recorded = self.state.dir().join("run.json");
2805
2806 let seats = match self.state.config.advisors() {
2807 Ok(seats) if !seats.is_empty() => seats,
2808 Ok(_) => {
2809 self.state.event(
2810 "advise",
2811 format!(
2812 "[graph] advisors is 0; skipping the design-deliberation \
2813 stage and continuing without a synthesis brief (see {})",
2814 where_recorded.display()
2815 ),
2816 );
2817 self.state.advise_attempted = true;
2818 self.state.save()?;
2819 return Ok(());
2820 }
2821 Err(e) => {
2822 self.state.event(
2823 "advise",
2824 format!(
2825 "could not resolve advisor seats ({e:#}); continuing \
2826 without a design-deliberation brief (see {})",
2827 where_recorded.display()
2828 ),
2829 );
2830 self.state.advise_attempted = true;
2831 self.state.save()?;
2832 return Ok(());
2833 }
2834 };
2835
2836 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2837 let artifacts = agent::artifacts_dir(&self.state.dir());
2838 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2839
2840 let mut jobs = Vec::new();
2841 for (i, spec) in seats.iter().cloned().enumerate() {
2842 let seat_key = format!("advisor-{}", i + 1);
2843 let seat = self.seat(&seat_key, &spec.id);
2844 jobs.push(SeatJob {
2845 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2846 spec,
2847 seat,
2848 cwd: worktrees[i % worktrees.len()].clone(),
2849 timeout,
2850 allow_write: false,
2851 sessions: false,
2852 artifacts: artifacts.clone(),
2853 stem: seat_key,
2854 handover: None,
2855 });
2856 }
2857
2858 self.state.event(
2859 "advise",
2860 format!(
2861 "{} advisor seat(s) sketching a design in parallel",
2862 jobs.len()
2863 ),
2864 );
2865 let mut quota_losses = Vec::new();
2866 let cache = self.state.config.cache_dir();
2867 let ctx = WaveCtx {
2868 carry_seats: false,
2869 pauses: Some((&self.pause, &self.interrupt)),
2870 run: &run_id,
2871 node: "advise",
2872 prompts: &prompts,
2873 cache: cache.as_deref(),
2874 round: None,
2875 };
2876 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2877 let results = ask_json_wave::<Proposal>(
2878 jobs,
2879 Arc::clone(&self.sem),
2880 self.state.config.graph.retries,
2881 &advisor_roster,
2882 &ctx,
2883 &mut quota_losses,
2884 &mut self.state,
2885 &|p: &Proposal| p.validate(),
2886 )
2887 .await;
2888 if self.state.resource_block.is_some() {
2889 return self.stop_for_resource();
2890 }
2891 self.state.quota.extend(quota_losses);
2892
2893 let mut records = Vec::with_capacity(results.len());
2894 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2895 let agent_id = seat.agent.clone();
2896 self.state.seats.insert(seat.key.clone(), seat);
2897 match res {
2898 Ok((proposal, out)) => {
2899 self.state
2900 .event("advise", format!("advisor-{} proposed a design", i + 1));
2901 records.push(advise::AdvisorRecord::proposed(
2902 i + 1,
2903 agent_id,
2904 proposal,
2905 out.duration_ms,
2906 ));
2907 }
2908 Err(e) => {
2909 self.state.event(
2910 "advise",
2911 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2912 );
2913 records.push(advise::AdvisorRecord::failed(
2914 i + 1,
2915 agent_id,
2916 e.to_string(),
2917 ));
2918 }
2919 }
2920 }
2921
2922 let mut advice = advise::Advice {
2923 records,
2924 synthesis: None,
2925 };
2926 if advice.proposals().is_empty() {
2927 self.state.event(
2928 "advise",
2929 "no advisor produced a usable proposal; continuing without a \
2930 synthesis brief"
2931 .to_owned(),
2932 );
2933 } else {
2934 match self
2935 .synthesize_brief(
2936 &advice,
2937 &instruction,
2938 &language,
2939 &worktrees[0],
2940 &artifacts,
2941 &run_id,
2942 &prompts,
2943 cache.as_deref(),
2944 )
2945 .await
2946 {
2947 Ok(Some(text)) => {
2948 self.state.event(
2949 "advise",
2950 "synthesized a design brief for the implementer".to_owned(),
2951 );
2952 advice.synthesis = Some(text);
2953 }
2954 Ok(None) => {
2955 self.state.event(
2956 "advise",
2957 "the synthesis seat produced nothing usable; continuing \
2958 without a design brief"
2959 .to_owned(),
2960 );
2961 }
2962 Err(e) => {
2963 if e.is::<crate::repo_lock::ResourceBlocked>() {
2964 return Err(e);
2965 }
2966 self.state.event(
2967 "advise",
2968 format!("could not synthesize a design brief: {e:#}"),
2969 );
2970 }
2971 }
2972 }
2973 advise::apply_reflection(&mut advice);
2974
2975 self.state.advice = Some(advice);
2976 self.state.advise_attempted = true;
2977 self.state.save()?;
2978 Ok(())
2979 }
2980
2981 #[allow(clippy::too_many_arguments)]
2993 async fn synthesize_brief(
2994 &mut self,
2995 advice: &advise::Advice,
2996 instruction: &str,
2997 language: &str,
2998 cwd: &Path,
2999 artifacts: &Path,
3000 run_id: &str,
3001 prompts: &Prompts,
3002 cache: Option<&Path>,
3003 ) -> Result<Option<String>> {
3004 let chain = agent::pick_chain(
3005 &self.state.config.agents,
3006 self.state.config.roles.synthesizer.as_ref(),
3007 &agent::installed,
3008 "synthesizer",
3009 )?;
3010 let proposals = advice.proposals();
3011 let mut prompt = prompt::with_overlay(
3012 prompt::synthesize_brief(instruction, &proposals, language),
3013 prompts.overlay("advise"),
3014 );
3015 if cache.is_some() {
3016 prompt.push('\n');
3021 prompt.push_str(&prompt::build_cache_note("advise", false));
3022 }
3023 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
3024 let mut last = None;
3029 for (n, spec) in chain.iter().enumerate() {
3030 if n > 0 {
3031 self.state
3032 .event("advise", format!("synthesis falling back to {}", spec.id));
3033 }
3034 let mut seat = self.seat("advise-synthesis", &spec.id);
3035 let wait = self.repository_wait();
3036 let outcome = wait
3037 .scope(agent::invoke(
3038 spec,
3039 &mut seat,
3040 &Invocation {
3041 coordination: self.state.config.coordination.as_ref(),
3042 fetched_base: None,
3043 cwd,
3044 prompt: &prompt,
3045 timeout,
3046 allow_write: false,
3047 sessions: false,
3048 artifacts,
3049 stem: &if n == 0 {
3050 "advise-synthesis".to_owned()
3051 } else {
3052 format!("advise-synthesis-{}", spec.id)
3053 },
3054 run: run_id,
3055 node: "advise",
3056 cache_dir: None,
3057 attachments: &[],
3058 writable: &[],
3059 },
3060 ))
3061 .await;
3062 if outcome.is_ok() {
3063 self.state.seats.insert(seat.key.clone(), seat);
3064 }
3065 let advance = agent::chain_advances(&outcome);
3066 last = Some(outcome);
3067 if !advance {
3068 break;
3069 }
3070 }
3071 let out = last.expect("a chain holds at least one agent")?;
3074 if !out.usable() {
3075 return Ok(None);
3076 }
3077 let text =
3078 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3079 Ok((!text.trim().is_empty()).then_some(text))
3080 }
3081
3082 async fn implement(&mut self) -> Result<()> {
3085 self.persist_candidates().await?;
3086 let run_id = self.state.id.clone();
3091 let prompts = self.state.config.prompts.clone();
3092 let todo: Vec<usize> = self
3093 .state
3094 .candidates
3095 .iter()
3096 .enumerate()
3097 .filter(|(_, c)| {
3098 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3099 && c.failed.is_none()
3100 && !c.empty
3101 })
3102 .map(|(i, _)| i)
3103 .collect();
3104 if todo.is_empty() {
3105 return self.after_implement();
3106 }
3107 self.state.status = RunStatus::Implementing;
3108
3109 let language = self.state.config.graph.language.clone();
3110 let github_language = self.state.config.graph.github_language.clone();
3111 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3112 let sessions = self.state.config.graph.sessions;
3113 let artifacts = agent::artifacts_dir(&self.state.dir());
3114 let brief = self
3118 .state
3119 .advice
3120 .as_ref()
3121 .and_then(|a| a.synthesis.as_deref())
3122 .map(str::to_owned);
3123 let attachments = self.state.attachments.clone();
3124
3125 let mut jobs = Vec::new();
3126 for &i in &todo {
3127 let (index, label, worktree) = {
3128 let c = &self.state.candidates[i];
3129 (c.index, c.label, c.worktree.clone())
3130 };
3131 let spec = self.roles.implementers[index].clone();
3132 let seat_key = format!("impl-{label}");
3133 let seat = self.seat(&seat_key, &spec.id);
3134 let instruction = seeded_instruction(&self.state);
3135 jobs.push(SeatJob {
3136 spec,
3137 seat,
3138 prompt: prompt::implement_in(
3139 &instruction,
3140 &worktree.to_string_lossy(),
3141 &language,
3142 &github_language,
3143 brief.as_deref(),
3144 &attachments,
3145 ),
3146 cwd: worktree,
3147 timeout,
3148 allow_write: true,
3149 sessions,
3150 artifacts: artifacts.clone(),
3151 stem: format!("impl-{label}"),
3152 handover: None,
3153 });
3154 }
3155
3156 self.state.event(
3157 "implement",
3158 format!("{} candidates in parallel", jobs.len()),
3159 );
3160 let mut sent = jobs.clone();
3166 let cache = self.state.config.cache_dir();
3167 let ctx = WaveCtx {
3168 carry_seats: false,
3169 pauses: Some((&self.pause, &self.interrupt)),
3170 run: &run_id,
3171 node: "implement",
3172 prompts: &prompts,
3173 cache: cache.as_deref(),
3174 round: None,
3175 };
3176 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3177 if !wave_blocked(&results) {
3178 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3179 .await;
3180 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3181 .await?;
3182 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3183 .await;
3184 }
3185 if !results.is_empty()
3188 && results
3189 .iter()
3190 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3191 {
3192 let losses: Vec<_> = results
3193 .iter()
3194 .map(|(_, seat, out)| {
3195 let AgentOutcome::Quota(o) = out else {
3196 unreachable!()
3197 };
3198 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3199 })
3200 .collect();
3201 self.state.quota.extend(losses.clone());
3202 for (_, seat, _) in results {
3203 self.state.seats.insert(seat.key.clone(), seat);
3204 }
3205 for &i in &todo {
3206 let label = self.state.candidates[i].label;
3207 self.state.interrupted_implementations.insert(label);
3208 self.state
3209 .candidate_preservation
3210 .insert(label, "machine outage awaiting persistence".into());
3211 }
3212 block_machine_outage(&mut self.state, "implement", &losses);
3215 self.state.save()?;
3216 self.persist_candidates().await?;
3217 for &i in &todo {
3218 self.state
3219 .interrupted_implementations
3220 .insert(self.state.candidates[i].label);
3221 }
3222 return self.stop_for_resource();
3223 }
3224 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3225
3226 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3227 && results
3228 .iter()
3229 .any(|(_, _, out)| FailClass::of(out).is_some());
3230 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3231 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3232 if self
3236 .state
3237 .candidate_preservation
3238 .contains_key(&self.state.candidates[i].label)
3239 {
3240 self.state.seats.insert(seat.key.clone(), seat);
3241 }
3242 continue;
3243 }
3244 let seat_key = seat.key.clone();
3245 let agent = seat.agent.clone();
3254 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3255 self.state.seats.insert(seat.key.clone(), seat);
3256 let label = self.state.candidates[i].label;
3257 self.state.interrupted_implementations.remove(&label);
3258
3259 let (summary, duration, failed, verified_claim) = match out {
3260 AgentOutcome::Ok(o) => {
3261 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3262 let failed = (!o.usable()).then(|| {
3263 if o.timed_out {
3264 "agent timed out".to_owned()
3265 } else {
3266 format!(
3267 "agent exited with {}",
3268 crate::notices::exit_status(o.exit_code)
3269 )
3270 }
3271 });
3272 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3273 (text, o.duration_ms, failed, verified_claim)
3274 }
3275 AgentOutcome::Dropped(o) => {
3281 let why = o
3282 .dropped
3283 .as_ref()
3284 .map(|d| d.why.as_str())
3285 .unwrap_or("the CLI ended the stream without delivering its answer");
3286 (
3287 String::new(),
3288 o.duration_ms,
3289 Some(format!("the CLI dropped the stream ({why})")),
3290 None,
3291 )
3292 }
3293 AgentOutcome::Quota(o) => {
3294 self.state.quota.push(QuotaLoss::from_report(
3295 seat_key,
3296 "implement",
3297 o.quota.as_ref(),
3298 ));
3299 (
3300 String::new(),
3301 o.duration_ms,
3302 Some("rate limited (quota); produced no change".to_owned()),
3303 None,
3304 )
3305 }
3306 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3307 (String::new(), 0, Some(e), None)
3308 }
3309 };
3310
3311 let c = &mut self.state.candidates[i];
3314 if !exhausted_the_fallback_chain {
3315 c.agent = agent;
3316 }
3317 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3318 c.duration_ms = duration;
3319 c.failed = failed;
3320 c.verified_noop = verified_claim;
3321 self.state
3322 .candidate_preservation
3323 .insert(label, "awaiting persistence".to_owned());
3324 }
3325 self.state.save()?;
3326 self.persist_candidates().await?;
3327
3328 if resource_blocked {
3329 return self.stop_for_resource();
3330 }
3331 if selected_failed {
3332 self.state.status = RunStatus::Blocked;
3333 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3334 self.state.save()?;
3335 bail!(
3336 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3337 );
3338 }
3339 self.after_implement()
3340 }
3341
3342 async fn persist_candidates(&mut self) -> Result<()> {
3344 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3345 for label in labels {
3346 let i = self
3347 .state
3348 .candidates
3349 .iter()
3350 .position(|c| c.label == label)
3351 .context("pending candidate is missing")?;
3352 let c = self.state.candidates[i].clone();
3353 let interrupted = self.state.interrupted_implementations.contains(&label);
3354 let result = async {
3355 let wait = crate::repo_lock::WaitPolicy::new(
3356 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3357 || false,
3358 );
3359 let _coordination =
3360 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3361 if let Err(e) = git::snapshot_uncommitted(
3364 &c.worktree,
3365 &self
3366 .state
3367 .dir()
3368 .join("artifacts")
3369 .join(format!("candidate-{label}-recovery")),
3370 )
3371 .await
3372 {
3373 self.state.event(
3374 "preserve",
3375 format!("candidate {label} snapshot incomplete: {e:#}"),
3376 );
3377 }
3378 let rescued = git::rescue_commit(
3379 &c.worktree,
3380 &format!("magi: candidate {label} (uncommitted work)"),
3381 )
3382 .await?;
3383 self.state.note_withheld("implement", &rescued.withheld);
3384 let base = &self.state.base_commit;
3385 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3386 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3387 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3388 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3389 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3390 if c.empty && commits > 0 && !patch.trim().is_empty()
3391 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3392 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3393 || self.state.tally.is_some())
3394 {
3395 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3399 }
3400 let c = &mut self.state.candidates[i];
3401 c.commits = commits;
3402 c.stat = stat;
3403 c.files = files;
3404 let no_changes = commits == 0 || patch.trim().is_empty();
3405 c.empty = no_changes && !interrupted;
3406 if interrupted && no_changes {
3407 c.failed = None;
3408 c.verified_noop = None;
3409 }
3410 if !c.empty {
3411 c.failed = None;
3412 c.verified_noop = None;
3413 }
3414 self.state.event(
3415 "preserve",
3416 format!("candidate {label}: {files} files, {commits} commits"),
3417 );
3418 Ok::<_, anyhow::Error>(())
3419 }
3420 .await;
3421 match result {
3422 Ok(()) => {
3423 self.state.candidate_preservation.remove(&label);
3424 if !interrupted || self.state.candidates[i].files > 0 {
3425 self.state.interrupted_implementations.remove(&label);
3426 }
3427 }
3428 Err(e) => {
3429 let reason = format!("candidate {label} preservation failed: {e:#}");
3430 self.state.event("preserve", &reason);
3431 self.state.candidate_preservation.insert(label, reason);
3432 }
3433 }
3434 if self.state.candidate_preservation.is_empty()
3435 && self.state.status == RunStatus::Blocked
3436 && self.state.candidates.iter().all(|c| c.failed.is_none())
3437 {
3438 self.state.status = RunStatus::Implementing;
3442 self.state.event(
3443 "preserve",
3444 "candidate persistence recovered; implementation can continue",
3445 );
3446 }
3447 self.state.save()?;
3448 }
3449 if !self.state.candidate_preservation.is_empty() {
3450 self.state.status = RunStatus::Blocked;
3451 self.state.save()?;
3452 bail!(
3453 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3454 );
3455 }
3456 Ok(())
3457 }
3458
3459 async fn resume_undelivered(
3487 &mut self,
3488 results: &mut [(usize, SeatState, AgentOutcome)],
3489 sent: &[SeatJob],
3490 prompts: &Prompts,
3491 run_id: &str,
3492 ) {
3493 for (wi, seat, out) in results.iter_mut() {
3494 let Some(dropped) = (match &*out {
3495 AgentOutcome::Dropped(o) => o.dropped.clone(),
3496 _ => None,
3497 }) else {
3498 continue;
3499 };
3500 let Some(job) = sent.get(*wi) else { continue };
3501 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3503 self.state.event(
3504 "implement",
3505 format!(
3506 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3507 work is in the tree",
3508 seat.key, dropped.output_tokens, dropped.why
3509 ),
3510 );
3511 continue;
3512 }
3513 if !has_context(&job.spec, seat, job.sessions) {
3521 self.state.event(
3522 "implement",
3523 format!(
3524 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3525 is no session left to resume",
3526 seat.key, dropped.output_tokens, dropped.why
3527 ),
3528 );
3529 continue;
3530 }
3531 self.state.event(
3532 "implement",
3533 format!(
3534 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3535 conversation",
3536 seat.key, dropped.output_tokens, dropped.why
3537 ),
3538 );
3539 let mut retry = job.clone();
3540 retry.seat = seat.clone();
3541 retry.prompt = prompt::resume_after_drop(&dropped.why);
3542 retry.timeout = retry_budget(job.timeout, true);
3543 retry.stem = format!("{}-resume", job.stem);
3544 let cache = self.state.config.cache_dir();
3545 let ctx = WaveCtx {
3546 carry_seats: false,
3547 pauses: Some((&self.pause, &self.interrupt)),
3548 run: run_id,
3549 node: "implement",
3550 prompts,
3551 cache: cache.as_deref(),
3552 round: None,
3553 };
3554 let (resumed_seat, resumed) =
3555 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3556 *seat = resumed_seat;
3557 *out = resumed;
3558 }
3559 }
3560
3561 async fn resume_seat_handovers(
3626 &mut self,
3627 results: &mut [(usize, SeatState, AgentOutcome)],
3628 sent: &mut [SeatJob],
3629 prompts: &Prompts,
3630 run_id: &str,
3631 ) -> Result<()> {
3632 let instruction = seeded_instruction(&self.state);
3633 let language = self.state.config.graph.language.clone();
3634 let github_language = self.state.config.graph.github_language.clone();
3635 let brief = self
3636 .state
3637 .advice
3638 .as_ref()
3639 .and_then(|a| a.synthesis.as_deref())
3640 .map(str::to_owned);
3641 let attachments = self.state.attachments.clone();
3642 for (wi, seat, out) in results.iter_mut() {
3643 let others: BTreeSet<String> = sent
3646 .iter()
3647 .enumerate()
3648 .filter(|(j, _)| j != wi)
3649 .map(|(_, j)| j.spec.id.clone())
3650 .collect();
3651 let Some(job) = sent.get_mut(*wi) else {
3652 continue;
3653 };
3654 let start = self
3659 .roles
3660 .implementer_roster
3661 .iter()
3662 .position(|s| s.id == job.spec.id)
3663 .unwrap_or(0);
3664 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3665 let mut fallback_attempt = 0usize;
3666 let mut prev: Option<FailClass> = None;
3667 while let Some(cur) = FailClass::of(&*out) {
3668 if !should_hand_over(prev.as_ref(), &cur) {
3669 break;
3670 }
3671 let Some(next) =
3672 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3673 .cloned()
3674 else {
3675 break;
3676 };
3677 tried.insert(next.id.clone());
3678 fallback_attempt += 1;
3679
3680 let coordination = self.coordinate(&job.cwd).await?;
3681 if let Ok(r) = git::rescue_commit(
3682 &job.cwd,
3683 &format!(
3684 "magi: candidate {} (uncommitted work before {} fallback)",
3685 seat.key,
3686 if cur == FailClass::Quota {
3687 "quota"
3688 } else {
3689 "handover"
3690 }
3691 ),
3692 )
3693 .await
3694 {
3695 self.state.note_withheld("implement", &r.withheld);
3696 }
3697 drop(coordination);
3698
3699 record_handover(
3700 &mut self.state,
3701 "implement",
3702 &seat.key,
3703 &seat.agent,
3704 &next.id,
3705 &cur,
3706 &fail_reason(&*out),
3707 );
3708 prev = Some(cur.clone());
3709
3710 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3711 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3712 job.spec = next.clone();
3720 let mut retry = job.clone();
3721 retry.seat = new_seat;
3722 retry.prompt = prompt::implement_in(
3723 &instruction,
3724 &job.cwd.to_string_lossy(),
3725 &language,
3726 &github_language,
3727 brief.as_deref(),
3728 &attachments,
3729 );
3730 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3731 let cache = self.state.config.cache_dir();
3732 let ctx = WaveCtx {
3733 carry_seats: false,
3734 pauses: Some((&self.pause, &self.interrupt)),
3735 run: run_id,
3736 node: "implement",
3737 prompts,
3738 cache: cache.as_deref(),
3739 round: None,
3740 };
3741 let (fallback_seat, fallback_out) = run_one(
3742 retry,
3743 Arc::clone(&self.sem),
3744 &ctx,
3745 &mut self.state,
3746 fallback_attempt,
3747 )
3748 .await;
3749 *seat = fallback_seat;
3750 *out = fallback_out;
3751 }
3752 }
3753 Ok(())
3754 }
3755
3756 async fn resume_unconfirmed_commands(
3780 &mut self,
3781 results: &mut [(usize, SeatState, AgentOutcome)],
3782 sent: &[SeatJob],
3783 prompts: &Prompts,
3784 run_id: &str,
3785 ) {
3786 for (wi, seat, out) in results.iter_mut() {
3787 let AgentOutcome::Ok(o) = &*out else {
3788 continue;
3789 };
3790 if !has_unconfirmed_command(&o.commands) {
3791 continue;
3792 }
3793 let Some(job) = sent.get(*wi) else { continue };
3794 if !has_context(&job.spec, seat, job.sessions) {
3795 self.state.event(
3796 "implement",
3797 format!(
3798 "{}: the reply named a command whose own CLI never confirmed the exit \
3799 status of, but there is no session left to resume",
3800 seat.key
3801 ),
3802 );
3803 continue;
3804 }
3805 self.state.event(
3806 "implement",
3807 format!(
3808 "{}: the reply named a command whose own CLI never confirmed the exit \
3809 status of; resuming the conversation",
3810 seat.key
3811 ),
3812 );
3813 let mut retry = job.clone();
3814 retry.seat = seat.clone();
3815 retry.prompt = prompt::resume_incomplete(
3816 "a command in your last reply had no confirmed exit status",
3817 );
3818 retry.timeout = retry_budget(job.timeout, true);
3819 retry.stem = format!("{}-confirm", job.stem);
3820 let cache = self.state.config.cache_dir();
3821 let ctx = WaveCtx {
3822 carry_seats: false,
3823 pauses: Some((&self.pause, &self.interrupt)),
3824 run: run_id,
3825 node: "implement",
3826 prompts,
3827 cache: cache.as_deref(),
3828 round: None,
3829 };
3830 let (resumed_seat, resumed) =
3831 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3832 *seat = resumed_seat;
3833 *out = resumed;
3834 }
3835 }
3836
3837 async fn continue_fix_report(
3858 &mut self,
3859 mut seat: SeatState,
3860 parse_err: String,
3861 job: &SeatJob,
3862 prompts: &Prompts,
3863 run_id: &str,
3864 round: usize,
3865 ) -> (
3866 SeatState,
3867 Option<FixReport>,
3868 Option<String>,
3869 ContinuationRecord,
3870 ) {
3871 let mut last_err = parse_err;
3872 let mut cumulative_wait_ms = 0u64;
3873 let mut attempts = 0usize;
3874 loop {
3875 if !has_context(&job.spec, &seat, job.sessions) {
3876 self.state.event(
3877 "fix",
3878 format!(
3879 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3880 session left to resume into"
3881 ),
3882 );
3883 let outcome = if attempts == 0 {
3884 ContinuationOutcome::NoSession
3885 } else {
3886 ContinuationOutcome::Exhausted
3887 };
3888 return (
3889 seat,
3890 None,
3891 Some(format!("unparsable fix report: {last_err}")),
3892 ContinuationRecord {
3893 attempts,
3894 cumulative_wait_ms,
3895 outcome,
3896 },
3897 );
3898 }
3899 if attempts >= MAX_FIX_CONTINUATIONS {
3900 self.state.event(
3901 "fix",
3902 format!(
3903 "round {round}: fixer's reply still had no adoption report after \
3904 {attempts} continuation(s) ({last_err}); giving up"
3905 ),
3906 );
3907 return (
3908 seat,
3909 None,
3910 Some(format!(
3911 "unparsable fix report after {attempts} continuation(s): {last_err}"
3912 )),
3913 ContinuationRecord {
3914 attempts,
3915 cumulative_wait_ms,
3916 outcome: ContinuationOutcome::Exhausted,
3917 },
3918 );
3919 }
3920 attempts += 1;
3921 self.state.event(
3922 "fix",
3923 format!(
3924 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3925 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3926 ),
3927 );
3928 let mut retry = job.clone();
3929 retry.seat = seat.clone();
3930 retry.prompt = prompt::resume_incomplete(&last_err);
3931 retry.timeout = retry_budget(job.timeout, true);
3932 retry.stem = format!("{}-continue{attempts}", job.stem);
3933 let cache = self.state.config.cache_dir();
3934 let ctx = WaveCtx {
3935 carry_seats: false,
3936 pauses: Some((&self.pause, &self.interrupt)),
3937 run: run_id,
3938 node: "fix",
3939 prompts,
3940 cache: cache.as_deref(),
3941 round: Some(round),
3942 };
3943 let (resumed_seat, resumed_out) = run_one(
3944 retry,
3945 Arc::clone(&self.sem),
3946 &ctx,
3947 &mut self.state,
3948 attempts,
3949 )
3950 .await;
3951 seat = resumed_seat;
3952 match resumed_out {
3953 AgentOutcome::Ok(o) => {
3954 cumulative_wait_ms += o.duration_ms;
3955 match verdict::extract_json::<FixReport>(&o.text) {
3956 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3957 self.state.event(
3958 "fix",
3959 format!(
3960 "round {round}: fixer's adoption report recovered after \
3961 {attempts} continuation(s)"
3962 ),
3963 );
3964 return (
3965 seat,
3966 Some(report),
3967 None,
3968 ContinuationRecord {
3969 attempts,
3970 cumulative_wait_ms,
3971 outcome: ContinuationOutcome::Resumed,
3972 },
3973 );
3974 }
3975 Ok(_) => {
3983 last_err = "the reply parsed, but it reported a command whose own CLI \
3984 never confirmed an exit status"
3985 .to_owned();
3986 }
3987 Err(e) => last_err = e.to_string(),
3988 }
3989 }
3990 AgentOutcome::Quota(o) => {
3991 cumulative_wait_ms += o.duration_ms;
3992 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3993 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3994 self.state.quota.push(loss);
3995 self.state.event(
3996 "fix",
3997 format!(
3998 "round {round}: continuation rate limited (quota); not retrying now"
3999 ),
4000 );
4001 return (
4002 seat,
4003 None,
4004 Some("rate limited (quota) while recovering the fix report".to_owned()),
4005 ContinuationRecord {
4006 attempts,
4007 cumulative_wait_ms,
4008 outcome: ContinuationOutcome::QuotaLost,
4009 },
4010 );
4011 }
4012 AgentOutcome::Dropped(o) => {
4013 cumulative_wait_ms += o.duration_ms;
4014 let why = o
4015 .dropped
4016 .as_ref()
4017 .map(|d| d.why.as_str())
4018 .unwrap_or("the CLI ended the stream without delivering its answer");
4019 last_err = format!("the CLI dropped the stream ({why})");
4020 }
4021 AgentOutcome::Failed(e) => last_err = e,
4022 AgentOutcome::ResourceBlocked(e) => {
4025 return (
4026 seat,
4027 None,
4028 Some(format!("the continuation was not started: {e}")),
4029 ContinuationRecord {
4030 attempts: attempts.saturating_sub(1),
4031 cumulative_wait_ms,
4032 outcome: ContinuationOutcome::NoSession,
4033 },
4034 );
4035 }
4036 }
4037 }
4038 }
4039
4040 fn after_implement(&mut self) -> Result<()> {
4041 if !self.state.candidate_preservation.is_empty() {
4042 bail!("candidate preservation incomplete; judging is unsafe");
4043 }
4044 if self.state.leaks.is_empty() {
4046 let cfg = self.state.config.blind.clone();
4047 let mut leaks = Vec::new();
4048 for c in &self.state.candidates {
4049 let Some(patch) =
4050 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4051 else {
4052 continue;
4053 };
4054 leaks.extend(blind::scan(
4055 &format!("candidate {} patch", c.label),
4056 &patch,
4057 &cfg.vendor_tokens,
4058 ));
4059 }
4060 if !leaks.is_empty() {
4061 let summary = leaks
4062 .iter()
4063 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4064 .collect::<Vec<_>>()
4065 .join(", ");
4066 match cfg.on_leak {
4067 LeakPolicy::Fail => {
4068 self.state.status = RunStatus::Failed;
4069 self.state
4070 .event("blind", format!("vendor text in a patch: {summary}"));
4071 self.state.leaks = leaks;
4072 self.state.save()?;
4073 self.settle_questions();
4074 bail!(
4075 "blind.on_leak = \"fail\" and vendor text reached a \
4076 judged patch: {summary}"
4077 );
4078 }
4079 LeakPolicy::Redact => self.state.event(
4080 "blind",
4081 format!("redacting vendor text for judging: {summary}"),
4082 ),
4083 LeakPolicy::Warn => self.state.event(
4084 "blind",
4085 format!("vendor text present in a judged patch (shown as-is): {summary}"),
4086 ),
4087 }
4088 self.state.leaks = leaks;
4089 }
4090 }
4091
4092 if self.state.viable().is_empty() {
4093 if self.state.all_candidates_verified_noop() {
4094 self.state.status = RunStatus::VerifiedNoop;
4105 self.state.save()?;
4106 self.settle_questions();
4107 return Ok(());
4108 }
4109 self.state.status = RunStatus::Failed;
4110 self.state.save()?;
4111 self.settle_questions();
4112 bail!("no candidate produced a change; nothing to judge");
4113 }
4114 self.state.status = RunStatus::Judging;
4115 self.state.save()?;
4116 Ok(())
4117 }
4118
4119 async fn judge(&mut self) -> Result<()> {
4122 let run_id = self.state.id.clone();
4127 let prompts = self.state.config.prompts.clone();
4128 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4129 return Ok(());
4130 }
4131 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4132 if viable.len() == 1 {
4133 self.state.judge_skipped = true;
4140 self.state.event(
4141 "judge",
4142 format!(
4143 "only candidate {} produced a change; judging skipped",
4144 viable[0].label
4145 ),
4146 );
4147 self.state.save()?;
4148 return Ok(());
4149 }
4150 self.state.status = RunStatus::Judging;
4151
4152 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4153 let language = self.state.config.graph.language.clone();
4154 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4155 let sessions = self.state.config.graph.sessions;
4156 let artifacts = agent::artifacts_dir(&self.state.dir());
4157 let root = self.state.worktree_root();
4158 let base_short = short(&self.state.base_commit);
4159
4160 let mut jobs = Vec::new();
4161 let mut orders = Vec::new();
4162 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4163 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4164 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4165 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4166 let seat_key = format!("judge-{}", j + 1);
4167 let seat = self.seat(&seat_key, &spec.id);
4168 jobs.push(SeatJob {
4169 prompt: prompt::judge(
4170 &self.state.instruction,
4171 &views,
4172 self.roles.judges.len(),
4173 &base_short,
4174 &language,
4175 ),
4176 spec,
4177 seat,
4178 cwd: root.join(format!("judge-{}", j + 1)),
4179 timeout,
4180 allow_write: false,
4181 sessions,
4182 artifacts: artifacts.clone(),
4183 stem: format!("judge-{}", j + 1),
4184 handover: None,
4185 });
4186 }
4187
4188 self.state.event(
4189 "judge",
4190 format!(
4191 "{} judges ranking {} candidates blind",
4192 jobs.len(),
4193 viable.len()
4194 ),
4195 );
4196 let labels_for_check = labels.clone();
4197 let mut quota_losses = Vec::new();
4198 let cache = self.state.config.cache_dir();
4199 let ctx = WaveCtx {
4200 carry_seats: false,
4201 pauses: Some((&self.pause, &self.interrupt)),
4202 run: &run_id,
4203 node: "judge",
4204 prompts: &prompts,
4205 cache: cache.as_deref(),
4206 round: None,
4207 };
4208 let results = ask_json_wave::<Ranking>(
4209 jobs,
4210 Arc::clone(&self.sem),
4211 self.state.config.graph.retries,
4212 &self.roles.judge_roster,
4213 &ctx,
4214 &mut quota_losses,
4215 &mut self.state,
4216 &move |r: &Ranking| r.validate(&labels_for_check),
4217 )
4218 .await;
4219 if self.state.resource_block.is_some() {
4220 return self.stop_for_resource();
4221 }
4222 self.state.quota.extend(quota_losses);
4223
4224 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4225 let agent_id = seat.agent.clone();
4226 self.state.seats.insert(seat.key.clone(), seat);
4227 let mut record = Judgement {
4228 judge: j + 1,
4229 seat: format!("judge-{}", j + 1),
4230 agent: agent_id,
4231 ranking: Vec::new(),
4232 reasons: BTreeMap::new(),
4233 confidence: None,
4234 strengths: BTreeMap::new(),
4235 order: orders[j].clone(),
4236 failed: None,
4237 duration_ms: 0,
4238 };
4239 match res {
4240 Ok((ranking, out)) => {
4241 record.ranking = ranking.normalized();
4242 record.reasons = ranking.reasons;
4243 record.strengths = ranking.strengths;
4244 record.confidence = ranking.confidence;
4245 record.duration_ms = out.duration_ms;
4246 self.state.event(
4247 "judge",
4248 format!(
4249 "judge {} ranked {}",
4250 j + 1,
4251 record.ranking.iter().collect::<String>()
4252 ),
4253 );
4254 }
4255 Err(e) => {
4256 record.failed = Some(e.to_string());
4257 self.state
4258 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4259 }
4260 }
4261 self.state.judgements.push(record);
4262 self.state.save()?;
4263 }
4264 Ok(())
4265 }
4266
4267 async fn deliberate(&mut self) -> Result<()> {
4270 if self.hold_base_sync().await? {
4271 return Ok(());
4272 }
4273 let run_id = self.state.id.clone();
4278 let prompts = self.state.config.prompts.clone();
4279 let rounds = self.state.config.graph.deliberate_rounds;
4280 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4282 return Ok(());
4283 }
4284 let tops: Vec<char> = self
4285 .state
4286 .judgements
4287 .iter()
4288 .filter_map(|j| j.ranking.first().copied())
4289 .collect();
4290 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4291 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4292 self.state.event(
4293 "deliberate",
4294 format!("judges agreed on {} outright; no deliberation", tops[0]),
4295 );
4296 }
4297 self.state.status = RunStatus::Voting;
4298 self.state.save()?;
4299 return Ok(());
4300 }
4301
4302 self.state.status = RunStatus::Deliberating;
4303 self.state.event(
4304 "deliberate",
4305 format!(
4306 "split: first choices were {} — opening {rounds} round(s)",
4307 tops.iter().collect::<String>()
4308 ),
4309 );
4310
4311 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4312 let language = self.state.config.graph.language.clone();
4313 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4314 let sessions = self.state.config.graph.sessions;
4315 let artifacts = agent::artifacts_dir(&self.state.dir());
4316 let root = self.state.worktree_root();
4317 let base_short = short(&self.state.base_commit);
4318
4319 for round in self.state.deliberation.len() + 1..=rounds {
4323 let mut turns: Vec<DeliberationTurn> = Vec::new();
4324 let mut round_losses = Vec::new();
4325 let mut asked = 0;
4326 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4327 if self.state.judgements[j].failed.is_some() {
4328 continue;
4329 }
4330 let seat_key = format!("judge-{}", j + 1);
4331 let spec = self.occupant(&seat_key, spec);
4332 let mut seat = self.seat(&seat_key, &spec.id);
4333 let transcript = self.transcript(&turns, j);
4334 let build = |context: Option<&str>| {
4335 prompt::deliberate(
4336 &self.state.instruction,
4337 context,
4338 &transcript,
4339 round,
4340 rounds,
4341 &language,
4342 )
4343 };
4344 let block = self.candidate_block(&viable, &base_short);
4345 let full = build(Some(&block));
4346 let text = if has_context(&spec, &seat, sessions) {
4347 build(None)
4348 } else {
4349 full.clone()
4350 };
4351 let job = SeatJob {
4352 spec,
4353 seat: seat.clone(),
4354 prompt: text,
4355 cwd: root.join(format!("judge-{}", j + 1)),
4356 timeout,
4357 allow_write: false,
4358 sessions,
4359 artifacts: artifacts.clone(),
4360 stem: format!("delib-{round}-judge-{}", j + 1),
4361 handover: Some(full),
4362 };
4363 let cache = self.state.config.cache_dir();
4364 let ctx = WaveCtx {
4365 carry_seats: false,
4366 pauses: Some((&self.pause, &self.interrupt)),
4367 run: &run_id,
4368 node: "deliberate",
4369 prompts: &prompts,
4370 cache: cache.as_deref(),
4371 round: None,
4372 };
4373 let mut losses = Vec::new();
4377 asked += 1;
4378 let mut results = ask_wave_with::<String>(
4379 vec![job],
4380 Arc::clone(&self.sem),
4381 0,
4382 &self.roles.judge_roster,
4383 &ctx,
4384 &mut losses,
4385 &mut self.state,
4386 &|text: &str| {
4387 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4388 },
4389 )
4390 .await;
4391 if self.state.resource_block.is_some() {
4392 return self.stop_for_resource();
4393 }
4394 round_losses.extend(losses.iter().cloned());
4395 self.state.quota.extend(losses);
4396 let (updated, res, _) = results.pop().expect("one job in, one result out");
4397 seat = updated;
4398 let agent_id = seat.agent.clone();
4399 self.state.seats.insert(seat.key.clone(), seat);
4400 let body = match res {
4401 Ok((body, _)) => body,
4402 Err(e) => {
4405 self.state
4406 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4407 continue;
4408 }
4409 };
4410 let tentative = verdict::extract_json::<Position>(&body)
4411 .ok()
4412 .and_then(|p| p.tentative)
4413 .and_then(|s| s.trim().chars().next())
4414 .map(|c| c.to_ascii_uppercase());
4415 self.state.event(
4416 "deliberate",
4417 format!(
4418 "round {round}: judge {} now favours {}",
4419 j + 1,
4420 tentative.map_or("—".to_owned(), |c| c.to_string())
4421 ),
4422 );
4423 turns.push(DeliberationTurn {
4424 judge: j + 1,
4425 agent: agent_id,
4426 body: blind::sanitize_prose(&body, &self.state.config.blind),
4427 tentative,
4428 });
4429 }
4430 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4431 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4432 return self.stop_for_resource();
4433 }
4434 self.state
4435 .deliberation
4436 .push(DeliberationRound { round, turns });
4437 self.state.save()?;
4438 }
4439
4440 self.state.status = RunStatus::Voting;
4441 self.state.save()?;
4442 Ok(())
4443 }
4444
4445 async fn vote(&mut self) -> Result<()> {
4448 if self.hold_base_sync().await? {
4449 return Ok(());
4450 }
4451 let run_id = self.state.id.clone();
4456 let prompts = self.state.config.prompts.clone();
4457 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4458 return Ok(());
4459 }
4460 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4461 if viable.len() == 1 {
4462 return Ok(());
4463 }
4464 self.state.status = RunStatus::Voting;
4465
4466 let language = self.state.config.graph.language.clone();
4467 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4468 let sessions = self.state.config.graph.sessions;
4469 let artifacts = agent::artifacts_dir(&self.state.dir());
4470 let root = self.state.worktree_root();
4471 let base_short = short(&self.state.base_commit);
4472 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4473
4474 let mut jobs = Vec::new();
4475 let mut seats_at = Vec::new();
4476 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4477 if self
4478 .state
4479 .judgements
4480 .get(j)
4481 .is_some_and(|r| r.failed.is_some())
4482 {
4483 continue;
4484 }
4485 let seat_key = format!("judge-{}", j + 1);
4486 let spec = self.occupant(&seat_key, spec);
4487 let seat = self.seat(&seat_key, &spec.id);
4488 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4489 let text = if has_context(&spec, &seat, sessions) {
4490 prompt::final_vote(&viable, &language)
4491 } else {
4492 full.clone()
4493 };
4494 jobs.push(SeatJob {
4495 spec,
4496 seat,
4497 prompt: text,
4498 cwd: root.join(format!("judge-{}", j + 1)),
4499 timeout,
4500 allow_write: false,
4501 sessions,
4502 artifacts: artifacts.clone(),
4503 stem: format!("vote-judge-{}", j + 1),
4504 handover: Some(full),
4505 });
4506 seats_at.push(j);
4507 }
4508
4509 self.state.event(
4510 "vote",
4511 format!(
4512 "collecting {} final votes one by one, privately",
4513 jobs.len()
4514 ),
4515 );
4516 let allowed = viable.clone();
4517 let mut quota_losses = Vec::new();
4518 let cache = self.state.config.cache_dir();
4519 let ctx = WaveCtx {
4520 carry_seats: false,
4521 pauses: Some((&self.pause, &self.interrupt)),
4522 run: &run_id,
4523 node: "vote",
4524 prompts: &prompts,
4525 cache: cache.as_deref(),
4526 round: None,
4527 };
4528 let results = ask_json_wave::<FinalVote>(
4529 jobs,
4530 Arc::clone(&self.sem),
4531 self.state.config.graph.retries,
4532 &self.roles.judge_roster,
4533 &ctx,
4534 &mut quota_losses,
4535 &mut self.state,
4536 &move |v: &FinalVote| match v.label() {
4537 Some(c) if allowed.contains(&c) => Ok(()),
4538 other => bail!("vote {other:?} is not one of {allowed:?}"),
4539 },
4540 )
4541 .await;
4542 if self.state.resource_block.is_some() {
4543 return self.stop_for_resource();
4544 }
4545 self.state.quota.extend(quota_losses);
4546
4547 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4548 let agent_id = seat.agent.clone();
4549 self.state.seats.insert(seat.key.clone(), seat);
4550 let initial = self
4551 .state
4552 .judgements
4553 .get(j)
4554 .and_then(|r| r.ranking.first().copied());
4555 let mut record = VoteRecord {
4556 judge: j + 1,
4557 agent: agent_id,
4558 vote: None,
4559 reason: String::new(),
4560 changed: false,
4561 };
4562 match res {
4563 Ok((v, _)) => {
4564 record.vote = v.label();
4565 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4566 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4567 self.state.event(
4568 "vote",
4569 format!(
4570 "judge {} voted {}{}",
4571 j + 1,
4572 record.vote.unwrap_or('?'),
4573 if record.changed { " (changed)" } else { "" }
4574 ),
4575 );
4576 }
4577 Err(e) => {
4578 self.state
4579 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4580 }
4581 }
4582 self.state.votes.push(record);
4583 self.state.save()?;
4584 }
4585 Ok(())
4586 }
4587
4588 fn tally(&mut self) -> Result<()> {
4591 if self.state.tally.is_some() {
4592 return Ok(());
4593 }
4594 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4595 let tops: Vec<char> = self
4596 .state
4597 .judgements
4598 .iter()
4599 .filter_map(|j| j.ranking.first().copied())
4600 .collect();
4601 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4602
4603 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4606 let mut cast: Vec<char> = Vec::new();
4607 for (i, j) in self.state.judgements.iter().enumerate() {
4608 let vote = self
4609 .state
4610 .votes
4611 .iter()
4612 .find(|v| v.judge == i + 1)
4613 .and_then(|v| v.vote)
4614 .or_else(|| j.ranking.first().copied());
4615 if let Some(v) = vote {
4616 *first_choice.entry(v).or_insert(0) += 1;
4617 cast.push(v);
4618 }
4619 }
4620
4621 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4622 for j in &self.state.judgements {
4623 let n = j.ranking.len();
4624 for (pos, label) in j.ranking.iter().enumerate() {
4625 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4626 }
4627 }
4628
4629 let best = first_choice.values().copied().max().unwrap_or(0);
4630 let mut leaders: Vec<char> = first_choice
4631 .iter()
4632 .filter(|(_, v)| **v == best)
4633 .map(|(k, _)| *k)
4634 .collect();
4635 let mut tie_break = None;
4636 if leaders.len() > 1 {
4637 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4638 let borda_leaders: Vec<char> = leaders
4639 .iter()
4640 .copied()
4641 .filter(|l| borda[l] == top_borda)
4642 .collect();
4643 tie_break = Some(if borda_leaders.len() == 1 {
4644 format!(
4645 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4646 leaders.len()
4647 )
4648 } else {
4649 format!(
4650 "{} way tie on both first-choice votes and Borda points, broken by label order",
4651 leaders.len()
4652 )
4653 });
4654 leaders = borda_leaders;
4655 leaders.sort_unstable();
4656 }
4657 let winner = *leaders
4658 .first()
4659 .or(viable.first())
4660 .context("no candidate to declare a winner from")?;
4661
4662 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4663 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4664 let deliberated = !self.state.deliberation.is_empty();
4665
4666 let quota_seats: std::collections::BTreeSet<&str> =
4670 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4671 let mut present = 0usize;
4672 for (i, j) in self.state.judgements.iter().enumerate() {
4673 if quota_seats.contains(j.seat.as_str()) {
4674 continue;
4675 }
4676 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4677 let voted = self
4678 .state
4679 .votes
4680 .iter()
4681 .any(|v| v.judge == i + 1 && v.vote.is_some());
4682 if ranked || voted {
4683 present += 1;
4684 }
4685 }
4686 let needs_quorum = viable.len() > 1;
4692 let judges_total = if needs_quorum {
4693 self.roles.judges.len()
4694 } else {
4695 0
4696 };
4697 let quorum = if needs_quorum {
4698 judges_total / 2 + 1
4699 } else {
4700 0
4701 };
4702 let met_quorum = !needs_quorum || present >= quorum;
4703 let uncontested = (!needs_quorum).then(|| {
4704 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4705 });
4706
4707 self.state.event(
4708 "tally",
4709 match &uncontested {
4710 Some(reason) => format!("winner {winner} — {reason}"),
4711 None => format!(
4712 "winner {winner} — votes {} | initial {} | {} changed | \
4713 {present}/{judges_total} judges{}",
4714 first_choice
4715 .iter()
4716 .map(|(k, v)| format!("{k}:{v}"))
4717 .collect::<Vec<_>>()
4718 .join(" "),
4719 if unanimous_initial {
4720 "unanimous"
4721 } else {
4722 "split"
4723 },
4724 changed_votes,
4725 if met_quorum {
4726 String::new()
4727 } else {
4728 format!(" — below quorum ({quorum} required)")
4729 },
4730 ),
4731 },
4732 );
4733 if !met_quorum {
4734 self.state.event(
4735 "stall",
4736 format!(
4737 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4738 the run stops here, resumable"
4739 ),
4740 );
4741 }
4742 self.state.tally = Some(Tally {
4743 first_choice,
4744 borda,
4745 winner,
4746 rankings: tops.len(),
4747 unanimous_initial,
4748 deliberated,
4749 changed_votes,
4750 unanimous_final,
4751 tie_break,
4752 judges: judges_total,
4753 present,
4754 quorum,
4755 met_quorum,
4756 uncontested,
4757 });
4758 self.state.status = if met_quorum {
4759 RunStatus::Reviewing
4760 } else {
4761 RunStatus::Stalled
4762 };
4763 self.state.save()?;
4764 Ok(())
4765 }
4766
4767 #[allow(clippy::too_many_lines)]
4788 async fn recover_stall(&mut self) -> Result<bool> {
4789 let run_id = self.state.id.clone();
4794 let prompts = self.state.config.prompts.clone();
4795 let quota_seats: BTreeSet<&str> =
4800 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4801 let absent: Vec<String> = self
4802 .state
4803 .judgements
4804 .iter()
4805 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4806 .map(|j| j.seat.clone())
4807 .collect();
4808 if absent.is_empty() {
4809 return Ok(false);
4810 }
4811 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4812 if viable.len() <= 1 {
4813 return Ok(false);
4814 }
4815 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4816 let language = self.state.config.graph.language.clone();
4817 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4818 let sessions = self.state.config.graph.sessions;
4819 let artifacts = agent::artifacts_dir(&self.state.dir());
4820 let root = self.state.worktree_root();
4821 let base_short = short(&self.state.base_commit);
4822 let candidates: Vec<Candidate> = viable.clone();
4823
4824 let mut positions: Vec<usize> = absent
4826 .iter()
4827 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4828 .collect();
4829 if positions.is_empty() {
4830 return Ok(false);
4831 }
4832 positions.sort_unstable();
4833 positions.dedup();
4834
4835 let mut judge_jobs = Vec::new();
4837 for &j in &positions {
4838 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4839 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4840 let seat_key = format!("judge-{}", j + 1);
4841 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4842 let seat = self.seat(&seat_key, &spec.id);
4843 judge_jobs.push(SeatJob {
4844 spec,
4845 seat,
4846 prompt: prompt::judge(
4847 &self.state.instruction,
4848 &views,
4849 self.roles.judges.len(),
4850 &base_short,
4851 &language,
4852 ),
4853 cwd: root.join(seat_key),
4854 timeout,
4855 allow_write: false,
4856 sessions,
4857 artifacts: artifacts.clone(),
4858 stem: format!("judge-{}-recover", j + 1),
4859 handover: None,
4860 });
4861 }
4862
4863 let labels_for_check = labels.clone();
4864 let mut judge_losses = Vec::new();
4865 let retries = self.state.config.graph.retries;
4866 let cache = self.state.config.cache_dir();
4867 let ctx = WaveCtx {
4868 carry_seats: false,
4869 pauses: Some((&self.pause, &self.interrupt)),
4870 run: &run_id,
4871 node: "judge",
4872 prompts: &prompts,
4873 cache: cache.as_deref(),
4874 round: None,
4875 };
4876 let results = ask_json_wave::<Ranking>(
4877 judge_jobs,
4878 Arc::clone(&self.sem),
4879 retries,
4880 &self.roles.judge_roster,
4881 &ctx,
4882 &mut judge_losses,
4883 &mut self.state,
4884 &move |r: &Ranking| r.validate(&labels_for_check),
4885 )
4886 .await;
4887 if self.state.resource_block.is_some() {
4888 self.stop_for_resource()?;
4889 return Ok(false);
4890 }
4891
4892 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4894 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4895 let agent_id = seat.agent.clone();
4896 self.state.seats.insert(seat.key.clone(), seat);
4897 let record = &mut self.state.judgements[j];
4898 match res {
4899 Ok((ranking, out)) => {
4900 record.agent = agent_id;
4901 record.ranking = ranking.normalized();
4902 record.reasons = ranking.reasons;
4903 record.strengths = ranking.strengths;
4904 record.confidence = ranking.confidence;
4905 record.failed = None;
4906 record.duration_ms = out.duration_ms;
4907 recovered.insert(j);
4908 self.state.event(
4909 "recover",
4910 format!("judge {} ranked again after the limit", j + 1),
4911 );
4912 }
4913 Err(e) => {
4914 self.state
4915 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4916 }
4917 }
4918 }
4919
4920 let mut vote_jobs = Vec::new();
4922 let mut vote_pos: Vec<usize> = Vec::new();
4923 for &j in &recovered {
4924 let seat_key = format!("judge-{}", j + 1);
4925 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4926 let seat = self.seat(&seat_key, &spec.id);
4927 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4928 let text = if has_context(&spec, &seat, sessions) {
4929 prompt::final_vote(&labels, &language)
4930 } else {
4931 full.clone()
4932 };
4933 vote_jobs.push(SeatJob {
4934 spec,
4935 seat,
4936 prompt: text,
4937 cwd: root.join(seat_key),
4938 timeout,
4939 allow_write: false,
4940 sessions,
4941 artifacts: artifacts.clone(),
4942 stem: format!("vote-judge-{}-recover", j + 1),
4943 handover: Some(full),
4944 });
4945 vote_pos.push(j);
4946 }
4947 let allowed = labels.clone();
4948 let mut vote_losses = Vec::new();
4949 let vote_retries = self.state.config.graph.retries;
4950 let vote_cache = self.state.config.cache_dir();
4951 let ctx = WaveCtx {
4952 carry_seats: false,
4953 pauses: Some((&self.pause, &self.interrupt)),
4954 run: &run_id,
4955 node: "vote",
4956 prompts: &prompts,
4957 cache: vote_cache.as_deref(),
4958 round: None,
4959 };
4960 let votes = ask_json_wave::<FinalVote>(
4961 vote_jobs,
4962 Arc::clone(&self.sem),
4963 vote_retries,
4964 &self.roles.judge_roster,
4965 &ctx,
4966 &mut vote_losses,
4967 &mut self.state,
4968 &move |v: &FinalVote| match v.label() {
4969 Some(c) if allowed.contains(&c) => Ok(()),
4970 other => bail!("vote {other:?} is not one of {allowed:?}"),
4971 },
4972 )
4973 .await;
4974 if self.state.resource_block.is_some() {
4975 self.stop_for_resource()?;
4976 return Ok(false);
4977 }
4978 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4979 let agent_id = seat.agent.clone();
4980 self.state.seats.insert(seat.key.clone(), seat);
4981 match res {
4982 Ok((v, _)) => {
4983 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4984 rec.vote = v.label();
4985 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4986 } else {
4987 self.state.votes.push(VoteRecord {
4988 judge: j + 1,
4989 agent: agent_id,
4990 vote: v.label(),
4991 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4992 changed: false,
4993 });
4994 }
4995 self.state.event(
4996 "recover",
4997 format!("judge {} voted again after the limit", j + 1),
4998 );
4999 }
5000 Err(e) => {
5001 self.state
5002 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
5003 }
5004 }
5005 }
5006
5007 let recovered_keys: BTreeSet<String> = recovered
5011 .iter()
5012 .map(|&j| format!("judge-{}", j + 1))
5013 .collect();
5014 self.state
5015 .quota
5016 .retain(|q| !recovered_keys.contains(&q.seat));
5017 for loss in judge_losses.into_iter().chain(vote_losses) {
5021 if recovered_keys.contains(&loss.seat) {
5022 continue;
5023 }
5024 self.state.quota.retain(|q| q.seat != loss.seat);
5025 self.state.quota.push(loss);
5026 }
5027
5028 self.state.tally = None;
5030 self.tally()?;
5031 Ok(self
5032 .state
5033 .tally
5034 .as_ref()
5035 .map(|t| t.met_quorum)
5036 .unwrap_or(false))
5037 }
5038
5039 async fn graft(&mut self) -> Result<()> {
5055 let Err(e) = self.graft_inner().await else {
5056 return Ok(());
5057 };
5058 if e.is::<crate::repo_lock::ResourceBlocked>() {
5059 return Err(e);
5060 }
5061 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5062 self.state.status = RunStatus::Blocked;
5063 self.state.event("graft", format!("blocked: {e:#}"));
5064 self.state.save()?;
5065 return Err(e);
5066 }
5067 if let Some(rec) = self.state.graft.clone()
5072 && !rec.outcome.settled()
5073 {
5074 if let Some(winner) = self.state.winner().cloned() {
5075 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5076 if re.is::<crate::repo_lock::ResourceBlocked>() {
5077 return Err(re);
5078 }
5079 self.state.status = RunStatus::Blocked;
5080 self.state.event("graft", format!("blocked: {re:#}"));
5081 self.state.save()?;
5082 return Err(re);
5083 }
5084 }
5085 if let Some(g) = self.state.graft.as_mut() {
5086 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5087 }
5088 }
5089 self.state.event("graft", format!("skipped: {e:#}"));
5090 if let Err(e) = self.state.save() {
5091 tracing::warn!("save after graft: {e:#}");
5092 }
5093 Ok(())
5094 }
5095
5096 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5099 let _coordination = self.coordinate(wt).await?;
5100 crate::recovery::restore_repair(wt, pre)
5101 .await
5102 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5103 }
5104
5105 fn graft_skipped(&mut self, why: &str) {
5107 let message = format!("skipped: {why}");
5108 if !self
5109 .state
5110 .events
5111 .iter()
5112 .any(|e| e.node == "graft" && e.message == message)
5113 {
5114 self.state.event("graft", message);
5115 }
5116 }
5117
5118 async fn graft_inner(&mut self) -> Result<()> {
5119 if let Some(rec) = self.state.graft.clone()
5122 && !rec.outcome.settled()
5123 {
5124 if self.state.reviews.is_empty()
5128 && let Some(winner) = self.state.winner().cloned()
5129 {
5130 self.graft_restore(&winner.worktree, &rec.pre).await?;
5131 }
5132 if rec.outcome == GraftOutcome::WaitingForRestore {
5133 self.state.graft = None;
5134 self.state.save()?;
5135 } else {
5136 if let Some(g) = self.state.graft.as_mut() {
5137 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5138 }
5139 self.state.event(
5140 "graft",
5141 "rolled back: the previous round was interrupted; the winner is untouched",
5142 );
5143 self.state.save()?;
5144 return Ok(());
5145 }
5146 }
5147 let plan = match graft::plan(&self.state) {
5148 Ok(plan) => plan,
5149 Err(skip) => {
5150 if skip != graft::Skip::AlreadyDone {
5151 self.graft_skipped(skip.reason());
5152 self.state.save()?;
5153 }
5154 return Ok(());
5155 }
5156 };
5157 let Some(winner) = self.state.winner().cloned() else {
5158 return Ok(());
5159 };
5160 let wt = winner.worktree.clone();
5161 if !git::is_clean(&wt).await? {
5162 self.graft_skipped("the winner's worktree has uncommitted changes");
5163 self.state.save()?;
5164 return Ok(());
5165 }
5166 let pre = git::rev_parse(&wt, "HEAD").await?;
5167 self.state.graft = Some(GraftRecord {
5170 pre: pre.clone(),
5171 post: None,
5172 lines: None,
5173 limit: None,
5174 agent: None,
5175 outcome: GraftOutcome::Running,
5176 requested: graft::requested(&plan),
5177 adoption: None,
5178 adoption_error: None,
5179 });
5180 self.state.event(
5181 "graft",
5182 format!(
5183 "porting strengths of candidate(s) {} onto {}",
5184 plan.strengths
5185 .iter()
5186 .map(|(l, _, _)| l.to_string())
5187 .collect::<Vec<_>>()
5188 .join(", "),
5189 plan.winner
5190 ),
5191 );
5192 self.state.save()?;
5193
5194 let fix_prompt = prompt::graft(
5195 &self.state.instruction,
5196 plan.winner,
5197 &plan.strengths,
5198 &self.state.config.graph.language,
5199 );
5200 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5201 let sessions = self.state.config.graph.sessions;
5202 let artifacts = agent::artifacts_dir(&self.state.dir());
5203 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5204 let (_, seat, out) = self
5205 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5206 SeatJob {
5207 prompt: fix_prompt.clone(),
5208 spec,
5209 seat,
5210 cwd: wt.clone(),
5211 timeout,
5212 allow_write: true,
5213 sessions,
5214 artifacts: artifacts.clone(),
5215 stem: "graft".to_owned(),
5216 handover: None,
5217 }
5218 })
5219 .await;
5220 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5221 self.state.graft.as_mut().expect("graft record").outcome =
5224 GraftOutcome::WaitingForRestore;
5225 return self.stop_for_resource();
5226 }
5227 let mut why: Option<String> = None;
5228 match &out {
5229 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5230 why = Some("the fixer produced nothing usable".to_owned());
5231 }
5232 AgentOutcome::Ok(_) => {}
5233 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5234 AgentOutcome::Quota(o) => {
5235 self.state.quota.push(QuotaLoss::from_report(
5236 seat.key.clone(),
5237 "graft",
5238 o.quota.as_ref(),
5239 ));
5240 why = Some("rate limited (quota)".to_owned());
5241 }
5242 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5243 why = Some(format!(
5244 "the fixer failed: {}",
5245 e.lines()
5246 .next()
5247 .unwrap_or("")
5248 .chars()
5249 .take(160)
5250 .collect::<String>()
5251 ));
5252 }
5253 }
5254 let agent_id = seat.agent.clone();
5255 self.state.seats.insert(seat.key.clone(), seat);
5256
5257 let mut post = None;
5258 let mut lines = None;
5259 let _coordination = self.coordinate(&wt).await?;
5260 if why.is_none() {
5261 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5262 if let Ok(r) = &rescued {
5263 self.state.note_withheld("graft", &r.withheld);
5264 }
5265 let head = git::rev_parse(&wt, "HEAD").await?;
5266 let dirty = match &rescued {
5270 Ok(r) => git::status_porcelain(&wt)
5271 .await
5272 .map(|st| {
5273 st.lines().any(|l| {
5274 !r.withheld
5275 .iter()
5276 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5277 })
5278 })
5279 .unwrap_or(true),
5280 Err(_) => true,
5281 };
5282 if dirty {
5283 why = Some("the fixer left changes that could not be committed".to_owned());
5284 } else if head == pre {
5285 why = Some("the fixer changed nothing".to_owned());
5286 } else {
5287 lines = git::changed_lines(&wt, &pre, &head).await;
5288 post = Some(head);
5289 }
5290 }
5291 let (adoption, adoption_error) = match &out {
5292 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5293 _ => (None, None),
5294 };
5295 let requested = graft::requested(&plan);
5296 let outcome = match why {
5297 None => {
5298 self.state.event(
5299 "graft",
5300 format!(
5301 "applied: {}",
5302 lines.map_or_else(
5303 || "changed lines unavailable".to_owned(),
5304 |n| format!("{n} changed lines")
5305 )
5306 ),
5307 );
5308 GraftOutcome::Applied
5309 }
5310 Some(why) => {
5311 crate::recovery::restore_repair(&wt, &pre)
5314 .await
5315 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5316 self.state.event(
5317 "graft",
5318 format!("rolled back: {why}; the winner is untouched"),
5319 );
5320 GraftOutcome::RolledBack(why)
5321 }
5322 };
5323 self.state.graft = Some(GraftRecord {
5324 pre,
5325 post,
5326 lines,
5327 limit: None,
5328 agent: Some(agent_id),
5329 outcome,
5330 requested,
5331 adoption,
5332 adoption_error,
5333 });
5334 self.state.save()?;
5335 Ok(())
5336 }
5337
5338 async fn fold_losers(&mut self) -> Result<()> {
5341 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5342 return Ok(());
5343 };
5344 if !self.state.candidate_preservation.is_empty() {
5345 bail!("candidate preservation incomplete; refusing cleanup");
5346 }
5347 let repo = self.state.repo.clone();
5348 let _coordination = self.coordinate(&repo).await?;
5349 let home = crate::run::home();
5350 let root = self.state.worktree_root();
5351 let isolated = crate::build_target::isolated(&self.state.config);
5352 let mut folded = Vec::new();
5353 for i in 0..self.state.candidates.len() {
5354 let c = &self.state.candidates[i];
5355 if c.label == winner || c.folded {
5356 continue;
5357 }
5358 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5359 let target = fold_target(&home, &root, &wt, isolated);
5360 remove_preserved_worktree(&repo, &wt).await?;
5361 reclaim_fold_target(&home, target);
5362 if git::branch_exists(&repo, &branch).await?
5363 && !git::branch_delete(&repo, &branch).await?
5364 {
5365 bail!("candidate branch removal refused");
5366 }
5367 self.state.candidates[i].folded = true;
5368 self.state.save()?;
5369 folded.push(label.to_string());
5370 }
5371 for j in 1..=self.roles.judges.len() {
5373 let wt = root.join(format!("judge-{j}"));
5374 if wt.exists() {
5375 let target = fold_target(&home, &root, &wt, isolated);
5376 remove_read_seat_worktree(&mut self.state, &wt).await?;
5377 reclaim_fold_target(&home, target);
5378 self.state.save()?;
5379 }
5380 }
5381 if self.state.config.graph.advise {
5384 for k in 1..=self.state.config.graph.advisors {
5385 let wt = root.join(format!("advisor-{k}"));
5386 if wt.exists() {
5387 let target = fold_target(&home, &root, &wt, isolated);
5388 remove_read_seat_worktree(&mut self.state, &wt).await?;
5389 reclaim_fold_target(&home, target);
5390 self.state.save()?;
5391 }
5392 }
5393 }
5394 if !folded.is_empty() {
5395 self.state
5396 .event("fold", format!("folded candidates {}", folded.join(", ")));
5397 self.state.save()?;
5398 }
5399 Ok(())
5400 }
5401
5402 async fn hold_base_sync(&mut self) -> Result<bool> {
5405 self.hold_base_sync_as(None).await
5406 }
5407
5408 async fn hold_base_sync_kind(
5409 &mut self,
5410 kind: crate::base_sync_recovery::StopKind,
5411 ) -> Result<()> {
5412 self.hold_base_sync_as(Some(kind)).await?;
5413 Ok(())
5414 }
5415
5416 async fn hold_base_sync_as(
5417 &mut self,
5418 kind: Option<crate::base_sync_recovery::StopKind>,
5419 ) -> Result<bool> {
5420 if !self.state.base_sync_blocked() {
5421 self.state.base_sync_recovery.stop = None;
5422 return Ok(false);
5423 }
5424 let head = match self.state.winner() {
5425 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5426 .await
5427 .unwrap_or_default(),
5428 None => String::new(),
5429 };
5430 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5431 && !authorization.consumed
5432 && authorization.stop.head != head
5433 {
5434 authorization.refuse(format!(
5435 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5436 authorization.stop.head
5437 ));
5438 }
5439 self.state.record_base_sync_stop(&head);
5440 if let Some(kind) = kind {
5441 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5442 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5443 stop.kind = kind;
5444 }
5445 }
5446 self.state.save()?;
5447 Ok(true)
5448 }
5449
5450 async fn sync_to_base(&mut self) -> Result<()> {
5492 if self.state.status == RunStatus::AlreadyInBase {
5493 return Ok(());
5494 }
5495 self.hold_base_sync().await?;
5496 let conflicted = self
5497 .state
5498 .base_sync
5499 .as_ref()
5500 .is_some_and(|s| s.conflict.is_some());
5501 let Some(winner) = self.state.winner().cloned() else {
5502 return Ok(());
5503 };
5504
5505 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5506
5507 let repo = self.state.repo.clone();
5508 let remote = self.state.config.merge.remote.clone();
5509 let base_branch = self.state.base_branch.clone();
5510 let tracking = format!("{remote}/{base_branch}");
5511
5512 let fetched_base =
5513 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5514 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5519 return Ok(());
5520 };
5521
5522 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5523 self.state.ensure_base_episode();
5524 let mut pending = self
5525 .state
5526 .base_sync_recovery
5527 .operations
5528 .last()
5529 .filter(|o| !o.finished)
5530 .cloned();
5531 let scratch = self.state.dir().join("base-sync");
5532 if git::rebase_in_progress(&scratch).await {
5533 if pending.is_none()
5536 && self.state.base_sync_recovery.operations.is_empty()
5537 && !conflicted
5538 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5539 .await
5540 .ok()
5541 .as_deref()
5542 == Some(head.as_str())
5543 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5544 && git::is_clean(&winner.worktree).await?
5545 {
5546 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5547 if self.state.base_sync.is_none() {
5548 self.state.base_sync = Some(BaseSync {
5549 tip: saved_tip.clone(),
5550 behind: 0,
5551 attempts: 0,
5552 conflict: None,
5553 already_in: None,
5554 });
5555 }
5556 self.state.reserve_base_operation(&head, &saved_tip);
5557 self.state.event(
5558 "base-recovery",
5559 "imported standing legacy rebase as a spent synchronization reservation",
5560 );
5561 self.state.save()?;
5562 pending = self.state.base_sync_recovery.operations.last().cloned();
5563 }
5564 let expected = pending.as_ref().map(|op| op.tip.as_str());
5565 if expected.is_none()
5566 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5567 {
5568 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5569 if self.state.base_sync.is_none() {
5570 self.state.base_sync = Some(BaseSync {
5571 tip: tip.clone(),
5572 behind: 0,
5573 attempts: 0,
5574 conflict: None,
5575 already_in: None,
5576 });
5577 }
5578 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5579 self.state.event("land", why);
5580 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5581 .await?;
5582 return Ok(());
5583 }
5584 }
5585 if let Some(op) = &pending {
5586 let mut replay_proven =
5588 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5589 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5590 let wait = repository_wait(&self.state, None);
5591 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5592 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5593 Ok(true) => {
5594 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5595 self.state.save()?;
5596 git::worktree_remove(&repo, &scratch).await.ok();
5597 replay_proven = true;
5598 }
5599 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5600 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5601 }
5602 }
5603 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5604 if self.state.foreign_checkout.is_some() {
5605 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5606 .await
5607 .ok()
5608 .as_deref()
5609 == Some(op.head.as_str())
5610 && git::rev_parse(&repo, &format!("{head}^2"))
5611 .await
5612 .ok()
5613 .as_deref()
5614 == Some(op.tip.as_str());
5615 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5616 .await
5617 .is_ok_and(|m| m.is_empty())
5618 && let Ok(added) =
5619 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5620 {
5621 replay_proven = true;
5622 for commit in added {
5623 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5624 replay_proven = false;
5625 break;
5626 }
5627 }
5628 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5629 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5630 }
5631 }
5632 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5636 || (head != op.head
5637 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5638 .await?
5639 .is_empty())
5640 || (!git::is_clean(&winner.worktree).await?
5641 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5642 .await
5643 .is_ok_and(|o| o.ok()))
5644 || !replay_proven
5645 {
5646 let why = format!(
5647 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5648 op.id
5649 );
5650 let sync = self
5651 .state
5652 .base_sync
5653 .as_mut()
5654 .context("reserved synchronization has no lifetime accounting")?;
5655 sync.conflict = Some(why.clone());
5656 self.state.event("land", why);
5657 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5658 .await?;
5659 return Ok(());
5660 }
5661 if head != op.head {
5662 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5663 }
5664 tip = op.tip.clone();
5665 }
5666 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5667 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5668 if pending.is_none()
5669 && !conflicted
5670 && fetched_base
5671 && self
5672 .state
5673 .base_sync_recovery
5674 .episodes
5675 .last()
5676 .is_some_and(|e| e.boundary.is_some())
5677 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5678 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5679 && owner.execution_runs().last() == Some(&self.state.id)
5680 && owner.status != crate::queue::TaskStatus::Done
5681 && !owner.fresh_start
5682 && owner.review_branch.is_none()
5683 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5684 && let Some(approval) = owner.resumption.as_ref()
5685 && approval.instruction == owner.instruction
5686 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5687 && git::is_clean(&winner.worktree).await?
5688 && self.state.renew_base_episode(&task, approval, &head, &tip)
5689 {
5690 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5691 self.state.save()?;
5692 }
5693
5694 if (behind > 0 || conflicted || head == tip)
5703 && self
5704 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5705 .await?
5706 {
5707 return Ok(());
5708 }
5709 if conflicted && self.state.base_recovery_pending() {
5710 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5711 let owner =
5712 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5713 let owner_valid = owner.as_ref().is_ok_and(|t| {
5714 t.execution_runs().last() == Some(&self.state.id)
5715 && t.status != crate::queue::TaskStatus::Done
5716 && !t.fresh_start
5717 && t.review_branch.is_none()
5718 });
5719 let invalidation = if !owner_valid {
5720 Some("latest task ownership changed")
5721 } else if !owner.as_ref().is_ok_and(|t| {
5722 authorization
5723 .generation
5724 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5725 }) {
5726 Some("owner approval generation changed")
5727 } else if authorization.question.as_ref().is_some_and(|question| {
5728 !crate::base_recovery_control::decision_current(
5729 &crate::queue::Queue::at(crate::run::home().join("queue")),
5730 &self.state,
5731 &authorization.task,
5732 question,
5733 )
5734 }) {
5735 Some("recovery decision ownership, routing or stop evidence changed")
5736 } else if authorization.request.is_some()
5737 && !owner
5738 .as_ref()
5739 .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5740 {
5741 Some("owner decision changed")
5742 } else if let Some(scope) = &authorization.scope {
5743 if scope.tip != tip {
5744 Some("fetched base changed")
5745 } else if !self
5746 .state
5747 .base_sync_recovery
5748 .episodes
5749 .last()
5750 .is_some_and(|e| e.id == scope.episode)
5751 {
5752 Some("active synchronization interval changed")
5753 } else if !owner
5754 .as_ref()
5755 .is_ok_and(|t| t.instruction == scope.instruction)
5756 {
5757 Some("task instruction changed")
5758 } else {
5759 None
5760 }
5761 } else {
5762 None
5763 };
5764 if let Some(reason) = invalidation {
5765 let authorization = self
5766 .state
5767 .base_sync_recovery
5768 .authorizations
5769 .last_mut()
5770 .unwrap();
5771 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5772 self.state.save()?;
5773 return Ok(());
5774 }
5775 if !fetched_base
5776 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5777 || !git::is_clean(&winner.worktree).await?
5778 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5779 {
5780 return Ok(());
5781 }
5782 }
5783 let recovery = conflicted && self.state.consume_base_recovery(&head);
5784 if conflicted {
5785 self.state.save()?;
5787 if !recovery {
5788 return Ok(());
5789 }
5790 }
5791
5792 if behind == 0 {
5793 if let Some(from) = self
5802 .state
5803 .rebase_fixes
5804 .iter()
5805 .rev()
5806 .find_map(|r| r.from.clone())
5807 && from != head
5808 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5809 .await
5810 .is_ok_and(|o| o.ok())
5811 {
5812 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5813 }
5814 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5820 if let Some(why) = &conflict {
5821 self.state.status = RunStatus::Blocked;
5822 self.state.event("land", why.clone());
5823 }
5824 self.state.base_sync = Some(BaseSync {
5825 tip,
5826 behind: 0,
5827 attempts,
5828 conflict,
5829 already_in: None,
5830 });
5831 if pending.is_some() {
5832 self.state.finish_base_operation(Some(&head));
5833 }
5834 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5835 .await?;
5836 self.state.save()?;
5837 return Ok(());
5838 }
5839
5840 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5841 let why = format!(
5842 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5843 synchronization(s); synchronizing again would only race it",
5844 winner.branch
5845 );
5846 self.state.status = RunStatus::Blocked;
5847 self.state.base_sync = Some(BaseSync {
5848 tip,
5849 behind,
5850 attempts,
5851 conflict: Some(why.clone()),
5852 already_in: None,
5853 });
5854 self.state.event("land", why);
5855 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5856 .await?;
5857 self.state.save()?;
5858 return Ok(());
5859 }
5860
5861 if self.state.foreign_checkout.is_some() {
5862 let wait = repository_wait(&self.state, None);
5865 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5866 if self.state.base_sync.is_none() {
5867 self.state.base_sync = Some(BaseSync {
5868 tip: tip.clone(),
5869 behind,
5870 attempts: 0,
5871 conflict: None,
5872 already_in: None,
5873 });
5874 }
5875 self.state.reserve_base_operation(&head, &tip);
5876 self.state.save()?;
5877 let merged = git::git_raw(
5878 &winner.worktree,
5879 &[
5880 "-c",
5881 "core.hooksPath=/dev/null",
5882 "merge",
5883 "--no-ff",
5884 "--no-autostash",
5885 "--no-edit",
5886 "-m",
5887 &format!("Merge {tracking} before verifying"),
5888 &tip,
5889 ],
5890 )
5891 .await?;
5892 let conflict = if merged.ok() {
5893 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5894 self.state.event(
5895 "land",
5896 format!("merged {tracking} into {} before verifying", winner.branch),
5897 );
5898 None
5899 } else {
5900 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5901 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5902 }
5903 let why = format!(
5904 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5905 winner.branch,
5906 merged.stderr.chars().take(600).collect::<String>()
5907 );
5908 self.state.status = RunStatus::Blocked;
5909 self.state.event("land", &why);
5910 Some(why)
5911 };
5912 let synced = conflict.is_none();
5913 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5914 self.state
5915 .finish_base_operation(synced.then_some(fresh.as_str()));
5916 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5917 self.state.base_sync = Some(BaseSync {
5918 tip,
5919 behind: if synced { 0 } else { behind },
5920 attempts,
5921 conflict,
5922 already_in: None,
5923 });
5924 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5925 .await?;
5926 self.state.save()?;
5927 return Ok(());
5928 }
5929
5930 self.state.event(
5931 "land",
5932 format!(
5933 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5934 winner.branch
5935 ),
5936 );
5937 self.state.save()?;
5938
5939 let branch_tracking = format!("{remote}/{}", winner.branch);
5945 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5946 let remote_tip = if self.state.task_pr_branch().is_none()
5947 && matches!(&fetched_branch, Ok(o) if o.ok())
5948 {
5949 git::rev_parse(&repo, &branch_tracking).await.ok()
5950 } else {
5951 None
5952 };
5953 if let Some(theirs) = &remote_tip
5957 && !git::is_ancestor(&repo, theirs, &head).await
5958 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5959 .await
5960 .is_ok_and(|missing| missing.is_empty())
5961 {
5962 let why = format!(
5963 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5964 them",
5965 short(theirs),
5966 winner.branch
5967 );
5968 self.state.status = RunStatus::Blocked;
5969 self.state.base_sync = Some(BaseSync {
5970 tip,
5971 behind,
5972 attempts,
5973 conflict: Some(why.clone()),
5974 already_in: None,
5975 });
5976 self.state.event("land", why);
5977 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5978 .await?;
5979 self.state.save()?;
5980 return Ok(());
5981 }
5982
5983 if self.state.base_sync.is_none() {
5984 self.state.base_sync = Some(BaseSync {
5985 tip: tip.clone(),
5986 behind,
5987 attempts: 0,
5988 conflict: None,
5989 already_in: None,
5990 });
5991 }
5992 let scratch = self.state.dir().join("base-sync");
5993 if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5994 {
5995 if error.is::<crate::repo_lock::ResourceBlocked>() {
5996 return Ok(());
5997 }
5998 return Err(error);
5999 }
6000 self.state.reserve_base_operation(&head, &tip);
6001 self.state.save()?;
6002 let rebased =
6003 match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
6004 .await
6005 {
6006 Ok(crate::rebase::Rebased::Applied) => Ok(None),
6007 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
6008 Err(e) => Err(e),
6009 };
6010 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
6011 let stop_kind = match &rebased {
6012 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
6013 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
6014 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
6015 };
6016 match rebased {
6017 Ok(None) => {
6018 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
6022 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
6023 self.state.finish_base_operation(Some(&fresh));
6024 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
6025 let mut conflict = None;
6026 if let Some(pinned) = &remote_tip {
6027 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
6028 match pushed {
6029 Ok(o) if o.ok() => self.state.event(
6030 "land",
6031 format!("pushed rebased {} to {remote}", winner.branch),
6032 ),
6033 Ok(o) => {
6034 conflict = Some(format!(
6035 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
6036 winner.branch,
6037 short(pinned),
6038 o.stderr.chars().take(600).collect::<String>()
6039 ));
6040 }
6041 Err(e) => {
6042 conflict = Some(format!(
6043 "rebased {} locally but could not push it: {e:#}",
6044 winner.branch
6045 ));
6046 }
6047 }
6048 }
6049 if let Some(why) = &conflict {
6050 self.state.status = RunStatus::Blocked;
6051 self.state.event("land", why.clone());
6052 }
6053 self.state.base_sync = Some(BaseSync {
6054 tip: tip.clone(),
6055 behind: 0,
6056 attempts,
6057 conflict,
6058 already_in: None,
6059 });
6060 self.state
6061 .event("land", format!("rebased {} onto {tracking}", winner.branch));
6062 }
6063 Ok(Some(conflict)) => {
6064 self.state.finish_base_operation(None);
6065 let why = format!(
6066 "{} conflicts with {tracking} and did not rebase: {}",
6067 winner.branch,
6068 conflict.chars().take(600).collect::<String>()
6069 );
6070 self.state.status = RunStatus::Blocked;
6071 self.state.base_sync = Some(BaseSync {
6072 tip,
6073 behind,
6074 attempts,
6075 conflict: Some(why.clone()),
6076 already_in: None,
6077 });
6078 self.state.event("land", why);
6079 }
6080 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6084 self.state.save()?;
6085 return Ok(());
6086 }
6087 Err(e) => {
6088 self.state.finish_base_operation(None);
6089 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6090 self.state.status = RunStatus::Blocked;
6091 self.state.base_sync = Some(BaseSync {
6092 tip,
6093 behind,
6094 attempts,
6095 conflict: Some(why.clone()),
6096 already_in: None,
6097 });
6098 self.state.event("land", why);
6099 }
6100 }
6101 if !self.state.base_sync_blocked() {
6102 self.state.base_sync_recovery.stop = None;
6103 }
6104 self.hold_base_sync_kind(stop_kind).await?;
6105 self.state.save()?;
6106 Ok(())
6107 }
6108
6109 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6115 if self.state.task_pr_branch().is_some() {
6118 return None;
6119 }
6120 let repo = self.state.repo.clone();
6121 let remote = self.state.config.merge.remote.clone();
6122 let fetched = git::fetch(&repo, &remote, branch).await;
6123 if !matches!(&fetched, Ok(o) if o.ok()) {
6124 return None;
6125 }
6126 let branch_tracking = format!("{remote}/{branch}");
6127 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6128 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6129 return None;
6130 }
6131 if !crate::reconcile::origin_missing(&repo, head, &theirs)
6132 .await
6133 .is_ok_and(|missing| missing.is_empty())
6134 {
6135 return Some(format!(
6136 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6137 them",
6138 short(&theirs)
6139 ));
6140 }
6141 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6142 Ok(o) if o.ok() => {
6143 self.state
6144 .event("land", format!("pushed rebased {branch} to {remote}"));
6145 None
6146 }
6147 Ok(o) => Some(format!(
6148 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6149 someone may have pushed): {}",
6150 short(&theirs),
6151 o.stderr.chars().take(600).collect::<String>()
6152 )),
6153 Err(e) => Some(format!(
6154 "{branch} is rebased locally but could not be pushed: {e:#}"
6155 )),
6156 }
6157 }
6158
6159 async fn settle_already_in(
6176 &mut self,
6177 branch: &str,
6178 tip: &str,
6179 head: &str,
6180 attempts: usize,
6181 behind: usize,
6182 ) -> Result<bool> {
6183 let repo = self.state.repo.clone();
6184 let remote = self.state.config.merge.remote.clone();
6185 let start = self.state.base_commit.clone();
6186 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6187 Ok(Some(e)) => e,
6188 Ok(None) => return Ok(false),
6189 Err(e) => {
6190 tracing::warn!("already-in-base check for {branch}: {e:#}");
6191 return Ok(false);
6192 }
6193 };
6194 let mut verified = vec![head.to_owned()];
6195 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6196 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6197 if matches!(&fetched, Ok(o) if o.ok())
6198 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6199 && theirs != head
6200 {
6201 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6202 Ok(Some(_)) => verified.push(theirs),
6203 _ => return Ok(false),
6204 }
6205 }
6206 let base_branch = self.state.base_branch.clone();
6207 let message = format!(
6208 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6209 land",
6210 evidence.names(),
6211 evidence.proof.as_str()
6212 );
6213 let closed =
6214 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6215 .await;
6216 match closed {
6217 Ok(Ok(url)) => self
6218 .state
6219 .event("land", format!("closed {url}: superseded on {base_branch}")),
6220 Ok(Err(why)) => self
6221 .state
6222 .event("land", format!("did not close a pull request: {why}")),
6223 Err(e) => {
6224 let why = format!(
6225 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6226 not be closed ({e:#}); resume to retry"
6227 );
6228 self.state.status = RunStatus::Blocked;
6229 self.state.base_sync = Some(BaseSync {
6230 tip: tip.to_owned(),
6231 behind,
6232 attempts,
6233 conflict: Some(why.clone()),
6234 already_in: None,
6235 });
6236 self.state.event("land", why);
6237 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6238 .await?;
6239 self.state.save()?;
6240 return Ok(true);
6241 }
6242 }
6243 self.state.status = RunStatus::AlreadyInBase;
6244 self.state.base_sync_recovery.stop = None;
6245 self.state.base_sync = Some(BaseSync {
6246 tip: tip.to_owned(),
6247 behind,
6248 attempts,
6249 conflict: None,
6250 already_in: Some(evidence),
6251 });
6252 self.state.event("land", message);
6253 self.state.save()?;
6254 self.settle_questions();
6255 Ok(true)
6256 }
6257
6258 fn landing_base(&self) -> String {
6268 self.state
6269 .base_sync
6270 .as_ref()
6271 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6272 }
6273
6274 pub async fn fix_selected(
6307 &mut self,
6308 ids: &[String],
6309 reason: &str,
6310 allow_stale: bool,
6311 ) -> Result<()> {
6312 let reason = reason.trim();
6313 if reason.is_empty() {
6314 bail!("a fix request needs a reason — that is the operator's own record of why");
6315 }
6316 if ids.is_empty() {
6317 bail!("no finding id given");
6318 }
6319 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6320 bail!(
6321 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6322 has already concluded — can be given a targeted fix. A run still \
6323 in progress should simply be resumed; a `merged` run's branch has \
6324 already landed, so its answer is a fresh `magi review <branch>`, \
6325 not reopening this run's own record",
6326 self.state.id,
6327 self.state.status.as_str()
6328 );
6329 }
6330 let _claim = FixClaim::acquire(&self.state.dir())?;
6333 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6334 let Some(winner) = self.state.winner().cloned() else {
6335 bail!("run {} has no winning candidate to fix", self.state.id);
6336 };
6337 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6338 bail!(
6339 "branch `{}` no longer exists; this run cannot be extended",
6340 winner.branch
6341 );
6342 }
6343 let home = crate::run::home();
6344 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6345 bail!(
6346 "run {} is currently being worked on by another magi process",
6347 self.state.id
6348 );
6349 }
6350 let mut seen = BTreeSet::new();
6354 let mut findings = Vec::new();
6355 let mut missing = Vec::new();
6356 for id in ids {
6357 if !seen.insert(id.clone()) {
6358 continue;
6359 }
6360 match self.state.finding(id) {
6361 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6362 id: f.id.clone(),
6363 severity: f.severity,
6364 reviewer_vote: rec.vote,
6365 round: round.round,
6366 round_head: round.head.clone(),
6367 reviewer: rec.reviewer,
6368 agent: rec.agent.clone(),
6369 file: f.file.clone(),
6370 line: f.line,
6371 title: f.title.clone(),
6372 detail: f.detail.clone(),
6373 outcome: OperatorFixOutcome::Pending,
6374 }),
6375 None => missing.push(id.clone()),
6376 }
6377 }
6378 if !missing.is_empty() {
6379 bail!(
6380 "unknown finding id(s): {}; nothing was changed",
6381 missing.join(", ")
6382 );
6383 }
6384
6385 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6386 let stale_details: Vec<(String, String)> = findings
6387 .iter()
6388 .filter(|f| f.round_head != head_at_request)
6389 .map(|f| (f.id.clone(), f.round_head.clone()))
6390 .collect();
6391 let stale = !stale_details.is_empty();
6392 if stale && !allow_stale {
6393 bail!(
6394 "the branch has moved since some finding(s) were raised — {} — now \
6395 at {}; pass --allow-stale to fix anyway, or re-run review first",
6396 stale_details
6397 .iter()
6398 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6399 .collect::<Vec<_>>()
6400 .join(", "),
6401 short(&head_at_request)
6402 );
6403 }
6404
6405 let request = OperatorFixRequest {
6406 requested_at: Timestamp::now(),
6407 reason: reason.to_owned(),
6408 findings,
6409 head_at_request: head_at_request.clone(),
6410 allow_stale,
6411 stale,
6412 fix: None,
6413 result_head: None,
6414 follow_up_review_run: None,
6415 };
6416 self.state.event(
6417 "fix",
6418 format!(
6419 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6420 request.findings.len(),
6421 request
6422 .findings
6423 .iter()
6424 .map(|f| f.id.as_str())
6425 .collect::<Vec<_>>()
6426 .join(", "),
6427 ),
6428 );
6429 self.state.operator_fixes.push(request);
6436 self.state.save()?;
6437 let request_index = self.state.operator_fixes.len() - 1;
6438
6439 let repo = self.state.repo.clone();
6448 let coordination = self.coordinate(&repo).await?;
6449 if winner.worktree.exists() {
6450 let dirty = git::git(
6453 &winner.worktree,
6454 &["status", "--porcelain", "--untracked-files=all"],
6455 )
6456 .await?;
6457 let only_withheld = dirty.lines().all(|l| {
6458 l.strip_prefix("?? ")
6459 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6460 });
6461 if !only_withheld {
6462 bail!(
6463 "`{}` has uncommitted changes; refusing to touch it — commit or \
6464 discard them first",
6465 winner.worktree.display()
6466 );
6467 }
6468 git::worktree_remove(&self.state.repo, &winner.worktree)
6469 .await
6470 .ok();
6471 }
6472 let fix_worktree = self.state.worktree_root().join("operator-fix");
6473 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6474 git::git(
6475 &self.state.repo,
6476 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6477 )
6478 .await
6479 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6480 if !git::is_clean(&fix_worktree).await? {
6481 git::worktree_remove(&self.state.repo, &fix_worktree)
6482 .await
6483 .ok();
6484 bail!(
6485 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6486 winner.branch
6487 );
6488 }
6489 let repo_for_setup = self.state.repo.clone();
6490 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6491 Ok(()) => {
6492 worktree_setup::prepare_with(
6493 &self.state.config,
6494 &repo_for_setup,
6495 &fix_worktree,
6496 crate::build_target::Coordination::Held(coordination.as_ref()),
6497 )
6498 .await
6499 }
6500 Err(e) => Err(e),
6501 };
6502 if let Err(e) = setup {
6503 git::worktree_remove(&self.state.repo, &fix_worktree)
6504 .await
6505 .ok();
6506 return Err(e);
6507 }
6508
6509 self.state.resource_block = None;
6512
6513 let run_id = self.state.id.clone();
6514 let prompts = self.state.config.prompts.clone();
6515 let language = self.state.config.graph.language.clone();
6516 let sessions = self.state.config.graph.sessions;
6517 let artifacts = agent::artifacts_dir(&self.state.dir());
6518 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6519 .findings
6520 .iter()
6521 .map(|f| Finding {
6522 id: f.id.clone(),
6523 severity: f.severity,
6524 file: f.file.clone(),
6525 line: f.line,
6526 title: f.title.clone(),
6527 detail: f.detail.clone(),
6528 })
6529 .collect();
6530 let fix_prompt = prompt::operator_fix_in(
6531 &self.state.instruction,
6532 &finding_list,
6533 reason,
6534 &stale_details,
6535 &head_at_request,
6536 &language,
6537 &self.state.config.graph.github_language,
6538 );
6539 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6540 drop(coordination);
6541 let (job, seat, out) = self
6542 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6543 prompt: fix_prompt.clone(),
6544 spec,
6545 seat,
6546 cwd: fix_worktree.clone(),
6547 timeout,
6548 allow_write: true,
6549 sessions,
6550 artifacts: artifacts.clone(),
6551 stem: "operator-fix".to_owned(),
6552 handover: None,
6553 })
6554 .await;
6555 let agent_id = seat.agent.clone();
6556
6557 let mut fix = FixRecord {
6558 agent: agent_id,
6559 addressed: Vec::new(),
6560 rejected: Vec::new(),
6561 notes: String::new(),
6562 committed: false,
6563 failed: None,
6564 duration_ms: 0,
6565 continuation: None,
6566 };
6567 let mut final_seat = seat.clone();
6568 match out {
6569 AgentOutcome::Ok(o) => {
6570 fix.duration_ms = o.duration_ms;
6571 let parsed = verdict::extract_json::<FixReport>(&o.text);
6572 let incomplete_reason = match &parsed {
6573 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6574 "the reply parsed, but it reported a command whose own CLI \
6575 never confirmed an exit status"
6576 .to_owned(),
6577 ),
6578 Ok(_) => None,
6579 Err(e) => Some(e.to_string()),
6580 };
6581 match incomplete_reason {
6582 None => {
6583 let report = parsed.expect("checked Ok above");
6584 fix.addressed = report.addressed;
6585 fix.rejected = report.rejected;
6586 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6587 }
6588 Some(reason) => {
6589 let (resumed_seat, resolved, failure, cont) = self
6590 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6591 .await;
6592 fix.duration_ms += cont.cumulative_wait_ms;
6593 fix.continuation = Some(cont);
6594 final_seat = resumed_seat;
6595 match resolved {
6596 Some(report) => {
6597 fix.addressed = report.addressed;
6598 fix.rejected = report.rejected;
6599 fix.notes =
6600 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6601 }
6602 None => fix.failed = failure,
6603 }
6604 }
6605 }
6606 }
6607 AgentOutcome::Dropped(o) => {
6608 fix.duration_ms = o.duration_ms;
6609 let why = o
6610 .dropped
6611 .as_ref()
6612 .map(|d| d.why.as_str())
6613 .unwrap_or("the CLI ended the stream without delivering its answer");
6614 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6615 }
6616 AgentOutcome::Quota(o) => {
6617 self.state.quota.push(QuotaLoss::from_report(
6618 final_seat.key.clone(),
6619 "fix",
6620 o.quota.as_ref(),
6621 ));
6622 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6623 }
6624 AgentOutcome::Failed(e) => fix.failed = Some(e),
6625 AgentOutcome::ResourceBlocked(e) => {
6626 fix.failed = Some(format!("the fixer was not started: {e}"));
6627 }
6628 }
6629 if fix.continuation.is_none() {
6632 fix.continuation = Some(ContinuationRecord::not_needed());
6633 }
6634 self.state.seats.insert(final_seat.key.clone(), final_seat);
6635
6636 let rescue_message = format!(
6637 "magi: operator-selected fix ({}) (uncommitted work)",
6638 self.state.operator_fixes[request_index]
6639 .findings
6640 .iter()
6641 .map(|f| f.id.as_str())
6642 .collect::<Vec<_>>()
6643 .join(", ")
6644 );
6645 let repo = self.state.repo.clone();
6646 let coordination = self.coordinate(&repo).await?;
6647 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6648 self.state.note_withheld("fix", &r.withheld);
6649 }
6650 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6651 fix.committed = after != head_at_request;
6652 git::worktree_remove(&self.state.repo, &fix_worktree)
6653 .await
6654 .ok();
6655 drop(coordination);
6656
6657 self.state.event(
6658 "fix",
6659 match &fix.failed {
6660 Some(reason) => format!(
6661 "operator fix: adoption report was lost ({reason}); {}",
6662 if fix.committed {
6663 "committed"
6664 } else {
6665 "NO new commit"
6666 }
6667 ),
6668 None => format!(
6669 "operator fix: {} addressed, {} rejected, {}",
6670 fix.addressed.len(),
6671 fix.rejected.len(),
6672 if fix.committed {
6673 "committed"
6674 } else {
6675 "NO new commit"
6676 }
6677 ),
6678 },
6679 );
6680
6681 for f in &mut self.state.operator_fixes[request_index].findings {
6688 f.outcome = if fix.failed.is_some() {
6689 OperatorFixOutcome::Unreported
6690 } else if fix.addressed.contains(&f.id) {
6691 OperatorFixOutcome::Addressed
6692 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6693 OperatorFixOutcome::Rejected { why: r.why.clone() }
6694 } else {
6695 OperatorFixOutcome::Unreported
6696 };
6697 }
6698
6699 let committed = fix.committed;
6700 if committed {
6701 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6702 }
6703 self.state.operator_fixes[request_index].fix = Some(fix);
6704 self.state.save()?;
6707 if self.state.resource_block.is_some() {
6708 return self.stop_for_resource();
6709 }
6710
6711 if committed {
6712 self.state.event(
6713 "fix",
6714 format!(
6715 "operator fix committed {}; opening a follow-up review-only run",
6716 short(&after)
6717 ),
6718 );
6719 let origin =
6722 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6723 match Self::review(
6724 &self.state.repo,
6725 &winner.branch,
6726 self.state.config.clone(),
6727 origin,
6728 )
6729 .await
6730 {
6731 Ok(mut follow_up) => {
6732 follow_up.state.event(
6733 "start",
6734 format!(
6735 "requested by an operator fix on run {} for finding(s) {}",
6736 self.state.id,
6737 self.state.operator_fixes[request_index]
6738 .findings
6739 .iter()
6740 .map(|f| f.id.as_str())
6741 .collect::<Vec<_>>()
6742 .join(", "),
6743 ),
6744 );
6745 follow_up.state.save()?;
6746 let follow_up_id = follow_up.state.id.clone();
6747 let adopted = match crate::direct::adopt(
6750 &follow_up.state,
6751 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6752 ) {
6753 Ok(a) => a,
6754 Err(e) => {
6755 self.state.event(
6758 "fix",
6759 format!(
6760 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6761 ),
6762 );
6763 self.state.save()?;
6764 return Ok(());
6765 }
6766 };
6767 let executed = follow_up.execute().await;
6768 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6769 if let Some(a) = adopted {
6770 a.finish(&follow_up.state, executed);
6771 }
6772 if let Some(e) = failure {
6773 self.state.event(
6774 "fix",
6775 format!(
6776 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6777 ),
6778 );
6779 }
6780 self.state.operator_fixes[request_index].follow_up_review_run =
6781 Some(follow_up_id);
6782 }
6783 Err(e) => {
6784 self.state.event(
6785 "fix",
6786 format!("committed the fix but could not open a follow-up review: {e:#}"),
6787 );
6788 }
6789 }
6790 self.state.save()?;
6791 }
6792
6793 Ok(())
6794 }
6795
6796 async fn ask_fixer(
6816 &mut self,
6817 winner: &Candidate,
6818 node: &'static str,
6819 round: Option<usize>,
6820 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6821 ) -> (SeatJob, SeatState, AgentOutcome) {
6822 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6823 self.ask_fixer_among(attempts, node, round, None, build)
6824 .await
6825 }
6826
6827 async fn ask_fixer_among(
6832 &mut self,
6833 attempts: Vec<(AgentSpec, String)>,
6834 node: &'static str,
6835 round: Option<usize>,
6836 restore: Option<(&Path, &str)>,
6837 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6838 ) -> (SeatJob, SeatState, AgentOutcome) {
6839 let run_id = self.state.id.clone();
6840 let prompts = self.state.config.prompts.clone();
6841 let cache = self.state.config.cache_dir();
6842 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6843 let mut last = None;
6844 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6845 let seat = self.seat(&key, &spec.id);
6846 let mut job = build(spec.clone(), seat);
6847 if i > 0 {
6848 job.stem = format!("{}-{}", job.stem, spec.id);
6849 }
6850 let ctx = WaveCtx {
6851 carry_seats: false,
6852 pauses: Some((&self.pause, &self.interrupt)),
6853 run: &run_id,
6854 node,
6855 prompts: &prompts,
6856 cache: cache.as_deref(),
6857 round,
6858 };
6859 let (seat, out) =
6860 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6861 let advances = match &out {
6864 AgentOutcome::Ok(o) => agent::output_advances(o),
6865 AgentOutcome::ResourceBlocked(_) => false,
6866 _ => true,
6867 };
6868 if let Some(next) = ids.get(i + 1)
6869 && advances
6870 {
6871 self.state.seats.insert(seat.key.clone(), seat.clone());
6872 let class =
6873 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6874 let reason = match &out {
6875 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6876 other => fail_reason(other),
6877 };
6878 if let Some((wt, rev)) = restore
6882 && let Err(e) = self.graft_restore(wt, rev).await
6883 {
6884 let why =
6885 format!("could not restore the worktree before the next fixer: {e:#}");
6886 tracing::warn!("{why}");
6887 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6888 AgentOutcome::ResourceBlocked(why)
6889 } else {
6890 AgentOutcome::Failed(why)
6891 };
6892 last = Some((job, seat, out));
6893 break;
6894 }
6895 record_handover(
6896 &mut self.state,
6897 node,
6898 &seat.key,
6899 &spec.id,
6900 next,
6901 &class,
6902 &reason,
6903 );
6904 continue;
6905 }
6906 last = Some((job, seat, out));
6907 break;
6908 }
6909 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6910 if let AgentOutcome::Quota(output) = &out
6911 && node != "graft"
6912 {
6913 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6918 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6919 self.state.quota.push(loss);
6920 self.state.seats.insert(seat.key.clone(), seat.clone());
6921 self.state.status = RunStatus::Blocked;
6922 return (
6923 job,
6924 seat,
6925 AgentOutcome::ResourceBlocked(
6926 "fixer unavailable; restore agent availability and resume".into(),
6927 ),
6928 );
6929 }
6930 (job, seat, out)
6931 }
6932
6933 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6944 use crate::recovery::{Mode, Repair};
6945 if !self.state.reviews.is_empty() {
6946 return Ok(());
6947 }
6948 let Some(mut rec) = self.state.recovery.clone() else {
6949 return Ok(());
6950 };
6951 if rec.mode != Mode::VerifyRepair {
6952 return Ok(());
6953 }
6954 let wt = winner.worktree.clone();
6955 let mut head = git::rev_parse(&wt, "HEAD").await?;
6956 match rec.repair.clone() {
6957 Repair::Done { .. } => return Ok(()),
6958 Repair::WaitingForRestore { before } => {
6959 self.graft_restore(&wt, &before).await?;
6960 head = before;
6961 rec.repair = Repair::NotStarted;
6962 self.state.recovery = Some(rec.clone());
6963 self.state.save()?;
6964 }
6965 Repair::Started { before } => {
6966 if let Err(e) = self.graft_restore(&wt, &before).await {
6970 if e.is::<crate::repo_lock::ResourceBlocked>() {
6971 return Err(e);
6972 }
6973 return self
6974 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6975 .await;
6976 }
6977 rec.repair = Repair::Done {
6978 before: before.clone(),
6979 after: before,
6980 committed: false,
6981 failed: Some("interrupted; rolled back".to_owned()),
6982 };
6983 self.state
6984 .event("recovery", "the repair was interrupted and rolled back");
6985 self.state.recovery = Some(rec);
6986 self.state.save()?;
6987 return Ok(());
6988 }
6989 Repair::NotStarted => {}
6990 }
6991 if crate::recovery::recheck(&mut rec, &head) {
6992 self.state.event("recovery", rec.summary());
6993 self.state.recovery = Some(rec);
6994 self.state.save()?;
6995 return Ok(());
6996 }
6997 rec.repair = Repair::Started {
6998 before: head.clone(),
6999 };
7000 self.state.recovery = Some(rec.clone());
7001 self.state.save()?;
7002
7003 let language = self.state.config.graph.language.clone();
7004 let sessions = self.state.config.graph.sessions;
7005 let artifacts = agent::artifacts_dir(&self.state.dir());
7006 let fix_prompt = prompt::recovery_fix(
7007 &self.state.instruction,
7008 &rec,
7009 self.state.config.graph.review_rounds,
7010 &language,
7011 );
7012 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7013 self.state.event(
7014 "recovery",
7015 format!(
7016 "repair: the fixer starts from the failed verification on {}",
7017 short(&head)
7018 ),
7019 );
7020 let (_job, seat, out) = self
7021 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
7022 prompt: fix_prompt.clone(),
7023 spec,
7024 seat,
7025 cwd: wt.clone(),
7026 timeout,
7027 allow_write: true,
7028 sessions,
7029 artifacts: artifacts.clone(),
7030 stem: "fix-recovery".to_owned(),
7031 handover: None,
7032 })
7033 .await;
7034 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7035 rec.repair = Repair::WaitingForRestore { before: head };
7036 self.state.recovery = Some(rec);
7037 return self.stop_for_resource();
7038 }
7039 let failed = match &out {
7040 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
7041 AgentOutcome::Quota(o) => {
7042 self.state.quota.push(QuotaLoss::from_report(
7043 seat.key.clone(),
7044 "fix",
7045 o.quota.as_ref(),
7046 ));
7047 Some("rate limited (quota); the fixer could not run".to_owned())
7048 }
7049 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
7050 AgentOutcome::Failed(e) => Some(e.clone()),
7051 };
7052 self.state.seats.insert(seat.key.clone(), seat);
7053 let coordination = self.coordinate(&wt).await?;
7054 let failed = match failed {
7055 Some(why) => {
7057 if let Err(e) = git::restore_to(&wt, &head).await {
7058 return self
7059 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7060 .await;
7061 }
7062 Some(why)
7063 }
7064 None => {
7065 if let Ok(r) = git::rescue_commit(
7066 &wt,
7067 "magi: recovery repair of the failed verification (uncommitted work)",
7068 )
7069 .await
7070 {
7071 self.state.note_withheld("fix", &r.withheld);
7072 }
7073 None
7074 }
7075 };
7076 let after = git::rev_parse(&wt, "HEAD").await?;
7077 drop(coordination);
7078 self.state.event(
7079 "recovery",
7080 match &failed {
7081 Some(why) => format!("repair: nothing kept ({why})"),
7082 None if after != head => {
7083 format!("repair: committed {}", short(&after))
7084 }
7085 None => "repair: the fixer made no commit".to_owned(),
7086 },
7087 );
7088 rec.repair = Repair::Done {
7089 before: head.clone(),
7090 committed: after != head,
7091 after,
7092 failed,
7093 };
7094 self.state.recovery = Some(rec);
7095 self.state.save()?;
7096 Ok(())
7097 }
7098
7099 async fn stop_recovery(
7102 &mut self,
7103 mut rec: crate::recovery::RecoveryRecord,
7104 head: &str,
7105 why: String,
7106 ) -> Result<()> {
7107 rec.repair = crate::recovery::Repair::Started {
7109 before: head.to_owned(),
7110 };
7111 self.state.recovery = Some(rec);
7112 self.state
7113 .event("recovery", format!("repair could not be undone: {why}"));
7114 self.state.status = RunStatus::Blocked;
7115 self.state.save()?;
7116 bail!("recovery repair could not be rolled back: {why}")
7117 }
7118
7119 async fn review_loop(&mut self) -> Result<()> {
7120 if self.hold_base_sync().await? {
7121 return Ok(());
7122 }
7123 loop {
7124 if self
7125 .state
7126 .base_sync
7127 .as_ref()
7128 .is_some_and(|s| s.conflict.is_some())
7129 {
7130 return Ok(());
7131 }
7132 if self.catchup_repairable() {
7133 let record = self.state.reviews.last().expect("repairable round").clone();
7134 let winner = self
7135 .state
7136 .winner()
7137 .cloned()
7138 .context("catch-up repair has no winner")?;
7139 if !self.fix_review_round(&winner, record).await? {
7140 return Ok(());
7141 }
7142 }
7143 self.review_loop_pass().await?;
7144 if !self.catchup_repairable() {
7145 return Ok(());
7146 }
7147 }
7148 }
7149
7150 fn catchup_repairable(&self) -> bool {
7153 self.state.reviews.len() < self.state.config.graph.review_rounds
7154 && self.state.reviews.last().is_some_and(|r| {
7155 !r.clean
7156 && r.blocking == 0
7157 && r.fix.is_none()
7158 && r.e2e_status() == E2eStatus::Failed
7159 })
7160 }
7161
7162 async fn review_loop_pass(&mut self) -> Result<()> {
7163 if self
7168 .state
7169 .base_sync
7170 .as_ref()
7171 .is_some_and(|s| s.conflict.is_some())
7172 {
7173 return Ok(());
7174 }
7175 let run_id = self.state.id.clone();
7180 let prompts = self.state.config.prompts.clone();
7181 let Some(winner) = self.state.winner().cloned() else {
7182 return Ok(());
7183 };
7184 let granted_review = self
7185 .state
7186 .repair_grants
7187 .last()
7188 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7189 .cloned();
7190 let fresh_review = granted_review
7191 .as_ref()
7192 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7193 let max_rounds = granted_review
7194 .as_ref()
7195 .map_or(self.state.config.graph.review_rounds, |g| {
7196 g.review_start.saturating_add(1)
7197 });
7198 if let Some(status) =
7208 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7209 {
7210 if status == RunStatus::Gating {
7213 self.record_contested_handoff();
7214 }
7215 self.state.status = status;
7216 self.state.save()?;
7217 return Ok(());
7218 }
7219 self.state.status = RunStatus::Reviewing;
7220 if !fresh_review
7230 && self.state.reviews.last().is_some_and(|r| {
7231 r.e2e_status() == E2eStatus::ResourceBlocked
7232 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7233 })
7234 {
7235 let shell = self.state.config.shell();
7236 return self
7237 .stop_unresolved_review(
7238 "the last round's own verification never resolved",
7239 &shell,
7240 &winner.worktree,
7241 )
7242 .await;
7243 }
7244
7245 if granted_review.is_none() {
7246 self.recovery_repair(&winner).await?;
7247 }
7248 if self.resource_halt()? {
7249 return Ok(());
7250 }
7251
7252 let repo = self.state.repo.clone();
7253 let root = self.state.worktree_root();
7254 let language = self.state.config.graph.language.clone();
7255 let sessions = self.state.config.graph.sessions;
7256 let artifacts = self.merge_repair_artifacts();
7257 let base = self.landing_base();
7258 let base_short = short(&base);
7259 let reviewers = self.roles.reviewers.clone();
7260 let shell = self.state.config.shell();
7261
7262 for round in (self.state.reviews.len() + 1)..=max_rounds {
7263 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7264 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7265 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7266 let prev_verification = self
7275 .state
7276 .reviews
7277 .last()
7278 .and_then(|r| r.verification_summary(&head));
7279 let prior_section = match (&self.state.recovery, round) {
7282 (Some(rec), 1) if self.state.reviews.is_empty() => {
7283 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7284 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7285 Some(crate::recovery::head_of(
7286 &git::diff(&winner.worktree, reviewed, "HEAD")
7287 .await
7288 .unwrap_or_default(),
7289 crate::recovery::SINCE_DIFF_MAX,
7290 ))
7291 }
7292 _ => None,
7293 };
7294 Some(prompt::recovery_review(rec, since.as_deref()))
7295 }
7296 _ => None,
7297 };
7298
7299 let mut jobs = Vec::new();
7303 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7304 self.coordinate(&repo).await?
7305 } else {
7306 None
7307 };
7308 for (r, spec) in reviewers.iter().cloned().enumerate() {
7309 let wt = root.join(format!("review-{}", r + 1));
7310 if wt.exists() {
7313 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7314 } else {
7315 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7316 None
7317 } else {
7318 self.coordinate(&repo).await?
7319 };
7320 git::worktree_add_detached(&repo, &wt, &head).await?;
7321 }
7322 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7323 None
7324 } else {
7325 self.coordinate(&repo).await?
7326 };
7327 self.setup_seat_worktree(
7328 &repo,
7329 &wt,
7330 false,
7331 coordination.as_ref().or(_setup_lease.as_ref()),
7332 )
7333 .await?;
7334 let seat_key = format!("review-{}", r + 1);
7335 let spec = pick_start_spec(
7338 &self.roles.reviewer_roster,
7339 spec,
7340 self.state.seat_history.get(&seat_key),
7341 );
7342 let seat = self.seat(&seat_key, &spec.id);
7343 jobs.push(SeatJob {
7344 prompt: prompt::review(&prompt::ReviewCtx {
7345 instruction: &self.state.instruction,
7346 branch: &winner.branch,
7347 base_short: &base_short,
7348 stat: &stat,
7349 patch: &patch,
7350 verification: prev_verification.as_ref(),
7351 prior: prior_section.as_deref(),
7352 reviewers: reviewers.len(),
7353 round,
7354 rounds: max_rounds,
7355 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7358 lens: Lens::for_seat(r),
7359 language: &language,
7360 github_language: &self.state.config.graph.github_language,
7361 }),
7362 spec,
7363 seat,
7364 cwd: wt,
7365 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7366 allow_write: false,
7367 sessions,
7368 artifacts: artifacts.clone(),
7369 stem: format!("review-{round}-{}", r + 1),
7370 handover: None,
7371 });
7372 }
7373
7374 drop(coordination);
7375 self.state.event(
7376 "review",
7377 format!(
7378 "round {round}: {} reviewers on {}",
7379 jobs.len(),
7380 short(&head)
7381 ),
7382 );
7383 let mut quota_losses = Vec::new();
7384 let review_retries = self.state.config.graph.retries;
7385 let review_cache = self.state.config.cache_dir();
7386 let ctx = WaveCtx {
7387 carry_seats: true,
7388 pauses: Some((&self.pause, &self.interrupt)),
7389 run: &run_id,
7390 node: "review",
7391 prompts: &prompts,
7392 cache: review_cache.as_deref(),
7393 round: Some(round),
7394 };
7395 let results = ask_json_wave::<Review>(
7396 jobs,
7397 Arc::clone(&self.sem),
7398 review_retries,
7399 &self.roles.reviewer_roster,
7400 &ctx,
7401 &mut quota_losses,
7402 &mut self.state,
7403 &|_: &Review| Ok(()),
7404 )
7405 .await;
7406 if self.state.resource_block.is_some() {
7407 return self.stop_for_resource();
7408 }
7409 let round_quota_missing = quota_losses.len();
7413 self.state.quota.extend(quota_losses);
7414
7415 let mut records = Vec::new();
7416 let mut all_findings = Vec::new();
7417 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7418 let agent_id = seat.agent.clone();
7419 self.state.seats.insert(seat.key.clone(), seat);
7420 let mut record = ReviewRecord {
7421 reviewer: r + 1,
7422 agent: agent_id,
7423 summary: String::new(),
7424 findings: Vec::new(),
7425 vote: None,
7426 failed: None,
7427 duration_ms: 0,
7428 attempts,
7434 };
7435 match res {
7436 Ok((review, out)) => {
7437 record.summary =
7445 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7446 record.vote = Some(review.vote);
7447 record.duration_ms = out.duration_ms;
7448 for (n, mut f) in review.findings.into_iter().enumerate() {
7449 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7452 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7453 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7454 f.file = f
7460 .file
7461 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7462 all_findings.push(f.clone());
7463 record.findings.push(f);
7464 }
7465 self.state.event(
7466 "review",
7467 format!(
7468 "round {round}: reviewer {} voted {} with {} finding(s)",
7469 r + 1,
7470 review.vote.label(),
7471 record.findings.len()
7472 ),
7473 );
7474 }
7475 Err(e) => {
7476 record.failed = Some(e.to_string());
7477 self.state.event(
7478 "review",
7479 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7480 );
7481 }
7482 }
7483 records.push(record);
7484 }
7485
7486 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7493 let vote_split =
7494 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7495 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7496 if vote_split {
7497 self.state.event(
7498 "review",
7499 format!(
7500 "round {round}: votes split ({}) — one round of reconsideration",
7501 initial_votes
7502 .iter()
7503 .map(|v| v.label())
7504 .collect::<Vec<_>>()
7505 .join(", ")
7506 ),
7507 );
7508 let panel: Vec<ReviewSeatReport<'_>> = records
7511 .iter()
7512 .filter_map(|r| {
7513 r.vote.map(|vote| ReviewSeatReport {
7514 reviewer: r.reviewer,
7515 vote,
7516 summary: &r.summary,
7517 findings: &r.findings,
7518 })
7519 })
7520 .collect();
7521
7522 let mut jobs = Vec::new();
7523 let mut seats_at = Vec::new();
7524 for (r, spec) in reviewers.iter().cloned().enumerate() {
7525 if records[r].vote.is_none() {
7529 continue;
7530 }
7531 let wt = root.join(format!("review-{}", r + 1));
7532 let seat_key = format!("review-{}", r + 1);
7533 let spec = self.occupant(&seat_key, spec);
7534 let seat = self.seat(&seat_key, &spec.id);
7535 let build = |with_patch: bool| {
7543 prompt::review_reconsider(&ReviewReconsiderCtx {
7544 instruction: &self.state.instruction,
7545 reviewer: r + 1,
7546 lens: Lens::for_seat(r),
7547 panel: &panel,
7548 patch: with_patch.then_some(ReviewPatch {
7549 branch: &winner.branch,
7550 base_short: &base_short,
7551 stat: &stat,
7552 patch: &patch,
7553 }),
7554 round,
7555 rounds: max_rounds,
7556 language: &language,
7557 })
7558 };
7559 let full = build(true);
7560 let prompt = if has_context(&spec, &seat, sessions) {
7561 build(false)
7562 } else {
7563 full.clone()
7564 };
7565 jobs.push(SeatJob {
7566 prompt,
7567 spec,
7568 seat,
7569 cwd: wt,
7570 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7571 allow_write: false,
7572 sessions,
7573 artifacts: artifacts.clone(),
7574 stem: format!("review-{round}-reconsider-{}", r + 1),
7575 handover: Some(full),
7576 });
7577 seats_at.push(r);
7578 }
7579
7580 let mut recon_quota_losses = Vec::new();
7581 let recon_cache = self.state.config.cache_dir();
7582 let recon_ctx = WaveCtx {
7583 carry_seats: true,
7584 pauses: Some((&self.pause, &self.interrupt)),
7585 run: &run_id,
7586 node: "review",
7587 prompts: &prompts,
7588 cache: recon_cache.as_deref(),
7589 round: Some(round),
7590 };
7591 let recon_results = ask_json_wave::<ReviewRevote>(
7592 jobs,
7593 Arc::clone(&self.sem),
7594 review_retries,
7595 &self.roles.reviewer_roster,
7596 &recon_ctx,
7597 &mut recon_quota_losses,
7598 &mut self.state,
7599 &|_: &ReviewRevote| Ok(()),
7600 )
7601 .await;
7602 if self.state.resource_block.is_some() {
7603 return self.stop_for_resource();
7604 }
7605 self.state.quota.extend(recon_quota_losses);
7606
7607 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7608 let agent_id = seat.agent.clone();
7609 self.state.seats.insert(seat.key.clone(), seat);
7610 let mut rec = ReviewRevoteRecord {
7611 reviewer: r + 1,
7612 agent: agent_id,
7613 vote: None,
7614 reason: String::new(),
7615 failed: None,
7616 };
7617 match res {
7618 Ok((rv, _)) => {
7619 rec.vote = Some(rv.vote);
7620 rec.reason =
7621 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7622 self.state.event(
7623 "review",
7624 format!(
7625 "round {round}: reviewer {} revoted {}",
7626 r + 1,
7627 rv.vote.label()
7628 ),
7629 );
7630 }
7631 Err(e) => {
7632 rec.failed = Some(e.to_string());
7633 self.state.event(
7634 "review",
7635 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7636 );
7637 }
7638 }
7639 reconsideration.push(rec);
7640 }
7641 } else if initial_votes.len() > 1 {
7642 self.state.event(
7643 "review",
7644 format!(
7645 "round {round}: votes agreed ({}) — no reconsideration",
7646 initial_votes[0].label()
7647 ),
7648 );
7649 }
7650
7651 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7652 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7653 let defer_e2e =
7664 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7665 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7666 let reason =
7667 format!("{blocking} blocking finding(s) already required a fix this round");
7668 self.state.event(
7669 "verify",
7670 format!(
7671 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7672 {}); it will run once a round has none left",
7673 short(&head)
7674 ),
7675 );
7676 (Vec::new(), false, true, Some(reason))
7677 } else {
7678 let e2e_commands = self.state.config.verify.e2e.clone();
7679 let cache_dir = self.state.config.cache_dir();
7680 let context = format!("round {round}");
7681 let wait = self.repository_wait();
7682 let (e2e, verify_retried) = with_cache_lease(
7683 &mut self.state,
7684 cache_dir.as_deref(),
7685 "e2e",
7686 "e2e",
7687 &winner.worktree,
7688 &head,
7689 verify_timeout,
7690 &context,
7691 &wait,
7692 |state, budget| {
7693 let shell = shell.clone();
7694 let e2e_commands = e2e_commands.clone();
7695 let worktree = winner.worktree.clone();
7696 let context = context.clone();
7697 let winner = winner.clone();
7698 async move {
7699 if !e2e_commands.is_empty() {
7700 Self::run_pre_gate(state, &winner).await;
7704 }
7705 run_e2e_with_retry(
7706 state,
7707 &shell,
7708 &e2e_commands,
7709 &worktree,
7710 budget,
7711 &context,
7712 Some(round),
7713 )
7714 .await
7715 }
7716 },
7717 )
7718 .await?;
7719 (e2e, verify_retried, false, None)
7720 };
7721
7722 let expected = records.len();
7723 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7724 let incomplete = answered < expected;
7725 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7726 let policy = self.state.config.graph.incomplete_review;
7727 let clean = round_is_clean(
7728 blocking,
7729 e2e_ok,
7730 answered,
7731 expected,
7732 round_quota_missing,
7733 policy,
7734 );
7735
7736 let mut round_record = ReviewRound {
7737 round,
7738 head: head.clone(),
7739 verified_head: None,
7740 verified_at: None,
7741 reviews: records,
7742 e2e,
7743 verify_retried,
7744 e2e_deferred,
7745 e2e_defer_reason,
7746 fix: None,
7747 blocking,
7748 answered,
7749 expected,
7750 clean,
7751 progressed: false,
7752 vote_split,
7753 reconsideration,
7754 verdict: None,
7755 };
7756 round_record.verdict = ReviewVote::worst(
7760 round_record
7761 .final_votes()
7762 .into_iter()
7763 .map(|(_, _, vote)| vote),
7764 );
7765 if !matches!(
7776 round_record.e2e_status(),
7777 E2eStatus::Deferred | E2eStatus::NotConfigured
7778 ) {
7779 round_record.verified_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
7782 round_record.verified_at = Some(Timestamp::now());
7783 }
7784
7785 if incomplete {
7786 let missing: Vec<String> = round_record
7787 .reviews
7788 .iter()
7789 .filter(|r| r.failed.is_some())
7790 .map(|r| format!("review-{}", r.reviewer))
7791 .collect();
7792 self.state.event(
7793 "review",
7794 format!(
7795 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7796 missing.join(", ")
7797 ),
7798 );
7799 }
7800
7801 if clean {
7802 self.state.event(
7803 "review",
7804 if incomplete && policy == IncompleteReviewPolicy::Warn {
7805 format!(
7806 "round {round}: clean (warn policy, incomplete panel) — no \
7807 blocking findings from the seats that answered, verification green"
7808 )
7809 } else if incomplete {
7810 format!(
7811 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7812 quorum) — no blocking findings from the seats that answered, \
7813 verification green",
7814 expected - answered
7815 )
7816 } else {
7817 format!("round {round}: clean — no blocking findings, verification green")
7818 },
7819 );
7820 self.state.reviews.push(round_record);
7821 self.state.status = RunStatus::Gating;
7822 self.state.save()?;
7823 return Ok(());
7824 }
7825
7826 if incomplete && blocking == 0 && e2e_ok {
7834 self.state.reviews.push(round_record);
7835 self.state.save()?;
7836 if round == max_rounds {
7837 self.state.status = RunStatus::Blocked;
7838 self.state.event(
7839 "review",
7840 format!(
7841 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7842 refusing to call it clean",
7843 expected - answered
7844 ),
7845 );
7846 return Ok(());
7847 }
7848 continue;
7849 }
7850
7851 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7863 self.state.reviews.push(round_record);
7864 return self
7865 .stop_unresolved_review(
7866 "the round's own verification could not run",
7867 &shell,
7868 &winner.worktree,
7869 )
7870 .await;
7871 }
7872
7873 if round == max_rounds {
7874 self.state.reviews.push(round_record);
7875 return self
7876 .stop_reviewing(
7877 &format!(
7878 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7879 ),
7880 &shell,
7881 &winner.worktree,
7882 )
7883 .await;
7884 }
7885
7886 if !self.fix_review_round(&winner, round_record).await? {
7887 return Ok(());
7888 }
7889 }
7890 Ok(())
7891 }
7892
7893 async fn fix_review_round(
7896 &mut self,
7897 winner: &Candidate,
7898 mut round_record: ReviewRound,
7899 ) -> Result<bool> {
7900 let round = round_record.round;
7901 let shell = self.state.config.shell();
7902 let step = self.run_fix_round(winner, &mut round_record).await?;
7903 if matches!(step, FixStep::Resource) {
7904 self.stop_for_resource()?;
7905 return Ok(false);
7906 }
7907 if self.state.reviews.len() < round {
7908 self.state.reviews.push(round_record);
7909 } else {
7910 self.state.reviews[round - 1] = round_record;
7911 }
7912 self.state.save()?;
7913
7914 if matches!(step, FixStep::Recorded { report_lost: true }) {
7927 self.stop_reviewing(
7928 "the fixer's adoption report never came back, even after resuming its \
7929 own seat; refusing to start another round against the same worktree \
7930 while that is unresolved",
7931 &shell,
7932 &winner.worktree,
7933 )
7934 .await?;
7935 return Ok(false);
7936 }
7937
7938 let streak = self
7939 .state
7940 .reviews
7941 .iter()
7942 .rev()
7943 .take_while(|r| !r.progressed)
7944 .count();
7945 if streak >= STAGNANT_LIMIT {
7946 self.stop_reviewing(
7947 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7948 &shell,
7949 &winner.worktree,
7950 )
7951 .await?;
7952 return Ok(false);
7953 }
7954 Ok(true)
7955 }
7956
7957 async fn run_fix_round(
7959 &mut self,
7960 winner: &Candidate,
7961 round_record: &mut ReviewRound,
7962 ) -> Result<FixStep> {
7963 let round = round_record.round;
7964 let max_rounds = self.state.config.graph.review_rounds;
7965 let run_id = self.state.id.clone();
7966 let prompts = self.state.config.prompts.clone();
7967 let language = self.state.config.graph.language.clone();
7968 let sessions = self.state.config.graph.sessions;
7969 let artifacts = self.merge_repair_artifacts();
7970 let base = self.landing_base();
7971 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7972 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7973 let this_round_verification = round_record.verification_summary(&head);
7974 let all_findings: Vec<_> = round_record
7975 .reviews
7976 .iter()
7977 .flat_map(|r| r.findings.iter().cloned())
7978 .collect();
7979 let blocking_findings: Vec<_> = all_findings
7982 .iter()
7983 .filter(|f| f.severity.blocks())
7984 .cloned()
7985 .collect();
7986 let fix_prompt = prompt::fix_in(
7987 &self.state.instruction,
7988 &blocking_findings,
7989 this_round_verification.as_ref(),
7990 round,
7991 max_rounds,
7992 &language,
7993 &self.state.config.graph.github_language,
7994 );
7995 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7996 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7997 let (job, seat, out) = self
7998 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7999 prompt: fix_prompt.clone(),
8000 spec,
8001 seat,
8002 cwd: winner.worktree.clone(),
8003 timeout,
8004 allow_write: true,
8005 sessions,
8006 artifacts: artifacts.clone(),
8007 stem: format!("fix-{round}"),
8008 handover: None,
8009 })
8010 .await;
8011 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8014 return Ok(FixStep::Resource);
8015 }
8016 let agent_id = seat.agent.clone();
8017
8018 let mut fix = FixRecord {
8019 agent: agent_id,
8020 addressed: Vec::new(),
8021 rejected: Vec::new(),
8022 notes: String::new(),
8023 committed: false,
8024 failed: None,
8025 duration_ms: 0,
8026 continuation: None,
8027 };
8028 let mut continuation = ContinuationRecord::not_needed();
8029 let mut final_seat = seat.clone();
8030 match out {
8031 AgentOutcome::Ok(o) => {
8032 fix.duration_ms = o.duration_ms;
8033 let parsed = verdict::extract_json::<FixReport>(&o.text);
8034 let incomplete_reason = match &parsed {
8041 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
8042 "the reply parsed, but it reported a command whose own CLI never \
8043 confirmed an exit status"
8044 .to_owned(),
8045 ),
8046 Ok(_) => None,
8047 Err(e) => Some(e.to_string()),
8048 };
8049 match incomplete_reason {
8050 None => {
8051 let report = parsed.expect("checked Ok above");
8052 fix.addressed = report.addressed;
8053 fix.rejected = report.rejected;
8054 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8055 }
8056 Some(reason) => {
8057 let (resumed_seat, resolved, failure, cont) = self
8058 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
8059 .await;
8060 fix.duration_ms += cont.cumulative_wait_ms;
8061 continuation = cont;
8062 final_seat = resumed_seat;
8063 match resolved {
8064 Some(report) => {
8065 fix.addressed = report.addressed;
8066 fix.rejected = report.rejected;
8067 fix.notes =
8068 blind::sanitize_prose(&report.notes, &self.state.config.blind);
8069 }
8070 None => fix.failed = failure,
8071 }
8072 }
8073 }
8074 }
8075 AgentOutcome::Dropped(o) => {
8077 fix.duration_ms = o.duration_ms;
8078 let why = o
8079 .dropped
8080 .as_ref()
8081 .map(|d| d.why.as_str())
8082 .unwrap_or("the CLI ended the stream without delivering its answer");
8083 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8084 }
8085 AgentOutcome::Quota(o) => {
8086 self.state.quota.push(QuotaLoss::from_report(
8087 final_seat.key.clone(),
8088 "fix",
8089 o.quota.as_ref(),
8090 ));
8091 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8092 }
8093 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8094 }
8095 if self.state.resource_block.is_some() {
8096 self.state.seats.insert(final_seat.key.clone(), final_seat);
8097 return Ok(FixStep::Resource);
8098 }
8099 fix.continuation = Some(continuation);
8100 self.state.seats.insert(final_seat.key.clone(), final_seat);
8101 let coordination = self.coordinate(&winner.worktree).await?;
8102 if let Ok(r) = git::rescue_commit(
8103 &winner.worktree,
8104 &format!("magi: review round {round} fixes (uncommitted work)"),
8105 )
8106 .await
8107 {
8108 self.state.note_withheld("fix", &r.withheld);
8109 }
8110 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8111 fix.committed = after != before;
8112 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8120 drop(coordination);
8121 let progressed = diff_after != patch;
8122 let commit_note = if fix.committed {
8123 "committed"
8124 } else {
8125 "NO new commit"
8126 };
8127 let tree_note = if progressed {
8128 "changed vs base"
8129 } else {
8130 "unchanged vs base"
8131 };
8132 self.state.event(
8133 "fix",
8134 match &fix.failed {
8135 Some(reason) => {
8141 format!(
8142 "round {round}: fixer's adoption report was lost ({reason}); \
8143 {commit_note}, tree {tree_note}"
8144 )
8145 }
8146 None => format!(
8147 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8148 {tree_note}{}",
8149 fix.addressed.len(),
8150 fix.rejected.len(),
8151 if continuation.outcome == ContinuationOutcome::Resumed {
8152 format!(
8153 " (adoption report recovered after {} continuation(s))",
8154 continuation.attempts
8155 )
8156 } else {
8157 String::new()
8158 },
8159 ),
8160 },
8161 );
8162 round_record.fix = Some(fix);
8163 round_record.progressed = progressed;
8164 Ok(FixStep::Recorded {
8165 report_lost: matches!(
8166 continuation.outcome,
8167 ContinuationOutcome::Exhausted
8168 | ContinuationOutcome::QuotaLost
8169 | ContinuationOutcome::NoSession
8170 ),
8171 })
8172 }
8173
8174 fn record_contested_handoff(&mut self) {
8208 if self.state.contested_handoff.is_some() {
8209 return;
8210 }
8211 let Some(contested) = self
8212 .state
8213 .reviews
8214 .last()
8215 .and_then(ReviewRound::contested_handoff)
8216 else {
8217 return;
8218 };
8219 self.state.event(
8220 "review",
8221 format!(
8222 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8223 wait for the owner's approval",
8224 contested.findings.len(),
8225 contested.rejecters.len()
8226 ),
8227 );
8228 self.state.contested_handoff = Some(contested);
8229 }
8230
8231 async fn stop_unresolved_review(
8232 &mut self,
8233 why: &str,
8234 shell: &[String],
8235 worktree: &Path,
8236 ) -> Result<()> {
8237 self.stop_reviewing_inner(why, shell, worktree, true).await
8238 }
8239
8240 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8241 self.stop_reviewing_inner(why, shell, worktree, false).await
8242 }
8243
8244 async fn stop_reviewing_inner(
8245 &mut self,
8246 why: &str,
8247 shell: &[String],
8248 worktree: &Path,
8249 repairable: bool,
8250 ) -> Result<()> {
8251 let round_idx = self.state.reviews.len() - 1;
8252 let needs_catchup_run = matches!(
8260 self.state.reviews[round_idx].e2e_status(),
8261 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8262 );
8263 if needs_catchup_run {
8264 let round = self.state.reviews[round_idx].round;
8265 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8266 let commands = self.state.config.verify.e2e.clone();
8267 let before_head = git::rev_parse(worktree, "HEAD").await?;
8268 let winner = self.state.winner().cloned();
8269 let cache_dir = self.state.config.cache_dir();
8270 let context = format!(
8271 "round {round}: verification unresolved, catching up before the final decision"
8272 );
8273 let wait = self.repository_wait();
8274 let (outcomes, verify_retried) = with_cache_lease(
8275 &mut self.state,
8276 cache_dir.as_deref(),
8277 "e2e",
8278 "e2e",
8279 worktree,
8280 &before_head,
8281 timeout,
8282 &context,
8283 &wait,
8284 |state, budget| {
8285 let shell = shell.to_vec();
8286 let commands = commands.clone();
8287 let context = context.clone();
8288 async move {
8289 if let Some(winner) = winner.filter(|_| !commands.is_empty()) {
8290 Self::run_pre_gate(state, &winner).await;
8293 }
8294 run_e2e_with_retry(
8295 state,
8296 &shell,
8297 &commands,
8298 worktree,
8299 budget,
8300 &context,
8301 Some(round),
8302 )
8303 .await
8304 }
8305 },
8306 )
8307 .await?;
8308 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8310 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8311 self.state.record_base_progress(&attempted_head);
8312 }
8313 let last = &mut self.state.reviews[round_idx];
8314 last.e2e = outcomes;
8315 last.verify_retried = verify_retried;
8316 last.verified_head = Some(attempted_head);
8323 last.verified_at = Some(Timestamp::now());
8324 if verify_inconclusive(&last.e2e) {
8325 self.state.save()?;
8332 return Ok(());
8333 }
8334 last.e2e_deferred = false;
8335 if last.answered == last.expected {
8340 last.clean = round_is_clean(
8341 last.blocking,
8342 last.e2e.iter().all(CommandOutcome::ok),
8343 last.answered,
8344 last.expected,
8345 0,
8346 self.state.config.graph.incomplete_review,
8347 );
8348 }
8349 }
8350 let last = &self.state.reviews[round_idx];
8351 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8352
8353 match last.e2e_status() {
8354 E2eStatus::Failed => {
8355 let red: Vec<String> = last
8356 .e2e
8357 .iter()
8358 .filter(|o| !o.ok())
8359 .map(|o| {
8360 format!(
8361 "`{}` -> {:?}\n{}",
8362 o.command,
8363 o.code,
8364 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8365 )
8366 })
8367 .collect();
8368 let can_repair = last.blocking == 0 && last.fix.is_none();
8369 self.state.reviews[round_idx].clean = false;
8370 self.state
8371 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8372 self.state.status = RunStatus::Blocked;
8373 if repairable
8374 && can_repair
8375 && self.state.reviews.len() < self.state.config.graph.review_rounds
8376 {
8377 self.state.status = RunStatus::Reviewing;
8378 }
8379 }
8380 E2eStatus::ResourceBlocked => {
8385 self.state.event(
8386 "review",
8387 format!(
8388 "{why}; e2e could not run (resource unavailable); not \
8389 deciding yet"
8390 ),
8391 );
8392 }
8393 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8394 self.state.event(
8395 "review",
8396 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8397 );
8398 self.record_contested_handoff();
8399 self.state.status = RunStatus::Gating;
8400 }
8401 }
8402 self.state.save()?;
8403 Ok(())
8404 }
8405
8406 async fn gate(&mut self) -> Result<()> {
8409 if self.hold_base_sync().await? {
8410 return Ok(());
8411 }
8412 if self.state.status == RunStatus::Failed
8424 || self
8425 .state
8426 .base_sync
8427 .as_ref()
8428 .is_some_and(|s| s.conflict.is_some())
8429 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8430 != Some(RunStatus::Gating)
8431 {
8432 return Ok(());
8433 }
8434 let Some(winner) = self.state.winner().cloned() else {
8435 return Ok(());
8436 };
8437 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8438 self.state.reconcile_verification_head(&head);
8439 self.state.save()?;
8440 let wait = self.repository_wait();
8441 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8442 return Ok(());
8443 }
8444 if self.state.gate_ran {
8445 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8456 self.state.status = RunStatus::Blocked;
8457 self.state.save()?;
8458 }
8459 return Ok(());
8460 }
8461 self.state.status = RunStatus::Gating;
8462 let mut outcomes = self.run_gate(&winner).await?;
8463 let mut attempted_head = self.state.gate_head.clone();
8464 loop {
8465 if verify_inconclusive(&outcomes) {
8476 self.state.save()?;
8477 return Ok(());
8478 }
8479 if outcomes.iter().all(CommandOutcome::ok) {
8480 break;
8481 }
8482 let fixes_before = self.state.gate_fixes.len();
8483 match self.gate_fix_round(&winner, &outcomes).await? {
8484 GateFix::Retry => {
8485 outcomes = self.run_gate(&winner).await?;
8486 attempted_head = self.state.gate_head.clone();
8487 }
8488 GateFix::Stop => {
8489 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8490 if self.state.gate_fixes.len() > fixes_before
8491 && attempted_head.as_deref() != Some(current.as_str())
8492 && verify_current_head(&mut self.state, &winner, &wait).await?
8493 {
8494 outcomes = self.run_gate(&winner).await?;
8497 attempted_head = self.state.gate_head.clone();
8498 } else {
8499 break;
8500 }
8501 }
8502 GateFix::Defer => {
8503 self.state.save()?;
8504 return Ok(());
8505 }
8506 }
8507 }
8508 let passed = outcomes.iter().all(CommandOutcome::ok);
8509 self.state.gate = outcomes;
8510 self.state.gate_ran = true;
8511 self.state.gate_head = attempted_head;
8512 if !self.state.gate.is_empty()
8513 && !verify_inconclusive(&self.state.gate)
8514 && let Some(head) = self.state.gate_head.clone()
8515 {
8516 self.state.record_base_progress(&head);
8517 }
8518 self.state.save()?;
8519 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8520 return Ok(());
8521 }
8522 if !passed {
8523 self.state.status = RunStatus::Blocked;
8524 let spent = self.state.gate_fixes.len();
8525 self.state.event(
8526 "gate",
8527 if spent == 0 {
8528 "gate failed; not merging".to_owned()
8529 } else {
8530 format!("gate failed after {spent} gate-fix round(s); not merging")
8531 },
8532 );
8533 }
8534 self.state.save()?;
8535 Ok(())
8536 }
8537
8538 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8556 let commands = state.config.verify.pre_gate.clone();
8557 if commands.is_empty() {
8558 return;
8559 }
8560 let shell = state.config.shell();
8561 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8562 let (outcomes, _) = run_commands(
8563 state,
8564 "pre_gate",
8565 "pre_gate",
8566 0,
8567 &shell,
8568 &commands,
8569 &winner.worktree,
8570 timeout,
8571 )
8572 .await;
8573 for o in &outcomes {
8574 if !o.ok() {
8575 tracing::warn!(
8576 "pre_gate `{}` failed ({:?}); the gate decides",
8577 o.command,
8578 o.code
8579 );
8580 }
8581 state.event(
8582 "pre_gate",
8583 format!(
8584 "`{}` -> {}",
8585 o.command,
8586 if o.ok() {
8587 "pass".to_owned()
8588 } else {
8589 format!(
8590 "FAIL ({:?})\n{}",
8591 o.code,
8592 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8593 )
8594 }
8595 ),
8596 );
8597 }
8598 state.pre_gate = outcomes;
8599 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8600 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8601 Ok(head) => {
8602 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8603 state.pre_gate_commit = Some(head);
8604 }
8605 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8606 },
8607 Ok(false) => {}
8608 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8609 }
8610 if let Err(e) = state.save() {
8611 tracing::warn!("could not persist the pre_gate record: {e:#}");
8612 }
8613 }
8614
8615 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8618 let wait = self.repository_wait();
8619 run_landing_gate(&mut self.state, winner, &wait).await
8620 }
8621
8622 async fn gate_fix_round(
8635 &mut self,
8636 winner: &Candidate,
8637 outcomes: &[CommandOutcome],
8638 ) -> Result<GateFix> {
8639 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8640 g.phase == crate::repair_budget::Phase::Fixing
8641 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8642 }) {
8643 self.state.gate_fixes.len().saturating_add(1)
8644 } else {
8645 self.state.config.graph.gate_fix_rounds
8646 };
8647 let spent = self.state.gate_fixes.len();
8648 if spent >= cap {
8649 if cap > 0 {
8650 self.state.event(
8651 "gate",
8652 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8653 );
8654 }
8655 return Ok(GateFix::Stop);
8656 }
8657 if !gate_fixable(outcomes) {
8658 self.state.event(
8659 "gate",
8660 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8661 command or similar); not spending a fix round on it",
8662 );
8663 return Ok(GateFix::Stop);
8664 }
8665 let min_free = self.state.config.disk.min_free_bytes;
8666 if min_free > 0 {
8667 match crate::disk::free_bytes(&winner.worktree) {
8668 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8669 Ok(free) => {
8670 self.state.event(
8671 "gate",
8672 format!(
8673 "only {free} bytes free ({min_free} required by `[disk] \
8674 min_free_bytes`); not spending a fix round on a failure the disk \
8675 may explain"
8676 ),
8677 );
8678 return Ok(GateFix::Stop);
8679 }
8680 Err(e) => {
8681 self.state.event(
8682 "gate",
8683 format!("free disk space could not be measured ({e:#}); no fix round"),
8684 );
8685 return Ok(GateFix::Stop);
8686 }
8687 }
8688 }
8689
8690 let attempt = spent + 1;
8691 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8692 let base = self.landing_base();
8693 let fix_prompt = prompt::gate_fix_in(
8694 &self.state.instruction,
8695 &failed,
8696 attempt,
8697 cap,
8698 &self.state.config.graph.language,
8699 &self.state.config.graph.github_language,
8700 );
8701 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8702 let sessions = self.state.config.graph.sessions;
8703 let artifacts = self.merge_repair_artifacts();
8704 self.state.event(
8705 "gate",
8706 format!("gate failed; gate-fix round {attempt} of {cap}"),
8707 );
8708 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8709 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8710 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8713 head: before.clone(),
8714 passed: false,
8715 outcomes: Vec::new(),
8716 });
8717 self.state.save()?;
8718 let (_, seat, out) = self
8719 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8720 prompt: fix_prompt.clone(),
8721 spec,
8722 seat,
8723 cwd: winner.worktree.clone(),
8724 timeout,
8725 allow_write: true,
8726 sessions,
8727 artifacts: artifacts.clone(),
8728 stem: format!("gate-fix-{attempt}"),
8729 handover: None,
8730 })
8731 .await;
8732 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8735 return Ok(GateFix::Defer);
8736 }
8737 let mut record = GateFixRecord {
8738 agent: seat.agent.clone(),
8739 failed,
8740 notes: String::new(),
8741 committed: false,
8742 error: None,
8743 };
8744 match out {
8745 AgentOutcome::Ok(o) => {
8746 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8749 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8750 }
8751 }
8752 AgentOutcome::Dropped(_) => {
8753 record.error = Some("the CLI dropped the stream".to_owned());
8754 }
8755 AgentOutcome::Quota(o) => {
8756 self.state.quota.push(QuotaLoss::from_report(
8757 seat.key.clone(),
8758 "gate-fix",
8759 o.quota.as_ref(),
8760 ));
8761 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8762 }
8763 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8764 }
8765 self.state.seats.insert(seat.key.clone(), seat);
8766 let coordination = self.coordinate(&winner.worktree).await?;
8767 if let Ok(r) = git::rescue_commit(
8768 &winner.worktree,
8769 &format!("magi: gate fix {attempt} (uncommitted work)"),
8770 )
8771 .await
8772 {
8773 self.state.note_withheld("gate-fix", &r.withheld);
8774 }
8775 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8776 record.committed = after != before;
8777 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8778 let note = record.error.clone();
8779 self.state.gate_fixes.push(record);
8780 self.state.save()?;
8781 if !changed {
8782 self.state.event(
8783 "gate",
8784 match note {
8785 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8786 None => format!("gate-fix round {attempt}: the tree did not change"),
8787 },
8788 );
8789 return Ok(GateFix::Stop);
8790 }
8791 self.state.event(
8792 "gate",
8793 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8794 );
8795
8796 drop(coordination);
8797 let wait = self.repository_wait();
8798 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8799 }
8800
8801 async fn guarded_pr_message(
8808 &mut self,
8809 winner: &Candidate,
8810 facts: Option<&BranchFacts>,
8811 posting: bool,
8812 ) -> Result<Option<PrMessage>> {
8813 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8814 let head = match final_head.as_deref() {
8815 Some(h) => crate::pr_outcome::FinalHead::At(h),
8816 None => crate::pr_outcome::FinalHead::Unknown,
8817 };
8818 let summarized = if posting {
8820 Some(self.summarize_change(winner, final_head.as_deref()).await)
8821 } else {
8822 None
8823 };
8824 if self.state.resource_block.is_some() {
8829 self.state.event(
8830 "pr-summary",
8831 "a shared resource was busy; nothing was published, resume to summarize again",
8832 );
8833 return Ok(None);
8834 }
8835 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8836 anyhow::bail!(
8837 "the final commit moved during public description generation; retry before publishing"
8838 );
8839 }
8840 let note = match (&summarized, &self.state.change_summary) {
8841 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8842 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8843 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8844 (None, _) => ChangeNote::NotAsked,
8845 };
8846 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8847 if !posting {
8848 let identity = crate::scrub::Identity::current();
8849 return Ok(Some(PrMessage {
8850 title: crate::scrub::scrub(&pr.title, &identity),
8851 body: crate::scrub::scrub(&pr.body, &identity),
8852 }));
8853 }
8854 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8857 if let Some(known) = self
8858 .state
8859 .github_text
8860 .clone()
8861 .filter(|g| g.fingerprint == raw_fp)
8862 {
8863 return self.settle_withheld(winner, &pr, known).await;
8864 }
8865 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8866 let mut violations =
8867 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8868 let identity = crate::scrub::Identity::current();
8869 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8870 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8871 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8872 {
8873 violations.push(crate::github_text::Violation::SensitiveData);
8874 }
8875 let has_language = violations
8879 .iter()
8880 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8881 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8882 self.state.event(
8883 "github-text",
8884 if has_language {
8885 format!("description rejected: {violations:?}; requesting one rewrite")
8886 } else {
8887 format!("description rejected: {violations:?}; asking the owner")
8888 },
8889 );
8890 let mut rewritten = false;
8891 let mut rejected_rewrite = None;
8892 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8893 {
8894 let output = text;
8895 if let Some(title) = summary_title(&output) {
8896 let raw_body = summary_without_title(&output);
8899 let clean_title = crate::scrub::scrub(&title, &identity);
8900 let mut body = crate::scrub::scrub(&raw_body, &identity);
8901 let redacted = clean_title != title || body != raw_body;
8902 let title = clean_title;
8903 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8904 if !complete {
8905 self.state.event(
8906 "github-text",
8907 "description rewrite omitted required findings",
8908 );
8909 }
8910 if !body.trim().is_empty() {
8911 body.push_str("\n\n");
8912 body.push_str(&public_pr_evidence(&self.state, head));
8913 }
8914 let marker = format!("magi:run/{}", self.state.id);
8915 if !body.trim().is_empty() && !body.contains(&marker) {
8916 body.push_str(&format!("\n\n{marker}"));
8917 }
8918 let rewrite = PrMessage {
8919 title: title.clone(),
8920 body: body.clone(),
8921 };
8922 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8923 let mut failed = crate::github_text::check_for(
8924 &self.state.config.graph,
8925 &title,
8926 &body,
8927 redecision,
8928 );
8929 if (!crate::github_text::shareable(&title)
8930 || !crate::github_text::shareable(&body))
8931 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8932 {
8933 failed.push(crate::github_text::Violation::SensitiveData);
8934 }
8935 if complete
8936 && !title.trim().is_empty()
8937 && !body.trim().is_empty()
8938 && !failed.is_empty()
8939 {
8940 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8941 title: title.clone(),
8942 body: body.clone(),
8943 failed: failed
8944 .iter()
8945 .map(|v| crate::github_text::category(*v).to_owned())
8946 .collect(),
8947 });
8948 }
8949 if complete
8950 && !title.trim().is_empty()
8951 && !raw_body.trim().is_empty()
8952 && failed.is_empty()
8953 {
8954 decision = redecision;
8955 pr = PrMessage { title, body };
8956 rewritten = true;
8957 self.state
8958 .event("github-text", "description rewrite passed");
8959 if redacted {
8960 self.state.event(
8961 "github-text",
8962 "sensitive data removed from description rewrite",
8963 );
8964 }
8965 }
8966 }
8967 }
8968 if !rewritten {
8969 self.state.event(
8970 "github-text",
8971 "description rewrite unavailable or rejected; asking the owner",
8972 );
8973 if let Some(w) =
8974 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8975 {
8976 self.retire_github_text_question("the withheld text changed");
8979 let record = crate::run::GithubTextAsk {
8980 rewrite: rejected_rewrite,
8981 fingerprint: raw_fp,
8982 title: w.title,
8983 body: w.body,
8984 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8985 question: None,
8986 asks: 0,
8987 resolved: false,
8988 chosen_title: None,
8989 choice: None,
8990 publication: None,
8991 };
8992 return self.settle_withheld(winner, &pr, record).await;
8993 }
8994 }
8995 }
8996 let (title, body) =
8997 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8998 Ok(Some(PrMessage { title, body }))
8999 }
9000
9001 async fn summarize_change(
9012 &mut self,
9013 winner: &Candidate,
9014 head: Option<&str>,
9015 ) -> std::result::Result<(), String> {
9016 let Some(head) = head else {
9017 return Err("the final commit could not be read".to_owned());
9018 };
9019 let remote = self.state.config.merge.remote.clone();
9020 let tracking = format!("{remote}/{}", self.state.base_branch);
9021 if !git::rev_exists(&winner.worktree, &tracking).await {
9022 return Err(format!("the remote base `{tracking}` could not be read"));
9023 }
9024 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
9025 Ok(b) if !b.is_empty() => b,
9026 _ => return Err("the merge base with the remote base could not be read".to_owned()),
9027 };
9028 let evidence = crate::pr_outcome::summary_evidence(
9029 &self.state,
9030 crate::pr_outcome::FinalHead::At(head),
9031 );
9032 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
9033 if self
9034 .state
9035 .change_summary
9036 .as_ref()
9037 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
9038 {
9039 return Ok(());
9040 }
9041 if self.state.change_summary.take().is_some() {
9042 self.state.event(
9043 "pr-summary",
9044 "the final diff or evidence changed; the earlier change summary was discarded",
9045 );
9046 }
9047 if let Err(e) = write_artifact(
9049 &self.state,
9050 &format!("public-description-{fingerprint}.md"),
9051 &format!("Final diff: {base}..{head}\n\n{evidence}"),
9052 ) {
9053 tracing::warn!("could not save public description evidence: {e:#}");
9054 }
9055 let chain = match agent::pick_chain(
9056 &self.state.config.agents,
9057 self.state.config.roles.synthesizer.as_ref(),
9058 &agent::installed,
9059 "synthesizer",
9060 ) {
9061 Ok(chain) => chain,
9062 Err(e) => {
9063 tracing::warn!("no agent for the change summary: {e:#}");
9064 self.state.event("pr-summary", "no summary agent available");
9065 return Err("no agent was available to summarize it".to_owned());
9066 }
9067 };
9068 let subjects = git::subjects(&winner.worktree, &base, head)
9069 .await
9070 .unwrap_or_default();
9071 let stat = git::diff_stat(&winner.worktree, &base, head)
9072 .await
9073 .unwrap_or_default();
9074 let stat: String = stat.chars().take(6_000).collect();
9075 let mut prompt = crate::prompt::change_summary_in(
9076 &base,
9077 head,
9078 &subjects,
9079 stat.trim(),
9080 &self.state.config.graph.github_language,
9081 );
9082 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
9083 let sessions = self.state.config.graph.sessions;
9084 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
9085 let prompts = self.state.config.prompts.clone();
9086 let cache = self.state.config.cache_dir();
9087 let run = self.state.id.clone();
9088 for (i, spec) in chain.into_iter().enumerate() {
9089 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
9090 let github_spec = match agent::for_github_text(
9091 &spec,
9092 &self.state.config.graph.github_language,
9093 &winner.worktree,
9094 ) {
9095 Ok(spec) => spec,
9096 Err(e) => {
9097 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9098 continue;
9099 }
9100 };
9101 let job = SeatJob {
9102 spec: github_spec,
9103 seat,
9104 cwd: winner.worktree.clone(),
9105 prompt: prompt.clone(),
9106 timeout,
9107 allow_write: false,
9108 sessions,
9109 artifacts: agent::artifacts_dir(&self.state.dir()),
9110 stem: if i == 0 {
9111 "change-summary".to_owned()
9112 } else {
9113 format!("change-summary-{}", spec.id)
9114 },
9115 handover: None,
9116 };
9117 let ctx = WaveCtx {
9118 run: &run,
9119 node: "pr-summary",
9120 prompts: &prompts,
9121 cache: cache.as_deref(),
9122 round: None,
9123 carry_seats: false,
9124 pauses: Some((&self.pause, &self.interrupt)),
9125 };
9126 let (seat, outcome) =
9127 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9128 self.state.seats.insert(seat.key.clone(), seat);
9129 let text = match outcome {
9130 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9131 AgentOutcome::ResourceBlocked(why) => {
9132 if self.state.resource_block.is_none() {
9135 self.state.resource_block = Some(crate::run::ResourceBlock {
9136 cause: crate::run::ResourceBlockCause::Other,
9137 node: "pr-summary".to_owned(),
9138 kind: "shared resource".to_owned(),
9139 detail: why,
9140 at: Timestamp::now(),
9141 });
9142 }
9143 return Err("a shared resource was busy".to_owned());
9144 }
9145 _ => continue,
9146 };
9147 let blind = &self.state.config.blind;
9148 let Some(title) = summary_title(&text) else {
9149 continue;
9150 };
9151 let body: String = summary_without_title(&text)
9152 .lines()
9153 .filter(|l| !l.contains("magi:run/"))
9154 .collect::<Vec<_>>()
9155 .join("\n");
9156 let body = blind::sanitize_prose(body.trim(), blind);
9157 let title = blind::sanitize_prose(&title, blind);
9158 if body.trim().is_empty() || title.trim().is_empty() {
9159 continue;
9160 }
9161 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9162 continue;
9163 }
9164 if git::rev_parse(&winner.worktree, "HEAD")
9166 .await
9167 .ok()
9168 .as_deref()
9169 != Some(head)
9170 {
9171 return Err("the branch moved while the summary was written".to_owned());
9172 }
9173 if git::merge_base(&winner.worktree, &tracking, head)
9174 .await
9175 .ok()
9176 .as_deref()
9177 != Some(base.as_str())
9178 || crate::pr_outcome::summary_evidence(
9179 &self.state,
9180 crate::pr_outcome::FinalHead::At(head),
9181 ) != evidence
9182 {
9183 return Err("the summary evidence changed while it was written".to_owned());
9184 }
9185 self.state.change_summary = Some(crate::run::ChangeSummary {
9186 head: head.to_owned(),
9187 base,
9188 evidence: fingerprint,
9189 title,
9190 body,
9191 });
9192 self.state.event(
9193 "pr-summary",
9194 format!("change summary written by {} for the final diff", spec.id),
9195 );
9196 if let Err(e) = self.state.save() {
9197 tracing::warn!("could not save the change summary: {e:#}");
9198 }
9199 return Ok(());
9200 }
9201 self.state
9202 .event("pr-summary", "no agent produced a change summary");
9203 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9204 }
9205
9206 async fn rewrite_pr_text(
9216 &mut self,
9217 winner: &Candidate,
9218 pr: &PrMessage,
9219 violations: &[crate::github_text::Violation],
9220 ) -> Option<String> {
9221 let prompt = format!(
9222 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9223 pr.title, pr.body
9224 );
9225 let sessions = self.state.config.graph.sessions;
9226 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9227 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9230 let key = format!("impl-{}", winner.label);
9231 if let Some(spec) = self
9232 .state
9233 .config
9234 .agents
9235 .iter()
9236 .find(|a| a.id == winner.agent)
9237 .cloned()
9238 && let Some(existing) = self.state.seats.get(&key)
9239 && existing.agent == spec.id
9240 && agent::has_session(spec.kind, existing, sessions)
9241 {
9242 attempts.push((spec, Some(existing.clone())));
9243 }
9244 match agent::pick_chain(
9245 &self.state.config.agents,
9246 self.state.config.roles.language_judge.as_ref(),
9247 &agent::installed,
9248 "language_judge",
9249 ) {
9250 Ok(chain) => {
9251 for spec in chain {
9252 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9253 continue;
9254 }
9255 attempts.push((spec, None));
9256 }
9257 }
9258 Err(e) if attempts.is_empty() => {
9259 tracing::warn!("no agent for the description rewrite: {e:#}");
9260 self.state
9261 .event("github-text", "no rewrite agent available");
9262 return None;
9263 }
9264 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9265 }
9266 let prompts = self.state.config.prompts.clone();
9267 let cache = self.state.config.cache_dir();
9268 let run = self.state.id.clone();
9269 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9270 let resumed = resumed_seat.is_some();
9271 let seat = resumed_seat.unwrap_or_else(|| {
9272 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9273 });
9274 let github_spec = match agent::for_github_text(
9275 &spec,
9276 &self.state.config.graph.github_language,
9277 &winner.worktree,
9278 ) {
9279 Ok(spec) => spec,
9280 Err(e) => {
9281 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9282 continue;
9283 }
9284 };
9285 let job = SeatJob {
9286 spec: github_spec,
9287 seat,
9288 cwd: winner.worktree.clone(),
9289 prompt: prompt.clone(),
9290 timeout: retry_budget(timeout_full, resumed),
9291 allow_write: false,
9292 sessions,
9293 artifacts: agent::artifacts_dir(&self.state.dir()),
9294 stem: if i == 0 {
9295 "github-text-rewrite".to_owned()
9296 } else {
9297 format!("github-text-rewrite-{}", spec.id)
9298 },
9299 handover: None,
9300 };
9301 let ctx = WaveCtx {
9302 run: &run,
9303 node: "github-text",
9304 prompts: &prompts,
9305 cache: cache.as_deref(),
9306 round: None,
9307 carry_seats: false,
9308 pauses: Some((&self.pause, &self.interrupt)),
9309 };
9310 let (seat, outcome) =
9311 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9312 self.state.seats.insert(seat.key.clone(), seat);
9313 match outcome {
9314 AgentOutcome::Ok(o)
9315 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9316 {
9317 return Some(o.text);
9318 }
9319 AgentOutcome::ResourceBlocked(_) => return None,
9320 _ => {}
9321 }
9322 }
9323 None
9324 }
9325
9326 fn retire_github_text_question(&mut self, why: &str) {
9328 let Some(id) = self
9329 .state
9330 .github_text
9331 .as_ref()
9332 .and_then(|g| g.question.clone())
9333 else {
9334 return;
9335 };
9336 let _ = ask::Questions::open().update(&id, |q| {
9337 q.abandon(why);
9338 Ok(())
9339 });
9340 }
9341
9342 async fn settle_withheld(
9346 &mut self,
9347 winner: &Candidate,
9348 pr: &PrMessage,
9349 mut g: crate::run::GithubTextAsk,
9350 ) -> Result<Option<PrMessage>> {
9351 use crate::github_text as gt;
9352 let store = ask::Questions::open();
9353 let timeout = self.state.config.graph.answer_timeout;
9354 if !g.resolved {
9355 let mut standing = g
9356 .question
9357 .as_ref()
9358 .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9359 if let Some(q) = standing.as_ref()
9360 && q.status.open()
9361 && !store.held_reply(&q.id)
9362 && gt::expired(q, timeout)
9363 {
9364 let why = format!("no answer within {}s of asking", timeout.max(1));
9365 standing = store
9366 .update(&q.id, |q| {
9367 q.abandon(&why);
9368 Ok(())
9369 })
9370 .ok()
9371 .map(|(q, ())| q);
9372 }
9373 let mut ask_again: Option<Option<Vec<String>>> = None;
9374 match standing {
9375 None if g.asks <= 1 => {
9378 g.asks = 0;
9379 ask_again = Some(None);
9380 }
9381 None => {
9382 g.resolved = true;
9383 g.chosen_title = None;
9384 g.choice = Some(GithubTextChoice::Fallback);
9385 }
9386 Some(q) if q.status.open() => {
9387 self.park_on_github_text(&g)?;
9388 return Ok(None);
9389 }
9390 Some(q) => match gt::read_reply(&q) {
9391 gt::Reply::Fallback => {
9392 g.resolved = true;
9393 g.chosen_title = None;
9394 g.choice = Some(GithubTextChoice::Fallback);
9395 }
9396 gt::Reply::Original => {
9397 g.resolved = true;
9398 g.chosen_title = None;
9399 g.choice = Some(GithubTextChoice::Original);
9400 }
9401 gt::Reply::Rewrite => {
9402 g.resolved = true;
9403 g.chosen_title = None;
9404 g.choice = Some(if g.rewrite.is_some() {
9405 GithubTextChoice::Rewrite
9406 } else {
9407 GithubTextChoice::Fallback
9408 });
9409 }
9410 gt::Reply::Title(text) => {
9411 let first = text
9412 .lines()
9413 .map(str::trim)
9414 .find(|l| !l.is_empty())
9415 .unwrap_or("");
9416 let decision = if first.is_empty() {
9417 None
9418 } else {
9419 let candidate = PrMessage {
9420 title: first.to_owned(),
9421 body: String::new(),
9422 };
9423 self.judge_pr_language(&winner.worktree, &candidate).await
9424 };
9425 match gt::vet_title(
9426 &text,
9427 decision,
9428 self.state.config.graph.github_english(),
9429 ) {
9430 Ok(title) => {
9431 g.resolved = true;
9432 g.chosen_title = Some(title);
9433 g.choice = Some(GithubTextChoice::MyTitle);
9434 }
9435 Err(cats) if g.asks < 2 => {
9436 ask_again = Some(Some(
9437 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9438 ))
9439 }
9440 Err(_) => {
9441 g.resolved = true;
9442 g.chosen_title = None;
9443 g.choice = Some(GithubTextChoice::Fallback);
9444 }
9445 }
9446 }
9447 },
9448 }
9449 if let Some(rejected) = ask_again {
9450 let w = gt::Withheld {
9453 title: g.title,
9454 body: g.body,
9455 categories: g.categories.clone(),
9456 };
9457 let artifact = match crate::run::write_artifact(
9460 &self.state,
9461 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9462 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9463 ) {
9464 Ok(p) => Some(p),
9465 Err(e) => {
9466 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9467 None
9468 }
9469 };
9470 let (detail, comparison) = gt::question_content(
9471 &w,
9472 &pr.title,
9473 &pr.body,
9474 &self.state.id,
9475 rejected.as_deref(),
9476 artifact.as_deref(),
9477 g.rewrite.as_ref(),
9478 );
9479 let mut q = ask::Question::new(
9480 self.state.id.clone(),
9481 gt::ASK_NODE.to_owned(),
9482 gt::ASK_SEAT.to_owned(),
9483 gt::question_summary(&self.state.config.graph.language, &w),
9484 detail,
9485 gt::question_choices(&w, g.rewrite.as_ref()),
9486 );
9487 q.posting_comparison = Some(comparison);
9488 q.answer_timeout = timeout;
9489 g.question = Some(q.id.clone());
9492 g.asks += 1;
9493 self.state.github_text = Some(g.clone());
9494 self.state.event(
9495 "github-text",
9496 format!("asking the owner about the withheld text ({})", q.short()),
9497 );
9498 self.state.save()?;
9499 store
9500 .put_executable(&mut q)
9501 .context("file the posting-gate question")?;
9502 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9503 tracing::warn!(
9504 "could not notify about posting-gate question {}: {e:#}",
9505 q.short()
9506 );
9507 }
9508 self.park_on_github_text(&g)?;
9509 return Ok(None);
9510 }
9511 }
9512 let resolved = g.chosen_title.clone();
9513 let original = g.choice == Some(GithubTextChoice::Original);
9514 self.state.event(
9516 "github-text",
9517 if original {
9518 format!(
9519 "posting the original text for: {}",
9520 match (g.title, g.body) {
9521 (true, true) => "title, description",
9522 (true, false) => "title",
9523 _ => "description",
9524 }
9525 )
9526 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9527 "posting the saved rewrite title and description".to_owned()
9528 } else if resolved.is_some() {
9529 "posting the owner's replacement title".to_owned()
9530 } else {
9531 "posting the neutral text for the withheld fields".to_owned()
9532 },
9533 );
9534 self.state.github_text = Some(g.clone());
9535 self.state.save()?;
9536 if g.choice == Some(GithubTextChoice::Rewrite)
9537 && let Some(r) = g.rewrite.as_ref()
9538 {
9539 return Ok(Some(PrMessage {
9540 title: r.title.clone(),
9541 body: r.body.clone(),
9542 }));
9543 }
9544 let w = gt::Withheld {
9545 title: g.title,
9546 body: g.body,
9547 categories: g.categories.clone(),
9548 };
9549 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9550 if let Some(t) = resolved {
9551 title = t;
9552 }
9553 let english = gt::LanguageDecision {
9555 title_english: true,
9556 body_english: true,
9557 body_complete: true,
9558 };
9559 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9560 if original {
9561 if g.title {
9564 title = pr.title.clone();
9565 }
9566 if g.body {
9567 body = pr.body.clone();
9568 }
9569 }
9570 Ok(Some(PrMessage { title, body }))
9571 }
9572
9573 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9574 self.state.github_text = Some(g.clone());
9575 self.state.end_pass_parked(ParkCause::Question);
9576 self.state.parked = true;
9577 self.state.event(
9578 "github-text",
9579 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9580 );
9581 self.state.save()
9582 }
9583
9584 async fn judge_pr_language(
9587 &mut self,
9588 cwd: &Path,
9589 pr: &PrMessage,
9590 ) -> Option<crate::github_text::LanguageDecision> {
9591 self.state.config.roles.language_judge.as_ref()?;
9592 if !self.state.config.graph.language_gate() {
9593 return None;
9594 }
9595 let decision =
9596 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9597 self.state.event(
9598 "github-text",
9599 if decision.is_some() {
9600 "language decided by the language judge"
9601 } else {
9602 "language judge unavailable; using built-in heuristics"
9603 },
9604 );
9605 decision
9606 }
9607
9608 fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9609 Box::pin(self.merge_inner())
9610 }
9611
9612 async fn merge_inner(&mut self) -> Result<()> {
9613 if self.resource_halt()? || self.hold_base_sync().await? {
9614 return Ok(());
9615 }
9616 if self
9631 .state
9632 .base_sync
9633 .as_ref()
9634 .is_some_and(|s| s.conflict.is_some())
9635 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9636 != Some(RunStatus::Gating)
9637 || !self.state.gate_allows_progress()
9645 {
9646 return Ok(());
9647 }
9648 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9657 return Ok(());
9658 }
9659 let Some(winner) = self.state.winner().cloned() else {
9660 return Ok(());
9661 };
9662 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9663 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9664 self.state.status = RunStatus::Blocked;
9665 self.state.event(
9666 "gate",
9667 "winner changed after gate; fresh verification required",
9668 );
9669 self.state.save()?;
9670 return Ok(());
9671 }
9672 let wait = self.repository_wait();
9673 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9674 return Ok(());
9675 }
9676 let repo = self.state.repo.clone();
9677 let base = self.state.base_branch.clone();
9678 let mode = self.state.config.merge.mode;
9679 let style = self.state.config.merge.style;
9680 let facts = if is_review_run(&self.state) {
9681 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9682 let start = review_base(
9683 &repo,
9684 &self.state.config.merge.remote,
9685 &base,
9686 &self.state.base_commit,
9687 &winner.branch,
9688 )
9689 .await;
9690 branch_facts(&repo, &start, &winner.branch).await
9691 } else {
9692 None
9693 };
9694 let leaked = if is_review_run(&self.state) {
9697 leaked_subjects(&self.state, &winner.branch).await
9698 } else {
9699 Some(Vec::new())
9700 };
9701 let Some(pr) = self
9705 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9706 .await?
9707 else {
9708 return Ok(());
9709 };
9710 let message = pr.commit_message();
9711
9712 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9713 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9714 return Ok(());
9715 }
9716 let outcome = match mode {
9717 MergeMode::None => MergeOutcome {
9718 mode,
9719 ok: true,
9720 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9721 empty: false,
9722 },
9723 MergeMode::Pr | MergeMode::Local
9724 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9725 {
9726 MergeOutcome {
9727 mode,
9728 ok: false,
9729 detail: empty_candidate_detail(&self.state, &base),
9730 empty: true,
9731 }
9732 }
9733 MergeMode::Local => {
9734 let on = git::current_branch(&repo).await?;
9735 if on.as_deref() != Some(base.as_str()) {
9736 MergeOutcome {
9737 mode,
9738 ok: false,
9739 detail: format!(
9740 "{} has {} checked out, not the base branch {base}",
9741 repo.display(),
9742 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9743 ),
9744 empty: false,
9745 }
9746 } else if !git::is_clean(&repo).await? {
9747 MergeOutcome {
9748 mode,
9749 ok: false,
9750 detail: format!("{} is dirty; refusing to merge", repo.display()),
9751 empty: false,
9752 }
9753 } else {
9754 let out = match style {
9755 MergeStyle::Merge => {
9756 git::merge_no_ff(&repo, &winner.branch, &message).await?
9757 }
9758 MergeStyle::Squash => {
9759 git::merge_squash(&repo, &winner.branch, &message).await?
9760 }
9761 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9762 };
9763 MergeOutcome {
9764 mode,
9765 ok: out.ok(),
9766 detail: if out.ok() { out.stdout } else { out.stderr },
9767 empty: false,
9768 }
9769 }
9770 }
9771 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9772 self.state.pr_publication_pending = true;
9773 self.state.save()?;
9774 let out =
9775 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9776 self.state.pr_publication_pending = out.is_err();
9777 MergeOutcome {
9778 mode,
9779 ok: out.is_ok(),
9780 detail: match out {
9781 Ok(url) => url,
9782 Err(e) => format!("{e:#}"),
9783 },
9784 empty: false,
9785 }
9786 }
9787 MergeMode::Pr => {
9788 let remote = self.state.config.merge.remote.clone();
9789 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9790 let pushed = if self.state.foreign_checkout.is_some() {
9791 let refspec = format!("{delivery_head}:refs/heads/{public}");
9792 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9793 } else {
9794 git::push(&winner.worktree, &remote, &winner.branch).await?
9795 };
9796 if !pushed.ok() {
9797 MergeOutcome {
9798 mode,
9799 ok: false,
9800 detail: pushed.stderr,
9801 empty: false,
9802 }
9803 } else {
9804 if self.state.foreign_checkout.is_some() {
9805 self.state.pr_pushed_head = Some(delivery_head.clone());
9806 }
9807 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9813 let out = match pr_merge_plan(found) {
9814 PrPlan::Create => {
9815 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9816 .await
9817 }
9818 PrPlan::Adopt { url, title } => {
9819 self.state
9820 .event("merge", format!("Pr: adopted open pull request {url}"));
9821 if self.state.explicit_github_text_selection() {
9822 land::set_pr_text_verbatim(
9825 &winner.worktree,
9826 &url,
9827 &pr.title,
9828 &pr.body,
9829 )
9830 .await
9831 .map(|()| url)
9832 } else {
9833 if title != pr.title
9834 && (!is_review_run(&self.state)
9835 || leaked
9836 .as_deref()
9837 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9838 && let Err(e) =
9839 if self.state.github_text.as_ref().is_some_and(|g| {
9840 g.resolved
9841 && g.title
9842 && g.choice == Some(GithubTextChoice::Original)
9843 }) {
9844 land::set_pr_title_verbatim(
9846 &winner.worktree,
9847 &url,
9848 &pr.title,
9849 )
9850 .await
9851 } else {
9852 land::set_pr_title(
9853 &mut self.state,
9854 &winner.worktree,
9855 &url,
9856 &pr.title,
9857 )
9858 .await
9859 }
9860 {
9861 tracing::warn!("could not refresh title of {url}: {e:#}");
9862 self.state
9863 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9864 }
9865 Ok(url)
9866 }
9867 }
9868 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9869 };
9870 match out {
9871 Ok(url) => MergeOutcome {
9872 mode,
9873 ok: true,
9874 detail: url,
9875 empty: false,
9876 },
9877 Err(e) => MergeOutcome {
9878 mode,
9879 ok: false,
9880 detail: e.to_string(),
9881 empty: false,
9882 },
9883 }
9884 }
9885 }
9886 };
9887
9888 self.state.status = match (mode, outcome.ok) {
9889 (MergeMode::None, _) => RunStatus::Ready,
9890 (_, true) if outcome.empty => RunStatus::Ready,
9891 (MergeMode::Pr, true) => RunStatus::Ready,
9892 (_, true) => RunStatus::Merged,
9893 (_, false) => RunStatus::Blocked,
9894 };
9895 self.state.event(
9896 "merge",
9897 format!(
9898 "{:?}: {}",
9899 mode,
9900 outcome.detail.lines().next().unwrap_or("")
9901 ),
9902 );
9903 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9904 self.state.record_github_text_publication(&outcome.detail);
9905 }
9906 self.state.merge = Some(outcome);
9907 self.state.save()?;
9908
9909 if self.state.config.graph.land
9915 && mode == MergeMode::Pr
9916 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9917 {
9918 self.run_land().await?;
9919 }
9920 self.settle_questions();
9925 Ok(())
9926 }
9927
9928 fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9939 Box::pin(self.run_land_inner())
9940 }
9941
9942 async fn run_land_inner(&mut self) -> Result<()> {
9943 crate::pr_head::ensure_owner(&self.state)?;
9944 let url = self
9945 .state
9946 .pr
9947 .as_ref()
9948 .map(|pr| pr.url.clone())
9949 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9950 .unwrap_or_default();
9951 let url = url.lines().next().unwrap_or("").trim().to_owned();
9952 if !url.starts_with("http") {
9953 return Ok(());
9954 }
9955 loop {
9958 match land::land(&mut self.state, &url).await {
9959 Ok(_)
9960 if self
9961 .state
9962 .merge_repairs
9963 .last()
9964 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9965 {
9966 if self.repair_merge_review().await? {
9967 continue;
9968 }
9969 }
9970 Ok(_) if self.state.resource_block.is_some() => {
9971 return self.stop_for_resource();
9972 }
9973 Ok(pr) if self.state.parked => {
9974 let _ = pr;
9978 }
9979 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9980 self.state.status = RunStatus::Superseded;
9981 self.state.save()?;
9982 }
9983 Ok(pr) => {
9984 self.state.status = match pr.state {
9985 land::PrLifecycle::Merged => RunStatus::Merged,
9986 _ => RunStatus::Blocked,
9987 };
9988 if bump::should_release_bump(self.state.status)
9995 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9996 {
9997 self.state
10003 .event("bump", format!("release bump skipped: {e:#}"));
10004 }
10005 if self.state.status == RunStatus::Merged {
10008 crate::followup::after_merge(&mut self.state, &pr.url).await;
10009 }
10010 self.state.save()?;
10011 }
10012 Err(e) => {
10013 if e.is::<crate::repo_lock::ResourceBlocked>() {
10014 return Err(e);
10015 }
10016 self.state.status = RunStatus::Blocked;
10017 self.state.event("land", format!("gave up: {e}"));
10018 self.state.save()?;
10019 }
10020 }
10021 break;
10022 }
10023 Ok(())
10024 }
10025
10026 async fn repair_merge_review(&mut self) -> Result<bool> {
10029 use crate::run::MergeRepairPhase as Phase;
10030 crate::pr_head::ensure_owner(&self.state)?;
10031 let i = self.state.merge_repairs.len() - 1;
10032 let request = self.state.merge_repairs[i].clone();
10033 if request.phase == Phase::Fixing {
10034 let restored = match self.state.winner().cloned() {
10035 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
10036 None => Err(anyhow::anyhow!("repair has no winning branch")),
10037 };
10038 let why = match restored {
10039 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
10040 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
10041 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
10042 };
10043 self.hold_merge_repair(i, why)?;
10044 return Ok(false);
10045 }
10046 if request.phase == Phase::Reviewing && self.state.gate_ran {
10051 let winner = self
10052 .state
10053 .winner()
10054 .context("repair has no winning branch")?;
10055 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10056 if self.state.gate_head.as_deref() != Some(head.as_str())
10057 || !git::is_clean(&winner.worktree).await?
10058 {
10059 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10060 return Ok(false);
10061 }
10062 }
10063 let failure = if i >= self.state.config.graph.land_rounds
10064 || self.state.config.graph.review_rounds == 0
10065 {
10066 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
10067 } else {
10068 None
10069 };
10070 if let Some(why) = failure {
10071 self.hold_merge_repair(i, why)?;
10072 return Ok(false);
10073 }
10074 if request.phase == Phase::Requested {
10075 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
10076 Ok(pr) => pr,
10077 Err(e) => {
10078 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
10079 return Ok(false);
10080 }
10081 };
10082 self.state.merge_repairs[i].phase = Phase::Fixing;
10083 self.state.land_approval = None;
10084 self.state.land_armed_head = None;
10085 self.state.save()?;
10086 let evidence = serde_json::to_string_pretty(&request)?;
10089 let reason = format!(
10090 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
10091 );
10092 let budget = self.state.config.graph.land_rounds;
10093 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10094 match result {
10095 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10096 Ok(land::Fixed::Deferred) => {
10097 let worktree = self
10101 .state
10102 .winner()
10103 .context("repair has no winning branch")?
10104 .worktree
10105 .clone();
10106 self.graft_restore(&worktree, &request.head).await?;
10107 self.state.merge_repairs[i].phase = Phase::Requested;
10108 self.stop_for_resource()?;
10109 return Ok(false);
10110 }
10111 Ok(land::Fixed::Failed(why)) => {
10112 self.hold_merge_repair(
10113 i,
10114 format!(
10115 "repair failed: {why}; inspect the branch and run an explicit review"
10116 ),
10117 )?;
10118 return Ok(false);
10119 }
10120 Err(e) => {
10121 self.hold_merge_repair(
10122 i,
10123 format!(
10124 "repair failed: {e:#}; inspect the branch and run an explicit review"
10125 ),
10126 )?;
10127 return Ok(false);
10128 }
10129 }
10130 self.begin_merge_repair_review(i)?;
10134 }
10135 self.review_loop().await?;
10136 self.gate().await?;
10137 if self.resource_halt()? || self.state.parked {
10138 return Ok(false);
10139 }
10140 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10141 != Some(RunStatus::Gating)
10142 || !self.state.gate_allows_progress()
10143 {
10144 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10145 return Ok(false);
10146 }
10147 let winner = self
10150 .state
10151 .winner()
10152 .cloned()
10153 .context("repair has no winning branch")?;
10154 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10155 if self.state.gate_head.as_deref() != Some(after.as_str())
10156 || !git::is_clean(&winner.worktree).await?
10157 {
10158 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10159 return Ok(false);
10160 }
10161 let expected = self
10162 .state
10163 .pr_pushed_head
10164 .as_deref()
10165 .unwrap_or(&request.head)
10166 .to_owned();
10167 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10168 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10171 return Ok(false);
10172 }
10173 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10174 let refspec = format!("{after}:refs/heads/{public}");
10175 let push = git::git_raw(
10176 &winner.worktree,
10177 &["push", &lease, &self.state.config.merge.remote, &refspec],
10178 )
10179 .await;
10180 match push {
10181 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10182 Ok(out) => {
10183 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10184 return Ok(false);
10185 }
10186 Err(e) => {
10187 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10188 return Ok(false);
10189 }
10190 }
10191 self.state.merge_repairs[i].phase = Phase::Complete;
10192 self.state.status = RunStatus::Landing;
10193 self.state.event(
10194 "repair",
10195 "new review and gate completed; fresh owner merge decision required",
10196 );
10197 self.state.save()?;
10198 Ok(true)
10199 }
10200
10201 fn merge_repair_artifacts(&self) -> PathBuf {
10202 let artifacts = agent::artifacts_dir(&self.state.dir());
10203 match self
10204 .state
10205 .merge_repairs
10206 .last()
10207 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10208 {
10209 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10210 None => artifacts,
10211 }
10212 }
10213
10214 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10215 if let Some(e2e) = self.state.head_e2e.take() {
10216 self.state
10217 .verification_history
10218 .push(crate::run::VerificationEvidence {
10219 superseded_by: None,
10220 archived_at: Some(Timestamp::now()),
10221 fixes_spent: Some(self.state.gate_fixes.len()),
10222 gate_head: None,
10223 gate_ran: false,
10224 gate: Vec::new(),
10225 e2e: Some(e2e),
10226 });
10227 }
10228 self.state.reviews.clear();
10229 self.state.gate.clear();
10230 self.state.gate_ran = false;
10231 self.state.gate_head = None;
10232 self.state.gate_fixes.clear();
10233 self.state.gate_repair_e2e = None;
10234 self.state.pre_gate.clear();
10235 self.state.pre_gate_commit = None;
10236 self.state.contested_handoff = None;
10237 self.state.land_reruns = None;
10238 self.state.land_approval = None;
10239 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10240 self.state.status = RunStatus::Reviewing;
10241 self.state.save()
10242 }
10243
10244 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10245 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10246 self.state.merge_repairs[index].failure = Some(why.clone());
10247 self.state.status = RunStatus::Blocked;
10248 self.state.event("repair", why);
10249 self.state.save()
10250 }
10251
10252 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10256 if let Some(existing) = self.state.seats.get(key)
10257 && existing.agent == agent
10258 {
10259 return existing.clone();
10260 }
10261 let fresh = if self.state.seats.contains_key(key) {
10264 handover_seat(key, agent, self.state.next_seat_seed())
10265 } else {
10266 SeatState::new(key, agent, self.state.seed)
10267 };
10268 self.state.seats.insert(key.to_owned(), fresh.clone());
10269 fresh
10270 }
10271
10272 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10278 match self.state.seats.get(key) {
10279 Some(s) if s.agent != spec.id => {
10280 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10281 }
10282 _ => spec,
10283 }
10284 }
10285
10286 fn view(&self, c: &Candidate) -> CandidateView {
10288 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10289 .unwrap_or_default();
10290 let (patch, _) = blind::sanitize_patch(
10291 &format!("candidate {} patch", c.label),
10292 &raw,
10293 &self.state.config.blind,
10294 );
10295 CandidateView {
10296 label: c.label,
10297 branch: c.branch.clone(),
10298 summary: c.summary.clone(),
10299 stat: c.stat.clone(),
10300 patch,
10301 }
10302 }
10303
10304 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10306 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10307 prompt::judge(
10308 "(see above)",
10309 &views,
10310 self.roles.judges.len(),
10311 base_short,
10312 "en",
10313 )
10314 }
10315
10316 fn vote_prompt_full(
10322 &self,
10323 j: usize,
10324 labels: &[char],
10325 language: &str,
10326 candidates: &[Candidate],
10327 base_short: &str,
10328 ) -> String {
10329 let mut text = format!(
10330 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10331 prompt::final_vote(labels, language),
10332 self.state.instruction,
10333 self.candidate_block(candidates, base_short)
10334 );
10335 if let Some(own) = self
10336 .state
10337 .judgements
10338 .get(j)
10339 .filter(|r| !r.ranking.is_empty())
10340 {
10341 let reasons = own
10342 .reasons
10343 .iter()
10344 .map(|(k, v)| format!("- {k}: {v}"))
10345 .collect::<Vec<_>>()
10346 .join("\n");
10347 text.push_str(&format!(
10348 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10349 own.ranking.iter().collect::<String>(),
10350 if reasons.is_empty() {
10351 ""
10352 } else {
10353 ", because:\n"
10354 }
10355 ));
10356 }
10357 if !self.state.deliberation.is_empty() {
10358 text.push_str("\n# What was argued before this vote\n");
10359 for t in self.transcript(&[], j) {
10360 text.push_str(&format!(
10361 "\n## {}{}\n\n{}\n",
10362 t.who,
10363 if t.is_self { " (you)" } else { "" },
10364 t.body.trim()
10365 ));
10366 }
10367 }
10368 text
10369 }
10370
10371 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10378 let mut turns = Vec::new();
10379 for j in &self.state.judgements {
10380 if j.ranking.is_empty() {
10381 continue;
10382 }
10383 let reasons = j
10384 .reasons
10385 .iter()
10386 .map(|(k, v)| format!("- {k}: {v}"))
10387 .collect::<Vec<_>>()
10388 .join("\n");
10389 turns.push(Turn {
10390 who: format!("Judge {} (opening ranking)", j.judge),
10391 is_self: j.judge == self_idx + 1,
10392 body: format!(
10393 "Ranked {}{}{reasons}",
10394 j.ranking.iter().collect::<String>(),
10395 if reasons.is_empty() {
10396 ""
10397 } else {
10398 ", because:\n"
10399 }
10400 ),
10401 });
10402 }
10403 for t in self
10404 .state
10405 .deliberation
10406 .iter()
10407 .flat_map(|r| r.turns.iter())
10408 .chain(current)
10409 {
10410 turns.push(Turn {
10411 who: format!("Judge {}", t.judge),
10412 is_self: t.judge == self_idx + 1,
10413 body: t.body.clone(),
10414 });
10415 }
10416 turns
10417 }
10418}
10419
10420fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10422 agent::has_session(spec.kind, seat, sessions)
10423}
10424
10425fn next_untried_in_roster<'a>(
10446 roster: &'a [AgentSpec],
10447 start: usize,
10448 tried: &BTreeSet<String>,
10449) -> Option<&'a AgentSpec> {
10450 roster
10451 .get(start + 1..)?
10452 .iter()
10453 .find(|s| !tried.contains(&s.id))
10454}
10455
10456fn pick_successor<'a>(
10477 roster: &'a [AgentSpec],
10478 start: usize,
10479 tried: &BTreeSet<String>,
10480 carried: Option<&BTreeSet<String>>,
10481 others: &BTreeSet<String>,
10482) -> Option<&'a AgentSpec> {
10483 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10484 roster
10485 .get(start + 1..)
10486 .and_then(|tail| {
10487 tail.iter()
10488 .filter(free)
10489 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10490 })
10491 .or_else(|| {
10492 carried
10493 .is_some()
10494 .then(|| roster.iter().find(free))
10495 .flatten()
10496 })
10497 .or_else(|| match carried {
10498 Some(c) => next_for_seat(roster, start, tried, c),
10499 None => next_untried_in_roster(roster, start, tried),
10500 })
10501}
10502
10503fn next_for_seat<'a>(
10513 roster: &'a [AgentSpec],
10514 start: usize,
10515 round_tried: &BTreeSet<String>,
10516 carried_failed: &BTreeSet<String>,
10517) -> Option<&'a AgentSpec> {
10518 roster
10519 .get(start + 1..)?
10520 .iter()
10521 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10522 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10523}
10524
10525fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10531 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10532 return spec;
10533 };
10534 let ok = |id: &str| !h.failed.contains(id);
10535 if let Some(last) = h.last_ok.as_deref()
10536 && ok(last)
10537 && let Some(s) = roster.iter().find(|s| s.id == last)
10538 {
10539 return s.clone();
10540 }
10541 if ok(&spec.id) {
10542 return spec;
10543 }
10544 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10545 roster
10546 .get(from + 1..)
10547 .into_iter()
10548 .flatten()
10549 .chain(roster.iter())
10550 .find(|s| ok(&s.id))
10551 .cloned()
10552 .unwrap_or(spec)
10553}
10554
10555pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10559 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10560}
10561
10562const TIMED_OUT: &str = "timed out";
10565
10566impl FailClass {
10567 fn of(out: &AgentOutcome) -> Option<Self> {
10569 match out {
10570 AgentOutcome::Ok(_) => None,
10571 AgentOutcome::Quota(_) => Some(Self::Quota),
10572 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10573 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10574 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10575 AgentOutcome::ResourceBlocked(_) => None,
10578 }
10579 }
10580
10581 fn stem_word(&self) -> &'static str {
10583 match self {
10584 Self::Quota => "quota",
10585 _ => "handover",
10586 }
10587 }
10588}
10589
10590fn failure_signature(msg: &str) -> String {
10594 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10595 if let Some(code) = line
10598 .strip_prefix("process exited with no usable output (exit code ")
10599 .and_then(|s| s.strip_suffix(')'))
10600 .and_then(|s| s.parse::<i32>().ok())
10601 {
10602 line = format!("exited with some({code}) and no usable output");
10603 } else if matches!(
10604 line.as_str(),
10605 "process exited with no usable output (no exit code reported)"
10606 | "process exited with no usable output (exit code unavailable)"
10607 ) {
10608 line = "exited with none and no usable output".to_owned();
10609 }
10610 let mut out = Vec::new();
10611 for word in line.split_whitespace() {
10612 if word.contains('/') || word.contains('\\') {
10613 out.push("<path>".to_owned());
10614 continue;
10615 }
10616 let mut w = String::new();
10617 let mut in_digits = false;
10618 for c in word.chars() {
10619 if c.is_ascii_digit() {
10620 if !in_digits {
10621 w.push('#');
10622 }
10623 in_digits = true;
10624 } else {
10625 in_digits = false;
10626 w.push(c);
10627 }
10628 }
10629 out.push(w);
10630 }
10631 out.join(" ").chars().take(120).collect()
10632}
10633
10634fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10641 match cur {
10642 FailClass::Quota | FailClass::Timeout => true,
10643 FailClass::Other(_) => prev != Some(cur),
10644 }
10645}
10646
10647fn fail_reason(out: &AgentOutcome) -> String {
10649 match out {
10650 AgentOutcome::Ok(_) => String::new(),
10651 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10652 AgentOutcome::Dropped(o) => format!(
10653 "the CLI dropped the stream ({})",
10654 o.dropped
10655 .as_ref()
10656 .map(|d| d.why.as_str())
10657 .unwrap_or("it ended without delivering its answer")
10658 ),
10659 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10660 e.lines().next().unwrap_or("").chars().take(160).collect()
10661 }
10662 }
10663}
10664
10665pub(crate) fn record_handover(
10668 state: &mut RunState,
10669 node: &str,
10670 seat: &str,
10671 from: &str,
10672 to: &str,
10673 class: &FailClass,
10674 reason: &str,
10675) {
10676 let message = if *class == FailClass::Quota {
10677 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10678 } else {
10679 format!("{seat}: handed over {from} -> {to} ({reason})")
10680 };
10681 state.event(node, message);
10682 state.handovers.push(Handover {
10683 at: Timestamp::now(),
10684 node: node.to_owned(),
10685 seat: seat.to_owned(),
10686 from: from.to_owned(),
10687 to: to.to_owned(),
10688 reason: reason.to_owned(),
10689 });
10690}
10691
10692fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10706 commands.iter().any(|c| c.exit_code.is_none())
10707}
10708
10709fn verified_noop_claim(
10722 usable: bool,
10723 commands: &[agent::CommandEvidence],
10724 text: &str,
10725) -> Option<String> {
10726 (usable && !has_unconfirmed_command(commands))
10727 .then(|| verdict::verified_noop(text))
10728 .flatten()
10729}
10730
10731fn short(commit: &str) -> String {
10732 commit.chars().take(7).collect()
10733}
10734
10735fn make_executable(path: &Path) -> Result<()> {
10736 #[cfg(unix)]
10737 {
10738 use std::os::unix::fs::PermissionsExt as _;
10739 let mut perms = std::fs::metadata(path)?.permissions();
10740 perms.set_mode(0o755);
10741 std::fs::set_permissions(path, perms)?;
10742 }
10743 #[cfg(not(unix))]
10744 {
10745 let _ = path;
10746 }
10747 Ok(())
10748}
10749
10750struct WaveCtx<'a> {
10757 run: &'a str,
10760 node: &'a str,
10762 prompts: &'a Prompts,
10763 cache: Option<&'a Path>,
10765 round: Option<usize>,
10768 carry_seats: bool,
10772 pauses: Option<(&'a Pause, &'a Pause)>,
10775}
10776
10777fn repository_wait(
10779 state: &RunState,
10780 pauses: Option<(&Pause, &Pause)>,
10781) -> crate::repo_lock::WaitPolicy {
10782 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10783 let inherited = crate::repo_lock::WaitPolicy::current();
10784 crate::repo_lock::WaitPolicy::new(
10785 Duration::from_secs(state.config.graph.timeout_resource_wait),
10786 move || {
10787 inherited.cancelled()
10788 || pauses
10789 .as_ref()
10790 .is_some_and(|(a, b)| a.parked() || b.parked())
10791 },
10792 )
10793}
10794
10795fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10796 if let Some(timeout) = error.downcast_ref::<agent::IsolatedSeatWait>() {
10797 state.event("resource", timeout.0.description());
10798 state.resource_block = Some(timeout.0.clone());
10799 return;
10800 }
10801 if error.is::<crate::repo_lock::ResourceBlocked>() {
10802 state.resource_block = Some(crate::run::ResourceBlock {
10803 cause: crate::run::ResourceBlockCause::Other,
10804 node: "repository".into(),
10805 kind: if error.is::<crate::build_target::AdmissionDiskWait>() {
10806 "disk admission"
10807 } else if error.to_string().contains("disk pressure") {
10808 "disk pressure"
10809 } else if error.to_string().contains("disk admission") {
10810 "disk admission"
10811 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10812 "incremental cache admission"
10813 } else {
10814 "repository coordination"
10815 }
10816 .into(),
10817 detail: format!("{error:#}"),
10818 at: Timestamp::now(),
10819 });
10820 state.event("resource", error.to_string());
10821 }
10822}
10823
10824async fn coordinate_operation(
10825 state: &mut RunState,
10826 cwd: &Path,
10827 wait: &crate::repo_lock::WaitPolicy,
10828) -> Result<Option<crate::repo_lock::SeatLease>> {
10829 let result =
10830 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10831 .await;
10832 if let Err(e) = &result {
10833 record_repository_block(state, e);
10834 }
10835 result
10836}
10837
10838async fn run_one(
10840 job: SeatJob,
10841 sem: Arc<Semaphore>,
10842 ctx: &WaveCtx<'_>,
10843 state: &mut RunState,
10844 attempt: usize,
10845) -> (SeatState, AgentOutcome) {
10846 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10847 .await
10848 .pop()
10849 .expect("one job in, one result out");
10850 (seat, out)
10851}
10852
10853async fn wave(
10859 jobs: Vec<SeatJob>,
10860 sem: Arc<Semaphore>,
10861 ctx: &WaveCtx<'_>,
10862 state: &mut RunState,
10863 attempt: usize,
10864) -> Vec<(usize, SeatState, AgentOutcome)> {
10865 let WaveCtx {
10866 run,
10867 node,
10868 prompts,
10869 cache,
10870 round,
10871 carry_seats: _,
10872 pauses,
10873 } = *ctx;
10874 let cache = cache.filter(|_| !crate::build_target::isolated(&state.config));
10875 for job in &jobs {
10876 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10877 }
10878 if let Err(e) = state.save() {
10879 tracing::warn!("could not persist in-progress seats: {e:#}");
10884 }
10885 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10888 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10892 let cache_guard = if let Some(cache_dir) = cache {
10893 if jobs_had_a_writer {
10894 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10895 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10896 match acquire_cache_lease(
10897 state,
10898 cache_dir,
10899 &owner,
10900 resource_limit,
10901 node,
10902 node,
10903 &cancel,
10904 )
10905 .await
10906 {
10907 Ok(Some(g)) => Some(g),
10908 Ok(None) => {
10911 if let Some(block) = &mut state.resource_block {
10912 block.cause = crate::run::ResourceBlockCause::Cancelled {
10913 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10914 };
10915 }
10916 return block_wave(
10917 jobs,
10918 state,
10919 node,
10920 attempt,
10921 "parked while waiting for the shared build cache",
10922 );
10923 }
10924 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10927 Err(e) => {
10928 let why = format!("{e:#}");
10932 state.event(
10933 "resource",
10934 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10935 );
10936 return block_wave(jobs, state, node, attempt, &why);
10937 }
10938 }
10939 } else {
10940 None
10941 }
10942 } else {
10943 None
10944 };
10945 state.seats_spawning(node, attempt);
10948 if let Err(e) = state.save() {
10949 tracing::warn!("could not persist the start of a wave: {e:#}");
10950 }
10951 let slot_limit = resource_limit
10955 + jobs
10956 .iter()
10957 .map(|j| j.timeout)
10958 .max()
10959 .unwrap_or_default()
10960 .saturating_mul(jobs.len() as u32);
10961 let mut set = tokio::task::JoinSet::new();
10962 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10965 let overlay = prompts.overlay(node);
10966 let repository_wait = repository_wait(state, pauses);
10967 for (i, mut job) in jobs.into_iter().enumerate() {
10968 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10969 if cache.is_some() {
10970 job.prompt.push('\n');
10971 job.prompt
10972 .push_str(&prompt::build_cache_note(node, job.allow_write));
10973 }
10974 let sem = Arc::clone(&sem);
10975 let run = run.to_owned();
10976 let node = node.to_owned();
10977 let attachments = if node == "implement" {
10980 state.attachments.clone()
10981 } else {
10982 Vec::new()
10983 };
10984 let cache = cache
10995 .filter(|_| job.allow_write && cache_guard.is_some())
10996 .map(Path::to_path_buf);
10997 let spawned_tx = spawned_tx.clone();
10998 let repository_wait = repository_wait.clone();
10999 let coordination = state.config.coordination.clone();
11000 let fetched_base = state.fetched_base.clone();
11001 set.spawn(async move {
11002 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
11006 Ok(p) => p,
11007 Err(_) => {
11008 return (
11009 i,
11010 job.seat,
11011 AgentOutcome::ResourceBlocked(format!(
11012 "no parallel slot became free within {}s",
11013 slot_limit.as_secs()
11014 )),
11015 None,
11016 false,
11017 );
11018 }
11019 };
11020 spawned_tx.send(job.seat.key.clone()).ok();
11021 let mut seat = job.seat;
11022 let out = repository_wait
11023 .scope(agent::invoke(
11024 &job.spec,
11025 &mut seat,
11026 &Invocation {
11027 coordination: coordination.as_ref(),
11028 fetched_base: fetched_base.as_ref(),
11029 cwd: &job.cwd,
11030 prompt: &job.prompt,
11031 timeout: job.timeout,
11032 allow_write: job.allow_write,
11033 sessions: job.sessions,
11034 artifacts: &job.artifacts,
11035 stem: &job.stem,
11036 run: &run,
11037 node: &node,
11038 cache_dir: cache.as_deref(),
11039 attachments: &attachments,
11040 writable: &[],
11041 },
11042 ))
11043 .await;
11044 let cache_block = out
11045 .as_ref()
11046 .err()
11047 .and_then(|error| error.downcast_ref::<agent::IsolatedSeatWait>())
11048 .map(|timeout| timeout.0.clone());
11049 let interrupted = out
11050 .as_ref()
11051 .err()
11052 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
11053 let out = match out {
11054 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
11055 Ok(o) if o.usable() => AgentOutcome::Ok(o),
11056 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
11064 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
11065 Ok(o) => AgentOutcome::Failed(format!(
11066 "process exited with no usable output ({})",
11067 crate::notices::exit_status(o.exit_code)
11068 )),
11069 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
11070 AgentOutcome::ResourceBlocked(e.to_string())
11071 }
11072 Err(e) => AgentOutcome::Failed(e.to_string()),
11073 };
11074 (i, seat, out, cache_block, interrupted)
11075 });
11076 }
11077 drop(spawned_tx);
11078 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
11079 loop {
11080 let joined = tokio::select! {
11081 Some(key) = spawned_rx.recv() => {
11082 state.seat_spawned(&key);
11084 if let Err(e) = state.save() {
11085 tracing::warn!("could not persist a seat's start: {e:#}");
11086 }
11087 continue;
11088 }
11089 joined = set.join_next() => joined,
11090 };
11091 let Some(joined) = joined else {
11092 break;
11093 };
11094 let (i, seat, out, cache_block, interrupted) = match joined {
11095 Ok(v) => v,
11096 Err(e) => {
11100 tracing::error!("agent task panicked: {e}");
11101 continue;
11102 }
11103 };
11104 state.seat_finished(&seat.key);
11105 if interrupted
11106 && node == "implement"
11107 && let Some(label) = state
11108 .candidates
11109 .iter()
11110 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11111 .map(|candidate| candidate.label)
11112 {
11113 state.candidate_preservation.insert(
11114 label,
11115 "interrupted implementation awaiting persistence".into(),
11116 );
11117 state.interrupted_implementations.insert(label);
11118 }
11119 if let Some(block) = cache_block
11120 && state.resource_block.is_none()
11121 {
11122 state.event("resource", block.description());
11123 state.resource_block = Some(block);
11124 }
11125 if let AgentOutcome::ResourceBlocked(why) = &out
11126 && state.resource_block.is_none()
11127 {
11128 state.resource_block = Some(crate::run::ResourceBlock {
11129 cause: crate::run::ResourceBlockCause::Other,
11130 node: node.to_owned(),
11131 kind: if why.contains("disk pressure") {
11132 "disk pressure"
11133 } else if why.contains("disk admission") {
11134 "disk admission"
11135 } else if why.contains(crate::incremental::BLOCKED) {
11136 "incremental cache admission"
11137 } else if why.contains(crate::build_target::KIND) {
11138 crate::build_target::KIND
11139 } else if why.contains("repository coordination") {
11140 "repository coordination"
11141 } else {
11142 "parallel slot"
11143 }
11144 .to_owned(),
11145 detail: why.clone(),
11146 at: Timestamp::now(),
11147 });
11148 let activity = if interrupted {
11149 "agent execution interrupted"
11150 } else {
11151 "no agent was started"
11152 };
11153 state.event("resource", format!("{node}: {activity} - {why}"));
11154 }
11155 record_jobs(state, node, round, &seat.key, &out);
11156 if let Err(e) = state.save() {
11157 tracing::warn!("could not persist a seat's completion: {e:#}");
11158 }
11159 if collected.len() <= i {
11160 collected.resize_with(i + 1, || None);
11161 }
11162 collected[i] = Some((i, seat, out));
11163 }
11164 if state
11170 .active
11171 .values()
11172 .any(|a| a.node == node && a.attempt == attempt)
11173 {
11174 state
11175 .active
11176 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11177 if let Err(e) = state.save() {
11178 tracing::warn!("could not persist the end of a wave: {e:#}");
11179 }
11180 }
11181 if let Some(cache_dir) = cache
11188 && jobs_had_a_writer
11189 {
11190 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11191 }
11192 if let Some(guard) = cache_guard {
11193 guard.release();
11194 }
11195 collected.into_iter().flatten().collect()
11196}
11197
11198fn record_jobs(
11209 state: &mut RunState,
11210 node: &str,
11211 round: Option<usize>,
11212 seat: &str,
11213 out: &AgentOutcome,
11214) {
11215 let commands: &[agent::CommandEvidence] = match out {
11216 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11217 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11218 };
11219 let checked_at = Timestamp::now();
11220 for c in commands {
11221 state.jobs.push(JobRecord {
11222 node: node.to_owned(),
11223 round,
11224 seat: seat.to_owned(),
11225 id: c.id.clone(),
11226 description: c.description.clone(),
11227 checked_at,
11228 status: match c.exit_code {
11229 Some(0) => JobStatus::Completed,
11230 Some(_) => JobStatus::Failed,
11231 None => JobStatus::Unknown,
11232 },
11233 exit_code: c.exit_code,
11234 result_summary: c.result_summary.clone(),
11235 source: c.source.clone(),
11236 });
11237 }
11238}
11239
11240fn round_is_clean(
11261 blocking: usize,
11262 e2e_ok: bool,
11263 answered: usize,
11264 expected: usize,
11265 quota_missing: usize,
11266 policy: IncompleteReviewPolicy,
11267) -> bool {
11268 if blocking != 0 || !e2e_ok {
11269 return false;
11270 }
11271 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11272 return true;
11273 }
11274 answered > 0 && expected - answered <= quota_missing
11275}
11276
11277pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11301 if max_rounds == 0
11302 || reviews.last().is_some_and(|r| {
11303 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11304 })
11305 {
11306 return Some(RunStatus::Gating);
11307 }
11308 let last = reviews.last()?;
11309 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11310 if reviews.len() < max_rounds && !stagnant {
11311 return None;
11312 }
11313 if last.incomplete() && last.blocking == 0 {
11314 return Some(RunStatus::Blocked);
11315 }
11316 if last.e2e_status() == E2eStatus::ResourceBlocked
11317 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11318 {
11319 return None;
11320 }
11321 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11322 RunStatus::Gating
11323 } else {
11324 RunStatus::Blocked
11325 })
11326}
11327
11328pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11343 if nudged {
11344 (full / 4).max(Duration::from_secs(120)).min(full)
11345 } else {
11346 full
11347 }
11348}
11349
11350#[allow(clippy::too_many_arguments)]
11379async fn ask_json_wave<T>(
11380 jobs: Vec<SeatJob>,
11381 sem: Arc<Semaphore>,
11382 retries: usize,
11383 roster: &[AgentSpec],
11384 ctx: &WaveCtx<'_>,
11385 losses: &mut Vec<QuotaLoss>,
11386 state: &mut RunState,
11387 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11388) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11389where
11390 T: serde::de::DeserializeOwned + Send + 'static,
11391{
11392 ask_wave_with(
11393 jobs,
11394 sem,
11395 retries,
11396 roster,
11397 ctx,
11398 losses,
11399 state,
11400 &|text: &str| {
11401 let v = verdict::extract_json::<T>(text)?;
11402 validate(&v)?;
11403 Ok(v)
11404 },
11405 )
11406 .await
11407}
11408
11409#[allow(clippy::too_many_arguments)]
11418async fn ask_wave_with<T>(
11419 jobs: Vec<SeatJob>,
11420 sem: Arc<Semaphore>,
11421 retries: usize,
11422 roster: &[AgentSpec],
11423 ctx: &WaveCtx<'_>,
11424 losses: &mut Vec<QuotaLoss>,
11425 state: &mut RunState,
11426 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11427) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11428where
11429 T: Send + 'static,
11430{
11431 let n = jobs.len();
11432 let originals: Vec<SeatJob> = jobs;
11433 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11434 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11435 let mut nudges: Vec<usize> = vec![0; n];
11442 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11446 let mut tried: Vec<BTreeSet<String>> = specs
11447 .iter()
11448 .map(|s| BTreeSet::from([s.id.clone()]))
11449 .collect();
11450 let starts: Vec<usize> = specs
11451 .iter()
11452 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11453 .collect();
11454 let carry = ctx.carry_seats && !roster.is_empty();
11455 let carried: Vec<BTreeSet<String>> = originals
11458 .iter()
11459 .map(|j| {
11460 state
11461 .seat_history
11462 .get(&j.seat.key)
11463 .filter(|_| carry)
11464 .map(|h| h.failed.clone())
11465 .unwrap_or_default()
11466 })
11467 .collect();
11468 let mut prev: Vec<Option<FailClass>> = originals
11469 .iter()
11470 .map(|j| {
11471 state
11472 .seat_history
11473 .get(&j.seat.key)
11474 .filter(|_| carry)
11475 .and_then(|h| h.last_fail.clone())
11476 })
11477 .collect();
11478 let next_agent =
11479 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11480 let others: BTreeSet<String> = specs
11481 .iter()
11482 .enumerate()
11483 .filter(|(j, _)| *j != i)
11484 .map(|(_, s)| s.id.clone())
11485 .collect();
11486 pick_successor(
11487 roster,
11488 starts[i],
11489 tried,
11490 carry.then(|| &carried[i]),
11491 &others,
11492 )
11493 .cloned()
11494 };
11495 let mut fresh: Vec<Option<String>> = vec![None; n];
11496 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11497 let mut pending: Vec<usize> = (0..n).collect();
11498
11499 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11504 for round in 0..max_rounds {
11505 if pending.is_empty() {
11506 break;
11507 }
11508 let mut batch = Vec::with_capacity(pending.len());
11509 let mut renudged: Vec<&str> = Vec::new();
11510 for &i in &pending {
11511 let src = &originals[i];
11512 let full: &str = match &src.handover {
11517 Some(h) if specs[i].id != src.spec.id => h,
11518 _ => &src.prompt,
11519 };
11520 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11523 (
11524 full.to_owned(),
11525 src.timeout,
11526 format!("{}-{word}-{}", src.stem, specs[i].id),
11527 )
11528 } else if nudges[i] == 0 {
11529 (full.to_owned(), src.timeout, src.stem.clone())
11530 } else {
11531 renudged.push(src.seat.key.as_str());
11532 let why = done[i]
11533 .as_ref()
11534 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11535 .unwrap_or_else(|| "no parsable answer".to_owned());
11536 let nudge = prompt::nudge(&why);
11537 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11538 let prompt = if nudged {
11539 nudge
11540 } else {
11541 format!("{full}\n\n---\n\n{nudge}")
11542 };
11543 (
11544 prompt,
11545 retry_budget(src.timeout, nudged),
11546 format!("{}-retry{}", src.stem, nudges[i]),
11547 )
11548 };
11549 batch.push(SeatJob {
11550 spec: specs[i].clone(),
11551 seat: seats[i].clone(),
11552 cwd: src.cwd.clone(),
11553 prompt,
11554 timeout,
11555 allow_write: src.allow_write,
11556 sessions: src.sessions,
11557 artifacts: src.artifacts.clone(),
11558 stem,
11559 handover: None,
11560 });
11561 }
11562
11563 if !renudged.is_empty() {
11564 state.event(
11565 ctx.node,
11566 format!("retry {round}: re-asking {}", renudged.join(", ")),
11567 );
11568 }
11569 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11570 let mut still = Vec::new();
11571 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11572 seats[i] = seat;
11573 if let AgentOutcome::ResourceBlocked(why) = &out {
11576 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11577 continue;
11578 }
11579 let class = FailClass::of(&out);
11580 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11583 && nudges[i] < retries
11584 && !roster.is_empty();
11585 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11586 let next = should_hand_over(prev[i].as_ref(), &cur)
11587 .then(|| next_agent(i, &tried[i], &specs))
11588 .flatten();
11589 if carry {
11590 let h = state
11591 .seat_history
11592 .entry(originals[i].seat.key.clone())
11593 .or_default();
11594 h.failed.insert(specs[i].id.clone());
11595 h.last_fail = Some(cur.clone());
11596 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11597 h.last_ok = None;
11598 }
11599 if let Err(e) = state.save() {
11602 tracing::warn!("could not persist a seat's failure history: {e:#}");
11603 }
11604 }
11605 if let Some(next) = next {
11606 record_handover(
11607 state,
11608 ctx.node,
11609 &originals[i].seat.key,
11610 &specs[i].id,
11611 &next.id,
11612 &cur,
11613 &fail_reason(&out),
11614 );
11615 tried[i].insert(next.id.clone());
11616 prev[i] = Some(cur.clone());
11617 seats[i] =
11618 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11619 specs[i] = next;
11620 fresh[i] = Some(cur.stem_word().to_owned());
11621 nudges[i] = 0;
11622 done[i] = Some(Err(anyhow::anyhow!(
11623 "handed over after: {}",
11624 fail_reason(&out)
11625 )));
11626 still.push(i);
11627 continue;
11628 }
11629 }
11630 if carry
11631 && !nudge_first
11632 && let Some(cur) = class.clone()
11633 {
11634 let h = state
11636 .seat_history
11637 .entry(originals[i].seat.key.clone())
11638 .or_default();
11639 h.failed.insert(specs[i].id.clone());
11640 h.last_fail = Some(cur);
11641 }
11642 let parsed = match out {
11643 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11644 AgentOutcome::Quota(o) => {
11645 last_quota[i] = o.quota.clone();
11646 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11647 }
11648 AgentOutcome::Dropped(o) => {
11652 let why = o
11653 .dropped
11654 .as_ref()
11655 .map(|d| d.why.as_str())
11656 .unwrap_or("the CLI ended the stream without delivering its answer");
11657 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11658 }
11659 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11660 Err(anyhow::anyhow!(e))
11661 }
11662 };
11663 let quota = class == Some(FailClass::Quota);
11664 let failed = parsed.is_err();
11665 if !failed && matches!(ctx.node, "judge" | "vote") {
11666 state
11669 .quota
11670 .retain(|loss| loss.seat != originals[i].seat.key);
11671 }
11672 done[i] = Some(parsed);
11673 if carry && !failed {
11674 let h = state
11675 .seat_history
11676 .entry(originals[i].seat.key.clone())
11677 .or_default();
11678 h.failed.remove(&specs[i].id);
11679 h.last_ok = Some(specs[i].id.clone());
11680 h.last_fail = None;
11681 }
11682 let agent_failure = class.is_some()
11690 && !nudge_first
11691 && !roster.is_empty()
11692 && next_agent(i, &tried[i], &specs).is_some();
11693 if failed && !quota && !agent_failure && nudges[i] < retries {
11694 nudges[i] += 1;
11695 still.push(i);
11696 }
11697 }
11698 pending = still;
11699 }
11700
11701 for (i, q) in last_quota.into_iter().enumerate() {
11704 if let Some(quota) = q {
11705 losses.push(QuotaLoss::from_report(
11706 originals[i].seat.key.clone(),
11707 ctx.node,
11708 Some("a),
11709 ));
11710 }
11711 }
11712
11713 if state.status != RunStatus::Stalled
11716 && ctx.node != "deliberate"
11717 && n > 0
11718 && losses.len() == n
11719 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11720 {
11721 block_machine_outage(state, ctx.node, losses);
11722 state.quota.append(losses);
11723 }
11724
11725 seats
11726 .into_iter()
11727 .zip(done)
11728 .zip(nudges)
11729 .map(|((seat, res), attempts)| {
11730 (
11731 seat,
11732 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11733 attempts,
11734 )
11735 })
11736 .collect()
11737}
11738
11739pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11742 let mut auth: Vec<_> = losses
11743 .iter()
11744 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11745 .map(|loss| loss.agent.clone())
11746 .collect();
11747 auth.sort();
11748 auth.dedup();
11749 let (kind, detail) = if auth.is_empty() {
11750 (
11751 "agent usage limit",
11752 "All seats reached their agent usage limits; resume after usage is available"
11753 .to_owned(),
11754 )
11755 } else {
11756 (
11757 "agent authentication",
11758 format!(
11759 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11760 auth.join(", ")
11761 ),
11762 )
11763 };
11764 state.resource_block = Some(crate::run::ResourceBlock {
11765 node: node.to_owned(),
11766 kind: kind.to_owned(),
11767 cause: crate::run::ResourceBlockCause::Other,
11768 detail: detail.clone(),
11769 at: Timestamp::now(),
11770 });
11771 state.event(node, detail);
11772}
11773
11774fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11776 results
11777 .iter()
11778 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11779}
11780
11781fn block_wave(
11785 jobs: Vec<SeatJob>,
11786 state: &mut RunState,
11787 node: &str,
11788 attempt: usize,
11789 why: &str,
11790) -> Vec<(usize, SeatState, AgentOutcome)> {
11791 state.resource_wait = None;
11792 state
11793 .active
11794 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11795 if let Err(e) = state.save() {
11796 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11797 }
11798 jobs.into_iter()
11799 .enumerate()
11800 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11801 .collect()
11802}
11803
11804async fn acquire_cache_lease(
11817 state: &mut RunState,
11818 cache_dir: &Path,
11819 owner: &crate::cache::Owner,
11820 budget: Duration,
11821 node: &str,
11822 context: &str,
11823 cancel: &(dyn Fn() -> bool + Sync),
11824) -> Result<Option<crate::cache::Guard>> {
11825 let home = crate::run::home();
11826 let kind = if crate::build_target::isolated(&state.config) {
11827 crate::build_target::KIND
11828 } else {
11829 "shared build cache"
11830 };
11831 let started = Instant::now();
11832 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11833 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11834 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11835 Err(e) => {
11836 state.event(
11837 "verify",
11838 format!(
11839 "{context}: could not check the {kind} at {}: {e:#}",
11840 cache_dir.display()
11841 ),
11842 );
11843 if let Err(e2) = state.save() {
11844 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11845 }
11846 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11847 }
11848 };
11849 state.event(
11850 "verify",
11851 format!(
11852 "{context}: waiting for the {kind} at {} ({})",
11853 cache_dir.display(),
11854 busy.describe()
11855 ),
11856 );
11857 state.resource_wait = Some(crate::run::ResourceWait {
11860 node: context.to_owned(),
11861 kind: kind.to_owned(),
11862 holder: format!("{}: {}", cache_dir.display(), busy.describe()),
11863 since: Timestamp::now(),
11864 limit_secs: budget.as_secs(),
11865 });
11866 if let Err(e) = state.save() {
11867 tracing::warn!("could not persist a cache wait: {e:#}");
11868 }
11869 let remaining = budget.saturating_sub(started.elapsed());
11870 let waited = crate::cache::wait_for_unless(
11871 &home,
11872 cache_dir,
11873 owner,
11874 remaining,
11875 Duration::from_secs(5),
11876 cancel,
11877 )
11878 .await;
11879 state.resource_wait = None;
11880 let cause = match &waited {
11881 Ok(Some(_)) => None,
11882 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11883 Err(error)
11884 if error.is::<crate::cache::WaitTimeout>() && kind == crate::build_target::KIND =>
11885 {
11886 Some(crate::run::ResourceBlockCause::IsolatedCacheTimeout {
11887 target: cache_dir.to_owned(),
11888 waited_secs: started.elapsed().as_secs(),
11889 limit_secs: budget.as_secs(),
11890 })
11891 }
11892 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11893 Some(crate::run::ResourceBlockCause::Timeout {
11894 waited_secs: started.elapsed().as_secs(),
11895 limit_secs: budget.as_secs(),
11896 })
11897 }
11898 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11899 };
11900 if let Some(cause) = cause {
11901 let detail = match &waited {
11902 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11903 Err(error) => format!("{error:#}"),
11904 };
11905 state.resource_block = Some(crate::run::ResourceBlock {
11906 cause,
11907 node: node.to_owned(),
11908 kind: kind.into(),
11909 detail: format!("{}: {detail}", cache_dir.display()),
11910 at: Timestamp::now(),
11911 });
11912 state.event(
11913 "resource",
11914 state.resource_block.as_ref().unwrap().description(),
11915 );
11916 }
11917 if let Err(error) = state.save() {
11918 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11919 }
11920 waited
11921}
11922
11923#[derive(Debug)]
11928struct CacheCheckError(String);
11929
11930impl std::fmt::Display for CacheCheckError {
11931 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11932 f.write_str(&self.0)
11933 }
11934}
11935
11936impl std::error::Error for CacheCheckError {}
11937
11938fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11940 let kind = if crate::build_target::isolated(&state.config) {
11941 crate::build_target::KIND
11942 } else {
11943 "shared build cache"
11944 };
11945 state
11946 .resource_block
11947 .get_or_insert_with(|| crate::run::ResourceBlock {
11948 cause: crate::run::ResourceBlockCause::Other,
11949 node: node.to_owned(),
11950 kind: kind.into(),
11951 detail: format!("{error:#}"),
11952 at: Timestamp::now(),
11953 });
11954 if let Err(error) = state.save() {
11955 tracing::warn!("could not persist cache resource block: {error:#}");
11956 }
11957}
11958
11959tokio::task_local! {
11960 static VERIFY_TARGET: PathBuf;
11961}
11962
11963#[allow(clippy::too_many_arguments)]
11984async fn with_cache_lease<'s, F, Fut>(
11985 state: &'s mut RunState,
11986 cache_dir: Option<&Path>,
11987 node: &str,
11988 seat: &str,
11989 worktree: &Path,
11990 head: &str,
11991 budget: Duration,
11992 context: &str,
11993 wait: &crate::repo_lock::WaitPolicy,
11994 body: F,
11995) -> Result<(Vec<CommandOutcome>, bool)>
11996where
11997 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11998 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11999{
12000 let started = Instant::now();
12003 let home = crate::run::home();
12004 let isolated = crate::build_target::isolated(&state.config);
12005 let planned_target = if isolated {
12006 let plan = (|| -> Result<PathBuf> {
12007 crate::build_target::supports(&state.config)?;
12008 crate::build_target::resolve(&home, worktree)
12009 })();
12010 match plan {
12011 Ok(path) => Some(path),
12012 Err(error) => {
12013 record_cache_error(state, node, &error);
12014 return Ok((
12015 vec![CommandOutcome {
12016 command: "(planning isolated build cache)".into(),
12017 code: None,
12018 output_tail: format!("{error:#}"),
12019 duration_ms: 0,
12020 resource_blocked: true,
12021 }],
12022 false,
12023 ));
12024 }
12025 }
12026 } else {
12027 None
12028 };
12029 let cache_dir = planned_target.as_deref().or(cache_dir);
12030 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
12031 let guard = if let Some(cache_dir) = cache_dir {
12032 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
12033 match acquire_cache_lease(
12034 state,
12035 cache_dir,
12036 &owner,
12037 resource_limit,
12038 node,
12039 context,
12040 &|| wait.cancelled(),
12041 )
12042 .await
12043 .and_then(|g| {
12044 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
12045 }) {
12046 Ok(g) => Some(g),
12047 Err(e) => {
12048 record_cache_error(state, node, &e);
12049 return Ok((
12050 vec![CommandOutcome {
12051 command: if isolated {
12052 "(waiting for the isolated build cache)"
12053 } else {
12054 "(waiting for the shared build cache)"
12055 }
12056 .into(),
12057 code: None,
12058 output_tail: e.to_string(),
12059 duration_ms: started.elapsed().as_millis() as u64,
12060 resource_blocked: true,
12061 }],
12062 false,
12063 ));
12064 }
12065 }
12066 } else {
12067 None
12068 };
12069 let _coordination = coordinate_operation(state, worktree, wait).await?;
12070 if let Some(path) = &planned_target {
12071 state.event(node, format!("isolated build target: {}", path.display()));
12072 }
12073 if let Some(cache_dir) = cache_dir.filter(|_| !isolated) {
12074 let identity = crate::cache::Identity::new(worktree, head);
12075 let fresh = crate::cache::ensure_fresh(
12076 &home,
12077 cache_dir,
12078 &identity,
12079 &state.config.verify.cargo_manifests(),
12080 );
12081 if let Err(e) = fresh {
12082 record_cache_error(state, node, &e);
12083 state.event(
12084 "verify",
12085 format!(
12086 "{context}: could not confirm the build cache matches {} at {}: {e:#}",
12087 worktree.display(),
12088 short(head)
12089 ),
12090 );
12091 if let Some(guard) = guard {
12092 guard.release();
12093 }
12094 return Ok((
12095 vec![CommandOutcome {
12096 command: "(confirming the build cache is fresh)".into(),
12097 code: None,
12098 output_tail: e.to_string(),
12099 duration_ms: started.elapsed().as_millis() as u64,
12100 resource_blocked: true,
12101 }],
12102 false,
12103 ));
12104 }
12105 }
12106 let future = body(state, budget);
12107 let (outcomes, retried, timed_out_pids) = if let Some(target) = planned_target {
12108 VERIFY_TARGET.scope(target, future).await
12109 } else {
12110 future.await
12111 };
12112 if !timed_out_pids.is_empty() {
12113 wait_for_timed_out_children_to_die(&timed_out_pids).await;
12114 }
12115 drop(_coordination);
12118 if let Some(guard) = guard {
12119 guard.release();
12120 }
12121 Ok((outcomes, retried))
12122}
12123
12124async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
12136 wait_for_pids_with(
12137 pids,
12138 crate::proc::pid_alive,
12139 LEASE_RELEASE_POLL,
12140 LEASE_RELEASE_MAX_WAIT,
12141 )
12142 .await;
12143}
12144
12145async fn wait_for_pids_with<F: Fn(u32) -> bool>(
12151 pids: &[u32],
12152 alive: F,
12153 poll: Duration,
12154 max_wait: Duration,
12155) {
12156 let deadline = Instant::now() + max_wait;
12157 loop {
12158 if pids.iter().all(|&pid| !alive(pid)) {
12159 return;
12160 }
12161 if Instant::now() >= deadline {
12162 return;
12163 }
12164 tokio::time::sleep(poll).await;
12165 }
12166}
12167
12168fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12175 outcomes.iter().any(|o| o.resource_blocked)
12176}
12177
12178pub(crate) enum GateFix {
12180 Retry,
12182 Stop,
12185 Defer,
12188}
12189
12190pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12198 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12199 red.peek().is_some()
12200 && red.all(|o| {
12201 !o.resource_blocked
12202 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12203 && !o.output_tail.trim().is_empty()
12204 })
12205}
12206
12207fn e2e_outcome_label(o: &CommandOutcome) -> String {
12211 if o.ok() {
12212 return "pass".to_owned();
12213 }
12214 let reason = if o.build_failed() {
12215 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12216 } else {
12217 format!("FAIL ({:?})", o.code)
12218 };
12219 format!(
12220 "{reason}\n{}",
12221 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12222 )
12223}
12224
12225fn rerunnable_failure(outcomes: &[CommandOutcome]) -> bool {
12232 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12233 red.peek().is_some()
12234 && !verify_inconclusive(outcomes)
12235 && red.all(|o| {
12236 !o.build_failed() && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12237 })
12238}
12239
12240const FAILED_TWICE: &str = "[this command failed twice on the same commit; the output below is \
12243 the second attempt's]";
12244
12245async fn run_e2e_with_retry(
12265 state: &mut RunState,
12266 shell: &[String],
12267 commands: &[String],
12268 worktree: &Path,
12269 timeout: Duration,
12270 context: &str,
12271 round: Option<usize>,
12272) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12273 let (mut e2e, mut timed_out_pids) = run_commands(
12274 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12275 )
12276 .await;
12277 for o in &e2e {
12278 state.event(
12279 "verify",
12280 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12281 );
12282 }
12283 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12287 if verify_retried {
12288 state.event(
12289 "verify",
12290 format!(
12291 "{context}: verify could not build/link, not a test result — retrying once \
12292 before concluding"
12293 ),
12294 );
12295 let retried = run_commands(
12296 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12297 )
12298 .await;
12299 e2e = retried.0;
12300 timed_out_pids.extend(retried.1);
12303 for o in &e2e {
12304 state.event(
12305 "verify",
12306 format!(
12307 "{context}: retry `{}` -> {}",
12308 o.command,
12309 e2e_outcome_label(o)
12310 ),
12311 );
12312 }
12313 }
12314 if state.config.graph.rerun_failed_e2e && rerunnable_failure(&e2e) {
12315 let failed: Vec<usize> = (0..e2e.len()).filter(|&i| !e2e[i].ok()).collect();
12316 let rerun_commands: Vec<String> = failed.iter().map(|&i| e2e[i].command.clone()).collect();
12317 let head = git::rev_parse(worktree, "HEAD").await.unwrap_or_default();
12318 state.event(
12319 "verify",
12320 format!(
12321 "{context}: {} failed with a test result — running it once more on the same \
12322 commit {} before deciding",
12323 rerun_commands
12324 .iter()
12325 .map(|c| format!("`{c}`"))
12326 .collect::<Vec<_>>()
12327 .join(", "),
12328 short(&head)
12329 ),
12330 );
12331 let (second, pids) = run_commands(
12332 state,
12333 "verify",
12334 "e2e",
12335 1 + usize::from(verify_retried),
12336 shell,
12337 &rerun_commands,
12338 worktree,
12339 timeout,
12340 )
12341 .await;
12342 timed_out_pids.extend(pids);
12343 for o in &second {
12344 state.event(
12345 "verify",
12346 format!(
12347 "{context}: rerun `{}` -> {}",
12348 o.command,
12349 e2e_outcome_label(o)
12350 ),
12351 );
12352 }
12353 let mut reproduced = false;
12354 for (slot, &i) in failed.iter().enumerate() {
12355 let Some(again) = second.get(slot) else {
12356 break;
12357 };
12358 let first = std::mem::replace(&mut e2e[i], again.clone());
12359 if again.ok() {
12360 let signature = crate::flake::signature(&first.output_tail, first.code);
12361 let flake = crate::run::VerifyFlake {
12362 round,
12363 context: context.to_owned(),
12364 head: head.clone(),
12365 command: first.command.clone(),
12366 first_failure: summarize_verify(&first.output_tail, EVENT_OUTPUT_TAIL),
12367 fingerprint: crate::flake::fingerprint(&state.repo, &signature),
12368 at: Timestamp::now(),
12369 task: None,
12370 filed: false,
12371 filing_error: None,
12372 };
12373 crate::flake::record(state, flake, &signature.label, &crate::queue::Queue::open());
12374 } else if !again.resource_blocked {
12375 reproduced = true;
12376 e2e[i].output_tail = format!("{FAILED_TWICE}\n{}", again.output_tail);
12377 }
12378 }
12379 if reproduced {
12380 state.event(
12381 "verify",
12382 format!("{context}: the failure reproduced on the second run — it stands"),
12383 );
12384 }
12385 }
12386 (e2e, verify_retried, timed_out_pids)
12387}
12388
12389#[allow(clippy::too_many_arguments)]
12405async fn run_commands(
12406 state: &mut RunState,
12407 node: &str,
12408 task: &str,
12409 attempt: usize,
12410 shell: &[String],
12411 commands: &[String],
12412 cwd: &Path,
12413 timeout: Duration,
12414) -> (Vec<CommandOutcome>, Vec<u32>) {
12415 let isolated = crate::build_target::isolated(&state.config);
12416 if isolated && let Some(block) = &state.resource_block {
12417 return (
12418 vec![CommandOutcome {
12419 command: "(isolated build admission)".into(),
12420 code: None,
12421 output_tail: block.description(),
12422 duration_ms: 0,
12423 resource_blocked: true,
12424 }],
12425 Vec::new(),
12426 );
12427 }
12428 if commands.is_empty() {
12429 return (Vec::new(), Vec::new());
12434 }
12435 let mut out = Vec::new();
12436 let mut timed_out_pids = Vec::new();
12437 let total = commands.len();
12438 for (idx, command) in commands.iter().enumerate() {
12439 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12440 if let Err(e) = state.save() {
12441 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12442 }
12443 let started = Instant::now();
12444 let mut cmd = tokio::process::Command::new(&shell[0]);
12445 cmd.quiet();
12446 for key in crate::worker::INVOCATION_ENV {
12450 cmd.env_remove(key);
12451 }
12452 cmd.args(&shell[1..])
12453 .current_dir(cwd)
12454 .stdin(std::process::Stdio::null())
12455 .stdout(std::process::Stdio::piped())
12456 .stderr(std::process::Stdio::piped())
12457 .kill_on_drop(true);
12458 let mut outputs = Vec::new();
12459 let mut cache = None;
12460 let home = crate::run::home();
12461 let head = if isolated {
12462 Some(git::rev_parse(cwd, "HEAD").await)
12463 } else {
12464 None
12465 };
12466 let protected = (|| -> Result<()> {
12467 crate::incremental::configure(&mut cmd, cwd)?;
12468 cache = if isolated {
12469 let target = VERIFY_TARGET
12470 .try_with(Clone::clone)
12471 .context("isolated verification has no target lease")?;
12472 cmd.arg(crate::build_target::command::normalize(command)?);
12473 crate::build_target::command::configure(&mut cmd, &target)?;
12474 Some(target)
12475 } else {
12476 crate::build_target::command::clear(&mut cmd)?;
12477 cmd.arg(command);
12478 state.config.cache_dir()
12479 };
12480 let mut paths = vec![cwd.to_owned()];
12481 if let Some(cache) = &cache {
12482 paths.push(cache.clone());
12483 }
12484 let admit = if isolated {
12485 crate::retention::OutputLease::admit_isolated
12486 } else {
12487 crate::retention::OutputLease::admit
12488 };
12489 outputs.extend(admit(&home, &paths, state.config.disk.min_free_bytes)?);
12490 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12491 outputs.push(guard);
12492 }
12493 if let Some(cache) = &cache {
12494 outputs.push(crate::retention::OutputLease::cache(
12495 &crate::run::home(),
12496 cache,
12497 false,
12498 )?);
12499 if let Some(head) = head {
12500 crate::build_target::ensure_fresh(&home, cache, cwd, &head?, &state.config)?;
12501 if task == "pre_gate" {
12502 crate::cache::invalidate_identity(&home, cache);
12503 }
12504 }
12505 }
12506 Ok(())
12507 })();
12508 if let Err(e) = protected {
12509 let detail = format!("execution admission unavailable: {e:#}");
12510 state.resource_block = Some(crate::run::ResourceBlock {
12511 cause: crate::run::ResourceBlockCause::Other,
12512 node: node.to_owned(),
12513 kind: if detail.contains("disk admission") {
12514 "disk admission"
12515 } else {
12516 "output ownership"
12517 }
12518 .to_owned(),
12519 detail: detail.clone(),
12520 at: Timestamp::now(),
12521 });
12522 out.push(CommandOutcome {
12523 command: command.clone(),
12524 code: None,
12525 output_tail: detail,
12526 duration_ms: 0,
12527 resource_blocked: true,
12528 });
12529 break;
12530 }
12531 let spawned = if isolated {
12532 crate::seat_process::Running::spawn_build(cmd, None, outputs)
12533 } else {
12534 crate::seat_process::Running::spawn(cmd, None, outputs)
12535 };
12536 let (code, body) = match spawned {
12537 Ok(mut running) => {
12538 let pid = running.child().id();
12539 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12540 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12541 let mut paths = vec![cwd.to_owned()];
12542 if let Some(cache) = cache {
12543 paths.push(cache);
12544 }
12545 let mut pressure = None;
12546 let status = tokio::select! {
12547 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12548 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes, isolated) => {
12549 pressure = Some(reason); None
12550 }
12551 };
12552 let (code, failure) = match status {
12553 Some(Ok(Ok(status))) => (
12554 status.code(),
12555 status
12556 .code()
12557 .is_none()
12558 .then(|| format!("command terminated: {status}")),
12559 ),
12560 Some(Ok(Err(error))) => {
12561 (None, Some(format!("failed waiting for command: {error}")))
12562 }
12563 Some(Err(_)) => {
12564 if !running.coordinated()
12568 && let Some(pid) = pid
12569 {
12570 timed_out_pids.push(pid);
12571 }
12572 (
12573 None,
12574 Some(format!("timed out after {}s", timeout.as_secs())),
12575 )
12576 }
12577 None => (None, None),
12578 };
12579 running.finish().await;
12580 if let Some(reason) = pressure {
12581 state.resource_block = Some(crate::run::ResourceBlock {
12582 cause: crate::run::ResourceBlockCause::Other,
12583 node: node.to_owned(),
12584 kind: "disk pressure".to_owned(),
12585 detail: reason.clone(),
12586 at: Timestamp::now(),
12587 });
12588 out.push(CommandOutcome {
12589 command: command.clone(),
12590 code: None,
12591 output_tail: reason,
12592 duration_ms: started.elapsed().as_millis() as u64,
12593 resource_blocked: true,
12594 });
12595 break;
12596 }
12597 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12598 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12599 if let Some(failure) = failure {
12600 body.push('\n');
12601 body.push_str(&failure);
12602 }
12603 (code, body)
12604 }
12605 Err(e) if isolated => {
12606 let e = anyhow::Error::new(e)
12607 .context(format!("could not start managed build `{}`", shell[0]));
12608 record_cache_error(state, node, &e);
12609 out.push(CommandOutcome {
12610 command: command.clone(),
12611 code: None,
12612 output_tail: format!("{e:#}"),
12613 duration_ms: 0,
12614 resource_blocked: true,
12615 });
12616 break;
12617 }
12618 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12619 };
12620 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12621 if !body.is_empty() {
12622 match crate::run::write_verification_artifact(state, task, &body) {
12626 Ok(name) if summary != body => {
12627 summary = format!("[full output: artifacts/{name}]\n{summary}");
12628 }
12629 Ok(_) => {}
12630 Err(e) => {
12631 let warning = format!("could not save the full {task} output: {e:#}");
12632 tracing::warn!("{warning}");
12633 state.event(node, warning);
12634 if summary != body {
12635 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12636 }
12637 }
12638 }
12639 }
12640 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12641 out.push(CommandOutcome {
12642 command: command.clone(),
12643 code,
12644 output_tail: summary,
12645 duration_ms: started.elapsed().as_millis() as u64,
12646 resource_blocked,
12647 });
12648 if resource_blocked {
12649 state.resource_block = Some(crate::run::ResourceBlock {
12650 cause: crate::run::ResourceBlockCause::Other,
12651 node: node.to_owned(),
12652 kind: "incremental cache admission".into(),
12653 detail: body,
12654 at: Timestamp::now(),
12655 });
12656 break;
12657 }
12658 }
12659 state.task_finished(task);
12660 if let Err(e) = state.save() {
12661 tracing::warn!("could not persist the end of {task}: {e:#}");
12662 }
12663 (out, timed_out_pids)
12664}
12665
12666fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12678 let repo = repo.display();
12679 match style {
12680 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12681 MergeStyle::Squash => {
12682 let subject = message
12685 .lines()
12686 .next()
12687 .unwrap_or(branch)
12688 .replace(['\\', '"', '$', '`'], "");
12689 format!(
12690 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12691 )
12692 }
12693 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12694 }
12695}
12696
12697const PR_TITLE_MAX: usize = 240;
12708
12709fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12720 let line = queue::first_line(text)?;
12721 let chars: Vec<char> = line.chars().collect();
12722 let (kept, mark) = if chars.len() <= max {
12723 (line.to_owned(), "")
12724 } else if let Some(end) = sentence_end(&chars, max) {
12725 (chars[..end].iter().collect::<String>(), "")
12726 } else {
12727 let room = max.saturating_sub(3);
12728 let cut = if chars[room].is_whitespace() {
12731 room
12732 } else {
12733 chars[..room]
12734 .iter()
12735 .rposition(|c| c.is_whitespace())
12736 .unwrap_or(room)
12737 };
12738 let head: String = chars[..cut].iter().collect();
12739 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12740 (head.to_owned(), "...")
12741 };
12742 if kept.is_empty() {
12743 return None;
12744 }
12745 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12746 return None;
12747 }
12748 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12749 return None;
12750 }
12751 Some(format!("{kept}{mark}"))
12752}
12753
12754fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12758 const MIN: usize = 20;
12759 for i in MIN..max.min(chars.len()) {
12760 if !matches!(chars[i], '.' | '!' | '?') {
12761 continue;
12762 }
12763 let Some(&next) = chars.get(i + 1) else {
12764 continue;
12765 };
12766 if !next.is_whitespace() {
12767 continue;
12768 }
12769 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12770 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12771 continue;
12772 }
12773 let word: String = chars[..i]
12774 .iter()
12775 .rev()
12776 .take_while(|c| !c.is_whitespace())
12777 .collect::<Vec<_>>()
12778 .into_iter()
12779 .rev()
12780 .collect();
12781 let word = word.to_ascii_lowercase();
12782 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12783 continue;
12784 }
12785 let end = chars[..i]
12786 .iter()
12787 .rposition(|c| !c.is_whitespace())
12788 .map_or(i, |p| p + 1);
12789 return Some(end);
12790 }
12791 None
12792}
12793
12794struct PrMessage {
12799 title: String,
12800 body: String,
12801}
12802
12803impl PrMessage {
12804 fn commit_message(&self) -> String {
12808 format!("{}\n\n{}", self.title, self.body)
12809 }
12810}
12811
12812fn title_marker(line: &str) -> Option<&str> {
12814 let line = line.trim();
12815 let head = line.get(..6)?;
12816 head.eq_ignore_ascii_case("title:")
12817 .then(|| line[6..].trim())
12818}
12819
12820fn summary_title(summary: &str) -> Option<String> {
12825 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12826 let raw = title_marker(first)?;
12827 if raw.is_empty() {
12828 return None;
12829 }
12830 let title = queue::title_from(raw, PR_TITLE_MAX);
12831 let lower = title.to_ascii_lowercase();
12832 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12833 return None;
12834 }
12835 Some(title)
12836}
12837
12838const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12840
12841pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12843
12844pub(crate) fn is_review_run(state: &RunState) -> bool {
12848 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12849}
12850
12851fn review_title(state: &RunState) -> Option<String> {
12856 english_subject(
12857 state.reviewed_commits.as_ref()?.first()?,
12858 state.config.graph.github_english(),
12859 )
12860}
12861
12862fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12865 let raw = raw.trim();
12866 if raw.is_empty() {
12867 return None;
12868 }
12869 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12870 return None;
12871 }
12872 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12873 return None;
12874 }
12875 let title = queue::title_from(raw, PR_TITLE_MAX);
12876 let lower = title.to_ascii_lowercase();
12877 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12878 return None;
12879 }
12880 Some(title)
12881}
12882
12883#[derive(Debug, Clone, PartialEq, Eq)]
12886struct BranchFacts {
12887 commits: Vec<(String, String)>,
12889 stat: String,
12891}
12892
12893const STAT_FILE_LINES: usize = 25;
12895const COMMITS_MAX_CHARS: usize = 20_000;
12897
12898async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12901 let commits = git::commit_log(repo, base, branch).await.ok()?;
12902 if commits.is_empty() {
12903 return None;
12904 }
12905 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12906 let lines: Vec<&str> = stat.lines().collect();
12907 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12908 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12909 let more = format!(" ... {omitted} more file(s)");
12910 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12911 kept.push(&more);
12912 kept.push(lines[lines.len() - 1]);
12913 kept.join("\n")
12914 } else {
12915 lines.join("\n")
12916 };
12917 Some(BranchFacts { commits, stat })
12918}
12919
12920fn markdown_safe(text: &str) -> String {
12923 text.replace("</details>", "</details>")
12924 .replace("\x60\x60\x60", "~~~")
12925}
12926
12927fn neutral_title(state: &RunState, winner: char) -> String {
12928 format!(
12929 "chore: land candidate {} of run {}",
12930 winner.to_ascii_uppercase(),
12931 state.id
12932 )
12933}
12934
12935pub(crate) async fn repair_landing_gate(
12938 state: &mut RunState,
12939 winner: &Candidate,
12940 outcomes: &[CommandOutcome],
12941) -> Result<GateFix> {
12942 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12943 return Ok(GateFix::Stop);
12944 }
12945 let mut runner = Runner {
12946 roles: state.config.resolve_roles()?,
12947 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12948 state: state.clone(),
12949 pause: Pause::new(),
12950 interrupt: Pause::new(),
12951 };
12952 let result = runner.gate_fix_round(winner, outcomes).await;
12953 *state = runner.state;
12954 result
12955}
12956
12957pub(crate) async fn verify_current_head(
12960 state: &mut RunState,
12961 winner: &Candidate,
12962 wait: &crate::repo_lock::WaitPolicy,
12963) -> Result<bool> {
12964 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12965 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12967 state
12968 .verification_history
12969 .push(crate::run::VerificationEvidence {
12970 superseded_by: Some(head.clone()),
12971 archived_at: Some(Timestamp::now()),
12972 fixes_spent: Some(state.gate_fixes.len()),
12973 gate_head: None,
12974 gate_ran: false,
12975 gate: Vec::new(),
12976 e2e: state.head_e2e.take(),
12977 });
12978 }
12979 if let Some(e) = &state.head_e2e
12980 && e.head == head
12981 {
12982 if e.passed {
12983 return Ok(true);
12984 }
12985 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12986 state.status = RunStatus::Blocked;
12987 state.save()?;
12988 return Ok(false);
12989 }
12990 } else if state.reviews.last().is_some_and(|r| {
12991 r.verified_head.as_deref() == Some(head.as_str())
12992 && (r.e2e_status() == E2eStatus::Passed
12993 || (r.e2e_status() == E2eStatus::NotConfigured
12994 && state.config.verify.e2e.is_empty()))
12995 }) || state
12996 .gate_repair_e2e
12997 .as_ref()
12998 .is_some_and(|e| e.head == head && e.passed)
12999 {
13000 return Ok(true);
13001 }
13002 if let Some(e) = &state.gate_repair_e2e
13003 && e.head == head
13004 && !e.passed
13005 && !e.outcomes.is_empty()
13006 && !verify_inconclusive(&e.outcomes)
13007 {
13008 state.status = RunStatus::Blocked;
13009 state.save()?;
13010 return Ok(false);
13011 }
13012 state.head_e2e = Some(crate::run::GateRepairE2e {
13013 head: head.clone(),
13014 passed: false,
13015 outcomes: Vec::new(),
13016 });
13017 state.save()?;
13018 let commands = state.config.verify.e2e.clone();
13019 let shell = state.config.shell();
13020 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
13021 let cache_dir = state.config.cache_dir();
13022 let outcomes = if commands.is_empty() {
13023 Vec::new()
13024 } else {
13025 let (outcomes, _) = with_cache_lease(
13026 state,
13027 cache_dir.as_deref(),
13028 "e2e",
13029 "e2e",
13030 &winner.worktree,
13031 &head,
13032 timeout,
13033 "current head e2e",
13034 wait,
13035 |state, budget| async move {
13036 run_e2e_with_retry(
13037 state,
13038 &shell,
13039 &commands,
13040 &winner.worktree,
13041 budget,
13042 "current head e2e",
13043 None,
13044 )
13045 .await
13046 },
13047 )
13048 .await?;
13049 outcomes
13050 };
13051 let unresolved = verify_inconclusive(&outcomes);
13052 let passed = !unresolved
13053 && outcomes.iter().all(CommandOutcome::ok)
13054 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
13055 if !outcomes.is_empty() && !unresolved {
13056 state.record_base_progress(&head);
13057 }
13058 state.head_e2e = Some(crate::run::GateRepairE2e {
13059 head: head.clone(),
13060 passed,
13061 outcomes,
13062 });
13063 if !passed && !unresolved {
13064 state.status = RunStatus::Blocked;
13065 state.event(
13066 "e2e",
13067 format!("verify.e2e failed on head {head}; nothing published"),
13068 );
13069 }
13070 state.save()?;
13071 Ok(passed)
13072}
13073
13074pub(crate) async fn check_gate_repair_e2e(
13077 state: &mut RunState,
13078 winner: &Candidate,
13079 wait: &crate::repo_lock::WaitPolicy,
13080) -> Result<GateFix> {
13081 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
13082 return Ok(GateFix::Retry);
13083 }
13084 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
13085 if state
13086 .gate_repair_e2e
13087 .as_ref()
13088 .is_some_and(|r| r.head == head && r.passed)
13089 {
13090 return Ok(GateFix::Retry);
13091 }
13092 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
13093 head: head.clone(),
13094 passed: false,
13095 outcomes: Vec::new(),
13096 });
13097 state.save()?;
13098 let commands = state.config.verify.e2e.clone();
13099 let outcomes = if commands.is_empty() {
13100 Vec::new()
13101 } else {
13102 let shell = state.config.shell();
13103 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
13104 let cache_dir = state.config.cache_dir();
13105 let (outcomes, _) = with_cache_lease(
13106 state,
13107 cache_dir.as_deref(),
13108 "e2e",
13109 "e2e",
13110 &winner.worktree,
13111 &head,
13112 timeout,
13113 "gate repair e2e",
13114 wait,
13115 |state, budget| async move {
13116 run_e2e_with_retry(
13117 state,
13118 &shell,
13119 &commands,
13120 &winner.worktree,
13121 budget,
13122 "gate repair e2e",
13123 None,
13124 )
13125 .await
13126 },
13127 )
13128 .await?;
13129 outcomes
13130 };
13131 let deferred = verify_inconclusive(&outcomes);
13132 let passed = outcomes.iter().all(CommandOutcome::ok)
13133 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
13134 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
13135 head,
13136 passed,
13137 outcomes,
13138 });
13139 state.save()?;
13140 if deferred {
13141 Ok(GateFix::Defer)
13142 } else if !passed {
13143 state.event(
13144 "gate",
13145 "verify.e2e failed after gate repair; nothing published",
13146 );
13147 Ok(GateFix::Stop)
13148 } else {
13149 Ok(GateFix::Retry)
13150 }
13151}
13152
13153pub(crate) async fn run_landing_gate(
13155 state: &mut RunState,
13156 winner: &Candidate,
13157 wait: &crate::repo_lock::WaitPolicy,
13158) -> Result<Vec<CommandOutcome>> {
13159 let shell = state.config.shell();
13160 let gate_commands = state.config.verify.gate.clone();
13161 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
13162 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
13163 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
13164 Vec::new()
13165 } else {
13166 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
13167 let cache_dir = state.config.cache_dir();
13168 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
13169 let (outcomes, _) = with_cache_lease(
13170 state,
13171 cache_dir.as_deref(),
13172 "gate",
13173 "gate",
13174 &winner.worktree,
13175 &head,
13176 timeout,
13177 "final gate",
13178 wait,
13179 |state, budget| {
13180 let shell = shell.clone();
13181 let gate_commands = gate_commands.clone();
13182 let winner = winner.clone();
13183 async move {
13184 Runner::run_pre_gate(state, &winner).await;
13185 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
13187 let (outcomes, timed_out_pids) = run_commands(
13188 state,
13189 "gate",
13190 "gate",
13191 0,
13192 &shell,
13193 &gate_commands,
13194 &winner.worktree,
13195 budget,
13196 )
13197 .await;
13198 (outcomes, false, timed_out_pids)
13199 }
13200 },
13201 )
13202 .await?;
13203 outcomes
13204 };
13205 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
13206 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
13207 let mut outcomes = outcomes;
13208 outcomes.push(CommandOutcome {
13209 command: "gate head stability".into(), code: None,
13210 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
13211 duration_ms: 0, resource_blocked: false,
13212 });
13213 return Ok(outcomes);
13214 }
13215 if outcomes.is_empty() {
13216 state.event("gate", "no gate commands configured; not tested");
13221 }
13222 for o in &outcomes {
13223 state.event(
13224 "gate",
13225 format!(
13226 "`{}` -> {}",
13227 o.command,
13228 if o.ok() {
13229 "pass".to_owned()
13230 } else {
13231 format!(
13232 "FAIL ({:?})\n{}",
13233 o.code,
13234 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
13235 )
13236 }
13237 ),
13238 );
13239 }
13240 Ok(outcomes)
13241}
13242
13243pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
13249 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
13250 ""
13251 } else {
13252 pr_title
13253 }
13254}
13255
13256pub fn landing_subject_source(state: &RunState) -> String {
13260 if is_review_run(state) {
13261 let winner = state.candidates.first().map_or('A', |c| c.label);
13262 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
13263 }
13264 state.instruction.clone()
13265}
13266
13267fn summary_without_title(summary: &str) -> String {
13270 let mut lines = summary.trim().lines().peekable();
13271 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
13272 lines.next();
13273 }
13274 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
13275}
13276
13277#[cfg(test)]
13289fn pr_message(state: &RunState, winner: char) -> PrMessage {
13290 pr_message_with(state, winner, None)
13291}
13292
13293#[cfg(test)]
13296fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
13297 let raw = pr_message_raw(
13298 state,
13299 winner,
13300 facts,
13301 crate::pr_outcome::FinalHead::Unreported,
13302 ChangeNote::NotAsked,
13303 );
13304 let id = crate::scrub::Identity::current();
13305 PrMessage {
13306 title: crate::scrub::scrub(&raw.title, &id),
13307 body: crate::scrub::scrub(&raw.body, &id),
13308 }
13309}
13310
13311#[derive(Debug, Clone, Copy)]
13313enum ChangeNote<'a> {
13314 NotAsked,
13316 Summary(&'a crate::run::ChangeSummary),
13318 Unavailable(&'a str),
13320}
13321
13322fn pr_message_raw(
13323 state: &RunState,
13324 winner: char,
13325 facts: Option<&BranchFacts>,
13326 head: crate::pr_outcome::FinalHead<'_>,
13327 change: ChangeNote<'_>,
13328) -> PrMessage {
13329 if !matches!(change, ChangeNote::NotAsked) {
13330 let (title, mut body) = match change {
13331 ChangeNote::Summary(c) => (
13332 c.title.clone(),
13333 format!("## Summary\n\n{}\n\n", c.body.trim()),
13334 ),
13335 ChangeNote::Unavailable(why) => (
13336 neutral_title(state, winner),
13337 format!(
13338 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13339 ),
13340 ),
13341 ChangeNote::NotAsked => unreachable!(),
13342 };
13343 body.push_str(&public_pr_evidence(state, head));
13344 body.push_str(&format!(
13345 "\n---\nmagi:run/{} magi:candidate-{}\n",
13346 state.id,
13347 winner.to_ascii_lowercase()
13348 ));
13349 return PrMessage { title, body };
13350 }
13351 pr_preview(state, winner, facts, head)
13352}
13353
13354fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13357 let mut body = crate::pr_outcome::review_rounds(state);
13358 body.push_str(&crate::pr_outcome::final_verification(state, head));
13359 let open = state.last_round_findings();
13360 if !open.is_empty() {
13361 body.push_str("## Open review findings\n\n");
13362 for f in open {
13363 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13364 }
13365 body.push('\n');
13366 }
13367 if let Some(c) = state.change_summary.as_ref()
13368 && head == crate::pr_outcome::FinalHead::At(&c.head)
13369 {
13370 body.push_str(&format!(
13371 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13372 c.base, c.head
13373 ));
13374 }
13375 body
13376}
13377
13378fn pr_preview(
13380 state: &RunState,
13381 winner: char,
13382 facts: Option<&BranchFacts>,
13383 head: crate::pr_outcome::FinalHead<'_>,
13384) -> PrMessage {
13385 let summary = state
13386 .candidates
13387 .iter()
13388 .find(|c| c.label == winner)
13389 .map(|c| c.summary.as_str())
13390 .unwrap_or_default();
13391 let review = is_review_run(state);
13394 let title = if review {
13395 facts
13396 .and_then(|f| {
13397 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13398 })
13399 .or_else(|| review_title(state))
13400 .or_else(|| {
13401 state
13402 .candidates
13403 .iter()
13404 .find(|c| c.label == winner)
13405 .filter(|c| !c.branch.starts_with("magi/"))
13406 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13407 })
13408 .unwrap_or_else(|| neutral_title(state, winner))
13409 } else {
13410 summary_title(summary).unwrap_or_else(|| {
13411 english_title(
13412 &state.instruction,
13413 PR_TITLE_MAX,
13414 state.config.graph.github_english(),
13415 )
13416 .unwrap_or_else(|| neutral_title(state, winner))
13417 })
13418 };
13419
13420 let mut body = String::new();
13421 let what = summary_without_title(summary);
13422 if !what.is_empty() {
13423 body.push_str("## Summary\n\n");
13424 body.push_str(&what);
13425 body.push_str("\n\n");
13426 }
13427
13428 body.push_str(&crate::pr_outcome::review_rounds(state));
13430 body.push_str(&crate::pr_outcome::final_verification(state, head));
13431 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13432
13433 let open = state.open_findings();
13434 if !open.is_empty() {
13435 body.push_str("## Open review findings\n\n");
13436 for f in &open {
13437 body.push_str(&format!(
13438 "- `{}` [{:?}] {}\n",
13439 f.id, f.severity, "Context retained in local review records."
13440 ));
13441 }
13442 body.push('\n');
13443 }
13444
13445 if let Some(fix) = fix
13446 && !fix.rejected.is_empty()
13447 {
13448 body.push_str("## Declined by the fixer\n\n");
13449 for r in &fix.rejected {
13450 body.push_str(&format!(
13451 "- `{}`: {}\n",
13452 r.id, "Reason retained in local review records."
13453 ));
13454 }
13455 body.push('\n');
13456 }
13457
13458 if review {
13459 body.push_str("## Commits under review\n\n");
13461 if let Some(facts) = facts {
13462 let mut left = COMMITS_MAX_CHARS;
13463 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13464 let mut entry = format!("- {}\n", markdown_safe(subject));
13465 for l in markdown_safe(text).lines() {
13466 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13467 }
13468 if left == 0 {
13469 body.push_str(&format!(
13470 "- ... {} more commit(s)\n",
13471 facts.commits.len() - i
13472 ));
13473 break;
13474 }
13475 if entry.len() > left {
13476 let mut end = left;
13479 while !entry.is_char_boundary(end) {
13480 end -= 1;
13481 }
13482 entry.truncate(end);
13483 entry.push_str("\n ... (truncated)\n");
13484 left = 0;
13485 } else {
13486 left -= entry.len();
13487 }
13488 body.push_str(&entry);
13489 }
13490 if !facts.stat.trim().is_empty() {
13491 body.push_str(&format!(
13492 "\n## Diff stat\n\n```\n{}\n```\n",
13493 markdown_safe(facts.stat.trim())
13494 ));
13495 }
13496 } else {
13497 match &state.reviewed_commits {
13498 Some(subjects) => {
13499 for s in subjects {
13500 body.push_str(&format!("- {}\n", s.trim()));
13501 }
13502 }
13503 None => {
13504 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13507 body.push_str(rest.trim());
13508 body.push('\n');
13509 }
13510 }
13511 }
13512 } else {
13513 let task = state.instruction.trim();
13514 let task = if task.is_empty() {
13515 "(empty task)"
13516 } else {
13517 task
13518 };
13519 body.push_str(&format!(
13520 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13521 task.replace("</details>", "</details>")
13522 ));
13523 }
13524
13525 body.push_str(&format!(
13526 "\n---\nmagi:run/{} magi:candidate-{}\n",
13527 state.id,
13528 winner.to_ascii_lowercase()
13529 ));
13530
13531 PrMessage { title, body }
13532}
13533
13534fn seeded_instruction(state: &RunState) -> String {
13537 let mut inputs = Vec::new();
13538 if let Some(start) = &state.sources.start_from {
13539 inputs.push(format!(
13540 "Start from `{}` at `{}` (including ancestry).",
13541 start.token, start.sha
13542 ));
13543 }
13544 for pick in &state.sources.pick_commits {
13545 inputs.push(format!(
13546 "Pick `{}` at `{}` (single commit).",
13547 pick.token, pick.sha
13548 ));
13549 }
13550 if inputs.is_empty() {
13551 state.instruction.clone()
13552 } else {
13553 format!(
13554 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13555 state.instruction,
13556 inputs.join("\n")
13557 )
13558 }
13559}
13560
13561async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13565 let base = &state.base_branch;
13566 let mut against = base.clone();
13567 if mode == MergeMode::Pr {
13568 let remote = &state.config.merge.remote;
13572 let tracking = format!("{remote}/{base}");
13573 let fetched = git::fetch(repo, remote, base).await;
13574 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13575 against = tracking;
13576 } else {
13577 return false;
13578 }
13579 }
13580 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13581}
13582
13583fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13586 let mut detail = format!(
13587 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13588 nothing to open a pull request for"
13589 );
13590 detail.push_str(
13591 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13592 );
13593 detail
13594}
13595
13596#[derive(Debug, PartialEq, Eq)]
13599enum PrPlan {
13600 Create,
13601 Adopt { url: String, title: String },
13602 Stop(String),
13603}
13604
13605fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13608 match found {
13609 Ok(land::OpenPr::None) => PrPlan::Create,
13610 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13611 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13612 "several open pull requests exist for this branch, not picking one: {}",
13613 urls.join(" ")
13614 )),
13615 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13616 }
13617}
13618
13619pub(crate) async fn gh_pr_create(
13621 cwd: &Path,
13622 base: &str,
13623 head: &str,
13624 title: &str,
13625 body: &str,
13626) -> Result<String> {
13627 let out = tokio::process::Command::new("gh")
13628 .args([
13629 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13630 ])
13631 .current_dir(cwd)
13632 .quiet()
13633 .stdin(std::process::Stdio::null())
13634 .output()
13635 .await
13636 .context("spawn gh")?;
13637 if out.status.success() {
13638 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13639 } else {
13640 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13641 }
13642}
13643
13644async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13646 match std::fs::symlink_metadata(path) {
13647 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13648 git::worktree_remove_clean(repo, path).await?;
13651 return Ok(());
13652 }
13653 Err(e) => return Err(e.into()),
13654 Ok(_) => {}
13655 }
13656 if !git::is_clean(path).await? {
13657 bail!(
13658 "worktree has uncommitted work; refusing removal: {}",
13659 path.display()
13660 );
13661 }
13662 if !git::worktree_remove_clean(repo, path).await? {
13663 bail!("worktree removal refused: {}", path.display());
13664 }
13665 Ok(())
13666}
13667
13668async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13671 if !crate::retention::read_seat(path) || !path.exists() {
13672 return remove_preserved_worktree(&state.repo, path).await;
13673 }
13674 let owned = std::fs::symlink_metadata(path)?.is_dir()
13675 && path.parent().map(Path::canonicalize).transpose()?
13676 == Some(state.worktree_root().canonicalize()?)
13677 && crate::retention::worktrees(state)
13678 .await?
13679 .contains(&path.canonicalize()?);
13680 let common = crate::repo_lock::common_dir(&state.repo).await?;
13681 if !owned
13682 || common.is_none()
13683 || crate::repo_lock::common_dir(path).await? != common
13684 || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13685 .await?
13686 .ok()
13687 || state.candidates.iter().any(|c| c.worktree == path)
13688 {
13689 return remove_preserved_worktree(&state.repo, path).await;
13690 }
13691 let changes = git::status_porcelain(path).await?;
13692 if !git::worktree_remove(&state.repo, path).await? {
13693 bail!(
13694 "read-only seat worktree removal refused: {}",
13695 path.display()
13696 );
13697 }
13698 if !changes.is_empty() {
13699 let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13700 let paths: String = paths.chars().take(1024).collect();
13701 state.event(
13702 "fold",
13703 format!(
13704 "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13705 path.file_name().unwrap_or_default().to_string_lossy(),
13706 changes.lines().count()
13707 ),
13708 );
13709 }
13710 Ok(())
13711}
13712
13713fn fold_target(
13714 home: &Path,
13715 root: &Path,
13716 path: &Path,
13717 isolated: bool,
13718) -> Option<crate::build_target::FoldTarget> {
13719 if !isolated {
13720 return None;
13721 }
13722 match crate::build_target::FoldTarget::for_checkout(home, path, root) {
13723 Ok(target) => target,
13724 Err(error) => {
13725 tracing::warn!(checkout = %path.display(), %error, "isolated target binding not proved at fold");
13726 None
13727 }
13728 }
13729}
13730
13731fn reclaim_fold_target(home: &Path, target: Option<crate::build_target::FoldTarget>) {
13732 if let Some(target) = target
13733 && let Err(error) = target.reclaim_orphaned_target(home)
13734 {
13735 tracing::warn!(%error, "isolated target retained after fold");
13736 }
13737}
13738
13739pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13748 fold_run_with_policy(state, drop_winner, home, false).await
13749}
13750
13751pub async fn fold_run_overriding_recovery_retention(
13753 state: &mut RunState,
13754 drop_winner: bool,
13755 home: &Path,
13756) -> Result<Vec<String>> {
13757 fold_run_with_policy(state, drop_winner, home, true).await
13758}
13759
13760async fn fold_run_with_policy(
13761 state: &mut RunState,
13762 drop_winner: bool,
13763 home: &Path,
13764 override_recovery_retention: bool,
13765) -> Result<Vec<String>> {
13766 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state)
13767 && !override_recovery_retention
13768 {
13769 bail!("refusing fold: {reason}");
13770 }
13771 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13772 bail!("candidate preservation incomplete; refusing cleanup");
13773 }
13774 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13778 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13779 if override_recovery_retention {
13781 *state = RunState::load_under(&state.id, home)?;
13782 anyhow::ensure!(
13783 state.status.done() && !state.status.resumable(),
13784 "recovery retention override requires a finished, non-resumable run"
13785 );
13786 let claimed = crate::daemon::is_working_on_other(home, &state.id, Timestamp::now())?;
13787 anyhow::ensure!(
13788 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
13789 "run is live or its ownership is unknown"
13790 );
13791 anyhow::ensure!(
13792 state.candidate_preservation.is_empty() && state.interrupted_implementations.is_empty(),
13793 "candidate preservation incomplete; refusing cleanup"
13794 );
13795 }
13796 let retention_reason = crate::pr_recovery::source_retention_reason(home, state);
13797 if let Some(reason) = &retention_reason
13798 && !override_recovery_retention
13799 {
13800 bail!("refusing fold: {reason}");
13801 }
13802 let _driver = crate::driver::maintenance(home, &state.id)?;
13803 let mut output_guards = Vec::new();
13804 for entry in std::fs::read_dir(state.worktree_root())
13805 .into_iter()
13806 .flatten()
13807 .flatten()
13808 {
13809 if entry.path().join(".git").is_file() {
13810 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13811 }
13812 }
13813 let repo = state.repo.clone();
13814 let wait = repository_wait(state, None);
13817 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13818 Ok(lease) => lease,
13819 Err(e) => {
13820 state.save_under(home)?;
13821 return Err(e);
13822 }
13823 };
13824 let _source_outputs = if retention_reason.is_some()
13827 || (state.status == RunStatus::Merged && state.merge.as_ref().is_some_and(|merge| merge.ok))
13828 {
13829 verify_merged_fold_sources(state, drop_winner, home).await?
13830 } else {
13831 Vec::new()
13832 };
13833 if let Some(reason) = retention_reason {
13834 state.event(
13835 "fold",
13836 format!(
13837 "operator CLI --override-recovery-retention authorized deletion for run {}: bypassed {reason}",
13838 state.id
13839 ),
13840 );
13841 state.save_under(home)?;
13842 }
13843 let root = state.worktree_root();
13844 let winner = state.tally.as_ref().map(|t| t.winner);
13845 let mut removed = Vec::new();
13846 let isolated = crate::build_target::isolated(&state.config);
13847
13848 for i in 0..state.candidates.len() {
13849 let c = state.candidates[i].clone();
13850 let is_winner = Some(c.label) == winner;
13851 if is_winner && !drop_winner {
13852 continue;
13853 }
13854 let existed = c.worktree.exists();
13855 let target = fold_target(home, &root, &c.worktree, isolated);
13856 remove_preserved_worktree(&repo, &c.worktree).await?;
13859 reclaim_fold_target(home, target);
13860 if existed {
13861 removed.push(c.worktree.to_string_lossy().into_owned());
13862 }
13863 let handed_over = state.released_branches.contains(&c.branch);
13866 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13867 if !git::branch_delete(&repo, &c.branch).await? {
13868 bail!("candidate branch removal refused");
13869 }
13870 removed.push(c.branch.clone());
13871 }
13872 state.candidates[i].folded = true;
13873 }
13874
13875 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13876 let path = name.path();
13877 let keep = !drop_winner
13878 && winner.is_some_and(|w| {
13879 path.file_name()
13880 .is_some_and(|n| n == format!("cand-{w}").as_str())
13881 });
13882 if keep {
13883 continue;
13884 }
13885 let target = fold_target(home, &root, &path, isolated);
13886 remove_read_seat_worktree(state, &path).await?;
13887 reclaim_fold_target(home, target);
13888 removed.push(path.to_string_lossy().into_owned());
13889 }
13890
13891 remove_if_empty(&root);
13900
13901 if state.enabled_worktree_config && drop_winner {
13902 git::release_worktree_config(&repo).await.ok();
13906 state.enabled_worktree_config = false;
13907 }
13908 state.save_under(home)?;
13909 Ok(removed)
13910}
13911
13912async fn verify_merged_fold_sources(
13913 state: &RunState,
13914 drop_winner: bool,
13915 home: &Path,
13916) -> Result<Vec<std::fs::File>> {
13917 let winner = state.tally.as_ref().map(|t| t.winner);
13918 let mut heads = BTreeSet::new();
13919 for candidate in &state.candidates {
13920 if Some(candidate.label) == winner && !drop_winner {
13921 continue;
13922 }
13923 anyhow::ensure!(
13924 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13925 "candidate recovery source is outside the inventoried run root: {}",
13926 candidate.worktree.display()
13927 );
13928 match std::fs::symlink_metadata(&candidate.worktree) {
13929 Ok(_) => {}
13930 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13931 Err(error) => return Err(error.into()),
13932 }
13933 if !state.released_branches.contains(&candidate.branch)
13934 && git::branch_exists(&state.repo, &candidate.branch).await?
13935 {
13936 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13937 }
13938 }
13939 let mut cargo_guards = Vec::new();
13940 let entries = match std::fs::read_dir(state.worktree_root()) {
13941 Ok(entries) => Some(entries),
13942 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13943 Err(error) => return Err(error.into()),
13944 };
13945 for entry in entries.into_iter().flatten() {
13946 let path = entry?.path();
13947 let metadata = std::fs::symlink_metadata(&path)?;
13948 anyhow::ensure!(
13949 metadata.is_dir() && !metadata.file_type().is_symlink(),
13950 "non-directory or symlink recovery source retained: {}",
13951 path.display()
13952 );
13953 if !drop_winner
13954 && winner.is_some_and(|w| {
13955 path.file_name()
13956 .is_some_and(|name| name == format!("cand-{w}").as_str())
13957 })
13958 {
13959 continue;
13960 }
13961 anyhow::ensure!(
13962 path.join(".git").is_file(),
13963 "unregistered recovery source retained: {}",
13964 path.display()
13965 );
13966 anyhow::ensure!(
13967 git::git(
13968 &path,
13969 &[
13970 "status",
13971 "--porcelain",
13972 "--untracked-files=all",
13973 "--ignored",
13974 "--",
13975 ".",
13976 ":(exclude)target"
13977 ]
13978 )
13979 .await?
13980 .is_empty(),
13981 "source changes or ignored recovery files retained: {}",
13982 path.display()
13983 );
13984 match std::fs::symlink_metadata(path.join("target")) {
13985 Ok(_) => {
13986 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13987 }
13988 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13989 Err(error) => return Err(error.into()),
13990 }
13991 for checkpoint in [
13992 "rebase-merge",
13993 "rebase-apply",
13994 "sequencer",
13995 "CHERRY_PICK_HEAD",
13996 "MERGE_HEAD",
13997 "REBASE_HEAD",
13998 "REVERT_HEAD",
13999 "BISECT_LOG",
14000 "AUTO_MERGE",
14001 "MERGE_MSG",
14002 ] {
14003 let location = git::git(
14004 &path,
14005 &[
14006 "rev-parse",
14007 "--path-format=absolute",
14008 "--git-path",
14009 checkpoint,
14010 ],
14011 )
14012 .await?;
14013 anyhow::ensure!(
14014 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
14015 if e.kind() == std::io::ErrorKind::NotFound),
14016 "Git recovery checkpoint retained: {checkpoint}"
14017 );
14018 }
14019 heads.insert(git::rev_parse(&path, "HEAD").await?);
14020 }
14021 crate::fold_evidence::preserve(state, home, &heads).await?;
14022
14023 Ok(cargo_guards)
14024}
14025
14026fn remove_if_empty(dir: &Path) {
14037 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
14038 std::fs::remove_dir(dir).ok();
14039 }
14040}
14041
14042pub fn worst_open(state: &RunState) -> Option<Severity> {
14044 state
14045 .reviews
14046 .last()?
14047 .reviews
14048 .iter()
14049 .flat_map(|r| r.findings.iter())
14050 .map(|f| f.severity)
14051 .max()
14052}
14053
14054async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
14058 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
14059 git::acquire_worktree_config(repo).await?;
14060 state.enabled_worktree_config = true;
14061 }
14062 Ok(())
14063}
14064
14065#[cfg(test)]
14066mod tests {
14067 #[tokio::test]
14068 async fn launcher_base_fetch_requires_remote_success() {
14069 let tmp = tempfile::tempdir().unwrap();
14070 let repo = tmp.path().join("repo");
14071 let remote = tmp.path().join("remote");
14072 std::fs::create_dir(&repo).unwrap();
14073 std::fs::create_dir(&remote).unwrap();
14074 git::git(&remote, &["init", "--bare", "-b", "main"])
14075 .await
14076 .unwrap();
14077 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
14078 git::git(
14079 &repo,
14080 &[
14081 "-c",
14082 "user.name=Fixture",
14083 "-c",
14084 "user.email=fixture@example.invalid",
14085 "commit",
14086 "--allow-empty",
14087 "-m",
14088 "fixture",
14089 ],
14090 )
14091 .await
14092 .unwrap();
14093 git::git(
14094 &repo,
14095 &["remote", "add", "origin", remote.to_str().unwrap()],
14096 )
14097 .await
14098 .unwrap();
14099 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
14100 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14101 assert_eq!(
14102 resolve_base(&repo, "main", "origin", None)
14103 .await
14104 .unwrap()
14105 .commit,
14106 head
14107 );
14108 #[cfg(unix)]
14109 {
14110 let policy: crate::config::Coordination = toml::from_str(
14111 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
14112 ).unwrap();
14113 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
14114 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
14115 let held = crate::repo_lock::SeatLease::operation_with(
14116 &repo,
14117 Some(&policy),
14118 &crate::repo_lock::WaitPolicy::default(),
14119 )
14120 .await
14121 .unwrap();
14122 assert!(held.is_some());
14123 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
14124 tokio::pin!(fetch);
14125 assert!(
14126 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
14127 .await
14128 .is_err()
14129 );
14130 assert!(
14131 !repo.join(".git/FETCH_HEAD").exists(),
14132 "no fetch before admission"
14133 );
14134 drop(held);
14135 let fetched = fetch.await.unwrap();
14136 assert_eq!(fetched.commit, head);
14137 assert_eq!(fetched.remote, "origin");
14138 assert_eq!(fetched.branch, "main");
14139 }
14140 git::git(
14142 &repo,
14143 &[
14144 "remote",
14145 "set-url",
14146 "origin",
14147 tmp.path().join("missing").to_str().unwrap(),
14148 ],
14149 )
14150 .await
14151 .unwrap();
14152 let error = resolve_base(&repo, "main", "origin", None)
14153 .await
14154 .unwrap_err()
14155 .to_string();
14156 assert!(error.contains("cannot read origin/main"));
14157 assert!(error.contains("refusing to branch off"));
14158 }
14159
14160 #[test]
14161 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
14162 let leaked = vec!["chore(deps): update a crate".to_owned()];
14163 let own = "fix(daemon): apply a chosen action";
14164 assert!(should_retitle("", own, &leaked));
14165 assert!(should_retitle(
14166 &format!("{REVIEW_PROMPT_OPENING} `x`"),
14167 own,
14168 &leaked
14169 ));
14170 assert!(should_retitle(
14171 "chore: land candidate A of run 1",
14172 own,
14173 &leaked
14174 ));
14175 assert!(should_retitle(
14176 "magi: candidate A (uncommitted work)",
14177 own,
14178 &leaked
14179 ));
14180 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
14181 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
14182 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
14183 }
14184
14185 #[test]
14186 fn pr_merge_plan_creates_adopts_or_stops() {
14187 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
14188 assert_eq!(
14189 pr_merge_plan(Ok(land::OpenPr::One {
14190 url: "u".into(),
14191 title: "t".into()
14192 })),
14193 PrPlan::Adopt {
14194 url: "u".into(),
14195 title: "t".into()
14196 }
14197 );
14198 let PrPlan::Stop(many) =
14199 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
14200 else {
14201 panic!("many must stop");
14202 };
14203 assert!(many.contains('a') && many.contains('b'));
14204 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
14205 panic!("a failed lookup must stop");
14206 };
14207 assert!(err.contains("bad token"));
14208 }
14209
14210 use super::*;
14211 use crate::run::GateStatus;
14212 use std::collections::BTreeMap;
14213 use std::time::Duration;
14214
14215 fn conductor() -> AgentSpec {
14216 AgentSpec {
14217 id: "conductor".to_owned(),
14218 kind: crate::config::AgentKind::Command,
14219 model: None,
14220 command: vec!["true".to_owned()],
14221 extra_args: Vec::new(),
14222 env: BTreeMap::new(),
14223 prompt_delivery: None,
14224 max_turn_secs: None,
14225 }
14226 }
14227
14228 fn spec(id: &str) -> AgentSpec {
14229 AgentSpec {
14230 id: id.to_owned(),
14231 kind: crate::config::AgentKind::Command,
14232 model: None,
14233 command: vec!["true".to_owned()],
14234 extra_args: Vec::new(),
14235 env: BTreeMap::new(),
14236 prompt_delivery: None,
14237 max_turn_secs: None,
14238 }
14239 }
14240
14241 fn ids(xs: &[&str]) -> BTreeSet<String> {
14242 xs.iter().map(|s| (*s).to_owned()).collect()
14243 }
14244
14245 #[test]
14246 fn pick_successor_skips_an_agent_another_seat_holds() {
14247 let roster = [spec("a"), spec("b"), spec("c")];
14249 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14250 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14251 }
14252
14253 #[test]
14254 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
14255 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
14257 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14258 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14259 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
14260 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
14261 }
14262
14263 #[test]
14264 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
14265 let roster = [spec("a"), spec("b")];
14266 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14267 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14268 let carried = ids(&["b"]);
14269 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
14270 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14271 }
14272
14273 #[test]
14274 fn pick_successor_returns_none_without_any_untried_successor() {
14275 let roster = [spec("a"), spec("b")];
14276 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
14277 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
14278 let carried = ids(&["a"]);
14279 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
14280 }
14281
14282 #[test]
14283 fn pick_successor_rescue_avoids_another_seats_occupant() {
14284 let roster = [spec("a"), spec("b"), spec("c")];
14286 let carried = ids(&["b", "c"]);
14287 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
14288 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14289 }
14290
14291 #[test]
14292 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
14293 let roster = [spec("a"), spec("b"), spec("c")];
14294 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
14295 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14296 }
14297
14298 #[test]
14299 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
14300 let roster = [spec("a"), spec("b"), spec("c")];
14301 let failed = ids(&["a", "b", "c"]);
14302 let mut tried = ids(&["b"]);
14304 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
14305 assert_eq!(first.id, "a");
14306 tried.insert(first.id.clone());
14307 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
14308 assert_eq!(second.id, "c");
14309 tried.insert(second.id.clone());
14310 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
14311 }
14312
14313 #[test]
14314 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
14315 let roster = [spec("a"), spec("b")];
14316 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
14317 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14318 }
14319
14320 #[test]
14321 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
14322 let roster = [spec("a"), spec("b"), spec("c")];
14323 let h = SeatHistory {
14324 failed: ids(&["a"]),
14325 last_ok: Some("b".to_owned()),
14326 last_fail: None,
14327 };
14328 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
14329 let gone = SeatHistory {
14331 last_ok: Some("zzz".to_owned()),
14332 ..h.clone()
14333 };
14334 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
14335 let failed = SeatHistory {
14336 failed: ids(&["a", "b"]),
14337 last_ok: Some("b".to_owned()),
14338 last_fail: None,
14339 };
14340 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
14341 }
14342
14343 #[test]
14344 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
14345 let first = SeatState::new("review-1", "alpha", 7);
14346 let next = handover_seat("review-1", "gamma", 7);
14347 assert_ne!(first.claude_session, next.claude_session);
14348 }
14349
14350 #[test]
14351 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
14352 let mut state = state_with_summary("x", "y");
14353 let a = handover_seat("review-1", "beta", state.next_seat_seed());
14354 let b = handover_seat("review-1", "beta", state.next_seat_seed());
14355 assert_ne!(a.claude_session, b.claude_session);
14356 }
14357
14358 #[test]
14359 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
14360 let roster = [spec("a"), spec("b")];
14361 let h = SeatHistory {
14362 failed: ids(&["a", "b"]),
14363 ..SeatHistory::default()
14364 };
14365 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
14366 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
14367 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
14368 }
14369
14370 #[test]
14377 fn failure_signature_ignores_numbers_and_paths() {
14378 assert_eq!(
14379 failure_signature("exited with Some(2) and no usable output"),
14380 failure_signature("exited with Some(137) and no usable output")
14381 );
14382 assert_eq!(
14383 failure_signature("process exited with no usable output (exit code 2)"),
14384 failure_signature("process exited with no usable output (exit code 137)")
14385 );
14386 for (old, new) in [
14387 (
14388 "exited with Some(2) and no usable output",
14389 "process exited with no usable output (exit code 2)",
14390 ),
14391 (
14392 "exited with None and no usable output",
14393 "process exited with no usable output (exit code unavailable)",
14394 ),
14395 ] {
14396 let previous = FailClass::Other(failure_signature(old));
14397 let current = FailClass::Other(failure_signature(new));
14398 assert_eq!(previous, current, "resumption retains the failure class");
14399 assert!(!should_hand_over(Some(&previous), ¤t));
14400 }
14401 assert_eq!(
14402 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
14403 failure_signature("cannot open /var/x.txt: denied")
14404 );
14405 assert_ne!(failure_signature("boom"), failure_signature("bang"));
14406 }
14407
14408 #[test]
14409 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
14410 let other = FailClass::Other("x".into());
14411 assert!(should_hand_over(None, &FailClass::Quota));
14412 assert!(should_hand_over(Some(&other), &FailClass::Quota));
14413 assert!(should_hand_over(
14414 Some(&FailClass::Timeout),
14415 &FailClass::Timeout
14416 ));
14417 assert!(should_hand_over(None, &other));
14418 assert!(!should_hand_over(Some(&other), &other));
14419 assert!(should_hand_over(
14420 Some(&other),
14421 &FailClass::Other("y".into())
14422 ));
14423 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14425 assert!(should_hand_over(Some(&FailClass::Quota), &other));
14426 }
14427
14428 #[test]
14429 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14430 let a = SeatState::new("judge-1", "alpha", 7);
14431 let b = handover_seat("judge-1", "beta", 7);
14432 assert_ne!(a.claude_session, b.claude_session);
14433 assert_eq!(b.turns, 0);
14434 }
14435
14436 #[test]
14437 fn a_timeout_is_classified_apart_from_other_failures() {
14438 assert_eq!(
14439 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14440 Some(FailClass::Timeout)
14441 );
14442 assert!(matches!(
14443 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14444 Some(FailClass::Other(_))
14445 ));
14446 }
14447
14448 #[test]
14449 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14450 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14451 let tried = BTreeSet::from(["beta".to_owned()]);
14452 let next = next_untried_in_roster(&roster, 1, &tried);
14455 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14456 }
14457
14458 #[test]
14459 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14460 let roster = vec![spec("alpha"), spec("beta")];
14461 let tried = BTreeSet::from(["beta".to_owned()]);
14462 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14466 }
14467
14468 #[test]
14469 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14470 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14471 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14472 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14476 }
14477
14478 #[test]
14479 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14480 let roster = vec![spec("a"), spec("a"), spec("b")];
14481 let tried = BTreeSet::from(["a".to_owned()]);
14482 let next = next_untried_in_roster(&roster, 0, &tried);
14483 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14484 }
14485
14486 #[test]
14487 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14488 let roster = vec![spec("a"), spec("b")];
14489 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14490 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14491 }
14492
14493 #[test]
14494 fn remove_if_empty_only_ever_takes_a_bare_directory() {
14495 let dir = tempfile::tempdir().unwrap();
14496 let bay = dir.path().join("ffff");
14497
14498 remove_if_empty(&bay);
14500 assert!(!bay.exists());
14501
14502 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14505 remove_if_empty(&bay);
14506 assert!(bay.exists(), "non-empty directory must survive");
14507
14508 std::fs::remove_dir(bay.join("cand-A")).unwrap();
14510 remove_if_empty(&bay);
14511 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14512 }
14513
14514 #[test]
14523 fn a_full_panel_that_found_nothing_is_clean() {
14524 assert!(round_is_clean(
14525 0,
14526 true,
14527 2,
14528 2,
14529 0,
14530 IncompleteReviewPolicy::Block
14531 ));
14532 }
14533
14534 #[test]
14535 fn a_missing_seat_is_never_clean_under_the_default_policy() {
14536 assert!(!round_is_clean(
14537 0,
14538 true,
14539 1,
14540 2,
14541 0,
14542 IncompleteReviewPolicy::Block
14543 ));
14544 }
14545
14546 #[test]
14547 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14548 assert!(!round_is_clean(
14549 1,
14550 true,
14551 1,
14552 2,
14553 0,
14554 IncompleteReviewPolicy::Warn
14555 ));
14556 }
14557
14558 #[test]
14559 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14560 assert!(round_is_clean(
14561 0,
14562 true,
14563 1,
14564 2,
14565 0,
14566 IncompleteReviewPolicy::Warn
14567 ));
14568 }
14569
14570 #[test]
14571 fn a_full_panel_with_an_open_finding_is_not_clean() {
14572 assert!(!round_is_clean(
14573 1,
14574 true,
14575 2,
14576 2,
14577 0,
14578 IncompleteReviewPolicy::Block
14579 ));
14580 }
14581
14582 #[test]
14583 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14584 assert!(!round_is_clean(
14585 0,
14586 false,
14587 2,
14588 2,
14589 0,
14590 IncompleteReviewPolicy::Block
14591 ));
14592 }
14593
14594 #[test]
14601 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14602 assert!(round_is_clean(
14605 0,
14606 true,
14607 1,
14608 2,
14609 1,
14610 IncompleteReviewPolicy::Block
14611 ));
14612 }
14613
14614 #[test]
14615 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14616 assert!(!round_is_clean(
14619 0,
14620 true,
14621 1,
14622 2,
14623 0,
14624 IncompleteReviewPolicy::Block
14625 ));
14626 }
14627
14628 #[test]
14629 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14630 assert!(!round_is_clean(
14631 1,
14632 true,
14633 1,
14634 2,
14635 1,
14636 IncompleteReviewPolicy::Block
14637 ));
14638 assert!(!round_is_clean(
14639 0,
14640 false,
14641 1,
14642 2,
14643 1,
14644 IncompleteReviewPolicy::Block
14645 ));
14646 }
14647
14648 #[test]
14649 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14650 assert!(!round_is_clean(
14654 0,
14655 true,
14656 0,
14657 2,
14658 2,
14659 IncompleteReviewPolicy::Block
14660 ));
14661 }
14662
14663 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14664 CommandOutcome {
14665 command: "test".to_owned(),
14666 code,
14667 output_tail: String::new(),
14668 duration_ms: 0,
14669 resource_blocked,
14670 }
14671 }
14672
14673 #[cfg(unix)]
14674 #[tokio::test]
14675 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14676 ask_test_home();
14677 let temp = tempfile::tempdir().unwrap();
14678 let mut runner = runner_at(RunStatus::Gating);
14679 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14680 let commands = vec![
14681 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14682 "touch must-not-run".to_owned(),
14683 ];
14684 let (outcomes, retried, pids) = run_e2e_with_retry(
14685 &mut runner.state,
14686 &shell,
14687 &commands,
14688 temp.path(),
14689 Duration::from_secs(5),
14690 "fixture",
14691 None,
14692 )
14693 .await;
14694 assert_eq!(outcomes.len(), 1);
14695 assert!(outcomes[0].resource_blocked);
14696 assert!(!outcomes[0].build_failed());
14697 assert!(!retried);
14698 assert!(pids.is_empty());
14699 assert!(!temp.path().join("must-not-run").exists());
14700 assert_eq!(
14701 runner.state.resource_block.as_ref().unwrap().kind,
14702 "incremental cache admission"
14703 );
14704 }
14705
14706 #[test]
14707 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14708 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14709 assert!(
14710 !verify_inconclusive(&[outcome(Some(1), false)]),
14711 "an ordinary failure is still evidence about the patch"
14712 );
14713 assert!(verify_inconclusive(&[outcome(None, true)]));
14714 assert!(
14715 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14716 "one inconclusive outcome taints the whole batch"
14717 );
14718 assert!(!verify_inconclusive(&[]));
14719 }
14720
14721 #[tokio::test]
14722 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14723 let calls = std::sync::atomic::AtomicUsize::new(0);
14727 let started = Instant::now();
14728 wait_for_pids_with(
14729 &[123],
14730 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14731 Duration::from_millis(5),
14732 Duration::from_secs(5),
14733 )
14734 .await;
14735 assert!(
14736 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14737 "must keep checking rather than deciding on the first answer"
14738 );
14739 assert!(
14740 started.elapsed() < Duration::from_secs(1),
14741 "must return the moment it is confirmed dead, not wait out the ceiling"
14742 );
14743 }
14744
14745 #[tokio::test]
14746 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14747 let started = Instant::now();
14748 wait_for_pids_with(
14749 &[123],
14750 |_| true, Duration::from_millis(5),
14752 Duration::from_millis(30),
14753 )
14754 .await;
14755 let elapsed = started.elapsed();
14756 assert!(
14757 elapsed >= Duration::from_millis(30),
14758 "must not give up before its own ceiling: {elapsed:?}"
14759 );
14760 assert!(
14761 elapsed < Duration::from_secs(1),
14762 "must not wait past its own ceiling either: {elapsed:?}"
14763 );
14764 }
14765
14766 #[tokio::test]
14767 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14768 let started = Instant::now();
14769 wait_for_pids_with(
14770 &[],
14771 |_| true,
14772 Duration::from_secs(5),
14773 Duration::from_secs(5),
14774 )
14775 .await;
14776 assert!(
14777 started.elapsed() < Duration::from_millis(200),
14778 "an empty pid list has nothing to confirm"
14779 );
14780 }
14781
14782 fn review_round(
14788 clean: bool,
14789 blocking: usize,
14790 answered: usize,
14791 expected: usize,
14792 progressed: bool,
14793 e2e_ok: bool,
14794 ) -> ReviewRound {
14795 ReviewRound {
14796 round: 1,
14797 head: "h".to_owned(),
14798 verified_head: None,
14799 verified_at: None,
14800 reviews: Vec::new(),
14801 e2e: vec![CommandOutcome {
14802 command: "test".to_owned(),
14803 code: Some(if e2e_ok { 0 } else { 1 }),
14804 output_tail: String::new(),
14805 duration_ms: 0,
14806 resource_blocked: false,
14807 }],
14808 verify_retried: false,
14809 e2e_deferred: false,
14810 e2e_defer_reason: None,
14811 fix: None,
14812 blocking,
14813 answered,
14814 expected,
14815 clean,
14816 progressed,
14817 vote_split: false,
14818 reconsideration: Vec::new(),
14819 verdict: None,
14820 }
14821 }
14822
14823 #[test]
14824 fn review_conclusion_is_none_when_nothing_has_run() {
14825 assert_eq!(review_conclusion(&[], 3), None);
14826 }
14827
14828 #[test]
14829 fn review_conclusion_is_none_while_rounds_remain() {
14830 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14831 assert_eq!(review_conclusion(&rounds, 3), None);
14832 }
14833
14834 #[test]
14835 fn review_conclusion_is_gating_once_a_round_is_clean() {
14836 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14837 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14838 }
14839
14840 #[test]
14841 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14842 let rounds = vec![
14843 review_round(false, 1, 2, 2, true, true),
14844 review_round(false, 1, 2, 2, true, true),
14845 ];
14846 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14847 }
14848
14849 #[test]
14850 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14851 let rounds = vec![
14852 review_round(false, 1, 2, 2, true, true),
14853 review_round(false, 1, 2, 2, true, false),
14854 ];
14855 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14856 }
14857
14858 #[test]
14859 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14860 let mut blocked = review_round(false, 1, 2, 2, true, false);
14867 blocked.e2e[0].resource_blocked = true;
14868 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14869 assert_eq!(review_conclusion(&rounds, 2), None);
14870 }
14871
14872 #[test]
14873 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14874 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14876 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14877 }
14878
14879 #[test]
14880 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14881 let rounds = vec![
14882 review_round(false, 1, 2, 2, false, true),
14883 review_round(false, 1, 2, 2, false, true),
14884 ];
14885 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14886 }
14887
14888 fn secs(n: u64) -> Duration {
14889 Duration::from_secs(n)
14890 }
14891
14892 #[tokio::test]
14895 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14896 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14897 if std::env::var_os(CHILD).is_none() {
14898 let output = std::process::Command::new(std::env::current_exe().unwrap())
14902 .args([
14903 "--exact",
14904 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14905 "--nocapture",
14906 ])
14907 .env(CHILD, "1")
14908 .env_remove("MAGI_WORKER_HOME")
14909 .env_remove("MAGI_WORKER_ID")
14910 .output()
14911 .unwrap();
14912 assert!(
14913 output.status.success()
14914 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14915 "{}\n{}",
14916 String::from_utf8_lossy(&output.stdout),
14917 String::from_utf8_lossy(&output.stderr)
14918 );
14919 return;
14920 }
14921 ask_test_home();
14922 let tmp = tempfile::tempdir().unwrap();
14923 let repo = tmp.path().join("repo");
14924 std::fs::create_dir_all(&repo).unwrap();
14925 init_repo(&repo);
14926 let g = |args: &[&str]| {
14927 let out = std::process::Command::new("git")
14928 .args(args)
14929 .current_dir(&repo)
14930 .quiet()
14931 .output()
14932 .unwrap();
14933 assert!(
14934 out.status.success(),
14935 "{}",
14936 String::from_utf8_lossy(&out.stderr)
14937 );
14938 };
14939 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14940 g(&["checkout", "-b", "historical"]);
14941 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14942 g(&["add", "."]);
14943 g(&["commit", "-m", "old alternative"]);
14944 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14945 g(&["checkout", "-b", "magi/task-selected", &base]);
14946 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14947 g(&["add", "."]);
14948 g(&["commit", "-m", "measurement infrastructure"]);
14949 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14950 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14951 g(&["add", "."]);
14952 g(&["commit", "-m", "dependent implementation"]);
14953 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14954 g(&["checkout", "main"]);
14955 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14956 g(&["add", "."]);
14957 g(&["commit", "-m", "unrelated base change"]);
14958 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14959 let origin = tmp.path().join("origin.git");
14960 g(&[
14961 "clone",
14962 "--bare",
14963 repo.to_str().unwrap(),
14964 origin.to_str().unwrap(),
14965 ]);
14966 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14967 g(&["fetch", "origin"]);
14968 let selection = crate::pr_recovery::Selection {
14969 url: "https://example.test/project/pull/1".into(),
14970 branch: "magi/task-selected".into(),
14971 head: head.clone(),
14972 source_head: None,
14973 source_run: None,
14974 source_proof: None,
14975 disposition: Some(crate::pr_recovery::Disposition::Review),
14976 instruction: "preserve measured behavior".into(),
14977 approved_at: jiff::Timestamp::now(),
14978 };
14979 let mut template = runner_at(RunStatus::Reviewing);
14980 template.state.repo = repo.clone();
14981 template.state.base_commit = current.clone();
14982 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14983 template.state.origin = Some(crate::run::Origin::queue("selected"));
14984 template.state.pr_head = Some(selection.branch.clone());
14985 template.state.pr_recovery = Some(selection.clone());
14986 let mut runner =
14987 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14988 .await
14989 .unwrap();
14990 let wt = runner.state.winner().unwrap().worktree.clone();
14991 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14992 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14993 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14994 assert!(runner.state.seeds.is_empty());
14995 assert_eq!(runner.state.instruction, "preserve measured behavior");
14996 runner.sync_to_base().await.unwrap();
14997 assert_eq!(
14998 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14999 "prerequisite"
15000 );
15001 assert_eq!(
15002 std::fs::read_to_string(wt.join("implementation")).unwrap(),
15003 "measured implementation"
15004 );
15005 assert_eq!(
15006 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
15007 "current main change"
15008 );
15009 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
15010 assert_eq!(
15011 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
15012 head
15013 );
15014
15015 let mut partial = runner_at(RunStatus::Reviewing);
15017 partial.state.repo = repo.clone();
15018 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
15019 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
15020 run: Some("exit 1".into()),
15021 ..Default::default()
15022 }];
15023 let selection = runner.state.pr_recovery.clone().unwrap();
15024 partial.state.pr_recovery = Some(selection.clone());
15025 partial.state.pr_head = Some(selection.branch.clone());
15026 partial.state.origin = Some(crate::run::Origin::queue("selected"));
15027 let id = partial.state.id.clone();
15028 let error = match Runner::open_selected(
15029 &repo,
15030 partial.state,
15031 partial.roles,
15032 1,
15033 runner.state.base_commit.clone(),
15034 selection,
15035 )
15036 .await
15037 {
15038 Ok(_) => panic!("setup must fail"),
15039 Err(e) => e,
15040 };
15041 assert_eq!(
15042 error
15043 .downcast_ref::<crate::run::StartupFailure>()
15044 .unwrap()
15045 .run,
15046 id
15047 );
15048 let failed = RunState::load(&id).unwrap();
15049 assert_eq!(failed.status, RunStatus::Failed);
15050 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
15051 assert!(Runner::resume(&id).is_err());
15052 runner.state.status = RunStatus::Ready;
15053 runner.state.driver_exited = true;
15054 runner.state.save().unwrap();
15055 let mut retry = runner_at(RunStatus::Reviewing);
15056 retry.state.repo = repo.clone();
15057 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
15058 let mut reselected = runner.state.pr_recovery.clone().unwrap();
15059 reselected.approved_at = jiff::Timestamp::now();
15060 retry.state.origin = Some(crate::run::Origin::queue("selected"));
15061 retry.state.pr_head = Some(reselected.branch.clone());
15062 retry.state.pr_recovery = Some(reselected.clone());
15063 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
15064 .await
15065 .unwrap();
15066 let restarted = Runner::open_selected(
15067 &repo,
15068 retry.state,
15069 retry.roles,
15070 1,
15071 runner.state.base_commit.clone(),
15072 reselected,
15073 )
15074 .await
15075 .unwrap();
15076 assert_eq!(
15077 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
15078 .await
15079 .unwrap(),
15080 head
15081 );
15082 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
15083 assert_eq!(
15084 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
15085 head
15086 );
15087 }
15088
15089 #[tokio::test]
15090 #[cfg(unix)]
15091 async fn fallback_and_interrupted_restoration_require_coordination() {
15092 use std::os::fd::AsRawFd as _;
15093 ask_test_home();
15094 let tmp = tempfile::tempdir().unwrap();
15095 let mut runner = registered_runner(tmp.path()).await;
15096 runner.state.config.graph.timeout_resource_wait = 1;
15097 let winner = runner.state.candidates[0].clone();
15098 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15099 let leftover = winner.worktree.join("leftover");
15100 std::fs::write(&leftover, "untrusted edits").unwrap();
15101 let next = tmp.path().join("next-fixer");
15102 let mut failed = spec("first");
15103 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
15104 let mut successor = spec("second");
15105 successor.command = vec![
15106 "sh".into(),
15107 "-c".into(),
15108 format!("touch '{}'; echo done", next.display()),
15109 ];
15110 let attempts = vec![
15111 (failed.clone(), "first".into()),
15112 (successor.clone(), "second".into()),
15113 ];
15114 let lock_path = runner.state.repo.join(".git/updater.lock");
15115 let external = std::fs::File::create(&lock_path).unwrap();
15116 assert_eq!(
15117 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15118 0
15119 );
15120 let artifacts = tmp.path().join("artifacts");
15121 let build = |spec, seat| SeatJob {
15122 spec,
15123 seat,
15124 cwd: winner.worktree.clone(),
15125 prompt: "fixture".into(),
15126 timeout: Duration::from_secs(10),
15127 allow_write: true,
15128 sessions: false,
15129 artifacts: artifacts.clone(),
15130 stem: "fix".into(),
15131 handover: None,
15132 };
15133 let (_, _, out) = runner
15134 .ask_fixer_among(
15135 attempts.clone(),
15136 "graft",
15137 None,
15138 Some((&winner.worktree, &head)),
15139 build,
15140 )
15141 .await;
15142 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
15143 assert!(leftover.exists(), "contention must prevent clean");
15144 assert!(!next.exists());
15145 assert!(runner.state.handovers.is_empty());
15146 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15148 from_run: "earlier".into(),
15149 generation: 1,
15150 mode: crate::recovery::Mode::VerifyRepair,
15151 reasons: Vec::new(),
15152 reviewed_head: None,
15153 rounds: Vec::new(),
15154 open_findings: Vec::new(),
15155 evidence: None,
15156 criteria_inherited: true,
15157 repair: crate::recovery::Repair::Started {
15158 before: head.clone(),
15159 },
15160 });
15161 assert!(
15162 runner
15163 .recovery_repair(&winner)
15164 .await
15165 .unwrap_err()
15166 .is::<crate::repo_lock::ResourceBlocked>()
15167 );
15168 assert!(matches!(
15169 runner.state.recovery.as_ref().unwrap().repair,
15170 crate::recovery::Repair::Started { .. }
15171 ));
15172 assert!(leftover.exists());
15173 drop(external);
15174 runner.recovery_repair(&winner).await.unwrap();
15175 assert!(!leftover.exists());
15176 assert!(matches!(
15177 runner.state.recovery.as_ref().unwrap().repair,
15178 crate::recovery::Repair::Done { .. }
15179 ));
15180 std::fs::remove_file(&lock_path).unwrap();
15182 std::fs::create_dir(&lock_path).unwrap();
15183 std::fs::write(&leftover, "untrusted edits").unwrap();
15184 let (_, _, out) = runner
15185 .ask_fixer_among(
15186 attempts,
15187 "graft",
15188 None,
15189 Some((&winner.worktree, &head)),
15190 build,
15191 )
15192 .await;
15193 assert!(matches!(out, AgentOutcome::Failed(_)));
15194 assert!(leftover.exists());
15195 assert!(!next.exists());
15196 assert_eq!(
15197 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15198 head
15199 );
15200 }
15201
15202 #[cfg(unix)]
15203 async fn registered_runner(root: &Path) -> Runner {
15204 repository_runner(root, crate::config::test_coordination()).await
15205 }
15206
15207 async fn repository_runner(
15208 root: &Path,
15209 coordination: Option<crate::config::Coordination>,
15210 ) -> Runner {
15211 ask_test_home();
15212 let repo = root.join("repo");
15213 std::fs::create_dir_all(&repo).unwrap();
15214 init_repo(&repo);
15215 if coordination.is_some() {
15216 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15217 }
15218 let mut runner = runner_at(RunStatus::Prep);
15219 runner.state.repo = repo.clone();
15220 runner.state.config.coordination = coordination;
15221 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15222 runner.state.config.graph.worktree_root = Some(root.join("wt"));
15223 runner.state.config.graph.advise = false;
15224 runner.roles.implementers = vec![spec("implement")];
15225 runner.roles.judges = vec![spec("judge")];
15226 runner.prep().await.unwrap();
15227 runner
15228 }
15229
15230 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
15231
15232 #[tokio::test]
15233 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
15234 let tmp = tempfile::tempdir().unwrap();
15235 let mut runner = repository_runner(tmp.path(), None).await;
15236 let prompts = runner.state.config.prompts.clone();
15237 let run = runner.state.id.clone();
15238 let ctx = WaveCtx {
15239 carry_seats: false,
15240 pauses: None,
15241 run: &run,
15242 node: "advise",
15243 prompts: &prompts,
15244 cache: None,
15245 round: None,
15246 };
15247 let jobs: Vec<_> = ["alpha", "beta"]
15248 .into_iter()
15249 .map(|id| SeatJob {
15250 spec: scripted(id, AUTH_OUTAGE),
15251 seat: SeatState::new(id, id, 7),
15252 prompt: "propose".into(),
15253 cwd: runner.state.candidates[0].worktree.clone(),
15254 timeout: Duration::from_secs(10),
15255 allow_write: false,
15256 sessions: false,
15257 artifacts: agent::artifacts_dir(&runner.state.dir()),
15258 stem: id.into(),
15259 handover: None,
15260 })
15261 .collect();
15262 let mut losses = Vec::new();
15263 let results = ask_wave_with(
15264 jobs.clone(),
15265 runner.sem.clone(),
15266 2,
15267 &[],
15268 &ctx,
15269 &mut losses,
15270 &mut runner.state,
15271 &|_: &str| Ok(()),
15272 )
15273 .await;
15274 assert!(
15275 results
15276 .iter()
15277 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
15278 );
15279 assert_eq!(runner.state.quota.len(), 2);
15280 assert!(
15281 runner
15282 .state
15283 .quota
15284 .iter()
15285 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
15286 );
15287 assert!(!runner.state.advise_attempted);
15288 runner.stop_for_resource().unwrap();
15289 assert_eq!(runner.state.status, RunStatus::Blocked);
15290 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
15291 assert_eq!(
15292 runner.state.resource_block.as_ref().unwrap().kind,
15293 "agent authentication"
15294 );
15295 runner.state.resource_block = None;
15298 runner.state.quota.clear();
15299 let mut mixed = jobs;
15300 mixed[1].spec.command = vec!["false".into()];
15301 let results = ask_wave_with(
15302 mixed,
15303 runner.sem.clone(),
15304 0,
15305 &[],
15306 &ctx,
15307 &mut losses,
15308 &mut runner.state,
15309 &|_: &str| Ok(()),
15310 )
15311 .await;
15312 assert!(results.iter().all(|(_, res, _)| res.is_err()));
15313 assert!(runner.state.resource_block.is_none());
15314 assert_eq!(losses.len(), 1);
15315 }
15316
15317 #[tokio::test]
15318 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
15319 let tmp = tempfile::tempdir().unwrap();
15320 let mut runner = repository_runner(tmp.path(), None).await;
15321 let implementer = scripted("implement", AUTH_OUTAGE);
15322 runner.roles.implementers = vec![implementer.clone()];
15323 runner.state.config.agents = vec![implementer];
15324 runner.implement().await.unwrap();
15325 assert_eq!(runner.state.status, RunStatus::Blocked);
15326 assert_eq!(runner.state.quota.len(), 1);
15327 assert_eq!(
15328 runner.state.quota[0].cause,
15329 agent::QuotaCause::Authentication
15330 );
15331 let candidate = &runner.state.candidates[0];
15332 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
15333 assert_eq!(runner.state.seats["impl-A"].turns, 1);
15334 }
15335
15336 #[tokio::test]
15337 #[cfg(unix)]
15338 async fn parked_implementer_persists_completed_work() {
15339 for registered in [false, true] {
15340 let tmp = tempfile::tempdir().unwrap();
15341 let mut runner = registered_runner(tmp.path()).await;
15342 if !registered {
15343 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
15344 }
15345 let marker = tmp.path().join("started");
15346 let release = tmp.path().join("release");
15347 let mut implementer = spec("implement");
15348 implementer.command = vec![
15349 "sh".into(),
15350 "-c".into(),
15351 format!(
15352 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
15353 marker.display(),
15354 release.display()
15355 ),
15356 ];
15357 runner.roles.implementers = vec![implementer.clone()];
15358 runner.state.config.agents = vec![implementer];
15359 let pause = runner.pause.clone();
15360 let (result, ()) = tokio::join!(runner.implement(), async {
15361 tokio::time::timeout(Duration::from_secs(10), async {
15362 while !marker.exists() {
15363 tokio::time::sleep(Duration::from_millis(10)).await;
15364 }
15365 })
15366 .await
15367 .unwrap();
15368 pause.park();
15369 std::fs::write(&release, "go").unwrap();
15370 });
15371 result.unwrap();
15372 let candidate = &runner.state.candidates[0];
15373 assert!(candidate.commits > 0, "{candidate:?}");
15374 assert!(candidate.summary.contains("saved-result"));
15375 assert_eq!(candidate.files, 1);
15376 let before = candidate.commits;
15377 runner.implement().await.unwrap();
15378 assert_eq!(runner.state.candidates[0].commits, before);
15379 }
15380 }
15381
15382 #[tokio::test]
15383 #[cfg(unix)]
15384 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
15385 let tmp = tempfile::tempdir().unwrap();
15386 let mut runner = registered_runner(tmp.path()).await;
15387 let winner = runner.state.candidates[0].clone();
15388 let cache = tmp.path().join("cache");
15389 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
15390 runner.state.config.graph.timeout_resource_wait = 0;
15391 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
15392 let crate::cache::AcquireOutcome::Acquired(held) =
15393 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
15394 else {
15395 panic!("cache must be free")
15396 };
15397 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15398 from_run: "earlier".into(),
15399 generation: 1,
15400 mode: crate::recovery::Mode::VerifyRepair,
15401 reasons: Vec::new(),
15402 reviewed_head: None,
15403 rounds: Vec::new(),
15404 open_findings: Vec::new(),
15405 evidence: None,
15406 criteria_inherited: true,
15407 repair: crate::recovery::Repair::NotStarted,
15408 });
15409 runner.recovery_repair(&winner).await.unwrap();
15410 assert!(matches!(
15411 runner.state.recovery.as_ref().unwrap().repair,
15412 crate::recovery::Repair::WaitingForRestore { .. }
15413 ));
15414 drop(held);
15415 runner.state.resource_block = None;
15416 runner.state.config.graph.timeout_resource_wait = 1;
15417 runner.recovery_repair(&winner).await.unwrap();
15418 assert!(matches!(
15419 runner.state.recovery.as_ref().unwrap().repair,
15420 crate::recovery::Repair::Done { .. }
15421 ));
15422 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15423 }
15424
15425 #[tokio::test]
15426 #[cfg(unix)]
15427 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15428 use std::os::fd::AsRawFd as _;
15429 for graft in [false, true] {
15430 let tmp = tempfile::tempdir().unwrap();
15431 let mut runner = registered_runner(tmp.path()).await;
15432 runner.state.config.graph.timeout_resource_wait = 1;
15433 let winner = runner.state.candidates[0].clone();
15434 let marker = tmp.path().join("fixer-started");
15435 let mut fixer = spec("fixer");
15436 fixer.command = vec![
15437 "sh".into(),
15438 "-c".into(),
15439 format!("echo started >> '{}'; echo done", marker.display()),
15440 ];
15441 runner.roles.fixer = Some(vec![fixer]);
15442 if graft {
15443 let mut loser = winner.clone();
15444 loser.label = 'B';
15445 loser.index = 1;
15446 runner.state.candidates.push(loser);
15447 runner.state.config.graph.graft_runner_up = true;
15448 runner.state.tally = Some(Tally {
15449 first_choice: BTreeMap::new(),
15450 borda: BTreeMap::new(),
15451 winner: winner.label,
15452 rankings: 1,
15453 unanimous_initial: true,
15454 deliberated: false,
15455 changed_votes: 0,
15456 unanimous_final: true,
15457 tie_break: None,
15458 judges: 1,
15459 present: 1,
15460 quorum: 1,
15461 met_quorum: true,
15462 uncontested: None,
15463 });
15464 runner.state.judgements.push(crate::run::Judgement {
15465 judge: 1,
15466 seat: "judge-1".into(),
15467 agent: "judge".into(),
15468 ranking: vec![winner.label, 'B'],
15469 reasons: BTreeMap::new(),
15470 confidence: None,
15471 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15472 order: vec![0, 1],
15473 failed: None,
15474 duration_ms: 0,
15475 });
15476 } else {
15477 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15478 from_run: "earlier".into(),
15479 generation: 1,
15480 mode: crate::recovery::Mode::VerifyRepair,
15481 reasons: Vec::new(),
15482 reviewed_head: None,
15483 rounds: Vec::new(),
15484 open_findings: Vec::new(),
15485 evidence: None,
15486 criteria_inherited: true,
15487 repair: crate::recovery::Repair::NotStarted,
15488 });
15489 }
15490 let reader = crate::repo_lock::SeatLease::for_checkout(
15491 &winner.worktree,
15492 false,
15493 crate::config::test_coordination().as_ref(),
15494 )
15495 .await
15496 .unwrap();
15497 let pause = runner.pause.clone();
15498 let (result, ()) = tokio::join!(
15499 async {
15500 if graft {
15501 runner.graft().await
15502 } else {
15503 runner.recovery_repair(&winner).await
15504 }
15505 },
15506 async {
15507 tokio::time::sleep(Duration::from_millis(100)).await;
15508 pause.park();
15509 }
15510 );
15511 result.unwrap();
15512 assert!(!marker.exists(), "a blocked seat must not spawn");
15513 runner.state = RunState::load(&runner.state.id).unwrap();
15515 if graft {
15516 assert_eq!(
15517 runner.state.graft.as_ref().unwrap().outcome,
15518 GraftOutcome::WaitingForRestore
15519 );
15520 } else {
15521 assert!(matches!(
15522 runner.state.recovery.as_ref().unwrap().repair,
15523 crate::recovery::Repair::WaitingForRestore { .. }
15524 ));
15525 }
15526 drop(reader);
15527 runner.pause = Pause::new();
15528 runner.state.resource_block = None;
15529 let external =
15531 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15532 assert_eq!(
15533 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15534 0
15535 );
15536 let leftover = winner.worktree.join("leftover");
15537 std::fs::write(&leftover, "earlier fallback edits").unwrap();
15538 let result = if graft {
15539 runner.graft().await
15540 } else {
15541 runner.recovery_repair(&winner).await
15542 };
15543 assert!(
15544 result
15545 .unwrap_err()
15546 .is::<crate::repo_lock::ResourceBlocked>()
15547 );
15548 assert!(leftover.exists());
15549 assert!(!marker.exists());
15550 drop(external);
15551 runner.state.resource_block = None;
15552 let result = if graft {
15553 runner.graft().await
15554 } else {
15555 runner.recovery_repair(&winner).await
15556 };
15557 result.unwrap();
15558 assert!(!leftover.exists(), "restore must precede retry");
15559 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15560 if graft {
15561 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15562 } else {
15563 assert!(matches!(
15564 runner.state.recovery.as_ref().unwrap().repair,
15565 crate::recovery::Repair::Done { .. }
15566 ));
15567 }
15568 }
15569 }
15570
15571 #[tokio::test]
15572 #[cfg(unix)]
15573 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15574 use std::os::fd::AsRawFd as _;
15575 let tmp = tempfile::tempdir().unwrap();
15576 let mut runner = registered_runner(tmp.path()).await;
15577 runner.state.config.graph.timeout_resource_wait = 1;
15578 let winner = runner.state.candidates[0].clone();
15579 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15580 runner.state.tally = Some(Tally {
15581 first_choice: BTreeMap::new(),
15582 borda: BTreeMap::new(),
15583 winner: winner.label,
15584 rankings: 0,
15585 unanimous_initial: false,
15586 deliberated: false,
15587 changed_votes: 0,
15588 unanimous_final: false,
15589 tie_break: None,
15590 judges: 0,
15591 present: 0,
15592 quorum: 0,
15593 met_quorum: true,
15594 uncontested: Some("fixture".into()),
15595 });
15596 runner.state.graft = Some(GraftRecord {
15597 pre: pre.clone(),
15598 post: None,
15599 lines: None,
15600 limit: None,
15601 agent: None,
15602 outcome: GraftOutcome::Running,
15603 requested: Vec::new(),
15604 adoption: None,
15605 adoption_error: None,
15606 });
15607 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15608 git::commit_all(&winner.worktree, "unfinished graft")
15609 .await
15610 .unwrap();
15611 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15612 let leftover = winner.worktree.join("leftover");
15613 std::fs::write(&leftover, "untrusted edits").unwrap();
15614 let path = runner.state.repo.join(".git/updater.lock");
15615 let external = std::fs::File::create(&path).unwrap();
15616 assert_eq!(
15617 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15618 0
15619 );
15620 assert!(
15621 runner
15622 .graft()
15623 .await
15624 .unwrap_err()
15625 .is::<crate::repo_lock::ResourceBlocked>()
15626 );
15627 assert!(matches!(
15628 runner.state.graft.as_ref().unwrap().outcome,
15629 GraftOutcome::Running
15630 ));
15631 assert_eq!(
15632 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15633 unfinished
15634 );
15635 assert!(leftover.exists());
15636 drop(external);
15637 std::fs::remove_file(&path).unwrap();
15638 std::fs::create_dir(&path).unwrap();
15639 assert!(runner.graft().await.is_err());
15640 assert!(matches!(
15641 runner.state.graft.as_ref().unwrap().outcome,
15642 GraftOutcome::Running
15643 ));
15644 assert_eq!(
15645 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15646 unfinished
15647 );
15648 assert!(leftover.exists());
15649 std::fs::remove_dir(&path).unwrap();
15650 runner.graft().await.unwrap();
15651 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15652 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15653 assert!(!leftover.exists());
15654 }
15655
15656 #[tokio::test]
15657 #[cfg(unix)]
15658 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15659 use std::os::fd::AsRawFd as _;
15660 let tmp = tempfile::tempdir().unwrap();
15661 let mut runner = registered_runner(tmp.path()).await;
15662 runner.state.prep_complete = Some(false);
15663 runner.state.config.graph.timeout_resource_wait = 1;
15664 runner.state.save().unwrap();
15665 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15666 assert_eq!(
15667 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15668 0
15669 );
15670 runner.execute().await.unwrap();
15671 let disk = RunState::load(&runner.state.id).unwrap();
15672 assert!(disk.driver_exited);
15673 assert_eq!(disk.driver_pid, None);
15674 assert_eq!(runner.state.status, RunStatus::Blocked);
15675 assert_eq!(
15676 runner.state.resource_block.as_ref().unwrap().kind,
15677 "repository coordination"
15678 );
15679 assert!(runner.state.prep_failure.is_none());
15680 assert!(runner.state.seats.is_empty());
15681 runner.pause.park();
15682 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15685 .await
15686 .unwrap()
15687 .unwrap();
15688 assert!(runner.state.parked);
15689 assert!(runner.state.prep_failure.is_none());
15690 let disk = RunState::load(&runner.state.id).unwrap();
15691 assert!(disk.driver_exited);
15692 assert_eq!(disk.driver_pid, None);
15693 }
15694
15695 #[tokio::test]
15696 #[cfg(unix)]
15697 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15698 use std::os::fd::AsRawFd as _;
15699 let tmp = tempfile::tempdir().unwrap();
15700 let mut runner = registered_runner(tmp.path()).await;
15701 runner.state.prep_complete = Some(false);
15702 runner.state.config.graph.timeout_resource_wait = 30;
15703 runner.state.save().unwrap();
15704 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15705 assert_eq!(
15706 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15707 0
15708 );
15709 assert!(
15710 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15711 .await
15712 .is_err()
15713 );
15714 let disk = RunState::load(&runner.state.id).unwrap();
15715 assert!(disk.driver_exited);
15716 assert_eq!(disk.driver_pid, None);
15717 assert!(disk.status.resumable());
15718 assert!(runner.state.driver_exited);
15719 assert_eq!(runner.state.driver_pid, None);
15720 runner.state.save().unwrap();
15721 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15722 }
15723
15724 #[tokio::test]
15725 #[cfg(unix)]
15726 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15727 let tmp = tempfile::tempdir().unwrap();
15728 let mut runner = registered_runner(tmp.path()).await;
15729 let winner = runner.state.candidates[0].clone();
15730 let cache = tmp.path().join("cache");
15731 let started = tmp.path().join("pre-gate-started");
15732 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15733 runner.state.config.verify.gate =
15734 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15735 let home = crate::run::home();
15736 let owner =
15737 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15738 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15739 crate::cache::AcquireOutcome::Acquired(g) => g,
15740 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15741 };
15742 let mut gate = Box::pin(runner.run_gate(&winner));
15743 assert!(
15744 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15745 .await
15746 .is_err()
15747 );
15748 assert!(!started.exists(), "pre-gate must wait for the cache too");
15749 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15753 let editor = crate::repo_lock::SeatLease::checkout_with(
15754 &winner.worktree,
15755 true,
15756 crate::config::test_coordination().as_ref(),
15757 &wait,
15758 )
15759 .await
15760 .unwrap();
15761 drop(editor);
15762 guard.release();
15763 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15764 .await
15765 .unwrap()
15766 .unwrap();
15767 drop(gate);
15768 assert!(outcomes.iter().all(CommandOutcome::ok));
15769 assert!(started.exists());
15770 }
15771
15772 #[tokio::test]
15773 #[cfg(unix)]
15774 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15775 use std::os::fd::AsRawFd as _;
15776
15777 ask_test_home();
15778 let tmp = tempfile::tempdir().unwrap();
15779 let repo = tmp.path().join("repo");
15780 std::fs::create_dir_all(&repo).unwrap();
15781 init_repo(&repo);
15782 let common = repo.join(".git");
15783 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15784 let mut runner = runner_at(RunStatus::Prep);
15785 runner.state.repo = repo.clone();
15786 runner.state.config.coordination = crate::config::test_coordination();
15787 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15788 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15789 runner.state.config.graph.advise = false;
15790 runner.roles.implementers = vec![spec("implement")];
15791 runner.roles.judges = vec![spec("judge")];
15792 runner.prep().await.unwrap();
15793 let winner = runner.state.candidates[0].clone();
15794 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15797 let completed = tmp.path().join("seat-completed");
15798 runner.roles.implementers[0].command = vec![
15799 "sh".into(),
15800 "-c".into(),
15801 format!("touch '{}'; echo completed", completed.display()),
15802 ];
15803 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15804 assert_eq!(
15805 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15806 0
15807 );
15808 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15809 let mut implement = Box::pin(runner.implement());
15810 tokio::select! {
15811 result = &mut implement => panic!("rescue must wait: {result:?}"),
15812 result = tokio::time::timeout(Duration::from_secs(10), async {
15813 while !completed.exists() {
15814 tokio::time::sleep(Duration::from_millis(20)).await;
15815 }
15816 }) => result.unwrap(),
15817 }
15818 assert!(
15819 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15820 .await
15821 .is_err()
15822 );
15823 assert_eq!(
15824 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15825 before
15826 );
15827 assert_eq!(
15828 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15829 0
15830 );
15831 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15832 .await
15833 .unwrap()
15834 .unwrap();
15835 drop(implement);
15836 assert_ne!(
15837 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15838 before
15839 );
15840 assert!(git::is_clean(&winner.worktree).await.unwrap());
15841
15842 let started = tmp.path().join("pre-gate-started");
15843 let proceed = tmp.path().join("pre-gate-proceed");
15844 let checked = tmp.path().join("gate-ran");
15845 runner.state.config.verify.pre_gate = vec![format!(
15846 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15847 started.display(),
15848 proceed.display()
15849 )];
15850 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15851 assert_eq!(
15852 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15853 0
15854 );
15855 let mut gate = Box::pin(runner.run_gate(&winner));
15856 assert!(
15857 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15858 .await
15859 .is_err()
15860 );
15861 assert!(!started.exists());
15862 assert!(!checked.exists());
15863 assert!(git::is_clean(&winner.worktree).await.unwrap());
15864 assert_eq!(
15865 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15866 0
15867 );
15868 tokio::select! {
15869 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15870 result = tokio::time::timeout(Duration::from_secs(10), async {
15871 while !started.exists() {
15872 tokio::time::sleep(Duration::from_millis(20)).await;
15873 }
15874 }) => result.unwrap(),
15875 }
15876 assert_eq!(
15877 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15878 -1,
15879 "mechanical edits must hold the real lock"
15880 );
15881 std::fs::write(proceed, "ready").unwrap();
15882 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15883 .await
15884 .unwrap()
15885 .unwrap();
15886 drop(gate);
15887 assert!(outcomes.iter().all(CommandOutcome::ok));
15888 assert!(checked.exists());
15889 assert!(runner.state.pre_gate_commit.is_some());
15890 assert!(git::is_clean(&winner.worktree).await.unwrap());
15891 assert_eq!(
15892 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15893 0,
15894 "gate completion must release coordination"
15895 );
15896 drop(outsider);
15897 std::fs::remove_file(common.join("updater.lock")).unwrap();
15898 std::fs::create_dir(common.join("updater.lock")).unwrap();
15899 std::fs::remove_file(&started).unwrap();
15900 std::fs::remove_file(&checked).unwrap();
15901 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15902 let error = runner.run_gate(&winner).await.unwrap_err();
15903 assert!(
15904 error
15905 .to_string()
15906 .contains("acquire mandatory repository lock")
15907 );
15908 assert!(
15909 !started.exists(),
15910 "failed acquisition must prevent pre-gate"
15911 );
15912 assert!(!checked.exists(), "failed acquisition must prevent gate");
15913 assert!(git::is_clean(&winner.worktree).await.unwrap());
15914 assert_eq!(
15915 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15916 head
15917 );
15918 }
15919
15920 #[tokio::test]
15921 #[cfg(unix)]
15922 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15923 use std::os::fd::AsRawFd as _;
15924 use std::time::Duration;
15925
15926 ask_test_home();
15927 let tmp = tempfile::tempdir().unwrap();
15928 let repo = tmp.path().join("repo");
15929 std::fs::create_dir_all(&repo).unwrap();
15930 init_repo(&repo);
15931 let common = repo.join(".git");
15932 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15933 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15934 assert_eq!(
15935 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15936 0
15937 );
15938 let mut runner = runner_at(RunStatus::Prep);
15939 runner.state.repo = repo.clone();
15940 runner.state.config.coordination = crate::config::test_coordination();
15941 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15942 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15943 runner.state.config.graph.advise = true;
15944 runner.state.config.graph.advisors = 1;
15945 runner.roles.implementers = vec![spec("implement")];
15946 runner.roles.judges = vec![spec("judge")];
15947 let root = runner.state.worktree_root();
15948 let started = tmp.path().join("setup-started");
15949 let proceed = tmp.path().join("setup-proceed");
15950 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15951 run: Some(format!(
15952 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15953 started.display(),
15954 proceed.display()
15955 )),
15956 ..Default::default()
15957 }];
15958 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15959 let config = git::git(&repo, &["config", "--local", "--list"])
15960 .await
15961 .unwrap();
15962 let mut prep = Box::pin(runner.prep());
15963 assert!(
15964 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15965 .await
15966 .is_err()
15967 );
15968 assert!(
15969 !root.exists(),
15970 "contention must prevent even checkout creation"
15971 );
15972 assert!(!started.exists(), "contention must prevent setup");
15973 assert_eq!(
15974 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15975 branches
15976 );
15977 assert_eq!(
15978 git::git(&repo, &["config", "--local", "--list"])
15979 .await
15980 .unwrap(),
15981 config
15982 );
15983 drop(lock);
15984 tokio::select! {
15985 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15986 result = tokio::time::timeout(Duration::from_secs(10), async {
15987 while !started.exists() {
15988 tokio::time::sleep(Duration::from_millis(20)).await;
15989 }
15990 }) => result.unwrap(),
15991 }
15992 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15993 assert_eq!(
15994 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15995 -1
15996 );
15997 assert!(
15998 tokio::time::timeout(
15999 Duration::from_millis(100),
16000 crate::repo_lock::SeatLease::for_checkout(
16001 &repo,
16002 true,
16003 crate::config::test_coordination().as_ref()
16004 )
16005 )
16006 .await
16007 .is_err(),
16008 "preparation must also exclude local editors"
16009 );
16010 std::fs::write(proceed, "ready").unwrap();
16011 tokio::time::timeout(Duration::from_secs(10), &mut prep)
16012 .await
16013 .unwrap()
16014 .unwrap();
16015 drop(prep);
16016 assert_eq!(runner.state.prep_complete, Some(true));
16017 assert!(root.join("judge-1").exists());
16018 assert!(root.join("advisor-1").exists());
16019 assert_eq!(
16020 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
16021 0
16022 );
16023 }
16024
16025 #[tokio::test]
16026 #[cfg(unix)]
16027 async fn preparation_lock_failure_prevents_repository_changes() {
16028 ask_test_home();
16029 let tmp = tempfile::tempdir().unwrap();
16030 let repo = tmp.path().join("repo");
16031 std::fs::create_dir_all(&repo).unwrap();
16032 init_repo(&repo);
16033 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
16034 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
16037 let mut runner = runner_at(RunStatus::Prep);
16038 runner.state.repo = repo.clone();
16039 runner.state.config.coordination = crate::config::test_coordination();
16040 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16041 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16042 runner.roles.implementers = vec![spec("implement")];
16043 let error = runner.prep().await.unwrap_err();
16044 assert!(
16045 error
16046 .to_string()
16047 .contains("acquire mandatory repository lock")
16048 );
16049 assert!(!runner.state.worktree_root().exists());
16050 assert!(runner.state.candidates.is_empty());
16051 assert!(!repo.join(".git/config.worktree").exists());
16052 assert_eq!(
16053 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
16054 "* main"
16055 );
16056 }
16057
16058 #[tokio::test]
16059 async fn preparation_records_creation_intent_before_git_can_fail() {
16060 ask_test_home();
16061 let tmp = tempfile::tempdir().unwrap();
16062 let repo = tmp.path().join("repo");
16063 std::fs::create_dir_all(&repo).unwrap();
16064 init_repo(&repo);
16065 let mut runner = runner_at(RunStatus::Prep);
16066 runner.state.repo = repo.clone();
16067 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16068 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16069 runner.state.config.blind.commit_msg_hook = false;
16070 runner.roles.implementers = vec![spec("one")];
16071 let label = blind::assign_labels(1, runner.state.seed)[0];
16072 let branch = runner.state.branch_for(label);
16073 git::git(&repo, &["branch", &branch]).await.unwrap();
16076 assert!(runner.prep().await.is_err());
16077 let saved = RunState::load(&runner.state.id).unwrap();
16078 assert!(saved.candidates.is_empty());
16079 let event = saved
16080 .events
16081 .iter()
16082 .find(|e| e.node == "prep-intent")
16083 .unwrap();
16084 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
16085 assert_eq!(intent["branch"], branch);
16086 assert_eq!(intent["start"], runner.state.base_commit);
16087 assert_eq!(
16088 intent["worktree"],
16089 runner
16090 .state
16091 .worktree_root()
16092 .join(format!("cand-{label}"))
16093 .to_string_lossy()
16094 .as_ref()
16095 );
16096 assert_eq!(intent["picks"], serde_json::json!([]));
16097 }
16098
16099 #[tokio::test]
16100 async fn invalid_pinned_sources_create_no_partial_worktrees() {
16101 ask_test_home();
16102 let tmp = tempfile::tempdir().unwrap();
16103 let repo = tmp.path().join("repo");
16104 std::fs::create_dir_all(&repo).unwrap();
16105 init_repo(&repo);
16106 let mut runner = runner_at(RunStatus::Prep);
16107 runner.state.repo = repo.clone();
16108 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16109 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16110 runner.state.sources.pick_commits.push(refs::Input {
16111 token: "missing".into(),
16112 sha: "0".repeat(40),
16113 provenance: refs::Provenance::ExplicitCli,
16114 });
16115 assert!(runner.prep().await.is_err());
16116 assert!(runner.state.candidates.is_empty());
16117 assert!(!runner.state.worktree_root().exists());
16118 }
16119
16120 #[tokio::test]
16121 async fn prose_and_legacy_inference_never_import_into_candidates() {
16122 ask_test_home();
16123 let tmp = tempfile::tempdir().unwrap();
16124 let repo = tmp.path().join("repo");
16125 std::fs::create_dir_all(&repo).unwrap();
16126 init_repo(&repo);
16127 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16128 git::git(&repo, &["checkout", "-b", "magi/history/A"])
16129 .await
16130 .unwrap();
16131 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
16132 git::git(&repo, &["add", "."]).await.unwrap();
16133 git::git(&repo, &["commit", "-m", "historical implementation"])
16134 .await
16135 .unwrap();
16136 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
16137 git::git(&repo, &["checkout", "main"]).await.unwrap();
16138 for instruction in [
16139 format!("Example: magi/history/A at {source}"),
16140 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
16141 format!("Never import magi/history/A or {source}"),
16142 format!("Conversation answer: I mentioned {source} only as background"),
16143 ] {
16144 let mut runner = runner_at(RunStatus::Prep);
16145 runner.state.repo = repo.clone();
16146 runner.state.base_commit = base.clone();
16147 runner.state.instruction = instruction.clone();
16148 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
16149 runner.state.config.blind.commit_msg_hook = false;
16150 runner.state.seeds = vec![refs::Seed {
16151 token: "magi/history/A".into(),
16152 sha: source.clone(),
16153 kind: refs::SeedKind::Unmerged,
16154 branch: true,
16155 detail: String::new(),
16156 }];
16157 runner.roles.implementers = vec![spec("one")];
16158 assert_eq!(seeded_instruction(&runner.state), instruction);
16159 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16161 legacy.as_object_mut().unwrap().remove("sources");
16162 runner.state = serde_json::from_value(legacy).unwrap();
16163 assert_eq!(runner.state.sources, refs::Sources::default());
16164 runner.state.save().unwrap();
16165 runner.state = RunState::load(&runner.state.id).unwrap();
16166 runner.prep().await.unwrap();
16167 for candidate in &runner.state.candidates {
16168 assert_eq!(
16169 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
16170 base
16171 );
16172 assert!(!candidate.worktree.join("unwanted").exists());
16173 }
16174 }
16175 }
16176
16177 #[tokio::test]
16178 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
16179 ask_test_home();
16180 let tmp = tempfile::tempdir().unwrap();
16181 let repo = tmp.path().join("repo");
16182 std::fs::create_dir_all(&repo).unwrap();
16183 init_repo(&repo);
16184 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16185 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
16186 std::fs::write(repo.join("referenced"), "required commit").unwrap();
16187 git::git(&repo, &["add", "."]).await.unwrap();
16188 git::git(&repo, &["commit", "-m", "required seed"])
16189 .await
16190 .unwrap();
16191 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
16192 git::git(&repo, &["checkout", "main"]).await.unwrap();
16193 let mut runner = runner_at(RunStatus::Prep);
16194 runner.state.repo = repo.clone();
16195 runner.state.base_commit = base.clone();
16196 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16197 runner.state.config.graph.advise = false;
16198 runner.state.config.blind.commit_msg_hook = false;
16199 runner.state.sources =
16200 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
16201 .await
16202 .unwrap();
16203 runner.roles.implementers = vec![spec("one"), spec("two")];
16204 runner.roles.judges = vec![spec("judge")];
16205 let label = blind::assign_labels(2, runner.state.seed)[0];
16206 let branch = runner.state.branch_for(label);
16207 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
16208 git::worktree_add_branch(&repo, &worktree, &branch, &base)
16209 .await
16210 .unwrap();
16211 runner.state.candidates.push(Candidate {
16212 index: 0,
16213 label,
16214 agent: "one".into(),
16215 branch,
16216 worktree,
16217 summary: String::new(),
16218 stat: String::new(),
16219 files: 0,
16220 commits: 0,
16221 empty: false,
16222 failed: None,
16223 verified_noop: None,
16224 duration_ms: 0,
16225 folded: false,
16226 });
16227 runner.state.save().unwrap();
16228 runner.state = RunState::load(&runner.state.id).unwrap();
16230 assert_eq!(runner.state.prep_complete, Some(false));
16231 runner.prep().await.unwrap();
16232 assert_eq!(runner.state.prep_complete, Some(true));
16233 assert_eq!(runner.state.candidates.len(), 2);
16234 for candidate in &runner.state.candidates {
16235 assert_eq!(
16236 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
16237 git::tree_of(&repo, &seed).await.unwrap()
16238 );
16239 assert_eq!(
16240 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
16241 "required commit"
16242 );
16243 }
16244 assert!(runner.state.worktree_root().join("judge-1").exists());
16245 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
16246 runner.prep().await.unwrap();
16247 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
16248
16249 runner.state.prep_complete = Some(false);
16250 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
16251 run: Some(format!(
16252 "test -f '{}'",
16253 tmp.path().join("dependency-ready").display()
16254 )),
16255 ..Default::default()
16256 }];
16257 assert!(runner.prep().await.is_err());
16258 assert!(runner.state.prep_failure.is_some());
16259 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
16260 runner.state = RunState::load(&runner.state.id).unwrap();
16261 assert!(
16262 runner.prep().await.is_err(),
16263 "correcting the cause alone must not auto-retry"
16264 );
16265 let failed_at = runner.state.events.last().unwrap().at;
16266 assert!(
16267 runner.retry_preparation(failed_at).is_err(),
16268 "old approvals cannot unlock the hold"
16269 );
16270 let mut task = crate::queue::Task::new(
16271 "retry".into(),
16272 "retry setup".into(),
16273 repo.clone(),
16274 crate::queue::Source::Human,
16275 );
16276 task.start(runner.state.id.clone());
16277 task.hold_machine(Some("setup failed".into()));
16278 task.release_operator("cli", None);
16279 runner
16280 .retry_preparation(task.resumption.as_ref().unwrap().at)
16281 .unwrap();
16282 runner.prep().await.unwrap();
16283 assert!(runner.state.prep_failure.is_none());
16284 assert_eq!(runner.state.prep_complete, Some(true));
16285 assert_eq!(runner.state.candidates.len(), 2);
16286 assert!(runner.state.worktree_root().join("judge-1").exists());
16287 for candidate in &runner.state.candidates {
16288 assert_eq!(
16289 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
16290 git::tree_of(&repo, &seed).await.unwrap()
16291 );
16292 }
16293 }
16294
16295 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
16296 let mut heads = Vec::new();
16297 for candidate in candidates {
16298 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
16299 }
16300 heads
16301 }
16302
16303 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
16304 ask_test_home();
16305 let tmp = tempfile::tempdir().unwrap();
16306 let repo = tmp.path().join("repo");
16307 std::fs::create_dir(&repo).unwrap();
16308 init_repo(&repo);
16309 let mut runner = runner_at(RunStatus::Reviewing);
16310 runner.state.repo = repo.clone();
16311 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16312 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16313 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16314 let branch = format!("candidate-{label}");
16315 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
16316 git::worktree_add_branch(&repo, &worktree, &branch, "main")
16317 .await
16318 .unwrap();
16319 if label == 'B' {
16320 std::fs::write(worktree.join("change"), "winner").unwrap();
16321 git::rescue_commit(&worktree, "candidate change")
16322 .await
16323 .unwrap();
16324 }
16325 runner.state.candidates.push(Candidate {
16326 index,
16327 label,
16328 agent: "alpha".into(),
16329 branch,
16330 worktree,
16331 summary: "completed reply".into(),
16332 stat: String::new(),
16333 files: usize::from(label == 'B'),
16334 commits: usize::from(label == 'B'),
16335 empty: label == 'A',
16336 failed: None,
16337 verified_noop: None,
16338 duration_ms: 1,
16339 folded: false,
16340 });
16341 }
16342 runner.state.judge_skipped = true;
16343 runner.state.tally = Some(Tally {
16344 first_choice: BTreeMap::new(),
16345 borda: BTreeMap::new(),
16346 winner: 'B',
16347 rankings: 0,
16348 unanimous_initial: false,
16349 deliberated: false,
16350 changed_votes: 0,
16351 unanimous_final: false,
16352 tie_break: None,
16353 judges: 0,
16354 present: 0,
16355 quorum: 0,
16356 met_quorum: true,
16357 uncontested: Some("only B produced a change".into()),
16358 });
16359 runner.state.save().unwrap();
16360 (tmp, runner)
16361 }
16362
16363 #[tokio::test]
16364 async fn fold_losers_reclaims_isolated_outputs_without_a_pool_limit() {
16365 let (_tmp, mut runner) = preservation_resume_fixture().await;
16366 runner.state.config.verify.build_cache = crate::config::BuildCache::Isolated;
16367 runner.state.config.disk.cache_limit_bytes = 0;
16368 runner.state.config.graph.advise = true;
16369 runner.state.config.graph.advisors = 1;
16370 let advisor = runner.state.worktree_root().join("advisor-1");
16371 git::worktree_add_detached(&runner.state.repo, &advisor, "main")
16372 .await
16373 .unwrap();
16374 let home = crate::run::home();
16375 let loser =
16376 crate::build_target::resolve(&home, &runner.state.candidates[0].worktree).unwrap();
16377 let winner =
16378 crate::build_target::resolve(&home, &runner.state.candidates[1].worktree).unwrap();
16379 let advisor = crate::build_target::resolve(&home, &advisor).unwrap();
16380 for target in [&loser, &winner, &advisor] {
16381 std::fs::write(target.join("build-output"), "compiled").unwrap();
16382 }
16383
16384 runner.fold_losers().await.unwrap();
16385
16386 assert!(!loser.join("build-output").exists());
16387 assert!(!advisor.join("build-output").exists());
16388 assert!(winner.join("build-output").is_file());
16389 }
16390
16391 #[tokio::test]
16392 async fn fold_losers_persists_discard_before_later_teardown_failure() {
16393 let (_tmp, mut runner) = preservation_resume_fixture().await;
16394 runner.state.candidates[0].folded = true;
16395 runner.state.config.graph.advise = true;
16396 runner.state.config.graph.advisors = 2;
16397 let repo = runner.state.repo.clone();
16398 let root = runner.state.worktree_root();
16399 let discarded = root.join("advisor-1");
16400 git::worktree_add_detached(&repo, &discarded, "main")
16401 .await
16402 .unwrap();
16403 std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
16404 let protected = root.join("advisor-2");
16405 git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
16406 .await
16407 .unwrap();
16408 std::fs::write(protected.join("keep"), "uncommitted").unwrap();
16409 runner.state.save().unwrap();
16410
16411 assert!(runner.fold_losers().await.is_err());
16412 assert!(!discarded.exists());
16413 assert!(protected.join("keep").exists());
16414 let saved = RunState::load(&runner.state.id).unwrap();
16415 assert!(saved.events.iter().any(|event| {
16416 event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
16417 }));
16418 }
16419
16420 #[tokio::test]
16421 async fn read_seat_teardown_discards_only_owned_detached_seats() {
16422 let (tmp, mut runner) = preservation_resume_fixture().await;
16423 let repo = runner.state.repo.clone();
16424 let root = runner.state.worktree_root();
16425 for name in ["advisor-3", "judge-1", "review-2"] {
16426 let path = root.join(name);
16427 git::worktree_add_detached(&repo, &path, "main")
16428 .await
16429 .unwrap();
16430 std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
16431 std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
16432 remove_read_seat_worktree(&mut runner.state, &path)
16433 .await
16434 .unwrap();
16435 assert!(!path.exists());
16436 }
16437 let outside = tmp.path().join("advisor-1");
16438 let unknown = root.join("unknown");
16439 let attached = root.join("advisor-4");
16440 git::worktree_add_detached(&repo, &outside, "main")
16441 .await
16442 .unwrap();
16443 git::worktree_add_detached(&repo, &unknown, "main")
16444 .await
16445 .unwrap();
16446 git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16447 .await
16448 .unwrap();
16449 let foreign = tmp.path().join("foreign");
16450 std::fs::create_dir(&foreign).unwrap();
16451 init_repo(&foreign);
16452 let wrong_repo = root.join("judge-2");
16453 git::worktree_add_detached(&foreign, &wrong_repo, "main")
16454 .await
16455 .unwrap();
16456 let candidate = runner.state.candidates[0].worktree.clone();
16457 for path in [outside, unknown, attached, wrong_repo, candidate] {
16458 std::fs::write(path.join("uncommitted"), "keep").unwrap();
16459 assert!(
16460 remove_read_seat_worktree(&mut runner.state, &path)
16461 .await
16462 .is_err()
16463 );
16464 assert_eq!(
16465 std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16466 "keep"
16467 );
16468 }
16469 }
16470
16471 #[tokio::test]
16472 #[cfg(unix)]
16473 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16474 for (solo, failed_checkpoint) in
16475 [(true, false), (false, false), (true, true), (false, true)]
16476 {
16477 let (_tmp, mut runner) = preservation_resume_fixture().await;
16478 if solo {
16479 runner.state.candidates.retain(|c| c.label == 'A');
16480 }
16481 runner.state.candidates[0].empty = false;
16482 runner.state.candidates[0].summary.clear();
16483 runner.state.tally = None;
16484 runner.state.judge_skipped = false;
16485 runner.state.status = RunStatus::Implementing;
16486 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16487 let mut implementer = spec("alpha");
16488 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16489 runner.roles.implementers = vec![implementer];
16490 let patch = runner.state.dir().join("artifacts/cand-A.patch");
16491 if failed_checkpoint {
16492 std::fs::create_dir_all(&patch).unwrap();
16493 }
16494 runner.state.save().unwrap();
16495 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16496 drop(pressure);
16497 runner.state = RunState::load(&runner.state.id).unwrap();
16498 assert_eq!(runner.state.status, RunStatus::Blocked);
16499 assert!(runner.state.interrupted_implementations.contains(&'A'));
16500 assert_eq!(runner.state.candidates[0].commits, 0);
16501 assert!(!runner.state.candidates[0].empty);
16502 assert!(runner.state.candidates[0].failed.is_none());
16503 if failed_checkpoint {
16504 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16505 std::fs::remove_dir(patch).unwrap();
16506 }
16507 runner.state.resource_block = None;
16508 runner.state.config.disk.min_free_bytes = 0;
16509 runner.roles.implementers[0].command = vec![
16510 "sh".into(),
16511 "-c".into(),
16512 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16513 ];
16514 runner.implement().await.unwrap();
16515 assert!(runner.state.interrupted_implementations.is_empty());
16516 assert!(runner.state.candidate_preservation.is_empty());
16517 assert_eq!(runner.state.candidates[0].commits, 1);
16518 assert!(!runner.state.candidates[0].empty);
16519 assert!(
16520 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16521 .unwrap()
16522 .contains("retried empty interrupted seat")
16523 );
16524 if !solo {
16525 assert_eq!(runner.state.candidates[1].commits, 1);
16526 }
16527 }
16528 }
16529
16530 #[tokio::test]
16531 #[cfg(unix)]
16532 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16533 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16534 let (_tmp, mut runner) = preservation_resume_fixture().await;
16535 runner.state.candidates.retain(|c| c.label == 'A');
16536 runner.state.candidates[0].empty = false;
16537 runner.state.candidates[0].summary.clear();
16538 runner.state.tally = None;
16539 runner.state.judge_skipped = false;
16540 runner.state.status = RunStatus::Implementing;
16541 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16544 let wt = runner.state.candidates[0].worktree.clone();
16545 std::fs::write(
16546 wt.join("interrupted-source"),
16547 "retain implementation work\n",
16548 )
16549 .unwrap();
16550 if committed {
16551 git::rescue_commit(&wt, "implementation checkpoint")
16552 .await
16553 .unwrap();
16554 }
16555 let lock = git::git(
16556 &wt,
16557 &[
16558 "rev-parse",
16559 "--path-format=absolute",
16560 "--git-path",
16561 "index.lock",
16562 ],
16563 )
16564 .await
16565 .unwrap();
16566 if block_rescue {
16567 std::fs::write(&lock, "fixture-owned lock").unwrap();
16568 }
16569 let mut implementer = spec("alpha");
16570 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16571 runner.roles.implementers = vec![implementer];
16572 runner.state.save().unwrap();
16573 let result = runner.implement().await;
16574 drop(pressure);
16575 assert_eq!(result.is_err(), block_rescue);
16576 assert_eq!(
16577 runner.state.resource_block.as_ref().unwrap().kind,
16578 "disk pressure"
16579 );
16580 assert!(
16581 runner
16582 .state
16583 .events
16584 .iter()
16585 .any(|e| e.message.contains("agent execution interrupted"))
16586 );
16587 assert!(
16588 runner
16589 .state
16590 .dir()
16591 .join("artifacts/candidate-A-recovery/staged.patch")
16592 .exists()
16593 );
16594 if block_rescue {
16595 let saved = RunState::load(&runner.state.id).unwrap();
16596 assert!(saved.candidate_preservation.contains_key(&'A'));
16597 assert!(!saved.candidates[0].empty);
16598 assert_eq!(
16599 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16600 "retain implementation work\n"
16601 );
16602 std::fs::remove_file(lock).unwrap();
16603 runner.state = saved;
16604 } else {
16605 assert_eq!(runner.state.status, RunStatus::Blocked);
16606 assert_eq!(runner.state.candidates[0].commits, 1);
16607 }
16608 runner.roles.implementers.clear();
16611 runner.state.config.disk.min_free_bytes = 0;
16612 runner.state.resource_block = None;
16613 runner.implement().await.unwrap();
16614 assert!(runner.state.candidate_preservation.is_empty());
16615 assert_eq!(runner.state.candidates[0].commits, 1);
16616 assert!(!runner.state.candidates[0].empty);
16617 assert!(git::is_clean(&wt).await.unwrap());
16618 assert!(
16619 crate::run::read_artifact(&runner.state, "cand-A.patch")
16620 .unwrap()
16621 .contains("retain implementation work")
16622 );
16623 }
16624 }
16625
16626 #[tokio::test]
16627 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16628 let (_tmp, mut runner) = preservation_resume_fixture().await;
16629 runner.implement().await.unwrap();
16631 assert!(runner.state.candidate_preservation.is_empty());
16632 assert!(
16633 !runner
16634 .state
16635 .dir()
16636 .join("artifacts/candidate-A-recovery")
16637 .exists()
16638 );
16639 runner.fold_losers().await.unwrap();
16640 assert!(!runner.state.candidates[0].worktree.exists());
16641 for schema in [14, crate::run::SCHEMA] {
16642 runner.state.schema = schema;
16643 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16644 runner.state.save().unwrap();
16645 runner.state = RunState::load(&runner.state.id).unwrap();
16646 runner.implement().await.unwrap();
16647 assert!(runner.state.candidate_preservation.is_empty());
16648 assert!(runner.state.candidates[0].folded);
16649 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16650 }
16651 }
16652
16653 #[tokio::test]
16654 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16655 let (_tmp, mut runner) = preservation_resume_fixture().await;
16656 runner.state.candidates.retain(|c| c.label == 'A');
16657 runner.state.tally = None;
16658 runner.state.judge_skipped = false;
16659 let candidate = &mut runner.state.candidates[0];
16660 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16661 candidate.failed = Some("deployment validation unavailable".into());
16662 std::fs::write(
16663 candidate.worktree.join("repair"),
16664 "future launcher policy\n",
16665 )
16666 .unwrap();
16667 runner
16668 .state
16669 .candidate_preservation
16670 .insert('A', "awaiting persistence".into());
16671 runner.persist_candidates().await.unwrap();
16672 let candidate = &runner.state.candidates[0];
16673 assert!(!candidate.empty);
16674 assert_eq!(candidate.commits, 1);
16675 assert_eq!(candidate.files, 1);
16676 assert!(candidate.failed.is_none());
16677 assert!(candidate.summary.contains("EPERM"));
16678 assert!(
16679 crate::run::read_artifact(&runner.state, "cand-A.patch")
16680 .unwrap()
16681 .contains("future launcher policy")
16682 );
16683 assert!(runner.state.candidate_preservation.is_empty());
16684 }
16685
16686 #[tokio::test]
16687 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16688 let (_tmp, mut runner) = preservation_resume_fixture().await;
16689 let wt = runner.state.candidates[0].worktree.clone();
16690 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16691 runner.state.schema = 14;
16692 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16693 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16694 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16695 for _ in 0..2 {
16696 assert!(runner.implement().await.is_err());
16697 assert_eq!(runner.state.status, RunStatus::Blocked);
16698 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16699 assert!(
16700 runner.state.candidates[0].empty,
16701 "old verdict must remain unusable"
16702 );
16703 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16704 assert!(runner.state.judge_skipped);
16705 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16706 assert!(
16707 crate::run::read_artifact(&runner.state, "cand-A.patch")
16708 .unwrap()
16709 .contains("owned edits")
16710 );
16711 assert!(wt.exists());
16712 assert_eq!(
16713 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16714 .await
16715 .unwrap(),
16716 1
16717 );
16718 runner.state = RunState::load(&runner.state.id).unwrap();
16719 }
16720 }
16721
16722 #[tokio::test]
16723 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16724 let (_tmp, mut runner) = preservation_resume_fixture().await;
16725 let wt = runner.state.candidates[0].worktree.clone();
16726 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16727 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16728 let index_lock = PathBuf::from(
16729 git::git(
16730 &wt,
16731 &[
16732 "rev-parse",
16733 "--path-format=absolute",
16734 "--git-path",
16735 "index.lock",
16736 ],
16737 )
16738 .await
16739 .unwrap(),
16740 );
16741 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16742 runner.state.schema = 14;
16743 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16744 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16745 assert!(runner.implement().await.is_err());
16746 assert!(runner.after_implement().is_err());
16747 assert!(runner.fold_losers().await.is_err());
16748 assert_eq!(
16749 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16750 "failed seat edits"
16751 );
16752 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16753 runner.state = RunState::load(&runner.state.id).unwrap();
16754 std::fs::remove_file(index_lock).unwrap();
16755 assert!(runner.implement().await.is_err());
16757 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16758 assert!(runner.state.candidates[0].empty);
16759 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16760 assert!(
16761 crate::run::read_artifact(&runner.state, "cand-A.patch")
16762 .unwrap()
16763 .contains("failed seat edits")
16764 );
16765 assert!(wt.exists());
16766 }
16767
16768 #[tokio::test]
16769 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16770 let (_tmp, mut runner) = preservation_resume_fixture().await;
16771 let repo = runner.state.repo.clone();
16772 let common = git::git(
16773 &repo,
16774 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16775 )
16776 .await
16777 .unwrap();
16778 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16779 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16780 assert!(runner.fold_losers().await.is_err());
16781 assert!(!runner.state.candidates[0].worktree.exists());
16782 assert!(!runner.state.candidates[0].folded);
16783 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16784 runner.state = RunState::load(&runner.state.id).unwrap();
16785 std::fs::remove_file(branch_lock).unwrap();
16786 let judge = runner.state.worktree_root().join("judge-1");
16787 git::worktree_add_detached(&repo, &judge, "main")
16788 .await
16789 .unwrap();
16790 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16791 git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16792 .await
16793 .unwrap();
16794 runner.roles.judges = vec![spec("judge")];
16795 assert!(runner.fold_losers().await.is_err());
16796 runner.state = RunState::load(&runner.state.id).unwrap();
16797 assert!(
16798 runner.state.candidates[0].folded,
16799 "candidate progress saved before judge failure"
16800 );
16801 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16802 assert!(judge.join("uncommitted").exists());
16803 git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16804 .await
16805 .unwrap();
16806 runner.fold_losers().await.unwrap();
16807 assert!(!judge.exists());
16808 runner.fold_losers().await.unwrap();
16809 }
16810
16811 #[tokio::test]
16812 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16813 let (_tmp, mut runner) = preservation_resume_fixture().await;
16814 runner.state.candidates.retain(|c| c.label == 'B');
16815 runner.state.tally = None;
16816 runner.state.judge_skipped = false;
16817 runner.state.prep_complete = Some(true);
16818 runner.state.config.graph.advise = false;
16819 runner.state.config.graph.selected_implementer = Some("alpha".into());
16820 let wt = runner.state.candidates[0].worktree.clone();
16821 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16822 runner
16823 .state
16824 .candidate_preservation
16825 .insert('B', "awaiting persistence".into());
16826 let index_lock = git::git(
16827 &wt,
16828 &[
16829 "rev-parse",
16830 "--path-format=absolute",
16831 "--git-path",
16832 "index.lock",
16833 ],
16834 )
16835 .await
16836 .unwrap();
16837 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16838 runner.state.save().unwrap();
16839 assert!(runner.execute_graph().await.is_err());
16840 assert_eq!(runner.state.status, RunStatus::Blocked);
16841 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16842 std::fs::remove_file(index_lock).unwrap();
16843 for _ in 0..2 {
16846 runner.state = RunState::load(&runner.state.id).unwrap();
16847 runner.pause.park();
16848 runner.execute_graph().await.unwrap();
16849 assert_eq!(runner.state.status, RunStatus::Implementing);
16850 assert!(runner.state.candidate_preservation.is_empty());
16851 assert_eq!(runner.state.candidates[0].commits, 2);
16852 assert!(runner.state.tally.is_none());
16853 }
16854 runner.implement().await.unwrap();
16855 assert_eq!(runner.state.status, RunStatus::Judging);
16856 runner.state.status = RunStatus::Blocked;
16858 runner.state.candidates[0].failed = Some("implementation failed".into());
16859 runner.state.save().unwrap();
16860 let error = runner.execute_graph().await.unwrap_err();
16861 assert!(
16862 error
16863 .to_string()
16864 .contains("selected implementation is held")
16865 );
16866 }
16867
16868 #[tokio::test]
16869 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16870 let (_tmp, mut runner) = preservation_resume_fixture().await;
16871 let repo = runner.state.repo.clone();
16872 let candidate = runner.state.candidates[0].clone();
16873 let registered = git::worktree_holding(&repo, &candidate.branch)
16876 .await
16877 .unwrap()
16878 .unwrap();
16879 assert_eq!(
16880 registered.canonicalize().unwrap(),
16881 candidate.worktree.canonicalize().unwrap()
16882 );
16883 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16884 assert_eq!(
16885 git::worktree_holding(&repo, &candidate.branch)
16886 .await
16887 .unwrap(),
16888 Some(registered)
16889 );
16890 assert!(
16891 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16892 "the stale registration initially keeps the branch checked out"
16893 );
16894 runner.fold_losers().await.unwrap();
16895 assert!(runner.state.candidates[0].folded);
16896 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16897 assert_eq!(
16898 git::worktree_holding(&repo, &candidate.branch)
16899 .await
16900 .unwrap(),
16901 None
16902 );
16903 runner.state = RunState::load(&runner.state.id).unwrap();
16904 runner.fold_losers().await.unwrap();
16905 }
16906
16907 #[tokio::test]
16908 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16909 let (_tmp, mut runner) = preservation_resume_fixture().await;
16910 let repo = runner.state.repo.clone();
16911 let candidate = runner.state.candidates[0].clone();
16912 let winner = runner.state.candidates[1].clone();
16913 let registered = git::worktree_holding(&repo, &candidate.branch)
16914 .await
16915 .unwrap()
16916 .unwrap();
16917 assert_eq!(
16918 registered.canonicalize().unwrap(),
16919 candidate.worktree.canonicalize().unwrap()
16920 );
16921 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16922 assert_eq!(
16923 git::worktree_holding(&repo, &candidate.branch)
16924 .await
16925 .unwrap(),
16926 Some(registered)
16927 );
16928 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16929
16930 for _ in 0..2 {
16931 fold_run(&mut runner.state, false, &crate::run::home())
16932 .await
16933 .unwrap();
16934 runner.state = RunState::load(&runner.state.id).unwrap();
16935 assert!(runner.state.candidates[0].folded);
16936 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16937 assert_eq!(
16938 git::worktree_holding(&repo, &candidate.branch)
16939 .await
16940 .unwrap(),
16941 None
16942 );
16943 assert!(!runner.state.candidates[1].folded);
16944 assert!(winner.worktree.exists());
16945 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16946 }
16947 }
16948 #[tokio::test]
16949 async fn fold_recovery_override_archives_sources_and_keeps_safety_checks() {
16950 for sidecar in [false, true] {
16951 let tmp = tempfile::tempdir().unwrap();
16952 let home = tmp.path().join("home");
16953 let repo = tmp.path().join("repo");
16954 std::fs::create_dir(&repo).unwrap();
16955 init_repo(&repo);
16956 let mut state = runner_at(RunStatus::Superseded).state;
16957 state.repo = repo.clone();
16958 state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16959 state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16960 state.driver_exited = true;
16961 state.reserve_under(&home).unwrap();
16962 git::git(
16963 &repo,
16964 &["update-ref", "refs/remotes/origin/main", &state.base_commit],
16965 )
16966 .await
16967 .unwrap();
16968 let wt = state.worktree_root().join("cand-A");
16969 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16970 .await
16971 .unwrap();
16972 std::fs::write(wt.join("independent"), "retained implementation\n").unwrap();
16973 git::commit_all(&wt, "fixture: independent source")
16974 .await
16975 .unwrap();
16976 let head = git::rev_parse(&wt, "HEAD").await.unwrap();
16977 state.candidates.push(Candidate {
16978 index: 0,
16979 label: 'A',
16980 agent: "fixture".into(),
16981 branch: "candidate".into(),
16982 worktree: wt.clone(),
16983 summary: String::new(),
16984 stat: String::new(),
16985 files: 0,
16986 commits: 0,
16987 empty: false,
16988 failed: None,
16989 verified_noop: None,
16990 duration_ms: 0,
16991 folded: false,
16992 });
16993 let mut task = crate::queue::Task::new(
16994 "task".into(),
16995 "task".into(),
16996 repo.clone(),
16997 crate::queue::Source::Human,
16998 );
16999 task.runs.push(state.id.clone());
17000 crate::queue::Queue::at(home.join("queue"))
17001 .put(&mut task)
17002 .unwrap();
17003 if sidecar {
17004 crate::pr_recovery::retain_sources(&task, &home).unwrap();
17005 }
17006 state.save_under(&home).unwrap();
17007 let reason = crate::pr_recovery::source_retention_reason(&home, &state).unwrap();
17008 assert!(
17009 fold_run(&mut state, true, &home)
17010 .await
17011 .unwrap_err()
17012 .to_string()
17013 .contains(&reason)
17014 );
17015 let mut disk = state.config.disk.clone();
17016 disk.fold_grace_secs = 0;
17017 assert_eq!(
17018 crate::clean::fold_due(
17019 &home.join("runs"),
17020 &home,
17021 &state.worktree_root(),
17022 &disk,
17023 Timestamp::now() + std::time::Duration::from_secs(10),
17024 )
17025 .await
17026 .unwrap(),
17027 (0, 0)
17028 );
17029 assert!(wt.exists());
17030 state.status = RunStatus::Blocked;
17031 state.save_under(&home).unwrap();
17032 assert!(
17033 fold_run_overriding_recovery_retention(&mut state, true, &home)
17034 .await
17035 .is_err()
17036 );
17037 state.status = RunStatus::Superseded;
17038 state.driver_exited = false;
17039 state.driver_claimed = None;
17040 state.save_under(&home).unwrap();
17041 assert!(
17042 fold_run_overriding_recovery_retention(&mut state, true, &home)
17043 .await
17044 .is_err()
17045 );
17046 state.driver_pid = Some(std::process::id());
17047 state.driver_started_at = crate::proc::process_started_at(std::process::id());
17048 state.save_under(&home).unwrap();
17049 assert!(
17050 fold_run_overriding_recovery_retention(&mut state, true, &home)
17051 .await
17052 .is_err()
17053 );
17054 state.driver_exited = true;
17055 state.driver_pid = None;
17056 state.driver_started_at = None;
17057 state.save_under(&home).unwrap();
17058 let lease = crate::driver::maintenance(&home, &state.id).unwrap();
17059 assert!(
17060 fold_run_overriding_recovery_retention(&mut state, true, &home)
17061 .await
17062 .is_err()
17063 );
17064 drop(lease);
17065 #[cfg(unix)]
17066 {
17067 let output = crate::retention::OutputLease::take(&wt, false).unwrap();
17068 assert!(
17069 fold_run_overriding_recovery_retention(&mut state, true, &home)
17070 .await
17071 .is_err()
17072 );
17073 drop(output);
17074 }
17075 let owner = crate::pr_head::owner_lock(&home, &state.id).unwrap();
17076 assert!(
17077 fold_run_overriding_recovery_retention(&mut state, true, &home)
17078 .await
17079 .is_err()
17080 );
17081 drop(owner);
17082 let checkpoint = git::git(
17083 &wt,
17084 &[
17085 "rev-parse",
17086 "--path-format=absolute",
17087 "--git-path",
17088 "CHERRY_PICK_HEAD",
17089 ],
17090 )
17091 .await
17092 .unwrap();
17093 std::fs::write(&checkpoint, &head).unwrap();
17094 assert!(
17095 fold_run_overriding_recovery_retention(&mut state, true, &home)
17096 .await
17097 .is_err()
17098 );
17099 std::fs::remove_file(checkpoint).unwrap();
17100 std::fs::write(wt.join("dirty"), "keep").unwrap();
17101 assert!(
17102 fold_run_overriding_recovery_retention(&mut state, true, &home)
17103 .await
17104 .is_err()
17105 );
17106 assert!(wt.join("dirty").exists());
17107 std::fs::remove_file(wt.join("dirty")).unwrap();
17108 let evidence = home
17109 .join("runs")
17110 .join(&state.id)
17111 .join("artifacts/fold-evidence");
17112 std::fs::create_dir_all(evidence.parent().unwrap()).unwrap();
17113 std::fs::write(&evidence, "blocked archive").unwrap();
17114 assert!(
17115 fold_run_overriding_recovery_retention(&mut state, true, &home)
17116 .await
17117 .is_err()
17118 );
17119 assert!(wt.exists());
17120 assert!(
17121 !RunState::load_under(&state.id, &home)
17122 .unwrap()
17123 .events
17124 .iter()
17125 .any(|event| event.message.contains("--override-recovery-retention"))
17126 );
17127 std::fs::remove_file(&evidence).unwrap();
17128 fold_run_overriding_recovery_retention(&mut state, true, &home)
17129 .await
17130 .unwrap();
17131 assert!(!wt.exists());
17132 assert!(!git::branch_exists(&repo, "candidate").await.unwrap());
17133 assert!(
17134 git::worktree_holding(&repo, "candidate")
17135 .await
17136 .unwrap()
17137 .is_none()
17138 );
17139 assert!(
17140 std::fs::read_to_string(evidence.join(format!("{head}.patch")))
17141 .unwrap()
17142 .contains("retained implementation")
17143 );
17144 assert!(evidence.join(format!("{head}.index")).is_file());
17145 let saved = RunState::load_under(&state.id, &home).unwrap();
17146 assert_eq!(saved.status, RunStatus::Superseded);
17147 assert!(saved.candidates[0].folded);
17148 assert!(saved.events.iter().any(|event| {
17149 event.message.contains("--override-recovery-retention")
17150 && event.message.contains(&state.id)
17151 && event.message.contains(&reason)
17152 }));
17153 assert_eq!(
17154 crate::pr_recovery::explicit_sources_retained(&home, &state.id),
17155 sidecar
17156 );
17157 }
17158 }
17159
17160 #[tokio::test]
17161 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
17162 ask_test_home();
17163 let tmp = tempfile::tempdir().unwrap();
17164 let repo = tmp.path().join("repo");
17165 std::fs::create_dir(&repo).unwrap();
17166 init_repo(&repo);
17167 let mut runner = runner_at(RunStatus::Implementing);
17168 runner.state.repo = repo.clone();
17169 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
17170 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
17171 for (index, label) in ['A', 'B'].into_iter().enumerate() {
17172 let branch = format!("candidate-{label}");
17173 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
17174 git::worktree_add_branch(&repo, &wt, &branch, "main")
17175 .await
17176 .unwrap();
17177 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
17178 git::git(&wt, &["add", "README.md"]).await.unwrap();
17179 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
17180 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
17181 runner.state.candidates.push(Candidate {
17182 index,
17183 label,
17184 agent: "alpha".to_owned(),
17185 branch,
17186 worktree: wt,
17187 summary: "completed implementation".to_owned(),
17188 stat: String::new(),
17189 files: 0,
17190 commits: 0,
17191 empty: false,
17192 failed: None,
17193 verified_noop: None,
17194 duration_ms: 42,
17195 folded: false,
17196 });
17197 runner
17198 .state
17199 .candidate_preservation
17200 .insert(label, "awaiting persistence".to_owned());
17201 }
17202 let wt = runner.state.candidates[0].worktree.clone();
17203 let index_lock = git::git(
17204 &wt,
17205 &[
17206 "rev-parse",
17207 "--path-format=absolute",
17208 "--git-path",
17209 "index.lock",
17210 ],
17211 )
17212 .await
17213 .unwrap();
17214 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
17215 runner.state.save().unwrap();
17216 assert!(runner.persist_candidates().await.is_err());
17217 assert!(runner.state.candidate_preservation.contains_key(&'A'));
17218 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
17219 assert_eq!(
17220 runner.state.candidates[1].commits, 1,
17221 "other results are still collected"
17222 );
17223 assert!(
17224 !runner.state.candidates[0].empty,
17225 "failed rescue is not no change"
17226 );
17227 assert_eq!(
17228 runner.state.candidates[0].summary,
17229 "completed implementation"
17230 );
17231 assert!(runner.after_implement().is_err());
17232 let home = tmp.path().join("fold-home");
17233 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
17234 assert!(wt.join("new-file").exists());
17235 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
17236 assert!(recovery.join("staged.patch").exists());
17237 assert!(recovery.join("unstaged.patch").exists());
17238 assert_eq!(
17239 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
17240 "untracked edits\n"
17241 );
17242 runner.state.begin_pass();
17243 runner.state.end_pass_parked(ParkCause::Upgrade {
17244 from: "1.0".into(),
17245 to: Some("1.1".into()),
17246 });
17247 runner.state.save().unwrap();
17248 runner.state = RunState::load(&runner.state.id).unwrap();
17249 assert!(matches!(
17250 runner.state.passes.last().unwrap().end,
17251 Some(crate::run::PassEnd::Parked {
17252 cause: ParkCause::Upgrade { .. }
17253 })
17254 ));
17255 std::fs::remove_file(index_lock).unwrap();
17256 runner.implement().await.unwrap();
17258 assert!(runner.state.candidate_preservation.is_empty());
17259 assert!(
17260 runner
17261 .state
17262 .candidates
17263 .iter()
17264 .all(|c| c.commits == 1 && !c.empty)
17265 );
17266 assert!(git::is_clean(&wt).await.unwrap());
17267 assert!(
17268 crate::run::read_artifact(&runner.state, "cand-A.patch")
17269 .unwrap()
17270 .contains("later tracked edits")
17271 );
17272 }
17273
17274 #[tokio::test]
17275 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
17276 ask_test_home();
17277 let tmp = tempfile::tempdir().unwrap();
17278 let repo = tmp.path().join("repo");
17279 std::fs::create_dir(&repo).unwrap();
17280 init_repo(&repo);
17281 let wt = tmp.path().join("candidate");
17282 git::worktree_add_branch(&repo, &wt, "candidate", "main")
17283 .await
17284 .unwrap();
17285 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
17286 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
17287 assert!(wt.join("uncommitted").exists());
17288 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
17289 std::fs::remove_file(wt.join("uncommitted")).unwrap();
17290 let mut runner = runner_at(RunStatus::Implementing);
17291 runner.state.repo = repo.clone();
17292 runner.state.base_commit = "missing-base".to_owned();
17293 runner.state.candidates.push(Candidate {
17294 index: 0,
17295 label: 'A',
17296 agent: "alpha".into(),
17297 branch: "candidate".into(),
17298 worktree: wt.clone(),
17299 summary: "completed".into(),
17300 stat: String::new(),
17301 files: 0,
17302 commits: 0,
17303 empty: false,
17304 failed: None,
17305 verified_noop: None,
17306 duration_ms: 1,
17307 folded: false,
17308 });
17309 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
17310 assert!(
17311 fold_run(&mut runner.state, true, &tmp.path().join("home"))
17312 .await
17313 .is_err()
17314 );
17315 assert!(!runner.state.candidates[0].folded);
17316 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
17317 std::fs::remove_file(wt.join("uncommitted")).unwrap();
17318 runner
17319 .state
17320 .candidate_preservation
17321 .insert('A', "pending".into());
17322 assert!(runner.persist_candidates().await.is_err());
17323 assert!(!runner.state.candidates[0].empty);
17324 assert!(!runner.state.candidates[0].folded);
17325 assert!(wt.exists());
17326 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
17327 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
17328 std::fs::create_dir_all(&patch_path).unwrap();
17329 assert!(
17330 runner.persist_candidates().await.is_err(),
17331 "artifact failure stays pending"
17332 );
17333 assert!(runner.state.candidate_preservation.contains_key(&'A'));
17334 assert!(!runner.state.candidates[0].empty);
17335 std::fs::remove_dir(patch_path).unwrap();
17336 runner.persist_candidates().await.unwrap();
17337 assert!(
17338 runner.state.candidates[0].empty,
17339 "only successful inspection confirms no change"
17340 );
17341 }
17342
17343 fn init_repo(dir: &Path) {
17344 let run = |args: &[&str]| {
17345 let out = std::process::Command::new("git")
17346 .args(args)
17347 .current_dir(dir)
17348 .quiet()
17349 .output()
17350 .expect("spawn git");
17351 assert!(
17352 out.status.success(),
17353 "git {args:?} failed: {}",
17354 String::from_utf8_lossy(&out.stderr)
17355 );
17356 };
17357 run(&["init", "-b", "main"]);
17358 run(&["config", "user.name", "magi test"]);
17359 run(&["config", "user.email", "magi@example.com"]);
17360 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
17361 run(&["add", "-A"]);
17362 run(&["commit", "-m", "init"]);
17363 }
17364
17365 fn ask_test_home() {
17373 crate::run::pin_test_home();
17374 }
17375
17376 #[test]
17377 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
17378 ask_test_home();
17379 let mut runner = runner_at(RunStatus::Prep);
17380 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17381 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
17384 let error = guard
17385 .finish(
17386 &mut runner.state,
17387 Err(
17388 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
17389 .into(),
17390 ),
17391 )
17392 .unwrap_err();
17393 assert_eq!(
17394 error.downcast_ref::<std::io::Error>().unwrap().kind(),
17395 std::io::ErrorKind::PermissionDenied
17396 );
17397 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
17398 let message = format!("{error:#}");
17399 assert!(message.contains("original refusal"));
17400 assert!(!message.contains("could not save terminal"));
17401 assert_eq!(
17402 error.chain().count(),
17403 2,
17404 "only startup attribution wraps the original error"
17405 );
17406 let saved = RunState::load(&runner.state.id).unwrap();
17407 assert_eq!(saved.status, RunStatus::Blocked);
17408 assert!(!saved.startup.unwrap().complete);
17409 }
17410
17411 #[test]
17412 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
17413 ask_test_home();
17414 let mut runner = runner_at(RunStatus::Prep);
17415 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17416 let id = runner.state.id.clone();
17417 drop(guard);
17418 let saved = RunState::load(&id).unwrap();
17419 assert_eq!(saved.status, RunStatus::Blocked);
17420 assert!(!saved.startup.as_ref().unwrap().complete);
17421 assert!(
17422 saved
17423 .startup
17424 .unwrap()
17425 .failure
17426 .unwrap()
17427 .contains("interrupted")
17428 );
17429 assert!(Runner::resume(&id).is_err());
17430 }
17431
17432 #[test]
17433 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
17434 ask_test_home();
17435 let mut runner = runner_at(RunStatus::Prep);
17436 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17437 let error = guard
17438 .finish(
17439 &mut runner.state,
17440 Err(
17441 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
17442 .into(),
17443 ),
17444 )
17445 .unwrap_err();
17446 assert_eq!(
17447 error.downcast_ref::<std::io::Error>().unwrap().kind(),
17448 std::io::ErrorKind::PermissionDenied
17449 );
17450 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
17451 let mut task = crate::queue::Task::new(
17452 "review".into(),
17453 "criteria".into(),
17454 runner.state.repo.clone(),
17455 crate::queue::Source::Human,
17456 );
17457 task.record_startup_failure(failure.clone());
17458 task.record_startup_failure(failure.clone());
17459 assert_eq!(task.startup_failures.len(), 1);
17460 assert!(task.runs.is_empty());
17461 assert_eq!(task.attempts, 0);
17462 }
17463
17464 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
17465 ask_test_home();
17466 let tmp = tempfile::tempdir().unwrap();
17467 let repo = tmp.path().join("repo");
17468 std::fs::create_dir(&repo).unwrap();
17469 init_repo(&repo);
17470 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17471 let origin = tmp.path().join("origin.git");
17472 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
17473 .await
17474 .unwrap();
17475 git::git(
17476 &repo,
17477 &["remote", "add", "origin", origin.to_str().unwrap()],
17478 )
17479 .await
17480 .unwrap();
17481 git::git(&repo, &["checkout", "-b", "fix/retained"])
17482 .await
17483 .unwrap();
17484 std::fs::write(repo.join("retained"), "retained change").unwrap();
17485 git::git(&repo, &["add", "."]).await.unwrap();
17486 git::git(&repo, &["commit", "-m", "retained implementation"])
17487 .await
17488 .unwrap();
17489 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
17490 git::git(&repo, &["checkout", "main"]).await.unwrap();
17491 std::fs::write(repo.join("base-change"), "new base").unwrap();
17492 git::git(&repo, &["add", "."]).await.unwrap();
17493 git::git(&repo, &["commit", "-m", "base moved"])
17494 .await
17495 .unwrap();
17496 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17497 git::git(&repo, &["checkout", "fix/retained"])
17498 .await
17499 .unwrap();
17500 let mut runner = runner_at(RunStatus::Voting);
17501 runner.state.repo = repo.clone();
17502 runner.state.base_commit = base;
17503 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17504 runner.state.config.disk.min_free_bytes = 0;
17505 runner.state.config.graph.review_rounds = 1;
17506 runner.state.config.verify.e2e = vec!["true".into()];
17507 runner.state.config.verify.gate = vec!["true".into()];
17508 runner.state.candidates = vec![Candidate {
17509 index: 0,
17510 label: 'A',
17511 agent: "fixture".into(),
17512 branch: "fix/retained".into(),
17513 worktree: repo.clone(),
17514 summary: String::new(),
17515 stat: String::new(),
17516 files: 1,
17517 commits: 1,
17518 empty: false,
17519 failed: None,
17520 verified_noop: None,
17521 duration_ms: 0,
17522 folded: false,
17523 }];
17524 runner.state.tally = Some(Tally {
17525 first_choice: BTreeMap::new(),
17526 borda: BTreeMap::new(),
17527 winner: 'A',
17528 rankings: 0,
17529 unanimous_initial: true,
17530 deliberated: false,
17531 changed_votes: 0,
17532 unanimous_final: true,
17533 tie_break: None,
17534 judges: 0,
17535 present: 0,
17536 quorum: 0,
17537 met_quorum: true,
17538 uncontested: Some("review-only".into()),
17539 });
17540 let mut task = crate::queue::Task::new(
17541 "fixture".into(),
17542 "fixture".into(),
17543 repo,
17544 crate::queue::Source::Human,
17545 );
17546 task.link_run(&runner.state.id);
17547 crate::queue::Queue::open().put(&mut task).unwrap();
17548 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
17549 runner.state.base_sync = Some(BaseSync {
17550 tip: String::new(),
17551 behind: 1,
17552 attempts: BASE_SYNC_ROUNDS,
17553 conflict: None,
17554 already_in: None,
17555 });
17556 runner.sync_to_base().await.unwrap();
17557 assert_eq!(runner.state.status, RunStatus::Blocked);
17558 (tmp, runner, head, task.id)
17559 }
17560
17561 #[tokio::test]
17562 async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
17563 let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
17564 runner.state.status = RunStatus::Voting;
17565 runner.state.base_sync = None;
17566 runner.state.base_sync_recovery = Default::default();
17567 let scratch = runner.state.dir().join("base-sync");
17568 std::fs::create_dir_all(&scratch).unwrap();
17569 std::fs::write(scratch.join("obstruction"), "keep").unwrap();
17570 runner.sync_to_base().await.unwrap();
17571 assert!(runner.state.resource_block.is_some());
17572 assert!(!runner.state.base_sync_blocked());
17573 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
17574 assert_eq!(runner.state.base_episode_used(), 0);
17575 assert!(runner.state.base_sync_recovery.operations.is_empty());
17576 assert!(runner.state.rebase_fixes.is_empty());
17577 assert_eq!(
17578 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17579 head
17580 );
17581 std::fs::remove_dir_all(&scratch).unwrap();
17582 runner.state.resource_block = None;
17583 runner.sync_to_base().await.unwrap();
17584 assert!(!runner.state.base_sync_blocked());
17585 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17586 assert_eq!(runner.state.base_episode_used(), 1);
17587 assert!(runner.state.base_sync_recovery.operations[0].finished);
17588 assert!(runner.state.rebase_fixes.is_empty());
17589 assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
17590 assert!(git::is_clean(&runner.state.repo).await.unwrap());
17591 }
17592
17593 #[cfg(unix)]
17594 #[tokio::test]
17595 async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
17596 for routed in [false, true] {
17597 let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
17598 let home = crate::run::home();
17599 let queue = crate::queue::Queue::open();
17600 let task = queue.get(&owner).unwrap();
17601 runner.state.task_instruction = Some(task.instruction.clone());
17602 runner.state.driver_exited = true;
17603 runner.state.save().unwrap();
17604 let questions = crate::ask::Questions::at(home.join("questions"));
17605 let mut q = crate::ask::Question::new(
17606 runner.state.id.clone(),
17607 "base-recovery".into(),
17608 "owner".into(),
17609 "Recover bounded base movement?".into(),
17610 String::new(),
17611 vec!["recover".into()],
17612 );
17613 q.actions.insert(
17614 "recover".into(),
17615 crate::ask::ChoiceAction::RecoverBase {
17616 run: runner.state.id.clone(),
17617 stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
17618 episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
17619 },
17620 );
17621 q.routed_task = Some(owner.clone());
17622 q.answer(crate::ask::Answer::Choice("recover".into()))
17623 .unwrap();
17624 questions.put(&mut q).unwrap();
17625 crate::base_recovery_control::recover(
17626 &queue,
17627 &home,
17628 crate::base_recovery_control::Request {
17629 run: runner.state.id.clone(),
17630 head: head.clone(),
17631 actor: "decisions".into(),
17632 request: q.id.clone(),
17633 decision: Some((owner.clone(), q.answered_at.unwrap())),
17634 expected_stop: Some((
17635 runner.state.base_sync_recovery.stop.clone().unwrap(),
17636 runner.state.base_sync_recovery.episodes.last().unwrap().id,
17637 )),
17638 },
17639 )
17640 .await
17641 .unwrap();
17642 if routed {
17643 questions
17644 .update(&q.id, |q| {
17645 q.routed_task = Some("another-owner".into());
17646 Ok(())
17647 })
17648 .unwrap();
17649 } else {
17650 let mut task = queue.get(&owner).unwrap();
17651 task.hold_manual(Some("new owner decision".into()));
17652 task.status = crate::queue::TaskStatus::Queued;
17654 queue.put(&mut task).unwrap();
17655 }
17656 runner.state = RunState::load(&runner.state.id).unwrap();
17657 runner.sync_to_base().await.unwrap();
17658 let approval = runner
17659 .state
17660 .base_sync_recovery
17661 .authorizations
17662 .last()
17663 .unwrap();
17664 assert!(approval.consumed);
17665 assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17666 "routing"
17667 } else {
17668 "owner decision"
17669 }));
17670 assert_eq!(
17671 runner.state.base_sync.as_ref().unwrap().attempts,
17672 BASE_SYNC_ROUNDS
17673 );
17674 assert_eq!(
17675 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17676 head
17677 );
17678 }
17679 }
17680
17681 #[cfg(unix)]
17682 #[tokio::test]
17683 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17684 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17685 if std::env::var_os(CHILD).is_none() {
17686 let home = tempfile::tempdir().unwrap();
17687 let output = std::process::Command::new(std::env::current_exe().unwrap())
17688 .args([
17689 "--exact",
17690 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17691 "--nocapture",
17692 ])
17693 .env(CHILD, "1")
17694 .env("MAGI_HOME", home.path())
17695 .env("MAGI_WORKER_HOME", home.path())
17696 .env("MAGI_WORKER_ID", "abcdef")
17697 .output()
17698 .unwrap();
17699 assert!(
17700 output.status.success()
17701 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17702 "{}\n{}",
17703 String::from_utf8_lossy(&output.stdout),
17704 String::from_utf8_lossy(&output.stderr)
17705 );
17706 return;
17707 }
17708 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17709 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17710 let home = crate::run::home();
17711 let queue = crate::queue::Queue::open();
17712 let mut owner = queue.get(&task).unwrap();
17713 owner.release_operator("test-release", None);
17714 owner.start(runner.state.id.clone());
17715 queue.put(&mut owner).unwrap();
17716 runner.state.driver_exited = true;
17717 runner.state.task_instruction = Some(owner.instruction.clone());
17718 runner.state.save().unwrap();
17719 let _claim = queue.claim(&task).unwrap();
17720 let reserve = |id: &str| {
17721 let directory = home.join("workers/jobs").join(id);
17722 std::fs::create_dir_all(&directory).unwrap();
17723 let job = crate::worker::Job {
17724 id: id.into(),
17725 sequence: 1,
17726 kind: "task".into(),
17727 key: format!("task:{task}"),
17728 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17729 generation: "fixture-generation".into(),
17730 version: None,
17731 status: crate::worker::Status::Running,
17732 result: None,
17733 pid: Some(std::process::id()),
17734 created_at: 0,
17735 finished_at: None,
17736 };
17737 std::fs::write(
17738 directory.join("job.json"),
17739 serde_json::to_vec(&job).unwrap(),
17740 )
17741 .unwrap();
17742 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17743 };
17744 let _own = reserve("abcdef");
17745 let other = reserve("fedcba");
17746 assert!(
17747 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17748 .await
17749 .is_err()
17750 );
17751 assert!(
17752 RunState::load(&runner.state.id)
17753 .unwrap()
17754 .base_sync_recovery
17755 .authorizations
17756 .is_empty()
17757 );
17758 drop(other);
17759 assert!(crate::daemon::is_working_on(
17760 &home,
17761 &runner.state.id,
17762 Timestamp::now()
17763 ));
17764 assert!(
17765 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17766 );
17767 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17768 .await
17769 .unwrap();
17770 assert!(
17771 RunState::load(&runner.state.id)
17772 .unwrap()
17773 .base_recovery_pending()
17774 );
17775 }
17776
17777 #[cfg(unix)]
17778 #[tokio::test]
17779 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17780 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17781 let home = crate::run::home();
17782 runner.state.driver_pid = Some(std::process::id());
17783 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17784 runner.state.driver_exited = false;
17785 runner.state.begin_pass();
17786 runner.state.save().unwrap();
17787 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17788 driver.record(&runner.state).unwrap();
17789 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17790 drop(driver);
17792 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17793 assert_eq!(
17794 runner.state.liveness_under(&home, false),
17795 crate::run::Liveness::Dead
17796 );
17797 let queue = crate::queue::Queue::open();
17798 let _claim = queue.claim(&task).unwrap();
17799 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17800 .await
17801 .unwrap();
17802 let disk = RunState::load(&runner.state.id).unwrap();
17803 assert!(!disk.driver_exited);
17804 assert!(disk.base_recovery_pending());
17805 }
17806
17807 #[cfg(unix)]
17808 #[tokio::test]
17809 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17810 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17811 if std::env::var_os(CHILD).is_none() {
17812 use std::os::unix::fs::PermissionsExt;
17813 let tmp = tempfile::tempdir().unwrap();
17814 let gh = tmp.path().join("gh");
17815 std::fs::write(&gh, r#"#!/bin/sh
17816case "$1 $2" in
17817 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17818 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17819 *) exit 2 ;;
17820esac
17821"#).unwrap();
17822 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17823 let mut paths = vec![tmp.path().to_path_buf()];
17824 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17825 let output = std::process::Command::new(std::env::current_exe().unwrap())
17826 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17827 .env(CHILD, "1")
17828 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17829 .env("PATH", std::env::join_paths(paths).unwrap())
17830 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17831 .output().unwrap();
17832 assert!(
17833 output.status.success()
17834 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17835 "{}\n{}",
17836 String::from_utf8_lossy(&output.stdout),
17837 String::from_utf8_lossy(&output.stderr)
17838 );
17839 return;
17840 }
17841 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17842 let repo = runner.state.repo.clone();
17843 git::git(&repo, &["checkout", "main"]).await.unwrap();
17844 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17845 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17846 git::git(&repo, &["checkout", "fix/retained"])
17847 .await
17848 .unwrap();
17849 runner.state.pr = Some(crate::run::PrRecord {
17850 url: "https://example.test/project/pull/1".into(),
17851 number: 1,
17852 state: "open".into(),
17853 checks: "unknown".into(),
17854 round: 0,
17855 rounds: 1,
17856 red_at_merge: Vec::new(),
17857 });
17858 runner.sync_to_base().await.unwrap();
17859 assert_eq!(runner.state.status, RunStatus::Blocked);
17860 assert_eq!(
17861 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17862 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17863 );
17864 assert!(runner.state.base_sync_closure_pending());
17865 assert!(crate::handover::can_resume(&runner.state));
17866 let queue = crate::queue::Queue::open();
17867 assert!(!crate::base_sync_recovery::held_task(
17868 &queue,
17869 &queue.get(&task).unwrap()
17870 ));
17871 assert!(runner.state.authorize_base_sync(&task).is_err());
17872 runner.state.base_sync_recovery.stop = None;
17874 runner.state.status = RunStatus::Voting;
17875 runner.state.gate_ran = true;
17876 runner.state.config.graph.review_rounds = 0;
17877 runner.state.save().unwrap();
17878 assert!(crate::handover::can_resume(&runner.state));
17879 runner.state = RunState::load(&runner.state.id).unwrap();
17880 runner.execute_graph().await.unwrap();
17881 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17882 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17883 assert_eq!(
17884 runner.state.base_sync.as_ref().unwrap().attempts,
17885 BASE_SYNC_ROUNDS
17886 );
17887 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17888 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17889 assert!(runner.state.reviews.is_empty());
17890 assert!(runner.state.gate.is_empty());
17891 }
17892
17893 #[cfg(unix)]
17894 #[tokio::test]
17895 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17896 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17897 runner.state.driver_exited = true;
17898 runner.state.save().unwrap();
17899 let home = crate::run::home();
17900 let queue = crate::queue::Queue::open();
17901 let _claim = queue.claim(&task).unwrap();
17902 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17903 assert!(
17904 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17905 .await
17906 .is_err()
17907 );
17908 drop(lease);
17909 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17910 assert!(
17911 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17912 .await
17913 .is_err()
17914 );
17915 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17916 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17917 .await
17918 .unwrap();
17919 let disk = RunState::load(&runner.state.id).unwrap();
17920 assert!(disk.base_recovery_pending());
17921 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17922 assert!(
17923 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17924 .await
17925 .is_err()
17926 );
17927 assert_eq!(
17928 RunState::load(&runner.state.id)
17929 .unwrap()
17930 .base_sync_recovery
17931 .authorizations
17932 .len(),
17933 1
17934 );
17935 let mut owner = queue.get(&task).unwrap();
17936 owner.runs.push("20260101-000000-ffff".into());
17937 queue.put(&mut owner).unwrap();
17938 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17939 .await
17940 .unwrap_err();
17941 assert!(error.to_string().contains("run task ownership changed"));
17942 }
17943
17944 #[cfg(unix)]
17945 #[tokio::test]
17946 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17947 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17948 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17949 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17950 legacy["status"] = serde_json::json!("voting");
17951 runner.state = serde_json::from_value(legacy).unwrap();
17952 runner.state.save().unwrap();
17953 for _ in 0..3 {
17954 runner.execute_graph().await.unwrap();
17955 assert_eq!(runner.state.status, RunStatus::Blocked);
17956 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17957 assert!(!crate::handover::can_resume(&runner.state));
17958 assert_eq!(
17959 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17960 head
17961 );
17962 assert!(runner.state.reviews.is_empty());
17963 assert!(!runner.state.gate_ran);
17964 }
17965 let notice = crate::notices::run_ended(&runner.state).unwrap();
17966 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17967 runner.state.begin_pass();
17968 runner.state.status = RunStatus::Voting;
17969 runner.deliberate().await.unwrap();
17970 runner.vote().await.unwrap();
17971 runner.finish_after_tally().await.unwrap();
17972 assert_eq!(runner.state.status, RunStatus::Blocked);
17973 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17974 assert_eq!(
17975 crate::notices::run_ended(&runner.state).unwrap().since,
17976 notice.since
17977 );
17978 }
17979
17980 #[cfg(unix)]
17981 #[tokio::test]
17982 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17983 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17984 for detail in [
17985 "fix/retained conflicts with origin/main and did not rebase: conflict",
17986 "rebased fix/retained locally but origin refused the push",
17987 "unknown failure",
17988 ] {
17989 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17990 runner.state.record_base_sync_stop(&head);
17991 assert!(runner.state.authorize_base_sync(&task).is_err());
17992 runner.state.status = RunStatus::Voting;
17993 runner.execute_graph().await.unwrap();
17994 assert_eq!(runner.state.status, RunStatus::Blocked);
17995 assert!(!crate::handover::can_resume(&runner.state));
17996 assert_eq!(
17997 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17998 head
17999 );
18000 }
18001 }
18002
18003 #[cfg(unix)]
18004 #[tokio::test]
18005 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
18006 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18007 runner.state.authorize_base_sync(&task).unwrap();
18008 assert!(runner.state.consume_base_recovery(&head));
18009 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18010 runner.state.reserve_base_operation(&head, &tip);
18011 runner.state.save().unwrap();
18012 runner.state = RunState::load(&runner.state.id).unwrap();
18013 assert!(!runner.state.consume_base_recovery(&head));
18014 runner.sync_to_base().await.unwrap();
18015 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18016 assert_eq!(runner.state.base_episode_used(), 1);
18017 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
18018 assert!(
18019 runner
18020 .state
18021 .base_sync_recovery
18022 .operations
18023 .last()
18024 .unwrap()
18025 .finished
18026 );
18027 assert!(!runner.state.base_sync_blocked());
18028 runner.sync_to_base().await.unwrap();
18029 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18030 }
18031
18032 #[cfg(unix)]
18033 #[tokio::test]
18034 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
18035 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18036 let original = runner
18038 .state
18039 .base_sync
18040 .as_ref()
18041 .unwrap()
18042 .conflict
18043 .clone()
18044 .unwrap();
18045 runner.state.base_sync.as_mut().unwrap().conflict = Some(
18046 original
18047 .replace("synchronization(s)", "rebase(s)")
18048 .replace("synchronizing", "rebasing"),
18049 );
18050 runner.state.base_sync_recovery = Default::default();
18051 runner.state.record_base_sync_stop(&head);
18052 assert_eq!(
18053 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
18054 crate::base_sync_recovery::StopKind::Exhausted
18055 );
18056 runner.state.authorize_base_sync(&task).unwrap();
18057 let mut legacy = serde_json::to_value(&runner.state).unwrap();
18058 let auth = legacy["base_sync_recovery"]["authorizations"][0]
18059 .as_object_mut()
18060 .unwrap();
18061 auth.remove("interval");
18062 auth.remove("scope");
18063 runner.state = serde_json::from_value(legacy).unwrap();
18064 assert!(runner.state.consume_base_recovery(&head));
18065 assert_eq!(runner.state.base_episode_used(), 3);
18066 runner.sync_to_base().await.unwrap();
18067 assert_eq!(runner.state.base_episode_used(), 4);
18068 runner.state.resource_block = Some(crate::run::ResourceBlock {
18069 node: "gate".into(),
18070 kind: "cache".into(),
18071 detail: "legacy wait".into(),
18072 at: Timestamp::now(),
18073 cause: crate::run::ResourceBlockCause::Unrecorded,
18074 });
18075 runner.state.save().unwrap();
18076 assert!(
18077 runner
18078 .state
18079 .base_sync_recovery
18080 .episodes
18081 .last()
18082 .unwrap()
18083 .boundary
18084 .is_none()
18085 );
18086 runner.state.resource_block = None;
18087 advance_base_fixture(&runner).await;
18088 runner.sync_to_base().await.unwrap();
18089 assert!(runner.state.base_sync_blocked());
18090 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18091 }
18092
18093 #[cfg(unix)]
18094 async fn advance_base_fixture(runner: &Runner) {
18095 let repo = &runner.state.repo;
18096 for args in [
18097 vec!["checkout", "main"],
18098 vec!["commit", "--allow-empty", "-m", "base advanced"],
18099 vec!["push", "origin", "main"],
18100 vec!["checkout", "fix/retained"],
18101 ] {
18102 git::git(repo, &args).await.unwrap();
18103 }
18104 }
18105
18106 #[cfg(unix)]
18107 fn fresh_base_interval_fixture(runner: &mut Runner) {
18108 runner.state.base_sync.as_mut().unwrap().attempts = 0;
18109 runner.state.base_sync.as_mut().unwrap().conflict = None;
18110 runner.state.base_sync_recovery = Default::default();
18111 runner.state.status = RunStatus::Voting;
18112 }
18113
18114 #[cfg(unix)]
18115 #[tokio::test]
18116 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
18117 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
18118 fresh_base_interval_fixture(&mut runner);
18119 runner.state.begin_pass();
18120 runner.sync_to_base().await.unwrap();
18121 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18122 runner.state.resource_block = Some(crate::run::ResourceBlock {
18123 node: "gate".into(),
18124 kind: "build cache".into(),
18125 detail: "busy".into(),
18126 at: Timestamp::now(),
18127 cause: crate::run::ResourceBlockCause::Timeout {
18128 waited_secs: 1,
18129 limit_secs: 1,
18130 },
18131 });
18132 runner.state.save().unwrap();
18133 let queue = crate::queue::Queue::open();
18134 let mut owner = queue.get(&task).unwrap();
18135 runner.state.task_instruction = Some(owner.instruction.clone());
18136 owner.release_operator("fixture-owner", None);
18137 queue.put(&mut owner).unwrap();
18138 let approval = owner.resumption.clone().unwrap();
18139 runner.state.resource_block = None;
18140 runner.state.save().unwrap();
18141 runner.state = RunState::load(&runner.state.id).unwrap();
18142 advance_base_fixture(&runner).await;
18143 runner.sync_to_base().await.unwrap();
18144 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18145 assert_eq!(runner.state.base_episode_used(), 1);
18146 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
18147 assert!(
18148 runner.state.base_sync_recovery.episodes[0]
18149 .boundary
18150 .as_ref()
18151 .unwrap()
18152 .consumed
18153 );
18154 assert!(
18155 !runner
18156 .state
18157 .renew_base_episode(&task, &approval, &head, "base")
18158 );
18159 for _ in 0..3 {
18160 owner.release_operator("fixture-owner", None);
18161 queue.put(&mut owner).unwrap();
18162 runner.state = RunState::load(&runner.state.id).unwrap();
18163 runner.state.begin_pass();
18164 runner.sync_to_base().await.unwrap();
18165 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
18166 assert_eq!(runner.state.base_episode_used(), 1);
18167 }
18168 runner.state.resource_block = Some(crate::run::ResourceBlock {
18169 node: "gate".into(),
18170 kind: "disk pressure".into(),
18171 detail: "free space required".into(),
18172 at: Timestamp::now(),
18173 cause: crate::run::ResourceBlockCause::Other,
18174 });
18175 runner.state.end_pass_parked(ParkCause::Interrupt {
18176 reason: "disk pressure".into(),
18177 });
18178 runner.state.save().unwrap();
18179 owner.release_operator("fixture-owner-after-disk", None);
18180 queue.put(&mut owner).unwrap();
18181 runner.state.resource_block = None;
18182 advance_base_fixture(&runner).await;
18183 runner.sync_to_base().await.unwrap();
18184 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
18185 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
18186 assert_eq!(runner.state.base_episode_used(), 1);
18187 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
18188 }
18189
18190 #[cfg(unix)]
18191 #[tokio::test]
18192 async fn base_recovery_continuous_advancement_stops_at_four() {
18193 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18194 fresh_base_interval_fixture(&mut runner);
18195 for spent in 1..=BASE_SYNC_ROUNDS {
18196 runner.sync_to_base().await.unwrap();
18197 assert!(!runner.state.base_sync_blocked());
18198 assert_eq!(runner.state.base_episode_used(), spent);
18199 advance_base_fixture(&runner).await;
18200 }
18201 runner.sync_to_base().await.unwrap();
18202 assert_eq!(
18203 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
18204 crate::base_sync_recovery::StopKind::Exhausted
18205 );
18206 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
18207 assert!(stop.unverified_sync);
18208 let action = runner.state.base_sync_next_action();
18209 assert!(action.contains("never reached its gate"));
18210 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
18211 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18212 runner.state = RunState::load(&runner.state.id).unwrap();
18213 runner.sync_to_base().await.unwrap();
18214 assert_eq!(runner.state.base_episode_used(), 4);
18215 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
18216 }
18217
18218 #[cfg(unix)]
18219 #[tokio::test]
18220 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
18221 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
18222 runner.state.ensure_base_episode();
18223 runner.state.record_base_progress(&head);
18225 runner.state.base_sync_recovery.stop = None;
18226 runner.state.record_base_sync_stop(&head);
18227 assert!(
18228 !runner
18229 .state
18230 .base_sync_recovery
18231 .stop
18232 .as_ref()
18233 .unwrap()
18234 .unverified_sync
18235 );
18236
18237 runner.state.finish_base_operation(Some(&head));
18238 runner.state.gate_ran = true;
18240 runner.state.gate_head = Some("older-head".into());
18241 runner.state.base_sync_recovery.stop = None;
18242 runner.state.record_base_sync_stop(&head);
18243 runner.state.save().unwrap();
18244 runner.state = RunState::load(&runner.state.id).unwrap();
18245 assert!(
18246 runner
18247 .state
18248 .base_sync_recovery
18249 .stop
18250 .as_ref()
18251 .unwrap()
18252 .unverified_sync
18253 );
18254
18255 runner.state.gate_head = Some(head.clone());
18256 runner.state.base_sync_recovery.stop = None;
18257 runner.state.record_base_sync_stop(&head);
18258 assert!(
18259 !runner
18260 .state
18261 .base_sync_recovery
18262 .stop
18263 .as_ref()
18264 .unwrap()
18265 .unverified_sync
18266 );
18267 assert!(
18268 !runner
18269 .state
18270 .base_sync_next_action()
18271 .contains("never reached its gate")
18272 );
18273
18274 let mut legacy = serde_json::to_value(&runner.state).unwrap();
18276 legacy["base_sync_recovery"]["stop"]
18277 .as_object_mut()
18278 .unwrap()
18279 .remove("unverified_sync");
18280 legacy["base_sync_recovery"]["episodes"][0]
18281 .as_object_mut()
18282 .unwrap()
18283 .remove("last_sync_head");
18284 runner.state = serde_json::from_value(legacy).unwrap();
18285 assert!(
18286 !runner
18287 .state
18288 .base_sync_recovery
18289 .stop
18290 .as_ref()
18291 .unwrap()
18292 .unverified_sync
18293 );
18294 assert!(
18295 runner.state.base_sync_recovery.episodes[0]
18296 .last_sync_head
18297 .is_none()
18298 );
18299 }
18300
18301 #[cfg(unix)]
18302 #[tokio::test]
18303 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
18304 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18305 runner.state.authorize_base_sync(&task).unwrap();
18306 assert!(
18307 runner
18308 .state
18309 .base_sync_next_action()
18310 .contains("Approval recorded")
18311 );
18312 assert!(
18313 runner
18314 .state
18315 .base_sync_next_action()
18316 .contains("4/4 in episode 0, 4 lifetime")
18317 );
18318 assert!(runner.state.consume_base_recovery(&head));
18319 assert!(
18320 runner
18321 .state
18322 .base_sync_next_action()
18323 .contains("0/4 in episode 1, 4 lifetime")
18324 );
18325 }
18326
18327 #[cfg(unix)]
18328 #[tokio::test]
18329 async fn base_recovery_changed_head_invalidates_without_spending() {
18330 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
18331 runner.state.authorize_base_sync(&task).unwrap();
18332 git::git(
18333 &runner.state.repo,
18334 &["commit", "--allow-empty", "-m", "external head change"],
18335 )
18336 .await
18337 .unwrap();
18338 runner.sync_to_base().await.unwrap();
18339 let a = runner
18340 .state
18341 .base_sync_recovery
18342 .authorizations
18343 .last()
18344 .unwrap();
18345 assert!(a.consumed);
18346 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
18347 runner.state = RunState::load(&runner.state.id).unwrap();
18348 assert!(
18349 runner
18350 .state
18351 .base_sync_next_action()
18352 .contains("Approval invalidated")
18353 );
18354 assert!(!runner.state.base_recovery_pending());
18355 assert!(runner.state.base_sync_blocked());
18356 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18357 }
18358
18359 #[cfg(unix)]
18360 #[tokio::test]
18361 async fn base_recovery_stale_fetched_base_invalidates_approval() {
18362 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18363 runner.state.driver_exited = true;
18364 runner.state.save().unwrap();
18365 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18366 .await
18367 .unwrap();
18368 runner.state = RunState::load(&runner.state.id).unwrap();
18369 advance_base_fixture(&runner).await;
18370 runner.sync_to_base().await.unwrap();
18371 let a = runner
18372 .state
18373 .base_sync_recovery
18374 .authorizations
18375 .last()
18376 .unwrap();
18377 assert!(
18378 a.consumed
18379 && a.invalidated
18380 .as_deref()
18381 .unwrap()
18382 .contains("fetched base changed")
18383 );
18384 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18385 assert_eq!(
18386 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18387 head
18388 );
18389 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18390 .await
18391 .unwrap();
18392 runner.state = RunState::load(&runner.state.id).unwrap();
18393 runner.sync_to_base().await.unwrap();
18394 assert!(!runner.state.base_sync_blocked());
18395 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18396 }
18397
18398 #[cfg(unix)]
18399 #[tokio::test]
18400 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
18401 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18402 runner.state.authorize_base_sync(&task).unwrap();
18403 assert!(runner.state.consume_base_recovery(&head));
18404 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18405 runner.state.reserve_base_operation(&head, &tip);
18406 runner.state.save().unwrap();
18407 let scratch = runner.state.dir().join("base-sync");
18408 assert_eq!(
18409 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18410 .await
18411 .unwrap(),
18412 crate::rebase::Rebased::Applied
18413 );
18414 runner.state = RunState::load(&runner.state.id).unwrap();
18415 runner.sync_to_base().await.unwrap();
18416 assert!(
18417 runner
18418 .state
18419 .base_sync_recovery
18420 .operations
18421 .last()
18422 .unwrap()
18423 .finished
18424 );
18425 assert_eq!(runner.state.base_episode_used(), 1);
18426 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18427 assert!(!runner.state.base_sync_blocked());
18428 }
18429
18430 #[cfg(unix)]
18431 #[tokio::test]
18432 async fn base_recovery_crash_preserves_untracked_files() {
18433 for ignored in [false, true] {
18434 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18435 runner.state.authorize_base_sync(&task).unwrap();
18436 assert!(runner.state.consume_base_recovery(&head));
18437 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18438 runner.state.reserve_base_operation(&head, &tip);
18439 runner.state.save().unwrap();
18440 let scratch = runner.state.dir().join("base-sync");
18441 assert_eq!(
18442 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18443 .await
18444 .unwrap(),
18445 crate::rebase::Rebased::Applied
18446 );
18447 let repo = &runner.state.repo;
18448 let result = git::rev_parse(repo, "HEAD").await.unwrap();
18449 assert_ne!(result, head);
18450 if ignored {
18451 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
18452 .await
18453 .unwrap();
18454 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
18455 }
18456 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
18457 assert!(
18458 git::git_raw(repo, &["diff", "--quiet", &head])
18459 .await
18460 .unwrap()
18461 .ok()
18462 );
18463 runner.state = RunState::load(&runner.state.id).unwrap();
18465 runner.sync_to_base().await.unwrap();
18466 assert!(runner.state.base_sync_blocked());
18467 assert!(!runner.state.base_sync_recovery.operations[0].finished);
18468 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18469 assert_eq!(
18470 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18471 result
18472 );
18473 assert_eq!(
18474 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
18475 "retained work\n"
18476 );
18477 assert!(
18478 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
18479 .await
18480 .unwrap()
18481 .ok()
18482 );
18483 }
18484 }
18485
18486 #[cfg(unix)]
18487 #[tokio::test]
18488 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
18489 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18490 runner.state.authorize_base_sync(&task).unwrap();
18491 assert!(runner.state.consume_base_recovery(&head));
18492 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18493 runner.state.reserve_base_operation(&head, &tip);
18494 runner.state.save().unwrap();
18495 let scratch = runner.state.dir().join("base-sync");
18496 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18497 .await
18498 .unwrap();
18499 git::sync_to_head(&runner.state.repo).await.unwrap();
18500 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
18501 git::git(&runner.state.repo, &["add", "foreign"])
18502 .await
18503 .unwrap();
18504 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
18505 .await
18506 .unwrap();
18507 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18508 runner.state = RunState::load(&runner.state.id).unwrap();
18509 runner.sync_to_base().await.unwrap();
18510 assert!(runner.state.base_sync_blocked());
18511 assert!(
18512 !runner
18513 .state
18514 .base_sync_recovery
18515 .operations
18516 .last()
18517 .unwrap()
18518 .finished
18519 );
18520 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18521 assert_eq!(
18522 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18523 foreign_head
18524 );
18525 }
18526
18527 #[cfg(unix)]
18528 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
18529 {
18530 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18531 fresh_base_interval_fixture(&mut runner);
18532 let repo = &runner.state.repo;
18533 git::git(repo, &["checkout", "main"]).await.unwrap();
18534 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
18535 git::git(repo, &["add", "retained"]).await.unwrap();
18536 git::git(repo, &["commit", "-m", "upstream retained change"])
18537 .await
18538 .unwrap();
18539 git::git(repo, &["push", "origin", "main"]).await.unwrap();
18540 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
18541 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
18542 let scratch = runner.state.dir().join("base-sync");
18543 assert!(matches!(
18544 git::rebase_start(repo, &scratch, "fix/retained", &tip)
18545 .await
18546 .unwrap(),
18547 git::RebaseStart::Conflicted(_)
18548 ));
18549 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
18550 agent: "fixture".into(),
18551 paths: vec!["retained".into()],
18552 from: Some(head.clone()),
18553 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
18554 index_outside: None,
18555 finished: false,
18556 error: None,
18557 launcher: None,
18558 infra: None,
18559 outage: None,
18560 });
18561 (tmp, runner, head, tip, scratch)
18562 }
18563
18564 #[cfg(unix)]
18565 #[tokio::test]
18566 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
18567 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18568 std::fs::write(
18571 scratch.join("retained"),
18572 "upstream and retained resolution\n",
18573 )
18574 .unwrap();
18575 runner.state.resource_block = Some(crate::run::ResourceBlock {
18576 node: "rebase".into(),
18577 kind: "repository coordination".into(),
18578 detail: "busy".into(),
18579 at: Timestamp::now(),
18580 cause: crate::run::ResourceBlockCause::Other,
18581 });
18582 runner.state.save().unwrap();
18583 let newer = git::git(
18586 &runner.state.repo,
18587 &[
18588 "commit-tree",
18589 "main^{tree}",
18590 "-p",
18591 "main",
18592 "-m",
18593 "base advanced during wait",
18594 ],
18595 )
18596 .await
18597 .unwrap();
18598 git::git(
18599 &runner.state.repo,
18600 &[
18601 "push",
18602 "origin",
18603 &format!("{}:refs/heads/main", newer.trim()),
18604 ],
18605 )
18606 .await
18607 .unwrap();
18608 runner.state = RunState::load(&runner.state.id).unwrap();
18609 runner.state.resource_block = None;
18610 runner.sync_to_base().await.unwrap();
18611 assert!(!runner.state.base_sync_blocked());
18612 let op = &runner.state.base_sync_recovery.operations[0];
18613 assert_eq!(op.head, head);
18614 assert_eq!(op.tip, tip);
18615 assert!(op.finished);
18616 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18617 assert_eq!(runner.state.rebase_fixes.len(), 1);
18618 let latest = git::rev_parse(&runner.state.repo, "origin/main")
18619 .await
18620 .unwrap();
18621 assert_ne!(latest, tip);
18622 runner.state = RunState::load(&runner.state.id).unwrap();
18623 runner.sync_to_base().await.unwrap();
18624 assert!(!runner.state.base_sync_blocked());
18625 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
18626 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
18627 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18628 }
18629
18630 #[cfg(unix)]
18631 #[tokio::test]
18632 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
18633 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18634 runner.state.reserve_base_operation(&head, &tip);
18635 runner.state.save().unwrap();
18636 std::fs::write(
18637 scratch.join("retained"),
18638 "upstream and retained resolution\n",
18639 )
18640 .unwrap();
18641 git::git(&scratch, &["add", "retained"]).await.unwrap();
18642 git::git(
18643 &scratch,
18644 &["-c", "core.editor=true", "rebase", "--continue"],
18645 )
18646 .await
18647 .unwrap();
18648 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18649 assert!(
18650 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18651 .await
18652 .unwrap()
18653 .is_empty()
18654 );
18655 assert_eq!(
18656 crate::rebase::finish(
18657 &mut runner.state,
18658 &scratch,
18659 "fix/retained",
18660 &head,
18661 &tip,
18662 &["retained".into()],
18663 ""
18664 )
18665 .await
18666 .unwrap(),
18667 crate::rebase::Rebased::Applied
18668 );
18669 assert!(!scratch.exists());
18670 runner.state = RunState::load(&runner.state.id).unwrap();
18672 let op = &runner.state.base_sync_recovery.operations[0];
18673 assert!(!op.finished);
18674 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18675 runner.sync_to_base().await.unwrap();
18676 assert!(!runner.state.base_sync_blocked());
18677 assert!(runner.state.base_sync_recovery.operations[0].finished);
18678 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18679 assert_eq!(runner.state.rebase_fixes.len(), 1);
18680 let winner = runner.state.winner().unwrap().clone();
18681 let wait = runner.repository_wait();
18682 assert!(
18683 verify_current_head(&mut runner.state, &winner, &wait)
18684 .await
18685 .unwrap()
18686 );
18687 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18688 }
18689
18690 #[cfg(unix)]
18691 #[tokio::test]
18692 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18693 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18694 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18695 runner.state.reserve_base_operation(&head, &tip);
18696 runner.state.save().unwrap();
18697 std::fs::write(
18698 scratch.join("retained"),
18699 "upstream and retained resolution\n",
18700 )
18701 .unwrap();
18702 git::git(&scratch, &["add", "retained"]).await.unwrap();
18703 git::git(
18704 &scratch,
18705 &["-c", "core.editor=true", "rebase", "--continue"],
18706 )
18707 .await
18708 .unwrap();
18709 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18710 assert!(!git::rebase_in_progress(&scratch).await);
18711 assert!(
18712 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18713 .await
18714 .unwrap()
18715 .is_empty()
18716 );
18717 match damage {
18718 Some("record") => {
18719 let path =
18720 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18721 .await
18722 .unwrap();
18723 std::fs::write(scratch.join(path), "").unwrap();
18724 }
18725 Some("foreign") => {
18726 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18727 git::git(&scratch, &["add", "foreign"]).await.unwrap();
18728 git::git(&scratch, &["commit", "-m", "foreign work"])
18729 .await
18730 .unwrap();
18731 }
18732 Some("dirty") => {
18733 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18734 }
18735 _ => {}
18736 }
18737 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18738 runner.state = RunState::load(&runner.state.id).unwrap();
18740 assert!(
18741 runner.state.base_sync_recovery.operations[0]
18742 .validated_head
18743 .is_none()
18744 );
18745 runner.sync_to_base().await.unwrap();
18746 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18747 assert_eq!(runner.state.base_episode_used(), 1);
18748 assert_eq!(runner.state.rebase_fixes.len(), 1);
18749 assert_eq!(
18750 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18751 retained_head
18752 );
18753 if damage.is_some() {
18754 assert!(runner.state.base_sync_blocked());
18755 assert!(scratch.exists());
18756 assert!(!runner.state.base_sync_recovery.operations[0].finished);
18757 assert!(
18758 runner.state.base_sync_recovery.operations[0]
18759 .validated_head
18760 .is_none()
18761 );
18762 } else {
18763 assert!(!runner.state.base_sync_blocked());
18764 assert!(!scratch.exists());
18765 assert!(runner.state.base_sync_recovery.operations[0].finished);
18766 assert_eq!(
18767 runner.state.base_sync_recovery.operations[0]
18768 .validated_head
18769 .as_deref(),
18770 Some(result.as_str())
18771 );
18772 let winner = runner.state.winner().unwrap().clone();
18773 let wait = runner.repository_wait();
18774 assert!(
18775 verify_current_head(&mut runner.state, &winner, &wait)
18776 .await
18777 .unwrap()
18778 );
18779 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18780 runner.state = RunState::load(&runner.state.id).unwrap();
18781 runner.sync_to_base().await.unwrap();
18782 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18783 }
18784 }
18785 }
18786
18787 #[cfg(unix)]
18788 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18789 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18790 runner.state.prep_complete = Some(true);
18792 runner.state.base_sync = None;
18793 runner.state.base_sync_recovery = Default::default();
18794 runner.state.config.graph.gate_fix_rounds = 1;
18795 let mut review = review_round(true, 0, 1, 1, false, true);
18796 review.head = head.clone();
18797 review.verified_head = Some(head.clone());
18798 runner.state.reviews = vec![review];
18799 runner.state.gate = vec![CommandOutcome {
18800 command: "old gate".into(),
18801 code: Some(1),
18802 output_tail: "old failure".into(),
18803 duration_ms: 0,
18804 resource_blocked: false,
18805 }];
18806 runner.state.gate_head = Some(head.clone());
18807 runner.state.gate_ran = true;
18808 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18809 agent: "fixture".into(),
18810 failed: runner.state.gate.clone(),
18811 notes: "spent".into(),
18812 committed: false,
18813 error: None,
18814 }];
18815 runner.state.land_approval = Some(crate::run::LandApproval {
18816 question: "old approval".into(),
18817 head: head.clone(),
18818 });
18819 runner.state.status = RunStatus::Blocked;
18820 (tmp, runner, head)
18821 }
18822
18823 #[cfg(unix)]
18824 #[tokio::test]
18825 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18826 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18827 let (tmp, mut runner, old) = resumed_gate_fixture().await;
18828 let phases = tmp.path().join("phases");
18829 runner.state.config.verify.e2e = vec![format!(
18830 "echo e2e >> '{}'; {}",
18831 phases.display(),
18832 if e2e_ok {
18833 "true"
18834 } else {
18835 "echo new-e2e-failure; false"
18836 }
18837 )];
18838 runner.state.config.verify.gate = vec![format!(
18839 "echo gate >> '{}'; {}",
18840 phases.display(),
18841 if gate_ok {
18842 "true"
18843 } else {
18844 "echo new-gate-failure; false"
18845 }
18846 )];
18847 runner.state.save().unwrap();
18848 runner.state = RunState::load(&runner.state.id).unwrap();
18849 runner.sync_to_base().await.unwrap();
18850 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18851 assert_ne!(head, old);
18852 runner.state = RunState::load(&runner.state.id).unwrap();
18854 runner.execute_graph().await.unwrap();
18855 assert_eq!(runner.state.gate_fixes.len(), 1);
18856 assert_eq!(runner.state.reviews.len(), 1);
18857 assert_eq!(runner.state.reviews[0].head, old);
18858 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18859 assert_eq!(runner.state.verification_history.len(), 1);
18860 let history = &runner.state.verification_history[0];
18861 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18862 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18863 assert!(history.archived_at.is_some());
18864 assert_eq!(history.fixes_spent, Some(1));
18865 assert_eq!(history.gate[0].output_tail, "old failure");
18866 let archive_events: Vec<_> = runner
18867 .state
18868 .events
18869 .iter()
18870 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18871 .collect();
18872 assert_eq!(archive_events.len(), 1);
18873 assert!(archive_events[0].message.contains(&short(&old)));
18874 assert!(archive_events[0].message.contains(&short(&head)));
18875 assert!(
18876 archive_events[0]
18877 .message
18878 .contains("1 gate-fix round(s) already spent stay spent")
18879 );
18880 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18881 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18882 if e2e_ok {
18883 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18884 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18885 } else {
18886 assert!(!runner.state.gate_ran);
18887 }
18888 assert_eq!(
18889 runner.state.status,
18890 if e2e_ok && gate_ok {
18891 RunStatus::Ready
18892 } else {
18893 RunStatus::Blocked
18894 }
18895 );
18896 if !e2e_ok || !gate_ok {
18897 let cause = runner.state.verification_blocking_cause().unwrap();
18898 assert!(cause.contains(&head));
18899 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18900 runner.state.event("land", "rebased successfully");
18902 let notice = crate::notices::run_ended(&runner.state).unwrap();
18903 assert!(!format!("{notice:?}").contains("rebased successfully"));
18904 assert!(runner.state.merge.is_none());
18905 }
18906 let before = std::fs::read_to_string(&phases).unwrap();
18907 runner.state.save().unwrap();
18908 runner.state = RunState::load(&runner.state.id).unwrap();
18909 runner.review_loop().await.unwrap();
18910 runner.gate().await.unwrap();
18911 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18912 assert_eq!(runner.state.verification_history.len(), 1);
18913 let history = &runner.state.verification_history[0];
18914 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18915 assert!(history.archived_at.is_some());
18916 assert_eq!(history.fixes_spent, Some(1));
18917 }
18918 }
18919
18920 #[cfg(unix)]
18921 #[tokio::test]
18922 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18923 let (tmp, mut runner, head) = resumed_gate_fixture().await;
18924 let marker = tmp.path().join("gate-ran");
18925 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18926 runner.review_loop().await.unwrap();
18927 runner.gate().await.unwrap();
18928 assert_eq!(runner.state.status, RunStatus::Blocked);
18929 assert!(!marker.exists());
18930 assert!(runner.state.verification_history.is_empty());
18931 runner.state.gate_head = None;
18932 runner.gate().await.unwrap();
18933 assert!(marker.exists());
18934 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18935 assert_eq!(runner.state.verification_history.len(), 1);
18936 assert_eq!(runner.state.gate_fixes.len(), 1);
18937 }
18938
18939 #[cfg(unix)]
18940 #[tokio::test]
18941 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18942 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18943 runner.sync_to_base().await.unwrap();
18944 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18945 runner.state.reconcile_verification_head(&head);
18946 for outcomes in [
18948 Vec::new(),
18949 vec![CommandOutcome {
18950 command: "true".into(),
18951 code: None,
18952 output_tail: "cache unavailable".into(),
18953 duration_ms: 0,
18954 resource_blocked: true,
18955 }],
18956 ] {
18957 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18958 head: head.clone(),
18959 passed: false,
18960 outcomes,
18961 });
18962 runner.state.gate_ran = false;
18963 runner.state.gate.clear();
18964 runner.state.save().unwrap();
18965 runner.state = RunState::load(&runner.state.id).unwrap();
18966 runner.merge().await.unwrap();
18967 assert!(runner.state.merge.is_none());
18968 runner.review_loop().await.unwrap();
18969 runner.gate().await.unwrap();
18970 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18971 assert!(runner.state.gate_allows_progress());
18972 assert_eq!(runner.state.gate_fixes.len(), 1);
18973 assert!(runner.state.merge.is_none());
18974 }
18975 git::git(
18977 &runner.state.repo,
18978 &["commit", "--allow-empty", "-m", "external update"],
18979 )
18980 .await
18981 .unwrap();
18982 runner.merge().await.unwrap();
18983 assert!(runner.state.merge.is_none());
18984 assert_eq!(runner.state.status, RunStatus::Blocked);
18985 }
18986
18987 #[cfg(unix)]
18988 #[tokio::test]
18989 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18990 for waiting_on_gate in [false, true] {
18991 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18992 runner.sync_to_base().await.unwrap();
18993 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18994 runner.state.reconcile_verification_head(&head);
18995 if waiting_on_gate {
18996 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18997 head: head.clone(),
18998 passed: true,
18999 outcomes: Vec::new(),
19000 });
19001 }
19002 let cache = tmp.path().join("cache");
19003 runner.state.config.verify.e2e =
19004 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
19005 runner.state.config.verify.gate = vec!["true".into()];
19006 runner.state.config.graph.timeout_resource_wait = 1;
19007 let owner =
19008 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
19009 let guard =
19010 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
19011 crate::cache::AcquireOutcome::Acquired(g) => g,
19012 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
19013 };
19014 runner.review_loop().await.unwrap();
19015 runner.gate().await.unwrap();
19016 assert!(!runner.state.gate_ran);
19017 assert_eq!(runner.state.gate_fixes.len(), 1);
19018 assert!(runner.state.resource_block.is_some());
19019 runner.merge().await.unwrap();
19020 assert!(runner.state.merge.is_none());
19021 drop(guard);
19022 runner.state = RunState::load(&runner.state.id).unwrap();
19023 runner.state.resource_block = None;
19025 runner.review_loop().await.unwrap();
19026 runner.gate().await.unwrap();
19027 assert!(runner.state.gate_allows_progress());
19028 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
19029 assert_eq!(runner.state.gate_fixes.len(), 1);
19030 }
19031 }
19032
19033 #[cfg(unix)]
19034 #[tokio::test]
19035 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
19036 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
19037 runner.sync_to_base().await.unwrap();
19038 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
19039 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
19040 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
19042 runner.review_loop().await.unwrap();
19043 runner.gate().await.unwrap();
19044 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
19045 assert_ne!(before, after);
19046 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
19047 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
19048 assert!(!runner.state.gate_allows_progress());
19049 runner.merge().await.unwrap();
19050 assert!(runner.state.merge.is_none());
19051 assert!(
19052 runner
19053 .state
19054 .verification_history
19055 .iter()
19056 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
19057 );
19058
19059 runner.state.reviews[0].clean = false;
19061 runner.state.reviews[0].answered = 0;
19062 runner.state.config.verify.e2e = vec!["true".into()];
19063 runner.review_loop().await.unwrap();
19064 runner.gate().await.unwrap();
19065 runner.merge().await.unwrap();
19066 assert_eq!(runner.state.status, RunStatus::Blocked);
19067 assert!(runner.state.merge.is_none());
19068 assert!(
19069 runner
19070 .state
19071 .verification_blocking_cause()
19072 .unwrap()
19073 .contains("review blocked")
19074 );
19075 }
19076
19077 #[cfg(unix)]
19078 #[tokio::test]
19079 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
19080 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
19081 runner.state.gate_head = None;
19082 runner.state.config.verify.gate =
19083 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
19084 runner.review_loop().await.unwrap();
19085 runner.gate().await.unwrap();
19086 assert_ne!(
19087 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
19088 old
19089 );
19090 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
19091 assert!(!runner.state.gate_allows_progress());
19092 assert_eq!(runner.state.status, RunStatus::Blocked);
19093 runner.merge().await.unwrap();
19094 assert!(runner.state.merge.is_none());
19095 }
19096
19097 #[cfg(unix)]
19098 #[tokio::test]
19099 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
19100 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
19101 let mut old_review = review_round(true, 0, 1, 1, false, true);
19102 old_review.head = head.clone();
19103 old_review.verified_head = Some(head.clone());
19104 runner.state.reviews = vec![old_review];
19105 runner.state.gate = vec![CommandOutcome {
19106 command: "old gate".into(),
19107 code: None,
19108 output_tail: "timed out after 2400s".into(),
19109 duration_ms: 2_400_000,
19110 resource_blocked: false,
19111 }];
19112 runner.state.gate_ran = true;
19113 runner.state.gate_head = Some(head.clone());
19114 runner.state.land_approval = Some(crate::run::LandApproval {
19115 question: "old-head-approval".into(),
19116 head: head.clone(),
19117 });
19118 runner.state.authorize_base_sync(&task).unwrap();
19119 runner.sync_to_base().await.unwrap();
19120 assert!(!runner.state.base_sync_blocked());
19121 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
19122 assert_ne!(head, fresh);
19123 assert_eq!(runner.state.reviews.len(), 1);
19124 assert!(!runner.state.gate_ran);
19125 let saved = &runner.state.base_sync_recovery.authorizations[0];
19126 assert_eq!(saved.reviews[0].head, head);
19127 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
19128 assert!(saved.gate_ran);
19129 assert!(!saved.gate[0].ok());
19130 runner.review_loop().await.unwrap();
19131 runner.gate().await.unwrap();
19132 assert_eq!(runner.state.reviews[0].head, head);
19133 assert_eq!(
19134 runner.state.reviews[0].verified_head.as_deref(),
19135 Some(head.as_str())
19136 );
19137 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
19138 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
19139 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
19140 assert!(runner.state.gate_ran);
19141 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
19142 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
19143 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
19144 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
19145 assert_eq!(
19146 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
19147 "retained change"
19148 );
19149 }
19150
19151 #[cfg(unix)]
19152 #[tokio::test]
19153 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
19154 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
19155 let queue = crate::queue::Queue::open();
19156 let mut owner = queue.get(&task).unwrap();
19157 owner.release_operator("test-resume", None);
19158 queue.put(&mut owner).unwrap();
19159 let generation = owner.resumption.as_ref().unwrap().generation;
19160 runner.state.task_instruction = Some(owner.instruction.clone());
19161 runner.state.driver_exited = true;
19162 runner.state.save().unwrap();
19163 let home = crate::run::home();
19164 let _claim = queue.claim(&task).unwrap();
19165 owner.resumption.as_mut().unwrap().at =
19166 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
19167 queue.put(&mut owner).unwrap();
19168 assert!(
19169 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
19170 .await
19171 .is_err(),
19172 "an approval from before the stop grants nothing"
19173 );
19174 owner.resumption.as_mut().unwrap().at = Timestamp::now();
19175 queue.put(&mut owner).unwrap();
19176 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
19177 .await
19178 .unwrap();
19179 assert!(
19180 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
19181 .await
19182 .is_err()
19183 );
19184 runner.state = RunState::load(&runner.state.id).unwrap();
19185 assert_eq!(
19186 runner.state.base_sync_recovery.authorizations[0].generation,
19187 Some(generation)
19188 );
19189 runner.sync_to_base().await.unwrap();
19190 let repo = runner.state.repo.clone();
19191 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
19192 git::git(&repo, &["checkout", "main"]).await.unwrap();
19193 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
19194 git::git(&repo, &["add", "."]).await.unwrap();
19195 git::git(&repo, &["commit", "-m", "move base again"])
19196 .await
19197 .unwrap();
19198 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
19199 git::git(&repo, &["checkout", "fix/retained"])
19200 .await
19201 .unwrap();
19202 runner.sync_to_base().await.unwrap();
19203 assert!(!runner.state.base_sync_blocked());
19204 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
19205 assert_eq!(runner.state.base_episode_used(), 2);
19206 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
19207 assert_ne!(fresh, head);
19208 assert!(!runner.state.base_recovery_pending());
19209 owner.resumption.as_mut().unwrap().at = Timestamp::now();
19211 queue.put(&mut owner).unwrap();
19212 assert!(
19213 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
19214 .await
19215 .is_err()
19216 );
19217 runner.state = RunState::load(&runner.state.id).unwrap();
19218 runner.sync_to_base().await.unwrap();
19219 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
19220 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
19221 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
19222 }
19223
19224 #[cfg(unix)]
19225 #[tokio::test]
19226 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
19227 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
19228 runner.state.base_sync = None;
19229 runner.state.status = RunStatus::Blocked;
19230 let mut second = runner.state.candidates[0].clone();
19231 second.label = 'B';
19232 runner.state.candidates.push(second);
19233 assert!(runner.state.votes.is_empty());
19234 assert!(runner.state.deliberation.is_empty());
19235 runner.deliberate().await.unwrap();
19236 runner.vote().await.unwrap();
19237 assert_eq!(runner.state.status, RunStatus::Blocked);
19238 assert!(runner.state.votes.is_empty());
19239 assert!(runner.state.deliberation.is_empty());
19240 }
19241
19242 fn runner_at(status: RunStatus) -> Runner {
19245 let mut state = RunState::new(
19246 PathBuf::from("/nonexistent/repo"),
19247 "main".to_owned(),
19248 "deadbeef".to_owned(),
19249 "task".to_owned(),
19250 Config::default(),
19251 );
19252 state.config.disk.min_free_bytes = 0;
19254 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
19257 state.id = format!(
19258 "20000101-000000-{:04x}",
19259 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
19260 );
19261 state.status = status;
19262 Runner {
19263 state,
19264 roles: ResolvedRoles {
19265 implementers: Vec::new(),
19266 judges: Vec::new(),
19267 reviewers: Vec::new(),
19268 fixer: None,
19269 conductor: conductor(),
19270 implementer_roster: Vec::new(),
19271 judge_roster: Vec::new(),
19272 reviewer_roster: Vec::new(),
19273 },
19274 sem: Arc::new(Semaphore::new(1)),
19275 pause: Pause::new(),
19276 interrupt: Pause::new(),
19277 }
19278 }
19279
19280 #[tokio::test]
19281 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
19282 crate::run::pin_test_home();
19283 let temp = tempfile::tempdir().unwrap();
19284 let root = temp.path().canonicalize().unwrap();
19285 let repo = root.join("repo");
19286 std::fs::create_dir(&repo).unwrap();
19287 init_repo(&repo);
19288 git::git(&repo, &["config", "core.autocrlf", "false"])
19289 .await
19290 .unwrap();
19291 let mut runner = runner_at(RunStatus::Blocked);
19292 runner.state.repo = repo.clone();
19293 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
19294 runner.state.config.agents = vec![conductor()];
19295 runner.state.config.graph.selected_implementer = Some("conductor".into());
19296 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
19297 let (shell, command, setup_output) = if cfg!(windows) {
19298 (
19299 vec!["cmd".into(), "/C".into()],
19300 "echo ready>seat-ready",
19301 "ready\r\n",
19302 )
19303 } else {
19304 (
19305 vec!["sh".into(), "-c".into()],
19306 "printf ready > seat-ready",
19307 "ready",
19308 )
19309 };
19310 runner.state.config.verify.shell = Some(shell);
19311 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
19312 run: Some(command.into()),
19313 ..Default::default()
19314 }];
19315 runner.state.save().unwrap();
19316 let path = runner.state.worktree_root().join("judge-1");
19317 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
19318 .await
19319 .unwrap();
19320 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
19321 git::git(&path, &["add", "."]).await.unwrap();
19322 let tree = git::git(&path, &["write-tree"]).await.unwrap();
19323 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
19324 .await
19325 .unwrap();
19326 git::git(&path, &["checkout", "--detach", &head])
19327 .await
19328 .unwrap();
19329 let evidence = crate::fold_evidence::snapshot(
19330 &runner.state,
19331 &crate::run::home().canonicalize().unwrap(),
19332 &head,
19333 64 * 1024 * 1024,
19334 )
19335 .await
19336 .unwrap();
19337 let mut contents = BTreeMap::new();
19338 contents.insert(
19339 repo.join("logs/HEAD"),
19340 format!("{head} {head} fixture\n").into_bytes(),
19341 );
19342 let record = serde_json::json!({
19343 "version":1, "run":runner.state.id, "source":path, "head":head,
19344 "evidence":evidence, "admin":{}, "admin_contents":contents,
19345 "history":{head.clone():evidence}, "phase":"evicted"
19346 });
19347 std::fs::write(
19348 runner.state.dir().join("artifacts/eviction-judge-1.json"),
19349 serde_json::to_vec(&record).unwrap(),
19350 )
19351 .unwrap();
19352 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
19353 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
19354 .await
19355 .unwrap();
19356 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
19357 assert!(
19358 !git::git_raw(&repo, &["cat-file", "-e", &head])
19359 .await
19360 .unwrap()
19361 .ok()
19362 );
19363 let mut resumed = Runner::resume(&runner.state.id).unwrap();
19364 let error = resumed.execute().await.unwrap_err();
19365 assert!(
19366 error
19367 .to_string()
19368 .contains("selected implementation is held"),
19369 "{error:#}"
19370 );
19371 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
19372 assert_eq!(
19373 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
19374 setup_output
19375 );
19376 assert_eq!(
19377 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
19378 "unrelated seat"
19379 );
19380 let error = resumed.execute().await.unwrap_err();
19381 assert!(
19382 error
19383 .to_string()
19384 .contains("selected implementation is held"),
19385 "{error:#}"
19386 );
19387 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
19388 git::git(&path, &["add", "."]).await.unwrap();
19389 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
19390 let next = git::git(
19391 &path,
19392 &[
19393 "commit-tree",
19394 &next_tree,
19395 "-p",
19396 &head,
19397 "-m",
19398 "next review checkpoint",
19399 ],
19400 )
19401 .await
19402 .unwrap();
19403 git::git(&path, &["checkout", "--detach", &next])
19404 .await
19405 .unwrap();
19406 resumed.state.reviews.push(
19407 serde_json::from_value(serde_json::json!({
19408 "round":1, "head":next, "reviews":[]
19409 }))
19410 .unwrap(),
19411 );
19412 resumed.state.save().unwrap();
19413 let error = resumed.execute().await.unwrap_err();
19414 assert!(
19415 error
19416 .to_string()
19417 .contains("selected implementation is held"),
19418 "{error:#}"
19419 );
19420 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
19421 }
19422
19423 #[tokio::test]
19424 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
19425 crate::run::pin_test_home();
19426 let mut runner = runner_at(RunStatus::Gating);
19427 runner.state.save().unwrap();
19428 let id = runner.state.id.clone();
19429 let passes = runner.state.passes.len();
19430 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19431 let mut claiming = Box::pin(runner.claim_driver());
19432 std::future::poll_fn(|cx| {
19433 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
19434 std::task::Poll::Ready(())
19435 })
19436 .await;
19437 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
19438 drop(owner);
19439 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
19440 .await
19441 .unwrap()
19442 .unwrap();
19443 assert_eq!(execution.runner.state.passes.len(), passes + 1);
19444 assert!(execution.runner.state.resource_block.is_none());
19445 assert_eq!(
19446 execution.runner.state.liveness(false),
19447 crate::run::Liveness::Live
19448 );
19449 drop(execution);
19450 assert!(RunState::load(&id).unwrap().driver_exited);
19451 }
19452
19453 #[tokio::test]
19454 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
19455 crate::run::pin_test_home();
19456 let mut runner = runner_at(RunStatus::Gating);
19457 runner.state.save().unwrap();
19458 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
19461 assert!(runner.claim_driver().await.is_err());
19462 assert!(runner.state.driver_exited);
19463 assert_eq!(runner.state.driver_pid, None);
19464 let disk = RunState::load(&runner.state.id).unwrap();
19465 assert!(disk.driver_exited);
19466 assert_eq!(disk.driver_pid, None);
19467 assert_eq!(disk.status, RunStatus::Gating);
19468 }
19469
19470 #[tokio::test]
19471 async fn an_early_graph_error_releases_its_driver() {
19472 crate::run::pin_test_home();
19473 let mut runner = runner_at(RunStatus::Blocked);
19474 runner.state.config.graph.selected_implementer = Some("selected".into());
19475 runner.state.save().unwrap();
19476 assert!(runner.execute().await.is_err());
19477 let disk = RunState::load(&runner.state.id).unwrap();
19478 assert!(disk.driver_exited);
19479 assert_eq!(disk.driver_pid, None);
19480 assert_eq!(disk.status, RunStatus::Blocked);
19481 }
19482
19483 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
19484 crate::run::MergeRepair {
19485 question: "repair-question".into(),
19486 head: "old-head".into(),
19487 pr: "https://github.com/o/r/pull/42".into(),
19488 reviews: vec![review_round(false, 1, 1, 1, false, true)],
19489 gate: vec![outcome(Some(0), false)],
19490 gate_head: Some("old-head".into()),
19491 gate_fixes: Vec::new(),
19492 pre_gate: Vec::new(),
19493 phase,
19494 failure: None,
19495 }
19496 }
19497
19498 #[tokio::test]
19499 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
19500 crate::run::pin_test_home();
19501 let mut runner = runner_at(RunStatus::Landing);
19502 runner.state.merge_repairs.push(merge_repair_request(
19503 crate::run::MergeRepairPhase::Requested,
19504 ));
19505 runner.state.save().unwrap();
19506 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
19507 assert!(
19508 runner
19509 .execute()
19510 .await
19511 .unwrap_err()
19512 .to_string()
19513 .contains("already running")
19514 );
19515 assert_eq!(
19516 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19517 crate::run::MergeRepairPhase::Requested
19518 );
19519 drop(claim);
19520 let mut disk = runner.state.clone();
19521 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
19522 disk.save().unwrap();
19523 assert!(
19524 runner
19525 .execute()
19526 .await
19527 .unwrap_err()
19528 .to_string()
19529 .contains("decision changed")
19530 );
19531 assert_eq!(
19532 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19533 crate::run::MergeRepairPhase::Complete
19534 );
19535 }
19536
19537 #[test]
19538 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
19539 crate::run::pin_test_home();
19540 let mut runner = runner_at(RunStatus::Landing);
19541 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
19542 runner.state.reviews = request.reviews.clone();
19543 runner.state.gate = request.gate.clone();
19544 runner.state.gate_ran = true;
19545 runner.state.land_approval = Some(crate::run::LandApproval {
19546 question: "old".into(),
19547 head: "old-head".into(),
19548 });
19549 runner.state.merge_repairs.push(request);
19550 runner.begin_merge_repair_review(0).unwrap();
19551 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
19552 assert!(!runner.state.gate_status().ok());
19553 assert!(runner.state.land_approval.is_none());
19554 let disk = RunState::load(&runner.state.id).unwrap();
19555 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
19556 assert!(disk.merge_repairs[0].gate[0].ok());
19557 assert_eq!(
19558 disk.merge_repairs[0].phase,
19559 crate::run::MergeRepairPhase::Reviewing
19560 );
19561 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
19562 assert!(disk.reviews.is_empty());
19563 }
19564
19565 #[tokio::test]
19566 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
19567 crate::run::pin_test_home();
19568 for phase in [
19569 crate::run::MergeRepairPhase::Fixing,
19570 crate::run::MergeRepairPhase::Requested,
19571 ] {
19572 let mut runner = runner_at(RunStatus::Landing);
19573 runner.state.config.graph.land_rounds = 0;
19574 runner.state.merge_repairs.push(merge_repair_request(phase));
19575 runner.state.save().unwrap();
19576 assert!(!runner.repair_merge_review().await.unwrap());
19577 assert_eq!(runner.state.status, RunStatus::Blocked);
19578 assert_eq!(
19579 runner.state.merge_repairs[0].phase,
19580 crate::run::MergeRepairPhase::Failed
19581 );
19582 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
19583 assert!(
19584 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
19585 "interrupted"
19586 } else {
19587 "budget exhausted"
19588 })
19589 );
19590 assert!(runner.state.merge.is_none());
19591 }
19592 }
19593
19594 #[test]
19598 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
19599 crate::run::pin_test_home();
19600 let mut runner = runner_at(RunStatus::Implementing);
19601 let interrupt = Pause::new();
19602 runner.watch_interrupt(interrupt.clone());
19603
19604 interrupt.park_because("task a1b2 asked to run first");
19605
19606 assert!(runner.park_here().expect("park_here"));
19607 assert!(runner.state.parked);
19608 let last = runner.state.events.last().expect("a park event");
19609 assert_eq!(last.node, "park");
19610 assert!(
19611 last.message.contains("task a1b2 asked to run first"),
19612 "expected the interrupt reason in {:?}",
19613 last.message
19614 );
19615 }
19616
19617 #[test]
19620 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
19621 crate::run::pin_test_home();
19622 let mut runner = runner_at(RunStatus::Implementing);
19623 runner.state.begin_pass();
19624 let shutdown = Pause::new();
19625 runner.on_pause(shutdown.clone());
19626 let upgrade = ParkCause::Upgrade {
19627 from: "0.121.0".to_owned(),
19628 to: Some("0.121.1".to_owned()),
19629 };
19630 shutdown.park_for(upgrade.clone());
19631 shutdown.park_for(ParkCause::Operator);
19632
19633 assert!(runner.park_here().expect("park_here"));
19634 assert_eq!(
19635 runner.state.passes[0].end,
19636 Some(crate::run::PassEnd::Parked { cause: upgrade })
19637 );
19638 let last = runner.state.events.last().expect("a park event");
19639 assert!(last.message.contains("0.121.0"), "{}", last.message);
19640 }
19641
19642 #[tokio::test]
19643 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
19644 crate::run::pin_test_home();
19645 let mut runner = runner_at(RunStatus::Reviewing);
19646 runner.state.save().unwrap();
19647 let mut execution = runner.claim_driver().await.expect("first execution");
19648 assert_eq!(execution.runner.state.passes.len(), 1);
19649 execution
19650 .driver
19651 .finish(&mut execution.runner.state)
19652 .await
19653 .unwrap();
19654 }
19655
19656 #[tokio::test]
19657 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19658 crate::run::pin_test_home();
19659 let mut runner = runner_at(RunStatus::Reviewing);
19660 runner.state.config.agents = vec![conductor()];
19661 runner.state.config.graph.timeout_resource_wait = 0;
19662 runner.state.save().unwrap();
19663 let id = runner.state.id.clone();
19664 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19665 assert!(runner.claim_driver().await.is_err());
19666 drop(owner);
19667 drop(runner);
19668 let mut resumed = Runner::resume(&id).unwrap();
19669 assert_eq!(resumed.state.driver_claimed, Some(false));
19670 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19671 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19672 let mut execution = resumed
19673 .claim_driver()
19674 .await
19675 .expect("retry initial claim after reload");
19676 assert_eq!(execution.runner.state.driver_claimed, Some(true));
19677 assert_eq!(execution.runner.state.passes.len(), 1);
19678 execution
19679 .driver
19680 .finish(&mut execution.runner.state)
19681 .await
19682 .unwrap();
19683 }
19684
19685 #[tokio::test]
19686 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19687 crate::run::pin_test_home();
19688 let mut runner = runner_at(RunStatus::Reviewing);
19689 runner.state.config.agents = vec![conductor()];
19690 runner.state.save().unwrap();
19691 let id = runner.state.id.clone();
19692 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19693 let mut resumed = Runner::resume(&id).unwrap();
19694 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19695 assert!(resumed.claim_driver().await.is_err());
19696 assert!(RunState::load(&id).unwrap().passes.is_empty());
19697 drop(lease);
19698 let mut execution = resumed.claim_driver().await.unwrap();
19699 execution
19700 .driver
19701 .finish(&mut execution.runner.state)
19702 .await
19703 .unwrap();
19704 }
19705
19706 #[tokio::test]
19707 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19708 crate::run::pin_test_home();
19709 let mut runner = runner_at(RunStatus::Reviewing);
19710 let state = &mut runner.state;
19711 state.config.agents = vec![conductor()];
19712 state.driver_claimed = None;
19713 state.driver_pid = Some(std::process::id());
19714 state.driver_started_at = None;
19715 state.begin_pass();
19716 state.save().unwrap();
19717 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19718 std::fs::write(
19719 state.dir().join("driver.json"),
19720 serde_json::to_vec(&serde_json::json!({
19721 "pid": state.driver_pid, "started_at": null,
19722 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19723 }))
19724 .unwrap(),
19725 )
19726 .unwrap();
19727 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19728 drop(lease);
19729 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19730 let mut resumed = Runner::resume(&state.id).unwrap();
19731 assert!(!resumed.state.driver_exited);
19732 let mut execution = resumed
19733 .claim_driver()
19734 .await
19735 .expect("lease proves previous execution ended");
19736 assert_eq!(execution.runner.state.passes.len(), 2);
19737 execution
19738 .driver
19739 .finish(&mut execution.runner.state)
19740 .await
19741 .unwrap();
19742 }
19743
19744 #[tokio::test]
19745 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19746 crate::run::pin_test_home();
19747 let mut runner = runner_at(RunStatus::Reviewing);
19748 let mut json = serde_json::to_value(&runner.state).unwrap();
19749 json.as_object_mut().unwrap().remove("driver_claimed");
19750 runner.state = serde_json::from_value(json).unwrap();
19751 assert_eq!(runner.state.driver_claimed, None);
19752 runner.state.save().unwrap();
19753 let mut execution = runner
19754 .claim_driver()
19755 .await
19756 .expect("the execution lease, not a missing marker, excludes a second driver");
19757 assert_eq!(execution.runner.state.passes.len(), 1);
19758 execution
19759 .driver
19760 .finish(&mut execution.runner.state)
19761 .await
19762 .unwrap();
19763 }
19764
19765 #[tokio::test]
19766 async fn driver_admission_refuses_a_proven_live_driver() {
19767 crate::run::pin_test_home();
19768 let mut runner = runner_at(RunStatus::Reviewing);
19769 let pid = std::process::id();
19770 runner.state.driver_pid = Some(pid);
19771 runner.state.driver_started_at = crate::proc::process_started_at(pid);
19772 assert!(runner.state.driver_started_at.is_some());
19773 runner.state.save().unwrap();
19774 let error = runner
19775 .claim_driver()
19776 .await
19777 .err()
19778 .expect("a live driver must refuse");
19779 assert!(error.to_string().contains("live driver"), "{error:#}");
19780 let saved = RunState::load(&runner.state.id).unwrap();
19781 assert_eq!(saved.driver_pid, Some(pid));
19782 assert_eq!(saved.status, RunStatus::Reviewing);
19783 assert!(saved.passes.is_empty());
19784 }
19785
19786 #[test]
19787 fn parking_resource_halt_persists_the_pass_once() {
19788 crate::run::pin_test_home();
19789 let mut runner = runner_at(RunStatus::Reviewing);
19790 runner.state.begin_pass();
19791 runner.state.resource_block = Some(crate::run::ResourceBlock {
19792 node: "fix".into(),
19793 kind: "shared build cache".into(),
19794 detail: "cancelled".into(),
19795 at: Timestamp::now(),
19796 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19797 });
19798 let cause = ParkCause::Upgrade {
19799 from: "old".into(),
19800 to: Some("new".into()),
19801 };
19802 runner.pause.park_for(cause.clone());
19803 runner.stop_for_resource().unwrap();
19806 assert!(runner.resource_halt().unwrap());
19807 assert!(runner.park_here().unwrap());
19808 let saved = RunState::load(&runner.state.id).unwrap();
19809 assert!(saved.parked);
19810 assert_eq!(saved.status, RunStatus::Reviewing);
19811 assert!(saved.passes[0].ended_at.is_some());
19812 assert_eq!(
19813 saved.passes[0].end,
19814 Some(crate::run::PassEnd::Parked {
19815 cause: cause.clone()
19816 })
19817 );
19818 assert_eq!(
19819 saved.resource_block.unwrap().cause,
19820 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19821 );
19822 assert_eq!(
19823 saved
19824 .events
19825 .iter()
19826 .filter(|event| event.node == "park")
19827 .count(),
19828 1
19829 );
19830 }
19831
19832 #[test]
19840 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19841 crate::run::pin_test_home();
19842 let mut runner = runner_at(RunStatus::Implementing);
19843 let shutdown = Pause::new();
19844 runner.on_pause(shutdown.clone());
19845 let interrupt = Pause::new();
19846 runner.watch_interrupt(interrupt.clone());
19847
19848 assert!(!runner.park_here().expect("park_here"));
19850 assert!(!runner.state.parked);
19851
19852 interrupt.park_because("test");
19854 assert!(!shutdown.parked());
19855 assert!(runner.park_here().expect("park_here"));
19856 }
19857
19858 #[tokio::test]
19872 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19873 crate::run::pin_test_home();
19874 let mut runner = runner_at(RunStatus::Implementing);
19875 let interrupt = Pause::new();
19876 runner.watch_interrupt(interrupt.clone());
19877
19878 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19879 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19880
19881 let node = async move {
19885 started_tx.send(()).expect("send started");
19886 finish_rx.await.expect("recv finish");
19887 "node finished"
19888 };
19889
19890 let interrupter = async move {
19891 started_rx.await.expect("recv started");
19892 interrupt.park_because("higher-priority task waiting");
19894 tokio::task::yield_now().await;
19898 finish_tx.send(()).expect("send finish");
19899 };
19900
19901 let (node_result, ()) = tokio::join!(node, interrupter);
19902 assert_eq!(
19903 node_result, "node finished",
19904 "the in-flight call ran to completion"
19905 );
19906
19907 assert!(runner.park_here().expect("park_here"));
19910 assert!(runner.state.parked);
19911 }
19912
19913 #[test]
19919 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19920 crate::run::pin_test_home();
19921 let mut runner = runner_at(RunStatus::Judging);
19922 runner.state.config.agents = vec![conductor()];
19926 runner.state.candidates = vec![Candidate {
19927 index: 0,
19928 label: 'A',
19929 agent: "alpha".to_owned(),
19930 branch: "magi/x/A".to_owned(),
19931 worktree: PathBuf::from("/nonexistent/worktree"),
19932 summary: "did the thing".to_owned(),
19933 stat: "1 file changed".to_owned(),
19934 files: 1,
19935 commits: 1,
19936 empty: false,
19937 failed: None,
19938 verified_noop: None,
19939 duration_ms: 1234,
19940 folded: false,
19941 }];
19942 let run_id = runner.state.id.clone();
19943
19944 let interrupt = Pause::new();
19945 runner.watch_interrupt(interrupt.clone());
19946 interrupt.park_because("task c3d4 asked to run first");
19947 assert!(runner.park_here().expect("park_here"));
19948
19949 let resumed = Runner::resume(&run_id).expect("resume");
19950 assert_eq!(resumed.state.candidates.len(), 1);
19951 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19952 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19953 assert_eq!(resumed.state.status, runner.state.status);
19954 assert!(
19955 resumed.state.parked,
19956 "still parked until `execute` actually walks the graph again"
19957 );
19958 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19959 }
19960
19961 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19963 let mut q = ask::Question::new(
19964 run.to_owned(),
19965 "implement".to_owned(),
19966 "impl-A".to_owned(),
19967 "Which storage backend should the cache use?".to_owned(),
19968 String::new(),
19969 vec!["SQLite".to_owned(), "Redis".to_owned()],
19970 );
19971 store.put(&mut q).unwrap();
19972 q
19973 }
19974
19975 #[test]
19976 fn a_failed_runs_open_question_is_abandoned() {
19977 ask_test_home();
19978 let store = ask::Questions::open();
19979 let mut runner = runner_at(RunStatus::Failed);
19980 let run = runner.state.id.clone();
19981 let q = ask_open_question(&store, &run);
19982
19983 runner.settle_questions();
19984
19985 let back = store.get(&q.id).unwrap();
19986 assert!(
19987 !back.status.open(),
19988 "the seat that asked died with the run; nobody is left to read an answer"
19989 );
19990 assert!(
19991 back.detail.contains(&run) && back.detail.contains("failed"),
19992 "the reason names what the run became, not just that it is gone: {}",
19993 back.detail
19994 );
19995 }
19996
19997 #[test]
19998 fn a_merged_runs_open_question_is_abandoned_too() {
19999 ask_test_home();
20000 let store = ask::Questions::open();
20001 for status in [RunStatus::Merged, RunStatus::Ready] {
20004 let mut runner = runner_at(status);
20005 let run = runner.state.id.clone();
20006 let q = ask_open_question(&store, &run);
20007
20008 runner.settle_questions();
20009
20010 let back = store.get(&q.id).unwrap();
20011 assert!(
20012 !back.status.open(),
20013 "{status:?} run's question must not outlive the run"
20014 );
20015 }
20016 }
20017
20018 #[test]
20019 fn a_still_resumable_runs_open_question_is_left_alone() {
20020 ask_test_home();
20021 let store = ask::Questions::open();
20022 for status in [RunStatus::Blocked, RunStatus::Stalled] {
20028 let mut runner = runner_at(status);
20029 let run = runner.state.id.clone();
20030 let q = ask_open_question(&store, &run);
20031
20032 runner.settle_questions();
20033
20034 let back = store.get(&q.id).unwrap();
20035 assert!(
20036 back.status.open(),
20037 "{status:?} is still alive; the question must still be waiting"
20038 );
20039 }
20040 }
20041
20042 #[test]
20043 fn settle_questions_never_touches_an_already_answered_question() {
20044 ask_test_home();
20045 let store = ask::Questions::open();
20046 let mut runner = runner_at(RunStatus::Failed);
20047 let run = runner.state.id.clone();
20048 let mut q = ask_open_question(&store, &run);
20049 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
20050 .unwrap();
20051 store.put(&mut q).unwrap();
20052
20053 runner.settle_questions();
20058 runner.settle_questions();
20059
20060 let back = store.get(&q.id).unwrap();
20061 assert_eq!(
20062 back.status,
20063 ask::QuestionStatus::Answered,
20064 "a real answer is a decision on record, never overwritten by a sweep"
20065 );
20066 }
20067
20068 #[tokio::test]
20069 async fn fold_reclaims_only_proven_idle_isolated_outputs() {
20070 let tmp = tempfile::tempdir().unwrap();
20071 let home = tmp.path().join("home");
20072 let repo = tmp.path().join("repo");
20073 std::fs::create_dir(&home).unwrap();
20074 std::fs::create_dir(&repo).unwrap();
20075 init_repo(&repo);
20076
20077 let mut config = Config::default();
20078 config.graph.worktree_root = Some(tmp.path().join("wt"));
20079 config.verify.build_cache = crate::config::BuildCache::Isolated;
20080 let mut state = RunState::new(
20081 repo.clone(),
20082 "main".into(),
20083 "base".into(),
20084 "fold fixture".into(),
20085 config,
20086 );
20087 let root = state.worktree_root();
20088 let mut outputs = Vec::new();
20089 for name in ["idle", "busy"] {
20090 let checkout = root.join(name);
20091 git::worktree_add_detached(&repo, &checkout, "main")
20092 .await
20093 .unwrap();
20094 let target = crate::build_target::resolve(&home, &checkout).unwrap();
20095 let output = target.join("generated");
20096 std::fs::write(&output, name).unwrap();
20097 outputs.push((target, output));
20098 }
20099 let unknown = crate::build_target::root(&home)
20100 .join("00000000-0000-0000-0000-000000000000")
20101 .join("target");
20102 std::fs::create_dir_all(&unknown).unwrap();
20103 let unowned = unknown.join("generated");
20104 std::fs::write(&unowned, "keep").unwrap();
20105 let owner = crate::cache::Owner::here("fixture", "build", "busy", &root.join("busy"), "");
20106 let crate::cache::AcquireOutcome::Acquired(_lease) =
20107 crate::cache::try_acquire(&home, &outputs[1].0, &owner).unwrap()
20108 else {
20109 panic!("fixture target must be free");
20110 };
20111
20112 fold_run(&mut state, true, &home).await.unwrap();
20113 assert!(!root.exists());
20114 assert!(!outputs[0].1.exists());
20115 assert!(outputs[1].1.is_file());
20116 assert!(unowned.is_file());
20117 }
20118
20119 #[tokio::test]
20130 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
20131 crate::run::pin_test_home();
20132 let tmp = tempfile::tempdir().expect("tempdir");
20133 let repo = tmp.path().join("repo");
20134 std::fs::create_dir_all(&repo).unwrap();
20135 init_repo(&repo);
20136
20137 let mut config = Config::default();
20138 config.graph.worktree_root = Some(tmp.path().join("wt"));
20139
20140 let mut state = RunState::new(
20141 repo.clone(),
20142 "main".to_owned(),
20143 "deadbeef".to_owned(),
20144 "task".to_owned(),
20145 config,
20146 );
20147 let root = state.worktree_root();
20148 let wt_a = root.join("cand-A");
20149 let wt_b = root.join("cand-B");
20150 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
20151 .await
20152 .expect("worktree A");
20153 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
20154 .await
20155 .expect("worktree B");
20156
20157 state.candidates = vec![
20158 Candidate {
20159 index: 0,
20160 label: 'A',
20161 agent: "alpha".to_owned(),
20162 branch: "magi/x/A".to_owned(),
20163 worktree: wt_a.clone(),
20164 summary: String::new(),
20165 stat: String::new(),
20166 files: 0,
20167 commits: 0,
20168 empty: false,
20169 failed: None,
20170 verified_noop: None,
20171 duration_ms: 0,
20172 folded: false,
20173 },
20174 Candidate {
20175 index: 1,
20176 label: 'B',
20177 agent: "beta".to_owned(),
20178 branch: "magi/x/B".to_owned(),
20179 worktree: wt_b.clone(),
20180 summary: String::new(),
20181 stat: String::new(),
20182 files: 0,
20183 commits: 0,
20184 empty: false,
20185 failed: None,
20186 verified_noop: None,
20187 duration_ms: 0,
20188 folded: false,
20189 },
20190 ];
20191 state.tally = Some(Tally {
20192 first_choice: BTreeMap::from([('A', 1)]),
20193 borda: BTreeMap::new(),
20194 winner: 'A',
20195 rankings: 1,
20196 unanimous_initial: true,
20197 deliberated: false,
20198 changed_votes: 0,
20199 unanimous_final: true,
20200 tie_break: None,
20201 judges: 1,
20202 present: 1,
20203 quorum: 1,
20204 met_quorum: true,
20205 uncontested: None,
20206 });
20207 state.status = RunStatus::Ready;
20208
20209 fold_run(&mut state, false, &crate::run::home())
20210 .await
20211 .expect("fold_run");
20212
20213 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
20214 assert!(
20215 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
20216 "the unmerged winner's branch survives"
20217 );
20218 assert!(
20219 !state.candidates[0].folded,
20220 "the winner is not marked folded"
20221 );
20222
20223 assert!(!wt_b.exists(), "the loser's worktree is removed");
20224 assert!(
20225 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
20226 "the loser's branch is removed"
20227 );
20228 assert!(state.candidates[1].folded, "the loser is marked folded");
20229 }
20230
20231 #[tokio::test]
20232 #[cfg(unix)]
20233 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
20234 use std::os::fd::AsRawFd as _;
20235 use std::time::Duration;
20236
20237 let tmp = tempfile::tempdir().unwrap();
20238 let repo = tmp.path().join("repo");
20239 std::fs::create_dir_all(&repo).unwrap();
20240 init_repo(&repo);
20241 let common = repo.join(".git");
20242 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20243 let home = tmp.path().join("home");
20244 let mut config = Config::default();
20245 config.coordination = crate::config::test_coordination();
20246 config.graph.worktree_root = Some(tmp.path().join("wt"));
20247 config.graph.timeout_resource_wait = 1;
20248 let mut state = RunState::new(
20249 repo.clone(),
20250 "main".into(),
20251 "deadbeef".into(),
20252 "task".into(),
20253 config,
20254 );
20255 let root = state.worktree_root();
20256 let candidate = root.join("cand-A");
20257 let reviewer = root.join("review-1");
20258 let branch = state.branch_for('A');
20259 git::worktree_add_branch(&repo, &candidate, &branch, "main")
20260 .await
20261 .unwrap();
20262 git::worktree_add_detached(&repo, &reviewer, "main")
20263 .await
20264 .unwrap();
20265 git::acquire_worktree_config(&repo).await.unwrap();
20266 state.enabled_worktree_config = true;
20267 state.status = RunStatus::Ready;
20268 state.candidates.push(Candidate {
20269 index: 0,
20270 label: 'A',
20271 agent: "implement".into(),
20272 branch: branch.clone(),
20273 worktree: candidate.clone(),
20274 summary: String::new(),
20275 stat: String::new(),
20276 files: 0,
20277 commits: 0,
20278 empty: false,
20279 failed: None,
20280 verified_noop: None,
20281 duration_ms: 0,
20282 folded: false,
20283 });
20284 state.save_under(&home).unwrap();
20285 let config_before = std::fs::read(common.join("config")).unwrap();
20286 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
20287 .await
20288 .unwrap();
20289 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20292 assert_eq!(
20293 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20294 0
20295 );
20296 let mut fold = Box::pin(fold_run(&mut state, true, &home));
20297 assert!(
20298 tokio::time::timeout(Duration::from_millis(200), &mut fold)
20299 .await
20300 .is_err()
20301 );
20302 assert!(candidate.exists());
20303 assert!(reviewer.exists());
20304 assert!(git::branch_exists(&repo, &branch).await.unwrap());
20305 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
20306 assert_eq!(
20307 git::git(&repo, &["worktree", "list", "--porcelain"])
20308 .await
20309 .unwrap(),
20310 worktrees_before
20311 );
20312 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
20313 .await
20314 .unwrap()
20315 .unwrap_err();
20316 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
20317 drop(fold);
20318 assert!(candidate.exists() && reviewer.exists());
20319 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
20320 assert_eq!(
20321 state.resource_block.as_ref().unwrap().kind,
20322 "repository coordination"
20323 );
20324 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
20325 drop(external);
20326 let mut fold = Box::pin(fold_run(&mut state, true, &home));
20327 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
20328 .await
20329 .unwrap()
20330 .unwrap();
20331 drop(fold);
20332 assert!(removed.contains(&branch));
20333 assert!(!root.exists());
20334 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
20335 assert!(state.candidates[0].folded);
20336 assert!(!state.enabled_worktree_config);
20337 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
20338 let enabled = git::git_raw(
20339 &repo,
20340 &["config", "--local", "--get", "extensions.worktreeConfig"],
20341 )
20342 .await
20343 .unwrap();
20344 assert!(!enabled.ok(), "the last config reference was released");
20345 }
20346
20347 #[tokio::test]
20350 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
20351 let tmp = tempfile::tempdir().expect("tempdir");
20352 let repo = tmp.path().join("repo");
20353 std::fs::create_dir_all(&repo).unwrap();
20354 init_repo(&repo);
20355 let home = tmp.path().join("home");
20356
20357 let mut config = Config::default();
20358 config.graph.worktree_root = Some(tmp.path().join("wt"));
20359 let mut state = RunState::new(
20360 repo.clone(),
20361 "main".to_owned(),
20362 "deadbeef".to_owned(),
20363 "task".to_owned(),
20364 config,
20365 );
20366 git::git(&repo, &["branch", "magi/x/A", "main"])
20368 .await
20369 .expect("branch");
20370 state.candidates = vec![Candidate {
20371 index: 0,
20372 label: 'A',
20373 agent: "alpha".to_owned(),
20374 branch: "magi/x/A".to_owned(),
20375 worktree: state.worktree_root().join("cand-A"),
20376 summary: String::new(),
20377 stat: String::new(),
20378 files: 0,
20379 commits: 0,
20380 empty: false,
20381 failed: None,
20382 verified_noop: None,
20383 duration_ms: 0,
20384 folded: true,
20385 }];
20386 state.released_to = Some("20260901-000000-new1".to_owned());
20387 state.released_branches = vec!["magi/x/A".to_owned()];
20388
20389 fold_run(&mut state, true, &home).await.expect("fold_run");
20390 state.status = RunStatus::Merged;
20391 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
20392 state.merge = Some(crate::run::MergeOutcome {
20393 mode: MergeMode::Pr,
20394 ok: true,
20395 detail: "confirmed merged".into(),
20396 empty: false,
20397 });
20398 fold_run(&mut state, true, &home)
20399 .await
20400 .expect("merged fold");
20401
20402 assert!(
20403 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
20404 "the handed-over branch survives a fold"
20405 );
20406 }
20407
20408 #[tokio::test]
20412 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
20413 let tmp = tempfile::tempdir().expect("tempdir");
20414 let repo = tmp.path().join("repo");
20415 std::fs::create_dir_all(&repo).unwrap();
20416 init_repo(&repo);
20417 let run = |args: &[&str]| {
20418 let out = std::process::Command::new("git")
20419 .quiet()
20420 .args(args)
20421 .current_dir(&repo)
20422 .output()
20423 .expect("spawn git");
20424 assert!(out.status.success(), "git {args:?}");
20425 };
20426 run(&["branch", "magi/x/A"]);
20427 run(&["checkout", "-q", "-b", "magi/x/B"]);
20428 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
20429 run(&["add", "-A"]);
20430 run(&["commit", "-q", "-m", "work"]);
20431 run(&["checkout", "-q", "main"]);
20432
20433 let probe = RunState::new(
20437 repo.clone(),
20438 "main".to_owned(),
20439 "deadbeef".to_owned(),
20440 "task".to_owned(),
20441 Config::default(),
20442 );
20443 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
20444 let bare = tmp.path().join("origin.git");
20445 let out = std::process::Command::new("git")
20446 .quiet()
20447 .args(["init", "-q", "--bare"])
20448 .arg(&bare)
20449 .output()
20450 .expect("spawn git");
20451 assert!(out.status.success(), "git init --bare");
20452 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
20453 run(&["push", "-q", "origin", "main"]);
20454
20455 let mut state = RunState::new(
20456 repo.clone(),
20457 "main".to_owned(),
20458 "deadbeef".to_owned(),
20459 "task".to_owned(),
20460 Config::default(),
20461 );
20462 state.seeds = vec![refs::Seed {
20463 token: "magi/27b2/A".to_owned(),
20464 kind: refs::SeedKind::Unresolved,
20465 sha: String::new(),
20466 branch: true,
20467 detail: "no branch or commit named magi/27b2/A".to_owned(),
20468 }];
20469
20470 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
20471 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
20472 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
20473 let detail = empty_candidate_detail(&state, "main");
20474 assert!(detail.starts_with("empty candidate"), "{detail}");
20475 assert!(!detail.contains("magi/27b2/A"), "{detail}");
20476 assert!(detail.contains("--start-from"), "{detail}");
20477 }
20478
20479 #[tokio::test]
20488 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
20489 let tmp = tempfile::tempdir().expect("tempdir");
20490 let repo = tmp.path().join("repo");
20491 std::fs::create_dir_all(&repo).unwrap();
20492 init_repo(&repo);
20493
20494 let mut config = Config::default();
20495 config.merge.mode = MergeMode::Local;
20496
20497 let mut state = RunState::new(
20498 repo.clone(),
20499 "main".to_owned(),
20500 "deadbeef".to_owned(),
20501 "task".to_owned(),
20502 config,
20503 );
20504 state.candidates = vec![Candidate {
20505 index: 0,
20506 label: 'A',
20507 agent: "alpha".to_owned(),
20508 branch: "does-not-exist".to_owned(),
20509 worktree: repo.clone(),
20510 summary: String::new(),
20511 stat: String::new(),
20512 files: 0,
20513 commits: 0,
20514 empty: false,
20515 failed: None,
20516 verified_noop: None,
20517 duration_ms: 0,
20518 folded: false,
20519 }];
20520 state.tally = Some(Tally {
20521 first_choice: BTreeMap::from([('A', 1)]),
20522 borda: BTreeMap::new(),
20523 winner: 'A',
20524 rankings: 1,
20525 unanimous_initial: true,
20526 deliberated: false,
20527 changed_votes: 0,
20528 unanimous_final: true,
20529 tie_break: None,
20530 judges: 0,
20531 present: 0,
20532 quorum: 0,
20533 met_quorum: true,
20534 uncontested: Some("only candidate A produced a change".to_owned()),
20535 });
20536 state.reviews = vec![ReviewRound {
20537 round: 1,
20538 head: "deadbeef".to_owned(),
20539 verified_head: None,
20540 verified_at: None,
20541 reviews: Vec::new(),
20542 e2e: Vec::new(),
20543 fix: None,
20544 blocking: 0,
20545 answered: 0,
20546 expected: 0,
20547 clean: true,
20548 verify_retried: false,
20549 e2e_deferred: false,
20550 e2e_defer_reason: None,
20551 progressed: false,
20552 vote_split: false,
20553 reconsideration: Vec::new(),
20554 verdict: None,
20555 }];
20556 state.gate = vec![CommandOutcome {
20557 command: "test".to_owned(),
20558 code: Some(0),
20559 output_tail: String::new(),
20560 duration_ms: 0,
20561 resource_blocked: false,
20562 }];
20563 state.gate_ran = true;
20564 state.status = RunStatus::Ready;
20569 state.merge = Some(MergeOutcome {
20570 mode: MergeMode::Local,
20571 ok: false,
20572 detail: "already concluded".to_owned(),
20573 empty: false,
20574 });
20575
20576 let mut runner = Runner {
20577 state,
20578 roles: ResolvedRoles {
20579 implementers: Vec::new(),
20580 judges: Vec::new(),
20581 reviewers: Vec::new(),
20582 fixer: None,
20583 conductor: conductor(),
20584 implementer_roster: Vec::new(),
20585 judge_roster: Vec::new(),
20586 reviewer_roster: Vec::new(),
20587 },
20588 sem: Arc::new(Semaphore::new(1)),
20589 pause: Pause::new(),
20590 interrupt: Pause::new(),
20591 };
20592
20593 runner.merge().await.expect("merge");
20594
20595 assert_eq!(
20596 runner.state.status,
20597 RunStatus::Ready,
20598 "a concluded run's status must not change on reentry"
20599 );
20600 assert_eq!(
20601 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20602 Some("already concluded"),
20603 "merge must not run again once the node already recorded an outcome"
20604 );
20605 }
20606
20607 #[tokio::test]
20616 async fn merge_refuses_a_gate_that_has_not_actually_run() {
20617 let tmp = tempfile::tempdir().expect("tempdir");
20618 let repo = tmp.path().join("repo");
20619 std::fs::create_dir_all(&repo).unwrap();
20620 init_repo(&repo);
20621
20622 let mut config = Config::default();
20623 config.merge.mode = MergeMode::Local;
20624
20625 let mut state = RunState::new(
20626 repo.clone(),
20627 "main".to_owned(),
20628 "deadbeef".to_owned(),
20629 "task".to_owned(),
20630 config,
20631 );
20632 state.candidates = vec![Candidate {
20633 index: 0,
20634 label: 'A',
20635 agent: "alpha".to_owned(),
20636 branch: "does-not-exist".to_owned(),
20637 worktree: repo.clone(),
20638 summary: String::new(),
20639 stat: String::new(),
20640 files: 0,
20641 commits: 0,
20642 empty: false,
20643 failed: None,
20644 verified_noop: None,
20645 duration_ms: 0,
20646 folded: false,
20647 }];
20648 state.tally = Some(Tally {
20649 first_choice: BTreeMap::from([('A', 1)]),
20650 borda: BTreeMap::new(),
20651 winner: 'A',
20652 rankings: 1,
20653 unanimous_initial: true,
20654 deliberated: false,
20655 changed_votes: 0,
20656 unanimous_final: true,
20657 tie_break: None,
20658 judges: 0,
20659 present: 0,
20660 quorum: 0,
20661 met_quorum: true,
20662 uncontested: Some("only candidate A produced a change".to_owned()),
20663 });
20664 state.reviews = vec![ReviewRound {
20665 round: 1,
20666 head: "deadbeef".to_owned(),
20667 verified_head: None,
20668 verified_at: None,
20669 reviews: Vec::new(),
20670 e2e: Vec::new(),
20671 fix: None,
20672 blocking: 0,
20673 answered: 0,
20674 expected: 0,
20675 clean: true,
20676 verify_retried: false,
20677 e2e_deferred: false,
20678 e2e_defer_reason: None,
20679 progressed: false,
20680 vote_split: false,
20681 reconsideration: Vec::new(),
20682 verdict: None,
20683 }];
20684 state.gate = Vec::new();
20686 state.gate_ran = false;
20687 state.status = RunStatus::Gating;
20688
20689 let mut runner = Runner {
20690 state,
20691 roles: ResolvedRoles {
20692 implementers: Vec::new(),
20693 judges: Vec::new(),
20694 reviewers: Vec::new(),
20695 fixer: None,
20696 conductor: conductor(),
20697 implementer_roster: Vec::new(),
20698 judge_roster: Vec::new(),
20699 reviewer_roster: Vec::new(),
20700 },
20701 sem: Arc::new(Semaphore::new(1)),
20702 pause: Pause::new(),
20703 interrupt: Pause::new(),
20704 };
20705
20706 runner.merge().await.expect("merge");
20707
20708 assert!(
20709 runner.state.merge.is_none(),
20710 "an empty gate must never be read as a passing one: {:?}",
20711 runner.state.merge
20712 );
20713 }
20714
20715 #[tokio::test]
20722 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20723 let tmp = tempfile::tempdir().expect("tempdir");
20724 let repo = tmp.path().join("repo");
20725 std::fs::create_dir_all(&repo).unwrap();
20726 init_repo(&repo);
20727
20728 let config = Config::default();
20730
20731 let mut state = RunState::new(
20732 repo.clone(),
20733 "main".to_owned(),
20734 "deadbeef".to_owned(),
20735 "task".to_owned(),
20736 config,
20737 );
20738 state.candidates = vec![Candidate {
20739 index: 0,
20740 label: 'A',
20741 agent: "alpha".to_owned(),
20742 branch: "does-not-exist".to_owned(),
20743 worktree: repo.clone(),
20744 summary: String::new(),
20745 stat: String::new(),
20746 files: 0,
20747 commits: 0,
20748 empty: false,
20749 failed: None,
20750 verified_noop: None,
20751 duration_ms: 0,
20752 folded: false,
20753 }];
20754 state.tally = Some(Tally {
20755 first_choice: BTreeMap::from([('A', 1)]),
20756 borda: BTreeMap::new(),
20757 winner: 'A',
20758 rankings: 1,
20759 unanimous_initial: true,
20760 deliberated: false,
20761 changed_votes: 0,
20762 unanimous_final: true,
20763 tie_break: None,
20764 judges: 0,
20765 present: 0,
20766 quorum: 0,
20767 met_quorum: true,
20768 uncontested: Some("only candidate A produced a change".to_owned()),
20769 });
20770 state.reviews = vec![ReviewRound {
20771 round: 1,
20772 head: "deadbeef".to_owned(),
20773 verified_head: None,
20774 verified_at: None,
20775 reviews: Vec::new(),
20776 e2e: Vec::new(),
20777 fix: None,
20778 blocking: 0,
20779 answered: 0,
20780 expected: 0,
20781 clean: true,
20782 verify_retried: false,
20783 e2e_deferred: false,
20784 e2e_defer_reason: None,
20785 progressed: false,
20786 vote_split: false,
20787 reconsideration: Vec::new(),
20788 verdict: None,
20789 }];
20790
20791 let mut runner = Runner {
20792 state,
20793 roles: ResolvedRoles {
20794 implementers: Vec::new(),
20795 judges: Vec::new(),
20796 reviewers: Vec::new(),
20797 fixer: None,
20798 conductor: conductor(),
20799 implementer_roster: Vec::new(),
20800 judge_roster: Vec::new(),
20801 reviewer_roster: Vec::new(),
20802 },
20803 sem: Arc::new(Semaphore::new(1)),
20804 pause: Pause::new(),
20805 interrupt: Pause::new(),
20806 };
20807
20808 runner.gate().await.expect("gate");
20809 assert!(
20810 runner.state.gate_ran,
20811 "zero configured commands is still a real attempt, not an unrun gate"
20812 );
20813 assert!(runner.state.gate.is_empty());
20814 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20815 assert_ne!(
20816 runner.state.status,
20817 RunStatus::Blocked,
20818 "a gate with nothing to check must not read as failed"
20819 );
20820
20821 runner.merge().await.expect("merge");
20822 assert_eq!(
20823 runner.state.status,
20824 RunStatus::Ready,
20825 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20826 );
20827 }
20828
20829 #[tokio::test]
20840 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20841 crate::run::pin_test_home();
20842 let home = crate::run::home();
20843
20844 let tmp = tempfile::tempdir().expect("tempdir");
20845 let repo = tmp.path().join("repo");
20846 std::fs::create_dir_all(&repo).unwrap();
20847 init_repo(&repo);
20848 let cache_dir = tmp.path().join("target");
20851
20852 let mut config = Config::default();
20853 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20854 config.graph.timeout_verify = Some(2);
20858 config.graph.timeout_resource_wait = 0;
20859
20860 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20861 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20862 .expect("no io error acquiring directly")
20863 {
20864 crate::cache::AcquireOutcome::Acquired(g) => g,
20865 crate::cache::AcquireOutcome::Busy(b) => {
20866 panic!("expected the direct acquire to win the lease first: {b:?}")
20867 }
20868 };
20869
20870 let mut state = RunState::new(
20871 repo.clone(),
20872 "main".to_owned(),
20873 "deadbeef".to_owned(),
20874 "task".to_owned(),
20875 config,
20876 );
20877 state.candidates = vec![Candidate {
20878 index: 0,
20879 label: 'A',
20880 agent: "alpha".to_owned(),
20881 branch: "does-not-exist".to_owned(),
20882 worktree: repo.clone(),
20883 summary: String::new(),
20884 stat: String::new(),
20885 files: 0,
20886 commits: 0,
20887 empty: false,
20888 failed: None,
20889 verified_noop: None,
20890 duration_ms: 0,
20891 folded: false,
20892 }];
20893 state.tally = Some(Tally {
20894 first_choice: BTreeMap::from([('A', 1)]),
20895 borda: BTreeMap::new(),
20896 winner: 'A',
20897 rankings: 1,
20898 unanimous_initial: true,
20899 deliberated: false,
20900 changed_votes: 0,
20901 unanimous_final: true,
20902 tie_break: None,
20903 judges: 0,
20904 present: 0,
20905 quorum: 0,
20906 met_quorum: true,
20907 uncontested: Some("only candidate A produced a change".to_owned()),
20908 });
20909 state.reviews = vec![ReviewRound {
20910 round: 1,
20911 head: "deadbeef".to_owned(),
20912 verified_head: None,
20913 verified_at: None,
20914 reviews: Vec::new(),
20915 e2e: Vec::new(),
20916 fix: None,
20917 blocking: 0,
20918 answered: 0,
20919 expected: 0,
20920 clean: true,
20921 verify_retried: false,
20922 e2e_deferred: false,
20923 e2e_defer_reason: None,
20924 progressed: false,
20925 vote_split: false,
20926 reconsideration: Vec::new(),
20927 verdict: None,
20928 }];
20929
20930 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20931 state.reviews[0].e2e = vec![CommandOutcome {
20932 command: state.config.verify.e2e[0].clone(),
20933 code: Some(0),
20934 output_tail: String::new(),
20935 duration_ms: 0,
20936 resource_blocked: false,
20937 }];
20938 let mut runner = Runner {
20939 state,
20940 roles: ResolvedRoles {
20941 implementers: Vec::new(),
20942 judges: Vec::new(),
20943 reviewers: Vec::new(),
20944 fixer: None,
20945 conductor: conductor(),
20946 implementer_roster: Vec::new(),
20947 judge_roster: Vec::new(),
20948 reviewer_roster: Vec::new(),
20949 },
20950 sem: Arc::new(Semaphore::new(1)),
20951 pause: Pause::new(),
20952 interrupt: Pause::new(),
20953 };
20954
20955 runner.gate().await.expect("gate");
20956 assert!(runner.state.resource_block.is_none());
20957 assert!(
20958 !runner
20959 .state
20960 .events
20961 .iter()
20962 .any(|event| { event.message.contains("waiting for the shared build cache") }),
20963 "a gate with nothing to run must never request the held cache lease"
20964 );
20965 assert!(
20966 runner.state.gate_ran,
20967 "zero commands is still a real, immediate attempt"
20968 );
20969 assert!(runner.state.gate.is_empty());
20970 assert_ne!(
20971 runner.state.status,
20972 RunStatus::Blocked,
20973 "must not read as resource-blocked on a lease it never asked for"
20974 );
20975 }
20976
20977 #[cfg(unix)]
20980 #[tokio::test]
20981 async fn verification_is_standalone_inside_a_live_worker() {
20982 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20983 if std::env::var_os(CHILD).is_none() {
20984 let home = tempfile::tempdir().unwrap();
20985 let generation = crate::worker::pin_current(home.path()).unwrap();
20986 let output = std::process::Command::new(std::env::current_exe().unwrap())
20987 .args([
20988 "--exact",
20989 "graph::tests::verification_is_standalone_inside_a_live_worker",
20990 "--nocapture",
20991 ])
20992 .env(CHILD, "1")
20993 .env("MAGI_HOME", home.path())
20994 .env("MAGI_WORKER_HOME", home.path())
20995 .env("MAGI_WORKER_ID", "abcdef")
20996 .env("MAGI_WORKER_GENERATION", &generation.id)
20997 .env("MAGI_WORKER_EXE", &generation.executable)
20998 .env("MAGI_RUN", "verification-parent")
20999 .env("MAGI_NODE", "gate")
21000 .env("MAGI_SEAT", "worker-fixture")
21001 .env("MAGI_PROCESS_OWNER", "fixture-owner")
21002 .env("MAGI_NO_AUTOUPDATE", "0")
21003 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
21004 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
21005 .env(
21006 "VERIFY_EXPECT_TOOLCHAIN",
21007 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
21008 )
21009 .env(
21010 "VERIFY_EXPECT_CACHE",
21011 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
21012 )
21013 .output()
21014 .unwrap();
21015 assert!(
21016 output.status.success()
21017 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
21018 "{}\n{}",
21019 String::from_utf8_lossy(&output.stdout),
21020 String::from_utf8_lossy(&output.stderr)
21021 );
21022 return;
21023 }
21024 let home = crate::run::home();
21025 let directory = home.join("workers/jobs/abcdef");
21026 std::fs::create_dir_all(&directory).unwrap();
21027 let job = crate::worker::Job {
21028 id: "abcdef".into(),
21029 sequence: 1,
21030 kind: "task".into(),
21031 key: "task:verification".into(),
21032 payload: serde_json::json!({}),
21033 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
21034 version: Some(env!("CARGO_PKG_VERSION").into()),
21035 status: crate::worker::Status::Running,
21036 result: None,
21037 pid: Some(std::process::id()),
21038 created_at: 0,
21039 finished_at: None,
21040 };
21041 std::fs::write(
21042 directory.join("job.json"),
21043 serde_json::to_vec(&job).unwrap(),
21044 )
21045 .unwrap();
21046 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
21047 let assert_worker = || {
21048 assert_eq!(
21049 crate::worker::current_job(&home).unwrap().unwrap().id,
21050 job.id
21051 );
21052 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
21053 assert_eq!(
21054 crate::worker::helper_executable(&home).unwrap(),
21055 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
21056 );
21057 assert!(
21058 crate::updater::Checker::new(&crate::config::Update {
21059 mode: crate::config::UpdateMode::Notify,
21060 interval: None,
21061 auto_restart: false,
21062 })
21063 .is_none(),
21064 "the driver remains pinned, even while tests are standalone"
21065 );
21066 };
21067 assert_worker();
21068 let repo = home.join("repo");
21069 std::fs::create_dir_all(&repo).unwrap();
21070 init_repo(&repo);
21071 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
21074 test -z "${MAGI_WORKER_GENERATION+x}" &&
21075 test -z "${MAGI_WORKER_HOME+x}" &&
21076 test -z "${MAGI_WORKER_EXE+x}" &&
21077 test "$MAGI_HOME/repo" -ef . &&
21078 test "$MAGI_RUN" = verification-parent &&
21079 test "$MAGI_NODE" = gate &&
21080 test "$MAGI_SEAT" = worker-fixture &&
21081 test -n "$MAGI_PROCESS_OWNER" &&
21082 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
21083 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
21084 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
21085 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
21086 assert!(
21088 !std::process::Command::new("sh")
21089 .args(["-c", probe])
21090 .current_dir(&repo)
21091 .status()
21092 .unwrap()
21093 .success()
21094 );
21095 let unrelated_home = tempfile::tempdir().unwrap();
21096 assert!(
21097 crate::worker::current_job(unrelated_home.path()).is_err(),
21098 "an inherited worker id cannot be resolved in a unit fixture's home"
21099 );
21100 let mut runner = runner_at(RunStatus::Gating);
21101 runner.state.repo = repo.clone();
21102 runner.state.config.disk.min_free_bytes = 0;
21103 runner.state.config.coordination = crate::config::test_coordination();
21104 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
21105 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
21106 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
21107 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
21108 let winner = Candidate {
21109 index: 0,
21110 label: 'A',
21111 agent: "fixture".into(),
21112 branch: "main".into(),
21113 worktree: repo.clone(),
21114 summary: String::new(),
21115 stat: String::new(),
21116 files: 0,
21117 commits: 0,
21118 empty: false,
21119 failed: None,
21120 verified_noop: None,
21121 duration_ms: 0,
21122 folded: false,
21123 };
21124 let (e2e, retried, pids) = run_e2e_with_retry(
21125 &mut runner.state,
21126 &["sh".into(), "-c".into()],
21127 &[format!("{probe} && printf initial")],
21128 &repo,
21129 Duration::from_secs(10),
21130 "initial",
21131 None,
21132 )
21133 .await;
21134 assert!(!retried && pids.is_empty());
21135 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
21136 assert_eq!(e2e[0].output_tail, "initial");
21137 let gate = runner.run_gate(&winner).await.unwrap();
21138 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
21139 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
21142 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
21143 passed: false,
21144 outcomes: Vec::new(),
21145 });
21146 runner.state.save().unwrap();
21147 let mut resumed = RunState::load(&runner.state.id).unwrap();
21148 let wait = crate::repo_lock::WaitPolicy::default();
21149 assert!(matches!(
21150 check_gate_repair_e2e(&mut resumed, &winner, &wait)
21151 .await
21152 .unwrap(),
21153 GateFix::Retry
21154 ));
21155 let saved = RunState::load(&resumed.id).unwrap();
21156 let repair = saved.gate_repair_e2e.unwrap();
21157 assert!(repair.passed);
21158 assert_eq!(repair.outcomes.len(), 1);
21159 assert!(repair.outcomes[0].ok());
21160 let landing = run_landing_gate(&mut resumed, &winner, &wait)
21161 .await
21162 .unwrap();
21163 assert_eq!(landing.len(), 1);
21164 assert!(landing[0].ok(), "{landing:?}");
21165 assert_eq!(
21166 std::fs::read_to_string(repo.join("phases")).unwrap(),
21167 "pre\ngate\ne2e\npre\ngate\n"
21168 );
21169 assert_worker();
21170 }
21171
21172 #[cfg(unix)]
21173 #[tokio::test]
21174 async fn verification_keeps_the_panic_and_the_full_output() {
21175 crate::run::pin_test_home();
21176 let tmp = tempfile::tempdir().unwrap();
21177 let mut config = Config::default();
21178 config.disk.min_free_bytes = 0;
21179 let mut state = RunState::new(
21180 tmp.path().to_owned(),
21181 "main".into(),
21182 "base".into(),
21183 "verification evidence".into(),
21184 config,
21185 );
21186 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
21187 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
21188 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
21189 let mut names = Vec::new();
21190 for _ in 0..2 {
21191 let (outcomes, _) = run_commands(
21192 &mut state,
21193 "gate",
21194 "gate",
21195 0,
21196 &["sh".into(), "-c".into()],
21197 &[script.into()],
21198 tmp.path(),
21199 Duration::from_secs(60),
21200 )
21201 .await;
21202 let tail = &outcomes[0].output_tail;
21203 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
21204 assert!(tail.contains("boom assertion"));
21205 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
21206 let line = tail.lines().next().unwrap();
21207 names.push(line.to_owned());
21208 }
21209 assert_ne!(names[0], names[1], "the second run must not overwrite");
21210 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
21211 assert!(full.len() > 500_000 && full.contains("boom assertion"));
21212 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
21213
21214 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
21216 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
21217 let (outcomes, _) = run_commands(
21218 &mut state,
21219 "gate",
21220 "gate",
21221 0,
21222 &["sh".into(), "-c".into()],
21223 &["echo failure-output; exit 7".into()],
21224 tmp.path(),
21225 Duration::from_secs(10),
21226 )
21227 .await;
21228 assert_eq!(outcomes[0].code, Some(7));
21229 assert!(state.events.iter().any(|e| {
21230 e.node == "gate" && e.message.contains("could not save the full gate output:")
21231 }));
21232 }
21233
21234 #[cfg(unix)]
21235 #[tokio::test]
21236 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
21237 crate::run::pin_test_home();
21238 let tmp = tempfile::tempdir().unwrap();
21239 let mut config = Config::default();
21240 config.disk.min_free_bytes = 0;
21241 let mut state = RunState::new(
21242 tmp.path().to_owned(),
21243 "main".into(),
21244 "base".into(),
21245 "verification evidence".into(),
21246 config,
21247 );
21248 let (outcomes, pids) = run_commands(
21249 &mut state,
21250 "gate",
21251 "gate",
21252 1,
21253 &["sh".into(), "-c".into()],
21254 &[
21255 "printf before-timeout; exec sleep 30".into(),
21256 "kill -TERM $$".into(),
21257 "exit 7".into(),
21258 "printf success".into(),
21259 ],
21260 tmp.path(),
21261 Duration::from_secs(1),
21262 )
21263 .await;
21264 wait_for_timed_out_children_to_die(&pids).await;
21265 assert_eq!(
21266 pids.len(),
21267 1,
21268 "unleased timeout retains its death-check PID"
21269 );
21270 assert_eq!(outcomes.len(), 4);
21271 assert_eq!(outcomes[0].code, None);
21272 assert!(outcomes[0].output_tail.contains("before-timeout"));
21273 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
21274 assert_eq!(outcomes[1].code, None);
21275 assert!(outcomes[1].output_tail.contains("command terminated:"));
21276 assert!(!outcomes[1].output_tail.contains("timed out"));
21277 assert_eq!(outcomes[2].code, Some(7));
21278 assert!(!outcomes[2].output_tail.contains("timed out"));
21279 assert_eq!(outcomes[3].code, Some(0));
21280 assert_eq!(outcomes[3].output_tail, "success");
21281 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
21282 }
21283
21284 #[tokio::test]
21294 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
21295 crate::run::pin_test_home();
21296
21297 let tmp = tempfile::tempdir().expect("tempdir");
21298 let repo = tmp.path().join("repo");
21299 std::fs::create_dir_all(&repo).unwrap();
21300 init_repo(&repo);
21301
21302 let mut config = Config::default();
21303 config.disk.min_free_bytes = 0;
21304 config.verify.gate = vec![
21305 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
21306 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
21307 .to_owned(),
21308 ];
21309
21310 let mut state = RunState::new(
21311 repo.clone(),
21312 "main".to_owned(),
21313 "deadbeef".to_owned(),
21314 "task".to_owned(),
21315 config,
21316 );
21317 let run_id = state.id.clone();
21318 state.candidates = vec![Candidate {
21319 index: 0,
21320 label: 'A',
21321 agent: "alpha".to_owned(),
21322 branch: "does-not-exist".to_owned(),
21323 worktree: repo.clone(),
21324 summary: String::new(),
21325 stat: String::new(),
21326 files: 0,
21327 commits: 0,
21328 empty: false,
21329 failed: None,
21330 verified_noop: None,
21331 duration_ms: 0,
21332 folded: false,
21333 }];
21334 state.tally = Some(Tally {
21335 first_choice: BTreeMap::from([('A', 1)]),
21336 borda: BTreeMap::new(),
21337 winner: 'A',
21338 rankings: 1,
21339 unanimous_initial: true,
21340 deliberated: false,
21341 changed_votes: 0,
21342 unanimous_final: true,
21343 tie_break: None,
21344 judges: 0,
21345 present: 0,
21346 quorum: 0,
21347 met_quorum: true,
21348 uncontested: Some("only candidate A produced a change".to_owned()),
21349 });
21350 state.reviews = vec![ReviewRound {
21351 round: 1,
21352 head: "deadbeef".to_owned(),
21353 verified_head: None,
21354 verified_at: None,
21355 reviews: Vec::new(),
21356 e2e: Vec::new(),
21357 fix: None,
21358 blocking: 0,
21359 answered: 0,
21360 expected: 0,
21361 clean: true,
21362 verify_retried: false,
21363 e2e_deferred: false,
21364 e2e_defer_reason: None,
21365 progressed: false,
21366 vote_split: false,
21367 reconsideration: Vec::new(),
21368 verdict: None,
21369 }];
21370
21371 let mut runner = Runner {
21372 state,
21373 roles: ResolvedRoles {
21374 implementers: Vec::new(),
21375 judges: Vec::new(),
21376 reviewers: Vec::new(),
21377 fixer: None,
21378 conductor: conductor(),
21379 implementer_roster: Vec::new(),
21380 judge_roster: Vec::new(),
21381 reviewer_roster: Vec::new(),
21382 },
21383 sem: Arc::new(Semaphore::new(1)),
21384 pause: Pause::new(),
21385 interrupt: Pause::new(),
21386 };
21387
21388 let started_marker = repo.join("started.marker");
21389 let release_marker = repo.join("release.marker");
21390 let poller = tokio::spawn(async move {
21391 for _ in 0..100 {
21396 if started_marker.exists()
21397 && let Ok(s) = crate::run::RunState::load(&run_id)
21398 && let Some(a) = s.active.get("gate")
21399 {
21400 std::fs::write(&release_marker, b"go").expect("release marker");
21401 return Some(a.clone());
21402 }
21403 tokio::time::sleep(Duration::from_millis(50)).await;
21404 }
21405 None
21406 });
21407
21408 runner.gate().await.expect("gate");
21409 let captured = poller.await.expect("poller task");
21410 let captured = captured.expect(
21411 "the poller never saw a `gate` task entry in run.json while the command was \
21412 still blocked on its own release marker",
21413 );
21414
21415 assert_eq!(captured.task.as_deref(), Some("gate"));
21416 assert_eq!(captured.node, "gate");
21417 assert_eq!(captured.index, Some(1));
21418 assert_eq!(captured.total, Some(1));
21419 assert!(
21420 captured
21421 .command
21422 .as_deref()
21423 .is_some_and(|c| c.contains("started.marker")),
21424 "{captured:?}"
21425 );
21426
21427 assert!(
21428 runner.state.active.is_empty(),
21429 "the entry must be cleared once the command actually finished: {:?}",
21430 runner.state.active
21431 );
21432 assert!(runner.state.gate_ran);
21433 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
21434 }
21435
21436 #[tokio::test]
21439 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
21440 use crate::verdict::{Finding, ReviewVote, Severity};
21441 crate::run::pin_test_home();
21442 let tmp = tempfile::tempdir().expect("tempdir");
21443 let repo = tmp.path().join("repo");
21444 std::fs::create_dir_all(&repo).unwrap();
21445 init_repo(&repo);
21446
21447 for (severity, vote, expect) in [
21448 (Severity::Major, ReviewVote::Reject, true),
21449 (Severity::Minor, ReviewVote::Reject, false),
21450 (Severity::Major, ReviewVote::Approve, false),
21451 ] {
21452 let mut round = review_round(false, 1, 1, 1, false, true);
21453 round.reviews = vec![ReviewRecord {
21454 reviewer: 1,
21455 agent: "alpha".to_owned(),
21456 summary: String::new(),
21457 findings: vec![Finding {
21458 id: "R1-1-1".to_owned(),
21459 severity,
21460 file: None,
21461 line: None,
21462 title: "t".to_owned(),
21463 detail: String::new(),
21464 }],
21465 vote: Some(vote),
21466 failed: None,
21467 duration_ms: 0,
21468 attempts: 0,
21469 }];
21470 let mut state = RunState::new(
21471 repo.clone(),
21472 "main".to_owned(),
21473 "deadbeef".to_owned(),
21474 "task".to_owned(),
21475 Config::default(),
21476 );
21477 state.reviews = vec![round];
21478 let mut runner = Runner {
21479 state,
21480 roles: ResolvedRoles {
21481 implementers: Vec::new(),
21482 judges: Vec::new(),
21483 reviewers: Vec::new(),
21484 fixer: None,
21485 conductor: conductor(),
21486 implementer_roster: Vec::new(),
21487 judge_roster: Vec::new(),
21488 reviewer_roster: Vec::new(),
21489 },
21490 sem: Arc::new(Semaphore::new(1)),
21491 pause: Pause::new(),
21492 interrupt: Pause::new(),
21493 };
21494 let shell = runner.state.config.shell();
21495 runner
21496 .stop_reviewing("round budget spent", &shell, &repo)
21497 .await
21498 .expect("stop_reviewing");
21499 assert_eq!(runner.state.status, RunStatus::Gating);
21500 assert_eq!(
21501 runner.state.contested_handoff.is_some(),
21502 expect,
21503 "{severity:?} + {vote:?}"
21504 );
21505 }
21506 }
21507
21508 #[tokio::test]
21521 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
21522 crate::run::pin_test_home();
21523 let home = crate::run::home();
21524
21525 let tmp = tempfile::tempdir().expect("tempdir");
21526 let repo = tmp.path().join("repo");
21527 std::fs::create_dir_all(&repo).unwrap();
21528 init_repo(&repo);
21529 let head = crate::git::rev_parse(&repo, "HEAD")
21530 .await
21531 .expect("rev-parse");
21532 let cache_dir = tmp.path().join("target");
21535
21536 let mut config = Config::default();
21537 config.verify.e2e = vec![format!(
21538 "CARGO_TARGET_DIR='{}' test -f README.md",
21539 cache_dir.display()
21540 )];
21541 config.graph.review_rounds = 1;
21542 config.graph.timeout_verify = Some(2);
21545 config.graph.timeout_resource_wait = 2;
21546
21547 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21548 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21549 .expect("no io error acquiring directly")
21550 {
21551 crate::cache::AcquireOutcome::Acquired(g) => g,
21552 crate::cache::AcquireOutcome::Busy(b) => {
21553 panic!("expected the direct acquire to win the lease first: {b:?}")
21554 }
21555 };
21556
21557 let mut state = RunState::new(
21558 repo.clone(),
21559 "main".to_owned(),
21560 head.clone(),
21561 "task".to_owned(),
21562 config,
21563 );
21564 state.candidates = vec![Candidate {
21565 index: 0,
21566 label: 'A',
21567 agent: "alpha".to_owned(),
21568 branch: "does-not-exist".to_owned(),
21569 worktree: repo.clone(),
21570 summary: String::new(),
21571 stat: String::new(),
21572 files: 0,
21573 commits: 0,
21574 empty: false,
21575 failed: None,
21576 verified_noop: None,
21577 duration_ms: 0,
21578 folded: false,
21579 }];
21580 state.tally = Some(Tally {
21581 first_choice: BTreeMap::from([('A', 1)]),
21582 borda: BTreeMap::new(),
21583 winner: 'A',
21584 rankings: 1,
21585 unanimous_initial: true,
21586 deliberated: false,
21587 changed_votes: 0,
21588 unanimous_final: true,
21589 tie_break: None,
21590 judges: 0,
21591 present: 0,
21592 quorum: 0,
21593 met_quorum: true,
21594 uncontested: Some("only candidate A produced a change".to_owned()),
21595 });
21596 state.reviews = vec![ReviewRound {
21600 round: 1,
21601 head: head.clone(),
21602 verified_head: None,
21603 verified_at: None,
21604 reviews: Vec::new(),
21605 e2e: Vec::new(),
21606 fix: None,
21607 blocking: 1,
21608 answered: 1,
21609 expected: 1,
21610 clean: false,
21611 verify_retried: false,
21612 e2e_deferred: true,
21613 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
21614 progressed: false,
21615 vote_split: false,
21616 reconsideration: Vec::new(),
21617 verdict: None,
21618 }];
21619
21620 let mut runner = Runner {
21621 state,
21622 roles: ResolvedRoles {
21623 implementers: Vec::new(),
21624 judges: Vec::new(),
21625 reviewers: Vec::new(),
21626 fixer: None,
21627 conductor: conductor(),
21628 implementer_roster: Vec::new(),
21629 judge_roster: Vec::new(),
21630 reviewer_roster: Vec::new(),
21631 },
21632 sem: Arc::new(Semaphore::new(1)),
21633 pause: Pause::new(),
21634 interrupt: Pause::new(),
21635 };
21636
21637 let shell = runner.state.config.shell();
21638 runner
21639 .stop_reviewing("round budget spent", &shell, &repo)
21640 .await
21641 .expect("stop_reviewing");
21642
21643 let last = runner.state.reviews.last().expect("round record");
21644 assert_eq!(
21645 last.e2e_status(),
21646 E2eStatus::ResourceBlocked,
21647 "the shared cache is still held; the attempt must read as blocked, not deferred or \
21648 failed: {last:?}"
21649 );
21650 assert_eq!(
21651 last.verified_head.as_deref(),
21652 Some(head.as_str()),
21653 "which commit this attempt targeted is known even though nothing finished checking \
21654 it"
21655 );
21656 let first_attempt_at = last
21657 .verified_at
21658 .expect("when this attempt ran is known too");
21659 assert_ne!(
21660 runner.state.status,
21661 RunStatus::Blocked,
21662 "contention is evidence about the machine, not the patch — it must not settle the \
21663 run as blocked: {:?}",
21664 runner.state.status
21665 );
21666 assert!(
21667 !runner
21668 .state
21669 .events
21670 .iter()
21671 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
21672 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
21673 runner.state.events
21674 );
21675
21676 runner
21681 .stop_reviewing("round budget spent", &shell, &repo)
21682 .await
21683 .expect("stop_reviewing retry");
21684 assert_eq!(
21685 runner.state.reviews.len(),
21686 1,
21687 "no new round was started: {:?}",
21688 runner.state.reviews
21689 );
21690 let last = runner.state.reviews.last().expect("round record");
21691 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
21692 assert!(
21693 last.verified_at.expect("still known") > first_attempt_at,
21694 "a second reentry must be a fresh attempt, not a stale copy of the first"
21695 );
21696 assert_ne!(runner.state.status, RunStatus::Blocked);
21697
21698 held.release();
21699 }
21700
21701 #[cfg(unix)]
21702 #[tokio::test]
21703 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
21704 ask_test_home();
21705 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21706 let tmp = tempfile::tempdir().unwrap();
21707 let repo = tmp.path().join("repo");
21708 std::fs::create_dir(&repo).unwrap();
21709 init_repo(&repo);
21710 let run_git = |args: &[&str]| {
21711 let out = std::process::Command::new("git")
21712 .args(args)
21713 .current_dir(&repo)
21714 .output()
21715 .unwrap();
21716 assert!(
21717 out.status.success(),
21718 "{args:?}: {}",
21719 String::from_utf8_lossy(&out.stderr)
21720 );
21721 };
21722 let origin = tmp.path().join("origin.git");
21723 run_git(&["init", "--bare", origin.to_str().unwrap()]);
21724 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21725 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21726 run_git(&["add", "spec.rs"]);
21727 run_git(&["commit", "-m", "initial spec"]);
21728 run_git(&["push", "origin", "main"]);
21729 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21730 run_git(&["checkout", "-b", "fix/review"]);
21731 std::fs::write(
21732 repo.join("check.rs"),
21733 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21734 )
21735 .unwrap();
21736 run_git(&["add", "check.rs"]);
21737 run_git(&["commit", "-m", "use spec"]);
21738 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21739 run_git(&["checkout", "main"]);
21740 std::fs::write(
21741 repo.join("spec.rs"),
21742 "struct AgentSpec { max_turn_secs: u64 }\n",
21743 )
21744 .unwrap();
21745 run_git(&["add", "spec.rs"]);
21746 run_git(&["commit", "-m", "extend spec"]);
21747 run_git(&["push", "origin", "main"]);
21748 run_git(&["checkout", "fix/review"]);
21749
21750 let mut runner = runner_at(RunStatus::Reviewing);
21751 runner.state.repo = repo.clone();
21752 runner.state.base_commit = base;
21753 runner.state.pr_head = Some("fix/review".into());
21754 runner.state.config.graph.review_rounds = max_rounds;
21755 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21756 runner.state.config.graph.sessions = false;
21757 runner.state.config.disk.min_free_bytes = 0;
21758 runner.state.config.verify.e2e =
21759 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21760 runner.state.candidates = vec![Candidate {
21761 index: 0,
21762 label: 'A',
21763 agent: "fixture".into(),
21764 branch: "fix/review".into(),
21765 worktree: repo.clone(),
21766 summary: String::new(),
21767 stat: String::new(),
21768 files: 1,
21769 commits: 1,
21770 empty: false,
21771 failed: None,
21772 verified_noop: None,
21773 duration_ms: 0,
21774 folded: false,
21775 }];
21776 runner.state.tally = Some(Tally {
21777 first_choice: BTreeMap::new(),
21778 borda: BTreeMap::new(),
21779 winner: 'A',
21780 rankings: 0,
21781 unanimous_initial: true,
21782 deliberated: false,
21783 changed_votes: 0,
21784 unanimous_final: true,
21785 tie_break: None,
21786 judges: 0,
21787 present: 0,
21788 quorum: 0,
21789 met_quorum: true,
21790 uncontested: Some("review-only".into()),
21791 });
21792 let mut last = review_round(false, 0, 1, 1, false, false);
21793 last.head = reviewed.clone();
21794 last.e2e_deferred = true;
21795 last.e2e[0].resource_blocked = true;
21796 last.e2e[0].output_tail = "waiting for the shared build cache".into();
21797 if max_rounds == 1 {
21798 last.e2e.clear();
21799 }
21800 runner.state.reviews = vec![last];
21801 let calls = tmp.path().join("fix-calls");
21802 let mut fixer = spec("repair");
21803 fixer
21804 .env
21805 .insert("FIX_CALLS".into(), calls.display().to_string());
21806 fixer.command = vec!["sh".into(), "-c".into(), r#"
21807 echo fix >> "$FIX_CALLS"
21808 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21809 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21810 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21811 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21812 "#.into()];
21813 let mut reviewer = spec("reviewer");
21814 reviewer.command = vec![
21815 "sh".into(),
21816 "-c".into(),
21817 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21818 .into(),
21819 ];
21820 runner.roles.fixer = Some(vec![fixer]);
21821 runner.roles.reviewers = vec![reviewer];
21822
21823 runner.sync_to_base().await.unwrap();
21824 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21825 assert_ne!(rebased, reviewed);
21826 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21827 if reentry {
21828 let shell = runner.state.config.shell();
21831 runner
21832 .stop_unresolved_review("deferred verification", &shell, &repo)
21833 .await
21834 .unwrap();
21835 assert!(runner.catchup_repairable());
21836 assert!(!calls.exists());
21837 runner.state = RunState::load(&runner.state.id).unwrap();
21838 assert!(runner.catchup_repairable());
21839 }
21840 runner.review_loop().await.unwrap();
21841 let first = &runner.state.reviews[0];
21842 assert_eq!(first.head, reviewed);
21843 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21844 assert!(
21845 first.verify_retried,
21846 "a reproduced build failure gets exactly one retry: {:?}",
21847 first.e2e
21848 );
21849 assert!(first.e2e[0].output_tail.contains("E0063"));
21850 assert!(!runner.catchup_repairable());
21851 if max_rounds == 2 {
21852 assert_eq!(
21853 runner.state.status,
21854 RunStatus::Gating,
21855 "{}",
21856 crate::report::run(&runner.state)
21857 );
21858 assert_eq!(runner.state.reviews.len(), 2);
21859 assert!(first.fix.as_ref().unwrap().committed);
21860 assert!(runner.state.reviews[1].clean);
21861 assert_eq!(
21862 runner
21863 .state
21864 .reviews
21865 .iter()
21866 .filter(|r| r.fix.is_some())
21867 .count(),
21868 1
21869 );
21870 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21871 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21872 } else {
21873 assert_eq!(runner.state.status, RunStatus::Blocked);
21874 assert!(first.fix.is_none());
21875 assert!(!calls.exists());
21876 let recovery = crate::recovery::assess(
21877 "prior",
21878 Some(&runner.state),
21879 &runner.landing_base(),
21880 crate::recovery::Relation::Diverged,
21881 );
21882 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21883 let reasons = recovery.reasons.join("\n");
21884 assert!(
21885 reasons.contains(&reviewed) && reasons.contains(&rebased),
21886 "{reasons}"
21887 );
21888 assert!(
21889 reasons.contains("base sync to")
21890 && reasons.contains("additional review is required"),
21891 "{reasons}"
21892 );
21893 }
21894 }
21895 }
21896
21897 #[tokio::test]
21909 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21910 crate::run::pin_test_home();
21911 let home = crate::run::home();
21912
21913 let tmp = tempfile::tempdir().expect("tempdir");
21914 let repo = tmp.path().join("repo");
21915 std::fs::create_dir_all(&repo).unwrap();
21916 init_repo(&repo);
21917 let head = crate::git::rev_parse(&repo, "HEAD")
21918 .await
21919 .expect("rev-parse");
21920 let cache_dir = tmp.path().join("target");
21921
21922 let mut config = Config::default();
21923 config.verify.e2e = vec![format!(
21924 "CARGO_TARGET_DIR='{}' test -f README.md",
21925 cache_dir.display()
21926 )];
21927 config.graph.review_rounds = 1;
21928 config.graph.timeout_verify = Some(2);
21929 config.graph.timeout_resource_wait = 2;
21930
21931 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21932 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21933 .expect("no io error acquiring directly")
21934 {
21935 crate::cache::AcquireOutcome::Acquired(g) => g,
21936 crate::cache::AcquireOutcome::Busy(b) => {
21937 panic!("expected the direct acquire to win the lease first: {b:?}")
21938 }
21939 };
21940
21941 let mut state = RunState::new(
21942 repo.clone(),
21943 "main".to_owned(),
21944 head.clone(),
21945 "task".to_owned(),
21946 config,
21947 );
21948 state.candidates = vec![Candidate {
21949 index: 0,
21950 label: 'A',
21951 agent: "alpha".to_owned(),
21952 branch: "does-not-exist".to_owned(),
21953 worktree: repo.clone(),
21954 summary: String::new(),
21955 stat: String::new(),
21956 files: 0,
21957 commits: 0,
21958 empty: false,
21959 failed: None,
21960 verified_noop: None,
21961 duration_ms: 0,
21962 folded: false,
21963 }];
21964 state.tally = Some(Tally {
21965 first_choice: BTreeMap::from([('A', 1)]),
21966 borda: BTreeMap::new(),
21967 winner: 'A',
21968 rankings: 1,
21969 unanimous_initial: true,
21970 deliberated: false,
21971 changed_votes: 0,
21972 unanimous_final: true,
21973 tie_break: None,
21974 judges: 0,
21975 present: 0,
21976 quorum: 0,
21977 met_quorum: true,
21978 uncontested: Some("only candidate A produced a change".to_owned()),
21979 });
21980 state.reviews = vec![ReviewRound {
21984 round: 1,
21985 head: head.clone(),
21986 verified_head: Some(head.clone()),
21987 verified_at: Some(jiff::Timestamp::now()),
21988 reviews: Vec::new(),
21989 e2e: vec![CommandOutcome {
21990 command: format!(
21991 "CARGO_TARGET_DIR='{}' test -f README.md",
21992 cache_dir.display()
21993 ),
21994 code: None,
21995 output_tail: "waiting for the shared build cache".to_owned(),
21996 duration_ms: 0,
21997 resource_blocked: true,
21998 }],
21999 fix: None,
22000 blocking: 1,
22001 answered: 1,
22002 expected: 1,
22003 clean: false,
22004 verify_retried: false,
22005 e2e_deferred: false,
22006 e2e_defer_reason: None,
22007 progressed: false,
22008 vote_split: false,
22009 reconsideration: Vec::new(),
22010 verdict: None,
22011 }];
22012
22013 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
22014 let mut runner = Runner {
22015 state,
22016 roles: ResolvedRoles {
22017 implementers: Vec::new(),
22018 judges: Vec::new(),
22019 reviewers: Vec::new(),
22020 fixer: None,
22021 conductor: conductor(),
22022 implementer_roster: Vec::new(),
22023 judge_roster: Vec::new(),
22024 reviewer_roster: Vec::new(),
22025 },
22026 sem: Arc::new(Semaphore::new(1)),
22027 pause: Pause::new(),
22028 interrupt: Pause::new(),
22029 };
22030
22031 runner.review_loop().await.expect("review_loop");
22036
22037 assert_eq!(
22038 runner.state.reviews.len(),
22039 1,
22040 "no new round was started on top of the unresolved one: {:?}",
22041 runner.state.reviews
22042 );
22043 let last = &runner.state.reviews[0];
22044 assert_eq!(
22045 last.e2e_status(),
22046 E2eStatus::ResourceBlocked,
22047 "still contended: {last:?}"
22048 );
22049 assert!(
22050 last.verified_at.expect("still known") > first_attempt_at,
22051 "review_loop must have actually retried the check, not left it exactly as found"
22052 );
22053 assert_ne!(
22054 runner.state.status,
22055 RunStatus::Blocked,
22056 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
22057 runner.state.status
22058 );
22059
22060 held.release();
22061 }
22062
22063 #[tokio::test]
22064 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
22065 crate::run::pin_test_home();
22066 for status in [RunStatus::Blocked, RunStatus::Gating] {
22067 let mut runner = runner_at(status);
22068 runner.state.config.graph.land = true;
22069 runner.state.gate_ran = true;
22070 runner.state.merge = Some(MergeOutcome {
22071 mode: MergeMode::Pr,
22072 ok: false,
22073 detail: "the fixer produced no commit".into(),
22074 empty: false,
22075 });
22076 let url = "https://example.invalid/o/r/pull/1";
22077 runner.state.pr = Some(crate::run::PrRecord {
22078 url: url.into(),
22079 number: 1,
22080 state: "open".into(),
22081 checks: "red".into(),
22082 round: 1,
22083 rounds: 1,
22084 red_at_merge: Vec::new(),
22085 });
22086 runner.state.save().unwrap();
22087 runner.execute_graph().await.unwrap();
22091 assert_eq!(runner.state.status, RunStatus::Blocked);
22092 assert!(runner.state.land_sync_pending);
22093 assert!(runner.state.events.iter().any(|event|
22094 event.node == "land" && event.message == format!("watching {url}")));
22095 assert_eq!(
22096 runner.state.merge.as_ref().unwrap().detail,
22097 "the fixer produced no commit"
22098 );
22099 }
22100 }
22101
22102 #[tokio::test]
22103 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
22104 crate::run::pin_test_home();
22105 let tmp = tempfile::tempdir().expect("tempdir");
22106 let repo = tmp.path().join("repo");
22107 std::fs::create_dir_all(&repo).unwrap();
22108 init_repo(&repo);
22109
22110 let mut config = Config::default();
22111 config.merge.mode = MergeMode::Pr;
22112 config.graph.land = true;
22113 config.graph.land_approval = false;
22114
22115 let mut state = RunState::new(
22116 repo.clone(),
22117 "main".to_owned(),
22118 "deadbeef".to_owned(),
22119 "task".to_owned(),
22120 config,
22121 );
22122 state.candidates = vec![Candidate {
22123 index: 0,
22124 label: 'A',
22125 agent: "alpha".to_owned(),
22126 branch: "does-not-exist".to_owned(),
22127 worktree: repo.clone(),
22128 summary: String::new(),
22129 stat: String::new(),
22130 files: 0,
22131 commits: 0,
22132 empty: false,
22133 failed: None,
22134 verified_noop: None,
22135 duration_ms: 0,
22136 folded: false,
22137 }];
22138 state.tally = Some(Tally {
22139 first_choice: BTreeMap::from([('A', 1)]),
22140 borda: BTreeMap::new(),
22141 winner: 'A',
22142 rankings: 1,
22143 unanimous_initial: true,
22144 deliberated: false,
22145 changed_votes: 0,
22146 unanimous_final: true,
22147 tie_break: None,
22148 judges: 0,
22149 present: 0,
22150 quorum: 0,
22151 met_quorum: true,
22152 uncontested: Some("only candidate A produced a change".to_owned()),
22153 });
22154 state.reviews = vec![ReviewRound {
22155 round: 1,
22156 head: "deadbeef".to_owned(),
22157 verified_head: None,
22158 verified_at: None,
22159 reviews: Vec::new(),
22160 e2e: Vec::new(),
22161 fix: None,
22162 blocking: 0,
22163 answered: 0,
22164 expected: 0,
22165 clean: true,
22166 verify_retried: false,
22167 e2e_deferred: false,
22168 e2e_defer_reason: None,
22169 progressed: false,
22170 vote_split: false,
22171 reconsideration: Vec::new(),
22172 verdict: None,
22173 }];
22174 state.gate = vec![CommandOutcome {
22175 command: "test".to_owned(),
22176 code: Some(0),
22177 output_tail: String::new(),
22178 duration_ms: 0,
22179 resource_blocked: false,
22180 }];
22181 state.gate_ran = true;
22182 state.status = RunStatus::Landing;
22186 state.merge = Some(MergeOutcome {
22187 mode: MergeMode::Pr,
22188 ok: true,
22189 detail: "https://example.invalid/x/y/pull/1".to_owned(),
22190 empty: false,
22191 });
22192
22193 ask_test_home();
22197 state.save().expect("save landing run");
22200 let store = ask::Questions::open();
22201 let q = ask_open_question(&store, &state.id);
22202
22203 let mut runner = Runner {
22204 state,
22205 roles: ResolvedRoles {
22206 implementers: Vec::new(),
22207 judges: Vec::new(),
22208 reviewers: Vec::new(),
22209 fixer: None,
22210 conductor: conductor(),
22211 implementer_roster: Vec::new(),
22212 judge_roster: Vec::new(),
22213 reviewer_roster: Vec::new(),
22214 },
22215 sem: Arc::new(Semaphore::new(1)),
22216 pause: Pause::new(),
22217 interrupt: Pause::new(),
22218 };
22219
22220 runner.execute().await.expect("execute");
22225
22226 assert_eq!(
22227 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
22228 Some("https://example.invalid/x/y/pull/1"),
22229 "reentry must not push again or open a second pull request over the \
22230 one `land` is already watching"
22231 );
22232 assert_ne!(
22233 runner.state.status,
22234 RunStatus::Landing,
22235 "land could not actually reach the fake pull request, so it must \
22236 have given up rather than left the run silently parked forever"
22237 );
22238 assert_eq!(runner.state.status, RunStatus::Blocked);
22242 assert!(
22243 store.get(&q.id).unwrap().status.open(),
22244 "Blocked is still alive; settle_questions must have been a no-op here"
22245 );
22246 }
22247
22248 fn state_with_round(round: ReviewRound) -> RunState {
22249 let mut s = RunState::new(
22250 PathBuf::from("/repo"),
22251 "main".to_owned(),
22252 "abc1234".to_owned(),
22253 "add retries".to_owned(),
22254 Config::default(),
22255 );
22256 s.reviews = vec![round];
22257 s
22258 }
22259
22260 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
22261 crate::verdict::Finding {
22262 id: id.to_owned(),
22263 severity,
22264 file: None,
22265 line: None,
22266 title: title.to_owned(),
22267 detail: String::new(),
22268 }
22269 }
22270
22271 #[test]
22272 fn pr_body_names_open_findings_and_declined_ones() {
22273 let round = ReviewRound {
22274 round: 2,
22275 head: "deadbee".to_owned(),
22276 verified_head: None,
22277 verified_at: None,
22278 reviews: vec![ReviewRecord {
22279 attempts: 0,
22280 reviewer: 1,
22281 agent: "alpha".to_owned(),
22282 summary: String::new(),
22283 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
22284 vote: None,
22285 failed: None,
22286 duration_ms: 0,
22287 }],
22288 e2e: vec![CommandOutcome {
22289 command: "cargo test".to_owned(),
22290 code: Some(0),
22291 output_tail: String::new(),
22292 duration_ms: 0,
22293 resource_blocked: false,
22294 }],
22295 verify_retried: false,
22296 e2e_deferred: false,
22297 e2e_defer_reason: None,
22298 fix: Some(FixRecord {
22299 agent: "alpha".to_owned(),
22300 addressed: Vec::new(),
22301 rejected: vec![crate::verdict::Rejection {
22302 id: "R1-1-1".to_owned(),
22303 why: "not reachable from any caller".to_owned(),
22304 }],
22305 notes: String::new(),
22306 committed: true,
22307 failed: None,
22308 duration_ms: 0,
22309 continuation: None,
22310 }),
22311 blocking: 0,
22312 answered: 1,
22313 expected: 1,
22314 clean: false,
22315 progressed: true,
22316 vote_split: false,
22317 reconsideration: Vec::new(),
22318 verdict: None,
22319 };
22320 let state = state_with_round(round);
22321 let body = pr_message(&state, 'A').body;
22322
22323 assert!(body.contains("add retries"), "the task must still be there");
22324 assert!(body.contains("R2-1-1"), "{body}");
22325 assert!(!body.contains("unused import"), "{body}");
22326 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
22327 assert!(
22328 !body.contains("not reachable from any caller"),
22329 "the reason it was declined: {body}"
22330 );
22331 }
22332
22333 #[test]
22334 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
22335 let round = ReviewRound {
22336 round: 1,
22337 head: "deadbee".to_owned(),
22338 verified_head: None,
22339 verified_at: None,
22340 reviews: vec![ReviewRecord {
22341 attempts: 0,
22342 reviewer: 1,
22343 agent: "alpha".to_owned(),
22344 summary: String::new(),
22345 findings: Vec::new(),
22346 vote: None,
22347 failed: None,
22348 duration_ms: 0,
22349 }],
22350 e2e: Vec::new(),
22351 verify_retried: false,
22352 e2e_deferred: false,
22353 e2e_defer_reason: None,
22354 fix: None,
22355 blocking: 0,
22356 answered: 1,
22357 expected: 1,
22358 clean: true,
22359 progressed: false,
22360 vote_split: false,
22361 reconsideration: Vec::new(),
22362 verdict: None,
22363 };
22364 let state = state_with_round(round);
22365 let body = pr_message(&state, 'A').body;
22366 assert!(!body.contains("Open review findings"), "{body}");
22367 assert!(!body.contains("Declined"), "{body}");
22368 }
22369
22370 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
22371 let mut state = RunState::new(
22372 PathBuf::from("/repo"),
22373 "main".to_owned(),
22374 "abc1234".to_owned(),
22375 instruction.to_owned(),
22376 Config::default(),
22377 );
22378 state.config.disk.min_free_bytes = 0;
22380 state.candidates.push(Candidate {
22381 index: 0,
22382 label: 'A',
22383 agent: "alpha".to_owned(),
22384 branch: "magi/x/A".to_owned(),
22385 worktree: PathBuf::from("/wt"),
22386 summary: summary.to_owned(),
22387 stat: String::new(),
22388 files: 1,
22389 commits: 1,
22390 empty: false,
22391 failed: None,
22392 verified_noop: None,
22393 folded: false,
22394 duration_ms: 0,
22395 });
22396 state
22397 }
22398
22399 fn review_state(subjects: &[&str]) -> RunState {
22400 let mut state = state_with_summary(
22401 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
22402 "",
22403 );
22404 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22405 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
22406 state
22407 }
22408
22409 #[tokio::test]
22412 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
22413 ask_test_home();
22414 let tmp = tempfile::tempdir().unwrap();
22415 let repo = tmp.path().join("repo");
22416 std::fs::create_dir_all(&repo).unwrap();
22417 init_repo(&repo);
22418 let origin = tmp.path().join("origin.git");
22419 let g = |dir: &Path, args: &[&str]| {
22420 let out = std::process::Command::new("git")
22421 .args(args)
22422 .current_dir(dir)
22423 .quiet()
22424 .output()
22425 .expect("spawn git");
22426 assert!(
22427 out.status.success(),
22428 "git {args:?}: {}",
22429 String::from_utf8_lossy(&out.stderr)
22430 );
22431 };
22432 g(
22433 tmp.path(),
22434 &[
22435 "clone",
22436 "--bare",
22437 "-q",
22438 repo.to_str().unwrap(),
22439 origin.to_str().unwrap(),
22440 ],
22441 );
22442 g(
22443 &repo,
22444 &["remote", "add", "origin", origin.to_str().unwrap()],
22445 );
22446 let c1 = git::rev_parse(&repo, "main").await.unwrap();
22447
22448 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
22450 g(&repo, &["add", "-A"]);
22451 g(
22452 &repo,
22453 &["commit", "-q", "-m", "chore(deps): update a crate"],
22454 );
22455 g(&repo, &["push", "-q", "origin", "main"]);
22456 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
22457 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22458 g(&repo, &["add", "-A"]);
22459 g(
22460 &repo,
22461 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
22462 );
22463 g(&repo, &["checkout", "-q", "main"]);
22464
22465 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
22466 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
22467 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
22469 assert_eq!(fallback, c1);
22470
22471 let mut state = review_state(&[
22472 "chore(deps): update a crate",
22473 "fix(daemon): apply a chosen action",
22474 ]);
22475 state.repo = repo.clone();
22476 state.base_branch = "main".to_owned();
22477 state.base_commit = c1;
22478 refresh_reviewed_commits(&mut state, "feat/own").await;
22479 assert_eq!(
22480 state.reviewed_commits,
22481 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
22482 );
22483 assert_eq!(
22484 review_title(&state).as_deref(),
22485 Some("fix(daemon): apply a chosen action")
22486 );
22487 assert_eq!(
22488 leaked_subjects(&state, "feat/own").await,
22489 Some(vec!["chore(deps): update a crate".to_owned()])
22490 );
22491 state.config.merge.remote = "nowhere".to_owned();
22494 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
22495 }
22496
22497 struct SummaryFixture {
22501 _tmp: tempfile::TempDir,
22502 repo: PathBuf,
22503 reply: PathBuf,
22504 runner: Runner,
22505 }
22506
22507 fn summary_fixture() -> SummaryFixture {
22508 ask_test_home();
22509 let tmp = tempfile::tempdir().unwrap();
22510 let repo = tmp.path().join("repo");
22511 std::fs::create_dir_all(&repo).unwrap();
22512 init_repo(&repo);
22513 let origin = tmp.path().join("origin.git");
22514 let g = |args: &[&str]| {
22515 let out = std::process::Command::new("git")
22516 .args(args)
22517 .current_dir(&repo)
22518 .quiet()
22519 .output()
22520 .expect("spawn git");
22521 assert!(
22522 out.status.success(),
22523 "git {args:?}: {}",
22524 String::from_utf8_lossy(&out.stderr)
22525 );
22526 };
22527 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
22528 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
22529 g(&["push", "-q", "origin", "main"]);
22530 g(&["checkout", "-q", "-b", "feat/own"]);
22531 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22532 g(&["add", "-A"]);
22533 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
22534 let reply = tmp.path().join("reply.txt");
22535 std::fs::write(
22536 &reply,
22537 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
22538 )
22539 .unwrap();
22540 let mut runner = runner_at(RunStatus::Gating);
22541 let mut state = review_state(&["feat(own): add the own file"]);
22542 state.repo = repo.clone();
22543 state.base_branch = "main".to_owned();
22544 state.candidates[0].branch = "feat/own".to_owned();
22545 state.candidates[0].worktree = repo.clone();
22546 let mut author = spec("alpha");
22547 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
22548 author
22549 .env
22550 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
22551 state.config.agents = vec![author];
22552 runner.state = state;
22553 SummaryFixture {
22554 _tmp: tmp,
22555 repo,
22556 reply,
22557 runner,
22558 }
22559 }
22560
22561 async fn posted(f: &mut SummaryFixture) -> PrMessage {
22562 let winner = f.runner.state.candidates[0].clone();
22563 f.runner
22564 .guarded_pr_message(&winner, None, true)
22565 .await
22566 .unwrap()
22567 .expect("an English description never parks")
22568 }
22569
22570 fn clean_round_with_minor_findings() -> ReviewRound {
22571 let mut round = passing_round(2, None);
22572 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
22573 round.reviews = vec![ReviewRecord {
22574 attempts: 0,
22575 reviewer: 1,
22576 agent: "alpha".into(),
22577 summary: "内部レビュー".into(),
22578 findings: vec![
22579 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
22580 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
22581 ],
22582 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
22583 failed: None,
22584 duration_ms: 0,
22585 }];
22586 round
22587 }
22588
22589 #[tokio::test]
22590 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
22591 for review_only in [false, true] {
22592 let mut f = summary_fixture();
22593 if !review_only {
22594 f.runner.state.candidates[0].agent = "alpha".into();
22595 }
22596 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
22597 assert!(f.runner.state.reviews[0].clean);
22598 assert!(f.runner.state.reviews[0].fix.is_none());
22599 assert!(f.runner.state.open_findings().is_empty());
22600 assert_eq!(
22601 crate::pr_outcome::required_finding_ids(&f.runner.state),
22602 ["R2-1-1", "R2-1-2"]
22603 );
22604 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22605 let evidence = crate::pr_outcome::summary_evidence(
22606 &f.runner.state,
22607 crate::pr_outcome::FinalHead::At(&head),
22608 );
22609 assert!(
22610 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
22611 );
22612 let omitted = posted(&mut f).await;
22614 assert!(omitted.body.contains("No description"));
22615 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
22616 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
22617 assert!(f.runner.state.change_summary.is_none());
22618 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
22619 let described = posted(&mut f).await;
22620 assert!(described.body.contains("improve documentation for callers"));
22621 assert!(described.body.contains("remaining edge case"));
22622 assert!(described.body.contains("## Open review findings"));
22623 assert!(!described.body.contains("未修正"));
22624 }
22625 }
22626
22627 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
22628 let mut f = summary_fixture();
22629 if !review_only {
22630 f.runner.state.candidates[0].agent = "alpha".into();
22631 }
22632 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22633 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
22634 fixed
22635 .fix
22636 .as_mut()
22637 .unwrap()
22638 .rejected
22639 .push(crate::verdict::Rejection {
22640 id: "R1-1-1".into(),
22641 why: "呼び出されないコードのため辞退".into(),
22642 });
22643 let mut clean = clean_round_with_minor_findings();
22644 clean.verified_head = Some(head.clone());
22645 f.runner.state.reviews = vec![fixed, clean];
22646 ok_gate(&mut f.runner.state, Some(&head));
22647 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
22648 f.runner.state.config.agents.push(scripted("beta", reply));
22649 f.runner.state.config.roles.language_judge =
22650 Some(crate::config::AgentChoice::One("beta".into()));
22651 f
22652 }
22653
22654 #[tokio::test]
22655 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
22656 for review_only in [false, true] {
22657 let mut f = rewrite_fixture_with_findings(review_only,
22658 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
22659 let m = posted(&mut f).await;
22660 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22661 let expected =
22662 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
22663 assert!(m.body.contains(&expected));
22664 assert!(m.body.contains("## Review rounds"));
22665 assert!(
22666 m.body
22667 .contains("Gate (`verify.gate`): passed on the final commit")
22668 );
22669 assert!(
22670 m.body
22671 .contains("Tests (`verify.e2e`): passed on the final commit")
22672 );
22673 let c = f.runner.state.change_summary.as_ref().unwrap();
22674 assert!(
22675 m.body
22676 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
22677 );
22678 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22679 assert!(crate::pr_outcome::covers_required_findings(
22680 &f.runner.state,
22681 &m.body
22682 ));
22683 assert!(m.body.contains("documentation remains incomplete"));
22684 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
22685 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
22686 assert!(f.runner.state.github_text.is_none());
22687 }
22688 }
22689
22690 #[tokio::test]
22691 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
22692 for review_only in [false, true] {
22693 for reply in [
22694 "TITLE: fix: public description\nA short English description.",
22695 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
22696 ] {
22697 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
22698 let winner = f.runner.state.candidates[0].clone();
22699 let posted = f
22700 .runner
22701 .guarded_pr_message(&winner, None, true)
22702 .await
22703 .unwrap();
22704 assert!(posted.is_none());
22705 assert!(f.runner.state.parked);
22706 let withheld = f.runner.state.github_text.as_ref().unwrap();
22707 assert!(
22708 withheld.rewrite.is_none(),
22709 "an incomplete rewrite must not be selectable"
22710 );
22711 let q = ask::Questions::open()
22712 .get(withheld.question.as_ref().unwrap())
22713 .unwrap();
22714 assert!(
22715 !q.choices
22716 .contains(&crate::github_text::USE_REWRITE.to_owned())
22717 );
22718 assert!(
22719 q.detail.contains("R1-1-1")
22720 && q.detail.contains("R2-1-1")
22721 && q.detail.contains("R2-1-2")
22722 );
22723 }
22724 }
22725 }
22726
22727 #[tokio::test]
22728 async fn japanese_reports_stay_local_in_both_public_run_modes() {
22729 for review_only in [false, true] {
22730 let mut f = summary_fixture();
22731 if !review_only {
22732 f.runner.state.candidates[0].agent = "alpha".into();
22733 }
22734 f.runner.state.instruction = "内部タスクの説明".into();
22735 f.runner.state.candidates[0].summary = "内部の候補報告".into();
22736 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22737 f.runner.state.reviews = vec![
22738 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22739 round_with_notes(2, Some("二回目の内部修正報告です。")),
22740 passing_round(3, Some(&head)),
22741 ];
22742 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22743 ok_gate(&mut f.runner.state, Some(&head));
22744 let captured = f.repo.join("public-prompt.txt");
22745 let author = &mut f.runner.state.config.agents[0];
22746 author.command = vec![
22747 "sh".into(),
22748 "-c".into(),
22749 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22750 ];
22751 author.env.insert(
22752 "PROMPT_FILE".into(),
22753 captured.to_string_lossy().into_owned(),
22754 );
22755 let m = posted(&mut f).await;
22756 assert!(m.body.contains("Adds `own.txt`"));
22757 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22758 assert!(m.body.contains("R1-1-1"));
22759 assert!(
22760 m.body
22761 .contains("Gate (`verify.gate`): passed on the final commit")
22762 );
22763 assert!(
22764 m.body
22765 .contains("Tests (`verify.e2e`): passed on the final commit")
22766 );
22767 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22768 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22769 let prompt = std::fs::read_to_string(captured).unwrap();
22770 assert!(prompt.contains(&format!(
22771 "git diff {} {head}",
22772 f.runner.state.change_summary.as_ref().unwrap().base
22773 )));
22774 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22775 assert_eq!(
22776 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22777 "二回目の内部修正報告です。"
22778 );
22779 }
22780 }
22781
22782 #[tokio::test]
22783 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22784 for review_only in [false, true] {
22785 let mut f = summary_fixture();
22786 if !review_only {
22787 f.runner.state.candidates[0].agent = "alpha".into();
22788 }
22789 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22790 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22791 std::fs::write(&f.reply, "no usable description").unwrap();
22792 let failed = posted(&mut f).await;
22793 assert!(
22794 failed
22795 .body
22796 .contains("No description of what this change does could be generated")
22797 );
22798 assert!(failed.body.is_ascii());
22799 assert!(f.runner.state.github_text.is_none());
22800 assert!(f.runner.state.change_summary.is_none());
22801 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22802 let evidence = crate::pr_outcome::summary_evidence(
22803 &f.runner.state,
22804 crate::pr_outcome::FinalHead::At(&head),
22805 );
22806 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22807 assert!(
22808 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22809 .exists()
22810 );
22811 f.runner.state =
22812 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22813 std::fs::write(
22814 &f.reply,
22815 "TITLE: feat(own): add the own file\nAdds the committed own file.",
22816 )
22817 .unwrap();
22818 let retried = posted(&mut f).await;
22819 assert!(retried.body.contains("Adds the committed own file."));
22820 assert_eq!(
22821 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22822 "日本語の修正報告"
22823 );
22824 }
22825 }
22826
22827 #[tokio::test]
22828 async fn same_head_verification_and_findings_invalidate_public_prose() {
22829 let mut f = summary_fixture();
22830 posted(&mut f).await;
22831 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22832 let original = f
22833 .runner
22834 .state
22835 .change_summary
22836 .as_ref()
22837 .unwrap()
22838 .evidence
22839 .clone();
22840 ok_gate(&mut f.runner.state, Some(&head));
22841 std::fs::write(
22842 &f.reply,
22843 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22844 )
22845 .unwrap();
22846 let verified = posted(&mut f).await;
22847 assert!(verified.body.contains("now verified"));
22848 assert_ne!(
22849 f.runner.state.change_summary.as_ref().unwrap().evidence,
22850 original
22851 );
22852 let mut round = round_with_notes(1, Some("内部報告"));
22853 round.reviews.push(ReviewRecord {
22854 attempts: 0,
22855 reviewer: 1,
22856 agent: "alpha".into(),
22857 summary: "内部レビュー".into(),
22858 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22859 vote: None,
22860 failed: None,
22861 duration_ms: 0,
22862 });
22863 round
22864 .fix
22865 .as_mut()
22866 .unwrap()
22867 .rejected
22868 .push(crate::verdict::Rejection {
22869 id: "R1-1-1".into(),
22870 why: "呼び出し元から到達できない".into(),
22871 });
22872 f.runner.state.reviews = vec![round];
22873 let omitted = posted(&mut f).await;
22875 assert!(omitted.body.contains("No description"));
22876 assert!(omitted.body.contains("R1-1-1"));
22877 assert!(!omitted.body.contains("到達"));
22878 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22879 let described = posted(&mut f).await;
22880 assert!(described.body.contains("unreachable from callers"));
22881 assert!(described.body.contains("R1-1-1"));
22882 assert!(!described.body.contains("到達"));
22883 let evidence = f
22884 .runner
22885 .state
22886 .change_summary
22887 .as_ref()
22888 .unwrap()
22889 .evidence
22890 .clone();
22891 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22892 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22893 assert!(posted(&mut f).await.body.contains("rationale updated"));
22894 assert_ne!(
22895 f.runner.state.change_summary.as_ref().unwrap().evidence,
22896 evidence
22897 );
22898 f.runner
22900 .state
22901 .change_summary
22902 .as_mut()
22903 .unwrap()
22904 .evidence
22905 .clear();
22906 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22907 assert!(posted(&mut f).await.body.contains("current specification"));
22908 }
22909
22910 #[tokio::test]
22911 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22912 let mut f = summary_fixture();
22913 let m = posted(&mut f).await;
22914 assert_eq!(m.title, "feat(own): add the own file");
22915 assert!(
22916 m.body
22917 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22918 "{}",
22919 m.body
22920 );
22921 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22922 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22923 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22924 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22925 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22926 let kept = f.runner.state.change_summary.as_ref().unwrap();
22927 assert_eq!(
22928 (kept.head.as_str(), kept.base.as_str()),
22929 (head.as_str(), base.as_str())
22930 );
22931 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22932 }
22933
22934 #[tokio::test]
22935 async fn review_only_summary_receives_the_configured_github_language() {
22936 let mut f = summary_fixture();
22937 f.runner.state.config.graph.github_language = "ja".into();
22938 let captured = f.repo.join("summary-prompt.txt");
22939 let author = &mut f.runner.state.config.agents[0];
22940 author.command = vec![
22941 "sh".into(),
22942 "-c".into(),
22943 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22944 ];
22945 author.env.insert(
22946 "PROMPT_FILE".into(),
22947 captured.to_string_lossy().into_owned(),
22948 );
22949 std::fs::write(
22950 &f.reply,
22951 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22952 )
22953 .unwrap();
22954 let m = posted(&mut f).await;
22955 let prompt = std::fs::read_to_string(captured).unwrap();
22956 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22957 assert!(!prompt.contains("in English"));
22958 assert_eq!(m.title, "feat(own): ファイルを追加");
22959 assert!(m.body.contains("own.txt を追加します。"));
22960 }
22961
22962 #[tokio::test]
22963 async fn review_fixes_come_after_the_whole_change_summary() {
22964 let mut f = summary_fixture();
22965 f.runner.state.reviews = vec![
22966 round_with_notes(1, Some("handled the PATH case")),
22967 round_with_notes(2, None),
22968 ];
22969 let m = posted(&mut f).await;
22970 let summary = m.body.find("## Summary").expect("summary");
22971 let fixes = m.body.find("## Review rounds").expect("review rounds");
22972 assert!(summary < fixes, "{}", m.body);
22973 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22974 assert!(m.body.contains("## Final verification"), "{}", m.body);
22975 }
22976
22977 #[tokio::test]
22978 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22979 let mut f = summary_fixture();
22980 let first = posted(&mut f).await;
22981 let first_head = f.runner.state.change_summary.clone().unwrap().head;
22982 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22984 let again = posted(&mut f).await;
22985 assert_eq!((again.title, again.body), (first.title, first.body));
22986
22987 let g = |args: &[&str]| {
22988 let out = std::process::Command::new("git")
22989 .args(args)
22990 .current_dir(&f.repo)
22991 .quiet()
22992 .output()
22993 .expect("spawn git");
22994 assert!(out.status.success(), "git {args:?}");
22995 };
22996 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22997 g(&["add", "-A"]);
22998 g(&["commit", "-q", "-m", "fix(own): handle more"]);
22999 let second = posted(&mut f).await;
23000 let kept = f.runner.state.change_summary.clone().unwrap();
23001 assert_ne!(kept.head, first_head);
23002 assert!(second.body.contains("Not read yet."), "{}", second.body);
23003 assert!(!second.body.contains("so the first summary is read"));
23004 assert_eq!(second.title, "feat(own): other");
23005 }
23006
23007 #[tokio::test]
23008 async fn an_unavailable_summary_is_stated_and_never_invented() {
23009 let mut f = summary_fixture();
23010 std::fs::write(&f.reply, "no title line here").unwrap();
23011 let m = posted(&mut f).await;
23012 assert!(
23013 m.body
23014 .contains("No description of what this change does could be generated"),
23015 "{}",
23016 m.body
23017 );
23018 assert!(m.body.contains("internal reports remain in local records"));
23019 assert!(f.runner.state.change_summary.is_none());
23020 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
23021 let winner = f.runner.state.candidates[0].clone();
23023 let preview = f
23024 .runner
23025 .guarded_pr_message(&winner, None, false)
23026 .await
23027 .unwrap()
23028 .unwrap();
23029 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
23030 }
23031
23032 #[test]
23033 fn pr_message_review_single_commit_uses_its_subject() {
23034 let state = review_state(&["feat(nats): per-role user"]);
23035 let m = pr_message(&state, 'A');
23036 assert_eq!(m.title, "feat(nats): per-role user");
23037 assert!(!m.body.contains("Review the work already"), "{}", m.body);
23038 assert!(m.body.contains("## Commits under review"), "{}", m.body);
23039 }
23040
23041 #[test]
23042 fn pr_message_review_multi_commit_takes_the_oldest() {
23043 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
23044 let m = pr_message(&state, 'A');
23045 assert_eq!(m.title, "feat: the change");
23046 for s in ["feat: the change", "fix: typo", "fix: again"] {
23047 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
23048 }
23049 }
23050
23051 #[test]
23052 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
23053 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
23054 let mut state = review_state(&[first, "fix: later fixup"]);
23055 state.candidates[0].branch = "機能/ログイン".to_owned();
23056 let m = pr_message(&state, 'A');
23057 assert!(
23058 m.title.starts_with("chore: land candidate A of run"),
23059 "{}",
23060 m.title
23061 );
23062 }
23063 }
23064
23065 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
23066 BranchFacts {
23067 commits: commits
23068 .iter()
23069 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
23070 .collect(),
23071 stat: stat.to_owned(),
23072 }
23073 }
23074
23075 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
23076 let mut r = review_round(true, 0, 1, 1, true, true);
23077 r.round = round;
23078 r.fix = notes.map(|n| FixRecord {
23079 agent: "fixer".to_owned(),
23080 addressed: Vec::new(),
23081 rejected: Vec::new(),
23082 notes: n.to_owned(),
23083 committed: true,
23084 failed: None,
23085 duration_ms: 0,
23086 continuation: None,
23087 });
23088 r
23089 }
23090
23091 #[test]
23092 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
23093 let state = review_state(&["ignored"]);
23094 let f = facts(
23095 &[
23096 (
23097 "fix(login): resolve PATH on macOS",
23098 "Login shells skip rc files.",
23099 ),
23100 ("fix: address review", ""),
23101 ],
23102 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
23103 );
23104 let m = pr_message_with(&state, 'A', Some(&f));
23105 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
23106 assert!(
23107 m.body
23108 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
23109 "{}",
23110 m.body
23111 );
23112 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
23113 assert!(m.body.contains("## Diff stat"), "{}", m.body);
23114 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
23115 for banned in [
23116 "Review the work already",
23117 "no task statement",
23118 "Original task",
23119 ] {
23120 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
23121 }
23122 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
23123 }
23124
23125 #[test]
23126 fn pr_message_review_truncates_a_huge_first_commit_body() {
23127 let state = review_state(&["ignored"]);
23128 let f = facts(
23129 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
23130 "s",
23131 );
23132 let m = pr_message_with(&state, 'A', Some(&f));
23133 assert!(m.body.len() < 30_000, "{}", m.body.len());
23134 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
23135 assert!(
23136 m.body.contains("- ... 1 more commit(s)"),
23137 "{}",
23138 m.body.len()
23139 );
23140 assert!(m.body.ends_with("magi:candidate-a\n"));
23141 }
23142
23143 #[test]
23144 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
23145 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
23146 assert_eq!(m.title, "feat: x");
23147 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
23148 assert!(!m.body.contains("Diff stat"), "{}", m.body);
23149 assert!(!m.body.contains("no task statement"), "{}", m.body);
23150 }
23151
23152 #[test]
23153 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
23154 let mut state = review_state(&["日本語の件名"]);
23155 state.candidates[0].branch = "fix/macos-login-path".to_owned();
23156 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
23157 state.candidates[0].branch = "機能/ログイン".to_owned();
23158 assert!(
23159 pr_message(&state, 'A')
23160 .title
23161 .starts_with("chore: land candidate A")
23162 );
23163 }
23164
23165 #[test]
23166 fn pr_message_review_fixes_survive_a_clean_final_round() {
23167 let mut state = review_state(&["feat: x"]);
23168 state.reviews = vec![
23169 round_with_notes(1, Some("handled the PATH case")),
23170 round_with_notes(2, None),
23171 ];
23172 let body = pr_message(&state, 'A').body;
23173 assert!(
23174 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
23175 "{body}"
23176 );
23177 assert!(body.contains("### Round 1"), "{body}");
23178
23179 state.reviews = vec![
23180 round_with_notes(1, Some("first fix")),
23181 round_with_notes(2, Some("")),
23182 round_with_notes(3, Some("second fix")),
23183 round_with_notes(4, None),
23184 ];
23185 let body = pr_message(&state, 'A').body;
23186 assert!(
23187 body.contains("### Round 1") && !body.contains("first fix"),
23188 "{body}"
23189 );
23190 assert!(
23191 body.contains("### Round 3") && !body.contains("second fix"),
23192 "{body}"
23193 );
23194 assert!(body.contains("### Round 2"), "{body}");
23195 }
23196
23197 const FINAL: &str = "abcdef1234567890";
23198
23199 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
23200 let mut r = round_with_notes(round, None);
23201 r.verified_head = verified.map(str::to_owned);
23202 r
23203 }
23204
23205 fn ok_gate(state: &mut RunState, head: Option<&str>) {
23206 state.gate = vec![CommandOutcome {
23207 command: "gate".to_owned(),
23208 code: Some(0),
23209 output_tail: String::new(),
23210 duration_ms: 0,
23211 resource_blocked: false,
23212 }];
23213 state.gate_ran = true;
23214 state.gate_head = head.map(str::to_owned);
23215 }
23216
23217 fn body_at(state: &RunState) -> String {
23218 pr_message_raw(
23219 state,
23220 'A',
23221 None,
23222 crate::pr_outcome::FinalHead::At(FINAL),
23223 ChangeNote::NotAsked,
23224 )
23225 .body
23226 }
23227
23228 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
23229
23230 #[test]
23231 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
23232 let mut state = review_state(&["feat: x"]);
23233 state.reviews = vec![
23234 round_with_notes(1, Some(SANDBOXED)),
23235 passing_round(2, Some(FINAL)),
23236 ];
23237 ok_gate(&mut state, Some(FINAL));
23238 let body = body_at(&state);
23239 assert!(!body.contains("Handled the PATH case."), "{body}");
23240 assert!(!body.contains("could not commit"), "{body}");
23241 assert!(!body.contains("could not run"), "{body}");
23242 assert!(
23243 body.contains("The fix is committed on the branch"),
23244 "{body}"
23245 );
23246 assert!(!body.contains("sandbox"), "{body}");
23247 assert!(
23248 body.contains("Tests (`verify.e2e`): passed on the final commit"),
23249 "{body}"
23250 );
23251 assert!(
23252 body.contains("Gate (`verify.gate`): passed on the final commit"),
23253 "{body}"
23254 );
23255 assert!(
23257 state.reviews[0]
23258 .fix
23259 .as_ref()
23260 .unwrap()
23261 .notes
23262 .contains("could not commit")
23263 );
23264 }
23265
23266 #[test]
23267 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
23268 let mut state = review_state(&["feat: x"]);
23269 let mut r1 = round_with_notes(1, Some(SANDBOXED));
23270 r1.fix.as_mut().unwrap().committed = false;
23271 let mut r2 = passing_round(2, Some(FINAL));
23272 r2.e2e[0].code = Some(1);
23273 state.reviews = vec![r1, r2];
23274 let body = body_at(&state);
23275 assert!(!body.contains("could not commit"), "{body}");
23276 assert!(!body.contains("could not run"), "{body}");
23277 assert!(body.contains("This round produced no commit"), "{body}");
23278 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
23279 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
23280 assert!(!body.contains("passed"), "{body}");
23281 }
23282
23283 #[test]
23284 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
23285 let mut state = review_state(&["feat: x"]);
23286 state.reviews = vec![
23287 round_with_notes(1, Some(SANDBOXED)),
23288 passing_round(2, Some("0123456789abcdef")),
23289 ];
23290 ok_gate(&mut state, Some("0123456789abcdef"));
23291 let body = body_at(&state);
23292 assert!(!body.contains("HTTP tests could not run"), "{body}");
23293 assert!(!body.contains("could not commit"), "{body}");
23295 assert!(
23296 body.contains("passed on an earlier commit (`0123456`)"),
23297 "{body}"
23298 );
23299 assert!(!body.contains("passed on the final commit"), "{body}");
23300
23301 ok_gate(&mut state, None);
23303 let body = body_at(&state);
23304 assert!(
23305 body.contains("passed on a commit that was not recorded"),
23306 "{body}"
23307 );
23308 }
23309
23310 #[test]
23311 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
23312 let state = state_with_summary(
23313 "add retries to the client",
23314 "TITLE: feat: retries\n\nDid it.",
23315 );
23316 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
23317 assert_eq!(m.title, "feat: retries");
23318 assert!(
23319 m.body.contains("<summary>Original task</summary>"),
23320 "{}",
23321 m.body
23322 );
23323 assert!(!m.body.contains("Commits under review"), "{}", m.body);
23324 assert!(
23325 m.body
23326 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
23327 "{}",
23328 m.body
23329 );
23330 }
23331
23332 #[test]
23333 fn pr_message_review_bounds_a_long_english_subject() {
23334 let long = format!("feat: {}", "word ".repeat(100));
23335 let m = pr_message(&review_state(&[&long]), 'A');
23336 assert!(m.title.starts_with("feat: word"), "{}", m.title);
23337 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23338 }
23339
23340 #[test]
23341 fn pr_message_implementation_run_is_unchanged_by_review_support() {
23342 let state = state_with_summary("add retries\n\ndetails", "- did some things");
23343 let m = pr_message(&state, 'A');
23344 assert_eq!(m.title, "add retries");
23345 assert!(m.body.contains("<summary>Original task</summary>"));
23346 assert!(!m.body.contains("Commits under review"));
23347 assert_eq!(landing_subject_source(&state), state.instruction);
23348 }
23349
23350 #[test]
23351 fn review_run_squash_subject_is_the_change_not_the_prompt() {
23352 let state = review_state(&["feat: the change", "fix: typo"]);
23353 let source = landing_subject_source(&state);
23354 assert_eq!(land::merge_subject("", &source), "feat: the change");
23355 assert_eq!(
23356 land::merge_subject("magi: candidate A (uncommitted work)", &source),
23357 "feat: the change"
23358 );
23359 assert_eq!(
23361 land::merge_subject("feat: renamed by hand", &source),
23362 "feat: renamed by hand"
23363 );
23364 let blank = review_state(&["日本語"]);
23365 assert!(
23366 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
23367 );
23368 }
23369
23370 #[test]
23371 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
23372 let state = review_state(&["feat: the change"]);
23373 let source = landing_subject_source(&state);
23374 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
23375 assert_eq!(
23376 land::merge_subject(landing_title(&state, old), &source),
23377 "feat: the change"
23378 );
23379 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
23380 let task = state_with_summary("add retries", "");
23381 assert_eq!(landing_title(&task, old), old);
23382 }
23383
23384 #[test]
23385 fn pr_message_describes_the_change_not_the_task() {
23386 let state = state_with_summary(
23387 "今回やってほしいこと: results projector を直す",
23388 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
23389 );
23390 let m = pr_message(&state, 'A');
23391 assert_eq!(m.title, "fix(web): batch the runs list reads");
23392 assert!(
23393 m.body.starts_with("## Summary\n\n- reads run.json once"),
23394 "{}",
23395 m.body
23396 );
23397 assert!(!m.body.contains("TITLE:"), "{}", m.body);
23398 let task_at = m.body.find("今回やってほしいこと").unwrap();
23399 let details_at = m.body.find("<details>").unwrap();
23400 assert!(
23401 details_at < task_at,
23402 "the task lives inside <details>: {}",
23403 m.body
23404 );
23405 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
23406 assert!(m.body.contains("magi:candidate-a"));
23407 }
23408
23409 #[test]
23410 fn pr_message_falls_back_to_the_task_without_a_title_line() {
23411 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
23412 let m = pr_message(&state, 'A');
23413 assert_eq!(m.title, "add retries");
23414 assert!(
23415 m.body.contains("## Summary\n\n- did some things"),
23416 "{}",
23417 m.body
23418 );
23419
23420 let none = RunState::new(
23421 PathBuf::from("/repo"),
23422 "main".to_owned(),
23423 "abc1234".to_owned(),
23424 "add retries".to_owned(),
23425 Config::default(),
23426 );
23427 let m = pr_message(&none, 'A');
23428 assert_eq!(m.title, "add retries");
23429 assert!(!m.body.contains("## Summary"), "{}", m.body);
23430 }
23431
23432 #[test]
23433 fn pr_message_refuses_the_candidate_commit_subject() {
23434 for bad in [
23435 "TITLE: magi: candidate A (uncommitted work)",
23436 "TITLE: chore: stuff (uncommitted work)",
23437 "TITLE: ",
23438 ] {
23439 let state = state_with_summary("add retries", bad);
23440 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
23441 }
23442 }
23443
23444 #[test]
23445 fn pr_message_bounds_a_very_long_task_and_title() {
23446 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
23447 let state = state_with_summary(&long, "- nothing");
23448 let m = pr_message(&state, 'A');
23449 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23450 assert!(!m.title.contains('\n'));
23451
23452 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
23453 let m = pr_message(&state, 'A');
23454 assert!(m.title.starts_with("feat: "));
23455 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23456 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
23457 }
23458
23459 fn long_title_of(instruction: &str) -> String {
23460 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
23461 }
23462
23463 #[test]
23464 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
23465 let first = "Make the landing path keep a readable title for long tasks";
23466 let line = format!(
23467 "{first}. {}",
23468 "And then keep going with more words ".repeat(20)
23469 );
23470 let t = long_title_of(&line);
23471 assert_eq!(t, first);
23472 assert!(!t.starts_with("chore: land"));
23473 }
23474
23475 #[test]
23476 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
23477 let line = "word ".repeat(200);
23478 let t = long_title_of(&line);
23479 assert!(t.ends_with("word..."), "{t}");
23480 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
23481 }
23482
23483 #[test]
23484 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
23485 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
23486 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
23487 }
23488 }
23489
23490 #[test]
23491 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
23492 let mut state = state_with_summary("再試行を追加する", "- nothing");
23493 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
23494 state.config.graph.github_language = "ja".into();
23495 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
23496 }
23497
23498 #[test]
23499 fn pr_message_title_limit_is_exact() {
23500 let at = "a".repeat(PR_TITLE_MAX);
23501 assert_eq!(long_title_of(&at), at);
23502 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
23503 assert!(over.ends_with("..."), "{over}");
23504 assert_eq!(over.chars().count(), PR_TITLE_MAX);
23505 }
23506
23507 #[test]
23508 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
23509 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
23510 let t = long_title_of(&line);
23511 assert!(t.ends_with("..."), "{t}");
23512 assert!(t.is_ascii(), "{t}");
23513 }
23514
23515 #[test]
23516 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
23517 let line = format!(
23518 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
23519 "plus more filler words ".repeat(20)
23520 );
23521 let t = long_title_of(&line);
23522 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
23523 }
23524
23525 #[test]
23526 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
23527 let line = format!("\n\n# {}", "title words ".repeat(40));
23528 let state = state_with_summary(&line, "- nothing");
23529 let m = pr_message(&state, 'A');
23530 assert!(m.title.starts_with("title words"), "{}", m.title);
23531 assert_eq!(
23532 land::merge_subject(&m.title, &landing_subject_source(&state)),
23533 m.title
23534 );
23535 assert_eq!(
23537 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
23538 "feat: renamed by hand"
23539 );
23540 }
23541
23542 #[tokio::test]
23543 async fn github_text_rewrite_is_bounded_and_falls_back() {
23544 crate::run::pin_test_home();
23545 for (reply, accepted) in [
23546 (
23547 "TITLE: fix: retries\nAdd retries for failed requests.",
23548 true,
23549 ),
23550 (
23551 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
23552 false,
23553 ),
23554 (
23555 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
23556 true,
23557 ),
23558 (
23559 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
23560 true,
23561 ),
23562 (
23563 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
23564 true,
23565 ),
23566 (
23567 "TITLE: fix: retries\n日本語の説明です。 token=secret",
23568 false,
23569 ),
23570 ("TITLE: fix: retries\n", false),
23571 ] {
23572 let dir = tempfile::tempdir().unwrap();
23573 let mut runner = runner_at(RunStatus::Gating);
23574 runner.state = state_with_summary(
23575 "add retries",
23576 "TITLE: fix: retries\n日本語の説明を書きます。",
23577 );
23578 runner.state.repo = dir.path().to_owned();
23579 runner.state.candidates[0].worktree = dir.path().to_owned();
23580 let mut author = spec("alpha");
23581 author.command = vec![
23582 "sh".into(),
23583 "-c".into(),
23584 "printf '%s' \"$REWRITE_REPLY\"".into(),
23585 ];
23586 author.env.insert("REWRITE_REPLY".into(), reply.into());
23587 runner.state.config.agents = vec![author];
23588 seed_public_summary(&mut runner);
23589 let winner = runner.state.candidates[0].clone();
23590 let first = runner
23591 .guarded_pr_message(&winner, None, true)
23592 .await
23593 .unwrap();
23594 let message = if accepted {
23595 first.expect("a passing rewrite never asks")
23596 } else {
23597 assert!(first.is_none());
23599 assert!(runner.state.parked);
23600 let store = ask::Questions::open();
23601 let asked: Vec<_> = store
23602 .list()
23603 .into_iter()
23604 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23605 .collect();
23606 assert_eq!(asked.len(), 1);
23607 assert!(!asked[0].detail.contains("secret"));
23608 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
23609 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
23610 assert_eq!(
23611 asked[0]
23612 .choices
23613 .contains(&crate::github_text::USE_REWRITE.to_owned()),
23614 rewrite.is_some()
23615 );
23616 store
23619 .update(&asked[0].id, |q| {
23620 q.abandon("test");
23621 Ok(())
23622 })
23623 .unwrap();
23624 let again = runner
23625 .guarded_pr_message(&winner, None, true)
23626 .await
23627 .unwrap()
23628 .expect("an abandoned question falls back");
23629 let n = store
23630 .list()
23631 .into_iter()
23632 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23633 .count();
23634 assert_eq!(n, 1, "asked once per rejected text");
23635 again
23636 };
23637 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
23638 assert!(!message.body.contains("secret"));
23639 assert!(!message.body.contains("prefix"));
23640 assert!(!message.title.contains("secret"));
23641 assert_eq!(
23642 message.body.contains("Add retries for failed requests."),
23643 accepted
23644 );
23645 assert_eq!(
23646 runner.state.seats["github-text-rewrite"].turns, 1,
23647 "only one rewrite invocation"
23648 );
23649 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
23650 if accepted {
23651 assert!(runner.state.github_text.is_none());
23652 assert!(!runner.state.parked);
23653 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
23654 assert!(message.body.contains("## Final verification"));
23655 assert!(message.body.contains("Evidence: final diff"));
23656 assert!(
23657 message
23658 .body
23659 .contains(&format!("magi:run/{}", runner.state.id))
23660 );
23661 }
23662 if !accepted {
23663 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23664 assert!(
23665 message
23666 .body
23667 .contains(&format!("magi:run/{}", runner.state.id))
23668 );
23669 }
23670 }
23671 }
23672
23673 fn scripted(id: &str, reply: &str) -> AgentSpec {
23674 let mut a = spec(id);
23675 a.command = vec![
23676 "sh".into(),
23677 "-c".into(),
23678 "printf '%s' \"$REWRITE_REPLY\"".into(),
23679 ];
23680 a.env.insert("REWRITE_REPLY".into(), reply.into());
23681 a
23682 }
23683
23684 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
23685 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
23686
23687 fn seed_public_summary(runner: &mut Runner) {
23689 let repo = &runner.state.repo;
23690 let g = |args: &[&str]| {
23691 let out = std::process::Command::new("git")
23692 .args(args)
23693 .current_dir(repo)
23694 .quiet()
23695 .output()
23696 .unwrap();
23697 assert!(out.status.success(), "git {args:?}");
23698 String::from_utf8_lossy(&out.stdout).trim().to_owned()
23699 };
23700 g(&["init", "-q", "-b", "main"]);
23701 g(&[
23702 "-c",
23703 "user.name=Test",
23704 "-c",
23705 "user.email=test@example.invalid",
23706 "commit",
23707 "-q",
23708 "--allow-empty",
23709 "-m",
23710 "feat: fixture",
23711 ]);
23712 let head = g(&["rev-parse", "HEAD"]);
23713 g(&["update-ref", "refs/remotes/origin/main", &head]);
23714 let evidence = crate::pr_outcome::summary_evidence(
23715 &runner.state,
23716 crate::pr_outcome::FinalHead::At(&head),
23717 );
23718 let generated = &runner.state.candidates[0].summary;
23719 runner.state.change_summary = Some(crate::run::ChangeSummary {
23720 base: head.clone(),
23721 head,
23722 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23723 title: summary_title(generated).unwrap(),
23724 body: summary_without_title(generated),
23725 });
23726 }
23727
23728 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23729 let mut runner = runner_at(RunStatus::Gating);
23730 runner.state = state_with_summary("add retries", JA_SUMMARY);
23731 if review_only {
23732 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23733 }
23734 runner.state.repo = dir.to_owned();
23735 runner.state.candidates[0].worktree = dir.to_owned();
23736 runner.state.config.agents = agents;
23737 seed_public_summary(&mut runner);
23738 runner
23739 }
23740
23741 fn gate_questions(runner: &Runner) -> usize {
23742 ask::Questions::open()
23743 .list()
23744 .into_iter()
23745 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23746 .count()
23747 }
23748
23749 #[tokio::test]
23750 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23751 crate::run::pin_test_home();
23752 let dir = tempfile::tempdir().unwrap();
23753 let mut runner = rewrite_runner(
23754 dir.path(),
23755 true,
23756 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23757 );
23758 runner.state.config.roles.language_judge =
23759 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23760 let winner = runner.state.candidates[0].clone();
23761 let msg = runner
23762 .guarded_pr_message(&winner, None, true)
23763 .await
23764 .unwrap()
23765 .expect("an English rewrite posts without asking");
23766 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23767 assert!(msg.body.contains("<details>kept</details>"));
23768 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23769 assert_eq!(gate_questions(&runner), 0);
23770 assert!(runner.state.github_text.is_none());
23771 assert!(!runner.state.seats.contains_key("impl-A"));
23772 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23773 }
23774
23775 #[tokio::test]
23776 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23777 crate::run::pin_test_home();
23778 let dir = tempfile::tempdir().unwrap();
23779 let mut runner = rewrite_runner(
23780 dir.path(),
23781 false,
23782 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23783 );
23784 runner.state.config.roles.language_judge =
23785 Some(crate::config::AgentChoice::One("beta".into()));
23786 runner.state.seats.insert(
23787 "impl-A".into(),
23788 SeatState::new("impl-A", "alpha", runner.state.seed),
23789 );
23790 let winner = runner.state.candidates[0].clone();
23791 let msg = runner
23792 .guarded_pr_message(&winner, None, true)
23793 .await
23794 .unwrap()
23795 .expect("rewritten");
23796 assert!(msg.body.contains("<details>kept</details>"));
23797 assert_eq!(runner.state.seats["impl-A"].turns, 0);
23798 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23799 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23800 }
23801
23802 #[tokio::test]
23803 async fn rewrite_reuses_a_resumable_implementer_session() {
23804 crate::run::pin_test_home();
23805 let dir = tempfile::tempdir().unwrap();
23806 let mut runner = rewrite_runner(
23807 dir.path(),
23808 false,
23809 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23810 );
23811 runner.state.config.roles.language_judge =
23812 Some(crate::config::AgentChoice::One("beta".into()));
23813 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23814 seat.turns = 1;
23815 runner.state.seats.insert("impl-A".into(), seat);
23816 let winner = runner.state.candidates[0].clone();
23817 runner
23818 .guarded_pr_message(&winner, None, true)
23819 .await
23820 .unwrap()
23821 .expect("rewritten");
23822 assert_eq!(runner.state.seats["impl-A"].turns, 2);
23823 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23824 }
23825
23826 #[tokio::test]
23827 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23828 crate::run::pin_test_home();
23831 let dir = tempfile::tempdir().unwrap();
23832 let mut exhausted = spec("alpha");
23833 exhausted.command = vec!["false".into()];
23834 let mut runner = rewrite_runner(
23835 dir.path(),
23836 false,
23837 vec![exhausted, scripted("beta", EN_REWRITE)],
23838 );
23839 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23841 "alpha".into(),
23842 "beta".into(),
23843 ]));
23844 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23845 seat.turns = 1;
23846 runner.state.seats.insert("impl-A".into(), seat);
23847 let winner = runner.state.candidates[0].clone();
23848 let msg = runner
23849 .guarded_pr_message(&winner, None, true)
23850 .await
23851 .unwrap()
23852 .expect("the role rewrites after the implementer fails");
23853 assert!(msg.body.contains("`src/net.rs`"));
23854 assert_eq!(
23855 runner.state.seats["impl-A"].turns, 2,
23856 "the implementer was tried once"
23857 );
23858 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23859 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23860 }
23861
23862 #[tokio::test]
23863 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23864 crate::run::pin_test_home();
23865 let dir = tempfile::tempdir().unwrap();
23866 let mut broken = spec("bad");
23867 broken.command = vec!["false".into()];
23868 let mut runner = rewrite_runner(
23869 dir.path(),
23870 true,
23871 vec![
23872 broken,
23873 scripted("noformat", "Add retries."),
23874 scripted("good", EN_REWRITE),
23875 ],
23876 );
23877 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23878 "bad".into(),
23879 "noformat".into(),
23880 "good".into(),
23881 ]));
23882 let winner = runner.state.candidates[0].clone();
23883 let msg = runner
23884 .guarded_pr_message(&winner, None, true)
23885 .await
23886 .unwrap()
23887 .expect("the third agent rewrites");
23888 assert!(msg.body.contains("`src/net.rs`"));
23889 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23890 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23891 }
23892
23893 #[tokio::test]
23894 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23895 crate::run::pin_test_home();
23896 for (agents, role, saved) in [
23898 (
23899 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23900 Some("beta"),
23901 true,
23902 ),
23903 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23904 (Vec::new(), None, false),
23905 ] {
23906 let dir = tempfile::tempdir().unwrap();
23907 let mut runner = rewrite_runner(dir.path(), true, agents);
23908 runner.state.config.roles.language_judge =
23909 role.map(|r| crate::config::AgentChoice::One(r.into()));
23910 let winner = runner.state.candidates[0].clone();
23911 let first = runner
23912 .guarded_pr_message(&winner, None, true)
23913 .await
23914 .unwrap();
23915 assert!(first.is_none(), "withheld text parks the run");
23916 assert!(runner.state.parked);
23917 assert_eq!(gate_questions(&runner), 1);
23918 let g = runner.state.github_text.as_ref().unwrap();
23919 assert_eq!(g.rewrite.is_some(), saved);
23920 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23921 }
23922 }
23923
23924 #[tokio::test]
23925 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23926 crate::run::pin_test_home();
23927 let dir = tempfile::tempdir().unwrap();
23928 let mut runner = runner_at(RunStatus::Gating);
23929 runner.state =
23930 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23931 runner.state.repo = dir.path().to_owned();
23932 runner.state.candidates[0].worktree = dir.path().to_owned();
23933 let mut author = spec("alpha");
23934 author.command = vec![
23935 "sh".into(),
23936 "-c".into(),
23937 "printf '%s' \"$REWRITE_REPLY\"".into(),
23938 ];
23939 author.env.insert(
23940 "REWRITE_REPLY".into(),
23941 format!(
23942 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23943 "English sentence. ".repeat(600)
23944 ),
23945 );
23946 runner.state.config.agents = vec![author];
23947 seed_public_summary(&mut runner);
23948 let winner = runner.state.candidates[0].clone();
23949 assert!(
23950 runner
23951 .guarded_pr_message(&winner, None, true)
23952 .await
23953 .unwrap()
23954 .is_none()
23955 );
23956 let record = runner.state.github_text.clone().unwrap();
23957 let rewrite = record.rewrite.clone().unwrap();
23958 assert_eq!(rewrite.failed, ["body-language"]);
23959 assert!(
23960 rewrite
23961 .body
23962 .ends_with(&format!("magi:run/{}", runner.state.id))
23963 );
23964 let store = ask::Questions::open();
23965 let q = store.get(record.question.as_ref().unwrap()).unwrap();
23966 assert!(
23967 q.choices
23968 .contains(&crate::github_text::USE_REWRITE.to_owned())
23969 );
23970 for label in [
23971 "ORIGINAL",
23972 "FALLBACK",
23973 "REWRITE",
23974 "Rules still failed: body-language",
23975 "truncated",
23976 ] {
23977 assert!(q.detail.contains(label), "{label}");
23978 }
23979 let artifact = std::fs::read_to_string(crate::run::artifact_path(
23980 &runner.state,
23981 &crate::github_text::artifact_name(&record.fingerprint),
23982 ))
23983 .unwrap();
23984 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23985 runner.state =
23987 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23988 store
23989 .update(&q.id, |q| {
23990 q.say("まだ日本語")?;
23991 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23992 })
23993 .unwrap();
23994 assert!(
23995 runner
23996 .guarded_pr_message(&winner, None, true)
23997 .await
23998 .unwrap()
23999 .is_none()
24000 );
24001 let retry = runner.state.github_text.as_ref().unwrap();
24002 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
24003 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
24004 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
24005 assert!(
24006 q.posting_comparison
24007 .as_ref()
24008 .unwrap()
24009 .matches_detail(&q.detail)
24010 );
24011 store
24012 .update(&q.id, |q| {
24013 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
24014 })
24015 .unwrap();
24016 let message = runner
24017 .guarded_pr_message(&winner, None, true)
24018 .await
24019 .unwrap()
24020 .unwrap();
24021 assert_eq!(message.title, rewrite.title);
24022 assert_eq!(message.body, rewrite.body);
24023 assert_eq!(
24024 runner.state.github_text.as_ref().unwrap().choice,
24025 Some(GithubTextChoice::Rewrite)
24026 );
24027 let again = runner
24028 .guarded_pr_message(&winner, None, true)
24029 .await
24030 .unwrap()
24031 .unwrap();
24032 assert_eq!(again.title, message.title);
24033 assert_eq!(again.body, message.body);
24034 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
24035 assert!(
24036 !runner
24037 .state
24038 .events
24039 .iter()
24040 .any(|e| e.message.contains(&rewrite.title)
24041 || e.message.contains("English sentence."))
24042 );
24043 }
24044
24045 #[tokio::test]
24046 async fn historical_posting_reply_never_expires_into_fallback() {
24047 crate::run::pin_test_home();
24048 let dir = tempfile::tempdir().unwrap();
24049 let mut runner = runner_at(RunStatus::Gating);
24050 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
24051 runner.state.repo = dir.path().to_owned();
24052 runner.state.candidates[0].worktree = dir.path().to_owned();
24053 seed_public_summary(&mut runner);
24054 let winner = runner.state.candidates[0].clone();
24055 assert!(
24056 runner
24057 .guarded_pr_message(&winner, None, true)
24058 .await
24059 .unwrap()
24060 .is_none()
24061 );
24062 let id = runner
24063 .state
24064 .github_text
24065 .as_ref()
24066 .unwrap()
24067 .question
24068 .clone()
24069 .unwrap();
24070 let store = ask::Questions::open();
24071 let mut q = store.get(&id).unwrap();
24072 q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
24073 q.answer(ask::Answer::Choice(
24074 crate::github_text::POST_ORIGINAL.into(),
24075 ))
24076 .unwrap();
24077 q.answer_delivered = true;
24078 store.put(&mut q).unwrap();
24079 let source = std::fs::read(store.path_of(&id)).unwrap();
24080 let migration = crate::run::home().join("notifications/task-chat-migration-v1");
24081 std::fs::create_dir_all(&migration).unwrap();
24082 std::fs::write(
24083 migration.join("state.json"),
24084 serde_json::to_vec(&serde_json::json!({
24085 "schema": 1, "sources": [id], "pending": [id]
24086 }))
24087 .unwrap(),
24088 )
24089 .unwrap();
24090 for _ in 0..2 {
24091 assert!(
24092 runner
24093 .guarded_pr_message(&winner, None, true)
24094 .await
24095 .unwrap()
24096 .is_none()
24097 );
24098 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
24099 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
24100 }
24101 let home = crate::run::home();
24102 crate::task_chat::sweep(
24103 &crate::queue::Queue::at(home.join("queue")),
24104 &crate::talk::Talks::at(home.join("talks")),
24105 &store,
24106 &home,
24107 );
24108 let outbox = crate::task_inquiry::Outbox::at(&home);
24109 outbox
24110 .update_decision(&id, |d| {
24111 d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
24112 d.source_answer_at = q.answered_at;
24113 Ok(true)
24114 })
24115 .unwrap();
24116 outbox
24117 .settle(
24118 &id,
24119 "owner-chat",
24120 "Owner confirms no publication remains necessary",
24121 None,
24122 )
24123 .unwrap();
24124 assert!(
24125 runner
24126 .guarded_pr_message(&winner, None, true)
24127 .await
24128 .unwrap()
24129 .is_none()
24130 );
24131 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
24132 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
24133 }
24134
24135 #[tokio::test]
24136 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
24137 crate::run::pin_test_home();
24138 let dir = tempfile::tempdir().unwrap();
24139 let mut runner = runner_at(RunStatus::Gating);
24140 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
24141 runner.state.repo = dir.path().to_owned();
24142 runner.state.candidates[0].worktree = dir.path().to_owned();
24143 seed_public_summary(&mut runner);
24144 let winner = runner.state.candidates[0].clone();
24145 assert!(
24146 runner
24147 .guarded_pr_message(&winner, None, true)
24148 .await
24149 .unwrap()
24150 .is_none()
24151 );
24152 let record = runner.state.github_text.as_ref().unwrap();
24153 assert!(record.rewrite.is_none());
24154 let mut old = serde_json::to_value(record).unwrap();
24156 old.as_object_mut().unwrap().remove("rewrite");
24157 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
24158 assert!(old.rewrite.is_none());
24159 let id = record.question.clone().unwrap();
24160 ask::Questions::open()
24161 .update(&id, |q| {
24162 q.choices.push(crate::github_text::USE_REWRITE.into());
24163 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
24164 })
24165 .unwrap();
24166 let message = runner
24167 .guarded_pr_message(&winner, None, true)
24168 .await
24169 .unwrap()
24170 .unwrap();
24171 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
24172 assert_eq!(
24173 runner.state.github_text.as_ref().unwrap().choice,
24174 Some(GithubTextChoice::Fallback)
24175 );
24176 }
24177
24178 #[tokio::test]
24179 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
24180 crate::run::pin_test_home();
24181 let dir = tempfile::tempdir().unwrap();
24182 let mut runner = runner_at(RunStatus::Gating);
24183 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
24184 runner.state.repo = dir.path().to_owned();
24185 runner.state.candidates[0].worktree = dir.path().to_owned();
24186 seed_public_summary(&mut runner);
24187 let winner = runner.state.candidates[0].clone();
24188 assert!(
24189 runner
24190 .guarded_pr_message(&winner, None, true)
24191 .await
24192 .unwrap()
24193 .is_none()
24194 );
24195 let store = ask::Questions::open();
24196 let find = |run: &str| {
24197 store
24198 .list()
24199 .into_iter()
24200 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
24201 .unwrap()
24202 };
24203 let run = runner.state.id.clone();
24204 let q = find(&run);
24205 let fp = runner
24206 .state
24207 .github_text
24208 .as_ref()
24209 .unwrap()
24210 .fingerprint
24211 .clone();
24212 let name = crate::github_text::artifact_name(&fp);
24213 let path = crate::run::artifact_path(&runner.state, &name);
24214 assert!(q.detail.contains(&path.display().to_string()));
24215 assert!(q.detail.contains("title-language: title"));
24216 let saved = std::fs::read_to_string(&path).expect("artifact written");
24217 assert!(saved.contains("日本語のタイトル"));
24218 assert!(q.cwd.is_none());
24219 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
24220 let say = |id: &str, text: &str, choice: &str| {
24222 store
24223 .update(id, |q| {
24224 q.thread.push(ask::Turn {
24225 who: ask::Who::Operator,
24226 body: text.to_owned(),
24227 at: jiff::Timestamp::now(),
24228 note: None,
24229 });
24230 q.answer(ask::Answer::Choice(choice.to_owned()))
24231 })
24232 .unwrap();
24233 };
24234 say(&q.id, "まだ日本語", "use my text");
24235 assert!(
24236 runner
24237 .guarded_pr_message(&winner, None, true)
24238 .await
24239 .unwrap()
24240 .is_none()
24241 );
24242 let second = find(&run);
24243 assert_ne!(second.id, q.id);
24244 assert!(second.detail.contains("did not pass"));
24245 assert!(second.detail.contains("日本語のタイトル"));
24248 assert!(!second.detail.contains("まだ日本語"));
24249 assert!(second.detail.contains(&path.display().to_string()));
24250 say(&second.id, "fix: retry failed requests", "use my text");
24251 let message = runner
24252 .guarded_pr_message(&winner, None, true)
24253 .await
24254 .unwrap()
24255 .expect("a vetted replacement is posted");
24256 assert_eq!(message.title, "fix: retry failed requests");
24257 assert_eq!(
24258 runner
24259 .state
24260 .github_text
24261 .as_ref()
24262 .unwrap()
24263 .chosen_title
24264 .as_deref(),
24265 Some("fix: retry failed requests")
24266 );
24267 assert!(runner.state.github_text.as_ref().unwrap().resolved);
24268 let again = runner
24270 .guarded_pr_message(&winner, None, true)
24271 .await
24272 .unwrap()
24273 .unwrap();
24274 assert_eq!(again.title, message.title);
24275 assert_eq!(
24276 store
24277 .list()
24278 .into_iter()
24279 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
24280 .count(),
24281 2
24282 );
24283 }
24284
24285 #[test]
24286 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
24287 let mut state = state_with_summary(
24291 "add retries",
24292 "TITLE: fix(web): batch reads\n- reads run.json once",
24293 );
24294 state.config.graph.language = "ja".to_owned();
24295 let m = pr_message(&state, 'A');
24296 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
24297 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
24298
24299 let task = "今回やってほしいこと: results projector を直す";
24302 let mut state = state_with_summary(task, "- no title line");
24303 state.config.graph.language = "ja".to_owned();
24304 let m = pr_message(&state, 'A');
24305 assert_eq!(
24306 m.title,
24307 format!("chore: land candidate A of run {}", state.id)
24308 );
24309 assert!(
24310 m.body.contains(&format!(
24311 "<summary>Original task</summary>\n\n{task}\n\n</details>"
24312 )),
24313 "{}",
24314 m.body
24315 );
24316 }
24317
24318 #[test]
24319 fn pr_message_scrubs_home_paths_and_addresses() {
24320 let state = state_with_summary(
24321 "fix it in /Users/someone/src/x",
24322 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
24323 );
24324 let m = pr_message(&state, 'A');
24325 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
24326 assert!(!m.body.contains(leak), "{}", m.body);
24327 }
24328 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
24329 }
24330
24331 #[test]
24332 fn pr_message_survives_a_task_that_closes_details() {
24333 let state = state_with_summary("a </details> b", "TITLE: fix: x");
24334 let m = pr_message(&state, 'A');
24335 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
24336 }
24337
24338 #[test]
24339 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
24340 let cmd = manual_merge_command(
24341 MergeStyle::Squash,
24342 Path::new("/repo"),
24343 "b",
24344 "fix: \"quoted\" $(x) `y`\n\nbody",
24345 );
24346 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
24347 }
24348
24349 #[test]
24350 fn manual_merge_command_matches_the_configured_style() {
24351 let repo = Path::new("/repo");
24352 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
24353
24354 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
24355 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
24356
24357 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
24358 assert_eq!(
24359 squash,
24360 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
24361 \"Merge magi run 0832 (candidate A)\""
24362 );
24363
24364 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
24365 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
24366 }
24367
24368 #[test]
24369 fn a_nudge_gets_a_quarter_of_the_budget() {
24370 assert_eq!(retry_budget(secs(1200), true), secs(300));
24372 assert_eq!(retry_budget(secs(3600), true), secs(900));
24373 }
24374
24375 #[test]
24376 fn a_resent_prompt_keeps_the_whole_budget() {
24377 assert_eq!(retry_budget(secs(1200), false), secs(1200));
24380 assert_eq!(retry_budget(secs(60), false), secs(60));
24381 }
24382
24383 #[test]
24384 fn the_floor_never_exceeds_the_original_budget() {
24385 assert_eq!(retry_budget(secs(60), true), secs(60));
24389 assert_eq!(retry_budget(secs(480), true), secs(120));
24390 assert_eq!(retry_budget(secs(0), true), secs(0));
24391 }
24392
24393 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
24394 agent::CommandEvidence {
24395 id: "item1".to_owned(),
24396 description: "cargo test".to_owned(),
24397 exit_code,
24398 result_summary: String::new(),
24399 source: "codex".to_owned(),
24400 }
24401 }
24402
24403 #[test]
24404 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
24405 assert!(!has_unconfirmed_command(&[]));
24409 }
24410
24411 #[test]
24412 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
24413 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
24417 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
24418 assert!(!has_unconfirmed_command(&[
24419 evidence(Some(0)),
24420 evidence(Some(101))
24421 ]));
24422 }
24423
24424 #[test]
24425 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
24426 assert!(has_unconfirmed_command(&[
24427 evidence(Some(0)),
24428 evidence(None)
24429 ]));
24430 }
24431
24432 #[test]
24433 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
24434 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24435 assert_eq!(
24436 verified_noop_claim(true, &[], text).as_deref(),
24437 Some("already fixed by b32cfc4, on main.")
24438 );
24439 }
24440
24441 #[test]
24442 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
24443 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24446 assert!(verified_noop_claim(false, &[], text).is_none());
24447 }
24448
24449 #[test]
24450 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
24451 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24452 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
24453 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
24455 }
24456
24457 #[test]
24458 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
24459 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
24460 }
24461
24462 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
24465 runner.state.candidates = shape
24466 .iter()
24467 .enumerate()
24468 .map(|(i, &(empty, verified))| Candidate {
24469 index: i,
24470 label: (b'A' + i as u8) as char,
24471 agent: "sonnet".to_owned(),
24472 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
24473 worktree: PathBuf::from(format!("/wt/{i}")),
24474 summary: String::new(),
24475 stat: String::new(),
24476 files: 0,
24477 commits: 0,
24478 empty,
24479 failed: None,
24480 verified_noop: verified.map(str::to_owned),
24481 duration_ms: 0,
24482 folded: false,
24483 })
24484 .collect();
24485 }
24486
24487 #[test]
24488 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
24489 ask_test_home();
24490 let mut runner = runner_at(RunStatus::Implementing);
24491 set_candidates(
24492 &mut runner,
24493 &[
24494 (true, Some("already on main at b32cfc4")),
24495 (true, Some("same fix, see the existing test")),
24496 ],
24497 );
24498
24499 runner
24500 .after_implement()
24501 .expect("a verified no-op is not an error");
24502
24503 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
24504 }
24505
24506 #[test]
24507 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
24508 ask_test_home();
24509 let mut runner = runner_at(RunStatus::Implementing);
24510 set_candidates(
24514 &mut runner,
24515 &[(true, Some("already on main at b32cfc4")), (true, None)],
24516 );
24517
24518 let err = runner
24519 .after_implement()
24520 .expect_err("an unverified empty candidate must still fail the run");
24521
24522 assert!(
24523 err.to_string().contains("no candidate produced a change"),
24524 "{err}"
24525 );
24526 assert_eq!(runner.state.status, RunStatus::Failed);
24527 }
24528
24529 #[test]
24530 fn after_implement_still_fails_an_ordinary_all_empty_run() {
24531 ask_test_home();
24532 let mut runner = runner_at(RunStatus::Implementing);
24533 set_candidates(&mut runner, &[(true, None), (true, None)]);
24534
24535 let err = runner
24536 .after_implement()
24537 .expect_err("no candidate declared anything; this is an ordinary failure");
24538
24539 assert!(
24540 err.to_string().contains("no candidate produced a change"),
24541 "{err}"
24542 );
24543 assert_eq!(runner.state.status, RunStatus::Failed);
24544 }
24545}
24546
24547#[derive(Debug)]
24550struct GraftRestoreFailed(String);
24551
24552impl std::fmt::Display for GraftRestoreFailed {
24553 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24554 write!(
24555 f,
24556 "could not restore the winner to its pre-graft commit: {}",
24557 self.0
24558 )
24559 }
24560}
24561
24562impl std::error::Error for GraftRestoreFailed {}
24563
24564#[cfg(test)]
24568mod resource_wait_tests {
24569 use super::*;
24570 use std::collections::BTreeMap;
24571
24572 fn sh(id: &str, script: &str) -> AgentSpec {
24573 AgentSpec {
24574 id: id.to_owned(),
24575 kind: crate::config::AgentKind::Command,
24576 model: None,
24577 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
24578 extra_args: Vec::new(),
24579 env: BTreeMap::new(),
24580 prompt_delivery: None,
24581 max_turn_secs: None,
24582 }
24583 }
24584
24585 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
24586 SeatJob {
24587 seat: SeatState::new("impl-A", &spec.id, 1),
24588 spec,
24589 cwd: dir.to_path_buf(),
24590 prompt: "do it".to_owned(),
24591 timeout,
24592 allow_write: true,
24593 sessions: false,
24594 artifacts: dir.join("art"),
24595 stem: "impl-A".to_owned(),
24596 handover: None,
24597 }
24598 }
24599
24600 fn state(resource_wait_secs: u64) -> RunState {
24601 let mut config = Config::default();
24602 config.graph.timeout_resource_wait = resource_wait_secs;
24603 RunState::new(
24604 PathBuf::from("/repo"),
24605 "main".to_owned(),
24606 "abc1234".to_owned(),
24607 "task".to_owned(),
24608 config,
24609 )
24610 }
24611
24612 fn hold(cache: &Path) -> crate::cache::Guard {
24613 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
24614 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
24615 crate::cache::AcquireOutcome::Acquired(g) => g,
24616 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
24617 }
24618 }
24619
24620 async fn run(
24621 jobs: Vec<SeatJob>,
24622 sem: Arc<Semaphore>,
24623 cache: Option<&Path>,
24624 st: &mut RunState,
24625 ) -> Vec<(usize, SeatState, AgentOutcome)> {
24626 let prompts = Prompts::default();
24627 let ctx = WaveCtx {
24628 run: "r",
24629 node: "implement",
24630 prompts: &prompts,
24631 cache,
24632 round: None,
24633 carry_seats: false,
24634 pauses: None,
24635 };
24636 wave(jobs, sem, &ctx, st, 0).await
24637 }
24638
24639 #[tokio::test]
24640 async fn isolated_editing_seat_wait_releases_only_after_its_own_target_is_free() {
24641 crate::run::pin_test_home();
24642 let home = crate::run::home();
24643 let tmp = tempfile::tempdir().unwrap();
24644 let repo = tmp.path().join("repo");
24645 let checkout = tmp.path().join("checkout");
24646 std::fs::create_dir(&repo).unwrap();
24647 git::git(&repo, &["init"]).await.unwrap();
24648 git::git(
24649 &repo,
24650 &[
24651 "-c",
24652 "user.name=Fixture",
24653 "-c",
24654 "user.email=fixture@example.invalid",
24655 "commit",
24656 "--allow-empty",
24657 "-m",
24658 "initial",
24659 ],
24660 )
24661 .await
24662 .unwrap();
24663 git::worktree_add_detached(&repo, &checkout, "HEAD")
24664 .await
24665 .unwrap();
24666 let target = crate::build_target::resolve(&home, &checkout).unwrap();
24667 let target_lease = hold(&target);
24668 let shared = tmp.path().join("unrelated");
24669 let _shared_lease = hold(&shared);
24670 let mut st = state(0);
24671 st.repo = repo.clone();
24672 st.config.verify.build_cache = crate::config::BuildCache::Isolated;
24673 st.config.verify.e2e = vec![format!(
24674 "CARGO_TARGET_DIR='{}' cargo test",
24675 git::path_arg(&shared)
24676 )];
24677 let config = tmp.path().join("config.toml");
24678 std::fs::write(&config, toml::to_string(&st.config).unwrap()).unwrap();
24679 let prompts = Prompts::default();
24680 let run_id = st.id.clone();
24681 let ctx = WaveCtx {
24682 run: &run_id,
24683 node: "implement",
24684 prompts: &prompts,
24685 cache: None,
24686 round: None,
24687 carry_seats: false,
24688 pauses: None,
24689 };
24690 let writer = job(sh("writer", "exit 99"), &checkout, Duration::from_secs(30));
24691 let results = wave(vec![writer], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0).await;
24692 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24693 assert_eq!(results[0].1.turns, 0);
24694 let saved = RunState::load_under(&run_id, &home).unwrap();
24695 let block = saved.resource_block.unwrap();
24696 assert!(matches!(
24697 block.cause,
24698 crate::run::ResourceBlockCause::IsolatedCacheTimeout { target: ref actual, .. }
24699 if actual == &target
24700 ));
24701 let queue = crate::queue::Queue::at(tmp.path().join("queue"));
24702 let mut task = crate::queue::Task::new(
24703 "build".into(),
24704 "build".into(),
24705 repo,
24706 crate::queue::Source::Human,
24707 );
24708 task.runs.push(run_id);
24709 task.hold_machine(Some(format!(
24710 "{}{}",
24711 crate::cache::RESOURCE_HOLD,
24712 block.description()
24713 )));
24714 queue.put(&mut task).unwrap();
24715 let questions = crate::ask::Questions::at(tmp.path().join("questions"));
24716 let triage =
24717 || crate::triage::run_once(&queue, &questions, Some(&config), Timestamp::now());
24718 let pending = triage();
24719 assert!(pending.resumed.is_empty() && pending.asked.is_empty());
24720 assert_eq!(
24721 queue.get(&task.id).unwrap().status,
24722 crate::queue::TaskStatus::Held
24723 );
24724 drop(target_lease);
24725 assert_eq!(triage().resumed, vec![task.id.clone()]);
24726 assert_eq!(
24727 queue.get(&task.id).unwrap().status,
24728 crate::queue::TaskStatus::Queued
24729 );
24730 assert!(crate::cache::in_use(&home, &shared));
24731 }
24732
24733 #[tokio::test]
24734 async fn verification_cache_wait_preserves_the_command_budget() {
24735 crate::run::pin_test_home();
24736 let tmp = tempfile::tempdir().unwrap();
24737 let cache = tmp.path().join("target");
24738 let guard = hold(&cache);
24739 let release = tokio::spawn(async move {
24740 tokio::time::sleep(Duration::from_millis(100)).await;
24741 guard.release();
24742 });
24743 let mut st = state(10);
24744 let budget = Duration::from_millis(50);
24745 let wait = repository_wait(&st, None);
24746 let (outcomes, _) = with_cache_lease(
24747 &mut st,
24748 Some(&cache),
24749 "gate",
24750 "gate",
24751 tmp.path(),
24752 "abc1234",
24753 budget,
24754 "test gate",
24755 &wait,
24756 |_, available| async move {
24757 assert_eq!(
24758 available, budget,
24759 "contention must not shorten verification"
24760 );
24761 (Vec::new(), false, Vec::new())
24762 },
24763 )
24764 .await
24765 .unwrap();
24766 release.await.unwrap();
24767 assert!(outcomes.is_empty(), "{outcomes:?}");
24768 assert!(
24769 st.events
24770 .iter()
24771 .any(|event| event.message.contains("waiting for"))
24772 );
24773 assert!(st.resource_wait.is_none());
24774 hold(&cache).release();
24776 }
24777
24778 #[tokio::test]
24779 async fn verification_cache_wait_expires_without_starting_commands() {
24780 crate::run::pin_test_home();
24781 let tmp = tempfile::tempdir().unwrap();
24782 let cache = tmp.path().join("target");
24783 let _held = hold(&cache);
24784 let mut st = state(0);
24785 let wait = repository_wait(&st, None);
24786 let (outcomes, _) = with_cache_lease(
24787 &mut st,
24788 Some(&cache),
24789 "gate",
24790 "gate",
24791 tmp.path(),
24792 "abc1234",
24793 Duration::from_secs(60),
24794 "test gate",
24795 &wait,
24796 |_, _| async { panic!("commands must not start without the cache lease") },
24797 )
24798 .await
24799 .unwrap();
24800 assert_eq!(outcomes.len(), 1);
24801 assert!(outcomes[0].resource_blocked);
24802 assert!(st.resource_wait.is_none());
24803 assert!(matches!(
24804 st.resource_block.unwrap().cause,
24805 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24806 ));
24807 }
24808
24809 #[tokio::test]
24810 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24811 crate::run::pin_test_home();
24812 let tmp = tempfile::tempdir().unwrap();
24813 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24814 let cache = tmp.path().join("cache");
24815 let mut state = state(21600);
24816 let wait = repository_wait(&state, None);
24817 let (outcomes, _) = with_cache_lease(
24818 &mut state,
24819 Some(&cache),
24820 "gate",
24821 "gate",
24822 tmp.path(),
24823 "head",
24824 Duration::from_secs(60),
24825 "gate",
24826 &wait,
24827 |_, _| async { panic!("freshness failure must not start verification") },
24828 )
24829 .await
24830 .unwrap();
24831 assert_eq!(outcomes.len(), 1);
24832 assert!(outcomes[0].resource_blocked);
24833 let block = state.resource_block.unwrap();
24834 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24835 assert!(!block.description().contains("timed out"));
24836 }
24837
24838 #[tokio::test]
24839 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24840 crate::run::pin_test_home();
24841 let tmp = tempfile::tempdir().unwrap();
24842 let cache = tmp.path().join("cache");
24843 let _held = hold(&cache);
24844 let mut state = state(21600);
24845 let pause = Pause::new();
24846 let interrupt = Pause::new();
24847 let cause = ParkCause::Operator;
24848 pause.park_for(cause.clone());
24849 let prompts = Prompts::default();
24850 let ctx = WaveCtx {
24851 run: "test",
24852 node: "fix",
24853 prompts: &prompts,
24854 cache: Some(&cache),
24855 round: None,
24856 carry_seats: false,
24857 pauses: Some((&pause, &interrupt)),
24858 };
24859 let result = wave(
24860 vec![job(
24861 sh("writer", "exit 99"),
24862 tmp.path(),
24863 Duration::from_secs(1),
24864 )],
24865 Arc::new(Semaphore::new(1)),
24866 &ctx,
24867 &mut state,
24868 0,
24869 )
24870 .await;
24871 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24872 assert_eq!(result[0].1.turns, 0);
24873 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24874 assert_eq!(
24875 state.resource_block.unwrap().cause,
24876 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24877 );
24878 assert!(state.active.is_empty());
24879 }
24880
24881 #[tokio::test]
24882 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24883 crate::run::pin_test_home();
24884 let tmp = tempfile::tempdir().unwrap();
24885 let cache = tmp.path().join("target-a");
24886 let guard = hold(&cache);
24887 tokio::spawn(async move {
24889 tokio::time::sleep(Duration::from_millis(1500)).await;
24890 guard.release();
24891 });
24892 let mut st = state(60);
24893 let results = run(
24894 vec![job(
24895 sh("alpha", "echo ok"),
24896 tmp.path(),
24897 Duration::from_secs(1),
24898 )],
24899 Arc::new(Semaphore::new(1)),
24900 Some(&cache),
24901 &mut st,
24902 )
24903 .await;
24904 assert!(
24905 matches!(results[0].2, AgentOutcome::Ok(_)),
24906 "the wait must not be charged to the agent"
24907 );
24908 assert!(st.resource_block.is_none());
24909 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24910 assert!(st.resource_wait.is_none() && st.active.is_empty());
24911 }
24912
24913 #[tokio::test]
24914 #[cfg(unix)]
24915 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24916 use std::os::fd::AsRawFd as _;
24917 crate::run::pin_test_home();
24918 let tmp = tempfile::tempdir().unwrap();
24919 crate::git::git(tmp.path(), &["init"]).await.unwrap();
24920 let common = tmp.path().join(".git");
24921 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24922 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24923 assert_eq!(
24924 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24925 0
24926 );
24927 let mut st = state(1);
24928 st.config.coordination = crate::config::test_coordination();
24929 let mut reader = job(
24930 sh("reader", "echo must-not-run"),
24931 tmp.path(),
24932 Duration::from_secs(30),
24933 );
24934 reader.allow_write = false;
24935 let results = run(
24936 vec![reader.clone()],
24937 Arc::new(Semaphore::new(1)),
24938 None,
24939 &mut st,
24940 )
24941 .await;
24942 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24943 assert_eq!(results[0].1.turns, 0);
24944 assert!(FailClass::of(&results[0].2).is_none());
24945 assert_eq!(
24946 st.resource_block.as_ref().unwrap().kind,
24947 "repository coordination"
24948 );
24949 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24950 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24951 drop(external);
24952 let editor = crate::repo_lock::SeatLease::for_checkout(
24953 tmp.path(),
24954 true,
24955 crate::config::test_coordination().as_ref(),
24956 )
24957 .await
24958 .unwrap();
24959 let pause = Pause::new();
24960 let interrupt = Pause::new();
24961 let prompts = Prompts::default();
24962 let ctx = WaveCtx {
24963 run: "r",
24964 node: "review",
24965 prompts: &prompts,
24966 cache: None,
24967 round: None,
24968 carry_seats: false,
24969 pauses: Some((&pause, &interrupt)),
24970 };
24971 let (results, ()) = tokio::join!(
24972 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24973 async {
24974 tokio::time::sleep(Duration::from_millis(50)).await;
24975 pause.park();
24976 }
24977 );
24978 assert!(
24979 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24980 );
24981 assert_eq!(results[0].1.turns, 0);
24982 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24983 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24984 drop(editor);
24985 }
24986
24987 #[tokio::test]
24988 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24989 crate::run::pin_test_home();
24990 let tmp = tempfile::tempdir().unwrap();
24991 let cache = tmp.path().join("target-b");
24992 let _held = hold(&cache);
24993 let mut st = state(1);
24994 let results = run(
24995 vec![job(
24996 sh("alpha", "echo ok"),
24997 tmp.path(),
24998 Duration::from_secs(1),
24999 )],
25000 Arc::new(Semaphore::new(1)),
25001 Some(&cache),
25002 &mut st,
25003 )
25004 .await;
25005 let (_, seat, out) = &results[0];
25006 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
25007 assert!(wave_blocked(&results));
25008 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
25009 assert_eq!(seat.turns, 0, "nothing was asked");
25010 assert!(
25011 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
25012 "no agent was spawned"
25013 );
25014 assert!(FailClass::of(out).is_none(), "not a failure class");
25015 let block = st.resource_block.as_ref().expect("recorded for the resume");
25016 assert_eq!(block.kind, "shared build cache");
25017 assert!(matches!(
25018 block.cause,
25019 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
25020 ));
25021 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
25022 assert!(st.active.is_empty());
25023 }
25024
25025 #[tokio::test]
25026 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
25027 crate::run::pin_test_home();
25028 let tmp = tempfile::tempdir().unwrap();
25029 let mut st = state(1);
25030 let results = run(
25031 vec![job(
25032 sh("alpha", "echo ok"),
25033 tmp.path(),
25034 Duration::from_secs(1),
25035 )],
25036 Arc::new(Semaphore::new(0)),
25037 None,
25038 &mut st,
25039 )
25040 .await;
25041 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
25042 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
25043 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
25044 }
25045
25046 #[tokio::test]
25047 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
25048 crate::run::pin_test_home();
25049 let tmp = tempfile::tempdir().unwrap();
25050 let mut st = state(60);
25051 let results = run(
25052 vec![job(
25053 sh("alpha", "sleep 30"),
25054 tmp.path(),
25055 Duration::from_secs(1),
25056 )],
25057 Arc::new(Semaphore::new(1)),
25058 None,
25059 &mut st,
25060 )
25061 .await;
25062 let out = &results[0].2;
25063 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
25064 let class = FailClass::of(out).expect("a failure");
25065 assert_eq!(class, FailClass::Timeout);
25066 assert!(should_hand_over(None, &class), "existing handover policy");
25067 assert!(st.resource_block.is_none());
25068 }
25069}